diff --git a/cmd/machine-config-controller/start.go b/cmd/machine-config-controller/start.go index f8de099ba8..769df54f7a 100644 --- a/cmd/machine-config-controller/start.go +++ b/cmd/machine-config-controller/start.go @@ -29,10 +29,10 @@ var ( } startOpts struct { - kubeconfig string - templates string - - resourceLockNamespace string + kubeconfig string + templates string + promMetricsListenAddress string + resourceLockNamespace string } ) @@ -40,6 +40,7 @@ func init() { rootCmd.AddCommand(startCmd) startCmd.PersistentFlags().StringVar(&startOpts.kubeconfig, "kubeconfig", "", "Kubeconfig file to access a remote cluster (testing only)") startCmd.PersistentFlags().StringVar(&startOpts.resourceLockNamespace, "resourcelock-namespace", metav1.NamespaceSystem, "Path to the template files used for creating MachineConfig objects") + startCmd.PersistentFlags().StringVar(&startOpts.promMetricsListenAddress, "metrics-listen-address", "127.0.0.1:8797", "Listen address for prometheus metrics listener") } func runStartCmd(cmd *cobra.Command, args []string) { @@ -56,6 +57,9 @@ func runStartCmd(cmd *cobra.Command, args []string) { run := func(ctx context.Context) { ctrlctx := ctrlcommon.CreateControllerContext(cb, ctx.Done(), componentName) + // Start the metrics handler + go ctrlcommon.StartMetricsListener(startOpts.promMetricsListenAddress, ctrlctx.Stop) + controllers := createControllers(ctrlctx) // Start the shared factory informers that you need to use in your controller @@ -140,6 +144,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle // The node controller consumes data written by the above node.New( ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), + ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(), ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(), ctx.KubeInformerFactory.Core().V1().Nodes(), ctx.ConfigInformerFactory.Config().V1().Schedulers(), diff --git a/install/0000_80_machine-config-operator_00_service.yaml b/install/0000_80_machine-config-operator_00_service.yaml index 055e9cdedc..d6eb1c7123 100644 --- a/install/0000_80_machine-config-operator_00_service.yaml +++ b/install/0000_80_machine-config-operator_00_service.yaml @@ -1,5 +1,26 @@ apiVersion: v1 kind: Service +metadata: + name: machine-config-controller + namespace: openshift-machine-config-operator + labels: + k8s-app: machine-config-controller + annotations: + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + include.release.openshift.io/single-node-developer: "true" + service.beta.openshift.io/serving-cert-secret-name: mcc-proxy-tls +spec: + type: ClusterIP + selector: + k8s-app: machine-config-controller + ports: + - name: metrics + port: 9001 + protocol: TCP +--- +apiVersion: v1 +kind: Service metadata: name: machine-config-daemon namespace: openshift-machine-config-operator diff --git a/install/0000_90_machine-config-operator_00_servicemonitor.yaml b/install/0000_90_machine-config-operator_00_servicemonitor.yaml index 5895bbc174..a622883dd2 100644 --- a/install/0000_90_machine-config-operator_00_servicemonitor.yaml +++ b/install/0000_90_machine-config-operator_00_servicemonitor.yaml @@ -1,5 +1,41 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor +metadata: + name: machine-config-controller + namespace: openshift-machine-config-operator + labels: + k8s-app: machine-config-controller + annotations: + include.release.openshift.io/self-managed-high-availability: "true" + include.release.openshift.io/single-node-developer: "true" +spec: + endpoints: + - interval: 30s + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: metrics + scheme: https + path: /metrics + relabelings: + - action: replace + regex: ;(.*) + replacement: $1 + separator: ";" + sourceLabels: + - node + - __meta_kubernetes_pod_node_name + targetLabel: node + tlsConfig: + caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt + serverName: machine-config-controller.openshift-machine-config-operator.svc + namespaceSelector: + matchNames: + - openshift-machine-config-operator + selector: + matchLabels: + k8s-app: machine-config-controller +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor metadata: name: machine-config-daemon namespace: openshift-machine-config-operator diff --git a/install/0000_90_machine-config-operator_01_prometheus-rules.yaml b/install/0000_90_machine-config-operator_01_prometheus-rules.yaml index aec8c85107..137a153e75 100644 --- a/install/0000_90_machine-config-operator_01_prometheus-rules.yaml +++ b/install/0000_90_machine-config-operator_01_prometheus-rules.yaml @@ -1,5 +1,41 @@ apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule +metadata: + name: machine-config-controller + namespace: openshift-machine-config-operator + labels: + k8s-app: machine-config-controller + annotations: + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + include.release.openshift.io/single-node-developer: "true" +spec: + groups: + - name: mcc-paused-pool-kubelet-ca + rules: + - alert: MachineConfigControllerPausedPoolKubeletCA + expr: | + max by (namespace,pool) (last_over_time(machine_config_controller_paused_pool_kubelet_ca[5m])) > 0 + for: 60m + labels: + severity: warning + annotations: + summary: "Paused machine configuration pool '{{$labels.pool}}' is blocking a necessary certificate rotation and must be unpaused before the current kube-apiserver-to-kubelet-signer certificate expires on {{ $value | humanizeTimestamp }}." + description: "Machine config pools have a 'pause' feature, which allows config to be rendered, but prevents it from being rolled out to the nodes. This alert indicates that a certificate rotation has taken place, and the new kubelet-ca certificate bundle has been rendered into a machine config, but because the pool '{{$labels.pool}}' is paused, the config cannot be rolled out to the nodes in that pool. You will notice almost immediately that for nodes in pool '{{$labels.pool}}', pod logs will not be visible in the console and interactive commands (oc log, oc exec, oc debug, oc attach) will not work. You must unpause machine config pool '{{$labels.pool}}' to let the certificates through before the kube-apiserver-to-kubelet-signer certificate expires on {{ $value | humanizeTimestamp }} or this pool's nodes will cease to function properly." + runbook_url: https://github.com/openshift/blob/master/alerts/machine-config-operator/MachineConfigControllerPausedPoolKubeletCA.md + - alert: MachineConfigControllerPausedPoolKubeletCA + expr: | + max by (namespace,pool) (last_over_time(machine_config_controller_paused_pool_kubelet_ca[5m]) - time()) < (86400 * 14) AND max by (namespace,pool) (last_over_time(machine_config_controller_paused_pool_kubelet_ca[5m])) > 0 + for: 60m + labels: + severity: critical + annotations: + summary: "Paused machine configuration pool '{{$labels.pool}}' is blocking a necessary certificate rotation and must be unpaused before the current kube-apiserver-to-kubelet-signer certificate expires in {{ $value | humanizeDuration }}." + description: "Machine config pools have a 'pause' feature, which allows config to be rendered, but prevents it from being rolled out to the nodes. This alert indicates that a certificate rotation has taken place, and the new kubelet-ca certificate bundle has been rendered into a machine config, but because the pool '{{$labels.pool}}' is paused, the config cannot be rolled out to the nodes in that pool. You will notice almost immediately that for nodes in pool '{{$labels.pool}}', pod logs will not be visible in the console and interactive commands (oc log, oc exec, oc debug, oc attach) will not work. You must unpause machine config pool '{{$labels.pool}}' to let the certificates through before the kube-apiserver-to-kubelet-signer certificate expires. You have approximately {{ $value | humanizeDuration }} remaining before this happens and nodes in '{{$labels.pool}}' cease to function properly." + runbook_url: https://github.com/openshift/blob/master/alerts/machine-config-operator/MachineConfigControllerPausedPoolKubeletCA.md +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule metadata: name: machine-config-daemon namespace: openshift-machine-config-operator diff --git a/lib/resourcemerge/core.go b/lib/resourcemerge/core.go index bd4fdcca4e..0922d32ed4 100644 --- a/lib/resourcemerge/core.go +++ b/lib/resourcemerge/core.go @@ -93,6 +93,12 @@ func ensureContainer(modified *bool, existing *corev1.Container, required corev1 setStringIfSet(modified, &existing.Name, required.Name) setStringIfSet(modified, &existing.Image, required.Image) + // This previously didn't properly sync the cpu and memory request fields, which caused a payload rejection + // https://github.com/openshift/machine-config-operator/pull/3027 + // cpu and memory are unfortunately not explicit fields, they are functions that reference keys in a map + // (a map that it really looks like we're not supposed to modify directly), so we just overwrite the whole map + setResourceListIfSet(modified, &existing.Resources.Requests, required.Resources.Requests) + // if you want modify the launch, you need to modify it in the config, not in the launch args setStringSliceIfSet(modified, &existing.Command, required.Command) setStringSliceIfSet(modified, &existing.Args, required.Args) @@ -306,6 +312,16 @@ func setStringSliceIfSet(modified *bool, existing *[]string, required []string) } } +func setResourceListIfSet(modified *bool, existing *corev1.ResourceList, required corev1.ResourceList) { + if required == nil { + return + } + if !equality.Semantic.DeepEqual(required, *existing) { + *existing = required + *modified = true + } +} + func mergeStringSlice(modified *bool, existing *[]string, required []string) { for _, required := range required { found := false diff --git a/manifests/machineconfigcontroller/clusterrole.yaml b/manifests/machineconfigcontroller/clusterrole.yaml index c4daf74f9c..4a04dc91b3 100644 --- a/manifests/machineconfigcontroller/clusterrole.yaml +++ b/manifests/machineconfigcontroller/clusterrole.yaml @@ -25,3 +25,16 @@ rules: - apiGroups: ["operator.openshift.io"] resources: ["etcds"] verbs: ["get", "list", "watch"] +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + - subjectaccessreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/manifests/machineconfigcontroller/clusterrolebinding.yaml b/manifests/machineconfigcontroller/clusterrolebinding.yaml index 36a46925f8..307f541137 100644 --- a/manifests/machineconfigcontroller/clusterrolebinding.yaml +++ b/manifests/machineconfigcontroller/clusterrolebinding.yaml @@ -10,3 +10,18 @@ subjects: - kind: ServiceAccount namespace: {{.TargetNamespace}} name: machine-config-controller +--- +# Bind auth-delegator role to the MCC service account +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: machine-config-controller + namespace: {{.TargetNamespace}} +roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:auth-delegator +subjects: +- kind: ServiceAccount + namespace: {{.TargetNamespace}} + name: machine-config-controller diff --git a/manifests/machineconfigcontroller/deployment.yaml b/manifests/machineconfigcontroller/deployment.yaml index e947db5307..868ded5d75 100644 --- a/manifests/machineconfigcontroller/deployment.yaml +++ b/manifests/machineconfigcontroller/deployment.yaml @@ -27,10 +27,42 @@ spec: cpu: 20m memory: 50Mi terminationMessagePolicy: FallbackToLogsOnError + - name: oauth-proxy + image: {{.Images.OauthProxy}} + ports: + - containerPort: 9001 + name: metrics + protocol: TCP + args: + - --https-address=:9001 + - --provider=openshift + - --openshift-service-account=machine-config-controller + - --upstream=http://127.0.0.1:8797 + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --cookie-secret-file=/etc/tls/cookie-secret/cookie-secret + - '--openshift-sar={"resource": "namespaces", "verb": "get"}' + - '--openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get"}}' + resources: + requests: + cpu: 20m + memory: 50Mi + volumeMounts: + - mountPath: /etc/tls/private + name: proxy-tls + - mountPath: /etc/tls/cookie-secret + name: cookie-secret serviceAccountName: machine-config-controller nodeSelector: node-role.kubernetes.io/master: "" priorityClassName: "system-cluster-critical" + volumes: + - name: proxy-tls + secret: + secretName: mcc-proxy-tls + - name: cookie-secret + secret: + secretName: cookie-secret restartPolicy: Always tolerations: - key: node-role.kubernetes.io/master diff --git a/pkg/controller/common/helpers.go b/pkg/controller/common/helpers.go index ddfb155400..99eebfee2c 100644 --- a/pkg/controller/common/helpers.go +++ b/pkg/controller/common/helpers.go @@ -4,6 +4,8 @@ import ( "bytes" "compress/gzip" "context" + "crypto/x509" + "encoding/pem" "fmt" "io" "io/ioutil" @@ -11,6 +13,7 @@ import ( "os" "reflect" "sort" + "strings" "github.com/clarketm/json" fcctbase "github.com/coreos/fcct/base/v0_1" @@ -779,3 +782,50 @@ func GetIgnitionFileDataByPath(config *ign3types.Config, path string) ([]byte, e } return nil, nil } + +// GetNewestCertificatesFromPEMBundle breaks a pem-encoded bundle out into its component certificates +func GetCertificatesFromPEMBundle(pemBytes []byte) ([]*x509.Certificate, error) { + var certs []*x509.Certificate + // There can be multiple certificates in the file + for { + // Decode a block to parse + block, rest := pem.Decode(pemBytes) + // Once we get no more blocks, we've read all the certs + if block == nil { + break + } + // Right now we just care about certificates, not keys + if block.Type == "CERTIFICATE" { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + // This isn't fatal, *this* cert could just be junk, next one could be okay + glog.Warningf("Failed to parse certificate: %v", err.Error()) + } else { + certs = append(certs, cert) + } + } + // Keep reading from where we left off + pemBytes = rest + } + return certs, nil +} + +// GetLongestValidCertificate returns the latest-expiring certificate from a given list of certificates +// whose Subject.CommonName also matches any of the given common-name prefixes +func GetLongestValidCertificate(certificateList []*x509.Certificate, subjectPrefixes []string) *x509.Certificate { + // Sort is smallest-to-largest, so we're putting the cert with the latest expiry date at the top + sort.Slice(certificateList, func(i, j int) bool { + return certificateList[i].NotAfter.After(certificateList[j].NotAfter) + }) + // For each certificate in our list + for _, certificate := range certificateList { + // Check it against our prefixes + for _, prefix := range subjectPrefixes { + // If it matches, this is the latest-expiring one since it's closest to the "top" + if strings.HasPrefix(certificate.Subject.CommonName, prefix) { + return certificate + } + } + } + return nil +} diff --git a/pkg/controller/common/metrics.go b/pkg/controller/common/metrics.go new file mode 100644 index 0000000000..c09b6ccba9 --- /dev/null +++ b/pkg/controller/common/metrics.go @@ -0,0 +1,68 @@ +package common + +import ( + "context" + "net/http" + + "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ( + // DefaultBindAddress is the port for the metrics listener + DefaultBindAddress = ":8797" +) + +var ( + // MachineConfigControllerPausedPoolKubeletCA logs when a certificate rotation is being held up by pause + MachineConfigControllerPausedPoolKubeletCA = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "machine_config_controller_paused_pool_kubelet_ca", + Help: "Set to the unix timestamp in utc of the current certificate expiry date if a certificate rotation is pending in specified paused pool", + }, []string{"pool"}) + + metricsList = []prometheus.Collector{ + MachineConfigControllerPausedPoolKubeletCA, + } +) + +func RegisterMCCMetrics() error { + for _, metric := range metricsList { + err := prometheus.Register(metric) + if err != nil { + return err + } + } + + return nil +} + +// StartMetricsListener is metrics listener via http on localhost +func StartMetricsListener(addr string, stopCh <-chan struct{}) { + if addr == "" { + addr = DefaultBindAddress + } + + glog.Info("Registering Prometheus metrics") + if err := RegisterMCCMetrics(); err != nil { + glog.Errorf("unable to register metrics: %v", err) + // No sense in continuing starting the listener if this fails + return + } + + glog.Infof("Starting metrics listener on %s", addr) + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + s := http.Server{Addr: addr, Handler: mux} + + go func() { + if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed { + glog.Errorf("metrics listener exited with error: %v", err) + } + }() + <-stopCh + if err := s.Shutdown(context.Background()); err != http.ErrServerClosed { + glog.Errorf("error stopping metrics listener: %v", err) + } +} diff --git a/pkg/controller/node/node_controller.go b/pkg/controller/node/node_controller.go index 178a949991..46b240c4be 100644 --- a/pkg/controller/node/node_controller.go +++ b/pkg/controller/node/node_controller.go @@ -2,11 +2,13 @@ package node import ( "context" + "crypto/x509" "encoding/json" "fmt" "reflect" "time" + ign3types "github.com/coreos/ignition/v2/config/v3_2/types" "github.com/golang/glog" configv1 "github.com/openshift/api/config/v1" cligoinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1" @@ -24,6 +26,7 @@ import ( goerrs "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -64,8 +67,15 @@ const ( // masterPoolName is the control plane MachineConfigPool name masterPoolName = "master" + + // kubeletCAFilePath is the expected file path for the kubelet ca + kubeletCAFilePath = "/etc/kubernetes/kubelet-ca.crt" ) +// kubeAPIToKubeletSignerNamePrefixes is the list of subject common names that are regarded as a kube-apiserver-to-kubelet-signer ca certificate +// Based on naming convention from https://github.com/openshift/library-go/blob/ed9bc958bd8a2fff079d52976806e4e0a8a7c315/pkg/operator/certrotation/signer.go#L132 +var kubeAPIToKubeletSignerNamePrefixes = []string{"openshift-kube-apiserver-operator_kube-apiserver-to-kubelet-signer@", "kube-apiserver-to-kubelet-signer"} + // Controller defines the node controller. type Controller struct { client mcfgclientset.Interface @@ -76,10 +86,12 @@ type Controller struct { enqueueMachineConfigPool func(*mcfgv1.MachineConfigPool) ccLister mcfglistersv1.ControllerConfigLister + mcLister mcfglistersv1.MachineConfigLister mcpLister mcfglistersv1.MachineConfigPoolLister nodeLister corelisterv1.NodeLister ccListerSynced cache.InformerSynced + mcListerSynced cache.InformerSynced mcpListerSynced cache.InformerSynced nodeListerSynced cache.InformerSynced @@ -92,6 +104,7 @@ type Controller struct { // New returns a new node controller. func New( ccInformer mcfginformersv1.ControllerConfigInformer, + mcInformer mcfginformersv1.MachineConfigInformer, mcpInformer mcfginformersv1.MachineConfigPoolInformer, nodeInformer coreinformersv1.NodeInformer, schedulerInformer cligoinformersv1.SchedulerInformer, @@ -128,9 +141,11 @@ func New( ctrl.enqueueMachineConfigPool = ctrl.enqueueDefault ctrl.ccLister = ccInformer.Lister() + ctrl.mcLister = mcInformer.Lister() ctrl.mcpLister = mcpInformer.Lister() ctrl.nodeLister = nodeInformer.Lister() ctrl.ccListerSynced = ccInformer.Informer().HasSynced + ctrl.mcListerSynced = mcInformer.Informer().HasSynced ctrl.mcpListerSynced = mcpInformer.Informer().HasSynced ctrl.nodeListerSynced = nodeInformer.Informer().HasSynced @@ -145,7 +160,7 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer ctrl.queue.ShutDown() - if !cache.WaitForCacheSync(stopCh, ctrl.ccListerSynced, ctrl.mcpListerSynced, ctrl.nodeListerSynced, ctrl.schedulerListerSynced) { + if !cache.WaitForCacheSync(stopCh, ctrl.ccListerSynced, ctrl.mcListerSynced, ctrl.mcpListerSynced, ctrl.nodeListerSynced, ctrl.schedulerListerSynced) { return } @@ -743,9 +758,17 @@ func (ctrl *Controller) syncMachineConfigPool(key string) error { if mcfgv1.IsMachineConfigPoolConditionTrue(pool.Status.Conditions, mcfgv1.MachineConfigPoolUpdating) { glog.Infof("Pool %s is paused and will not update.", pool.Name) } + + // Only check for pending files if we're out of sync + if pool.Spec.Configuration.Name != pool.Status.Configuration.Name { + ctrl.setPendingFileMetrics(pool) + } return ctrl.syncStatusOnly(pool) } + // We aren't paused anymore, so reset the metrics + ctrl.resetPendingFileMetrics(pool) + nodes, err := ctrl.getNodesForPool(pool) if err != nil { if syncErr := ctrl.syncStatusOnly(pool); syncErr != nil { @@ -1042,3 +1065,112 @@ func getErrorString(err error) string { } return "" } + +// setPendingFileMetrics checks to see if there are any important files in the +// machineconfig that the pool should be moving to, and sets metrics if there are +func (ctrl *Controller) setPendingFileMetrics(pool *mcfgv1.MachineConfigPool) { + // Retrieve and parse the pool's machine config + currentConfig, pendingConfig, err := ctrl.parseConvertMachineConfigFilesForPool(pool) + if err != nil { + glog.Warningf("Error converting pool configs for %s pool: %v", pool.Name, err) + return + } + + // Figure out what files differ between pool.Spec and pool.Status + fileDiff := ctrlcommon.CalculateConfigFileDiffs(currentConfig, pendingConfig) + + // Go through our files until we hit the kubelet CA bundle + for _, path := range fileDiff { + // We only care about the kubelet CA bundle + if path != kubeletCAFilePath { + continue + } + + // If it's there, get the *newest* (in case there have been multiple rotations) kube-apiserver-to-kubelet signer certifiate out of the bundle + newestSignerCertificate, err := ctrl.getNewestAPIToKubeletSignerCertificate(currentConfig) + if err != nil { + glog.Warningf("Error retrieving kubelet-ca certificates from pool %s: %v", pool.Name, err) + } else { + // Set the metric value to the UTC expiry date of that cert so we can count down to it + glog.V(2).Infof("Kubelet CA is stuck in paused pool %s. Setting metric to expiry date of %s (%s)", pool.Name, newestSignerCertificate.Subject.CommonName, newestSignerCertificate.NotAfter.UTC()) + ctrlcommon.MachineConfigControllerPausedPoolKubeletCA.WithLabelValues(pool.Name).Set(float64(newestSignerCertificate.NotAfter.UTC().Unix())) + } + break + } +} + +// resetPendingFileMetrics turns off any "paused file" metrics that were firing for the pool +func (ctrl *Controller) resetPendingFileMetrics(pool *mcfgv1.MachineConfigPool) { + // Set the metric for this pool back to zero + ctrlcommon.MachineConfigControllerPausedPoolKubeletCA.WithLabelValues(pool.Name).Set(0) +} + +// parseConvertMachineConfigFilesForPool retrieves the current and pending configurations for +// a pool, parses and converts them, and returns them as ignition v3 Config objects. The controller needs +// to retrieve and examine the actual configurations so it can diff the file lists and figure out which new +// files are "stuck" behind a paused pool. +func (ctrl *Controller) parseConvertMachineConfigFilesForPool(pool *mcfgv1.MachineConfigPool) (current, pending *ign3types.Config, err error) { + // The config we're in right now + currentName := pool.Status.Configuration.Name + // The config we would be going to + pendingName := pool.Spec.Configuration.Name + + // Get the machine config objects + currentConfig, err := ctrl.mcLister.Get(currentName) + if apierrors.IsNotFound(err) { + glog.V(2).Infof("MachineConfig %v has been deleted", currentName) + return nil, nil, err + } + + pendingConfig, err := ctrl.mcLister.Get(pendingName) + if apierrors.IsNotFound(err) { + glog.V(2).Infof("MachineConfigPool %v has been deleted", pendingName) + return nil, nil, err + } + + // Make sure we can coax the objects into ignitionv3 + currentIgnConfig, err := ctrlcommon.ParseAndConvertConfig(currentConfig.Spec.Config.Raw) + if err != nil { + return nil, nil, err + } + pendingIgnConfig, err := ctrlcommon.ParseAndConvertConfig(pendingConfig.Spec.Config.Raw) + if err != nil { + return nil, nil, err + } + + return ¤tIgnConfig, &pendingIgnConfig, nil +} + +// getNewestAPIToKubeletSignerCertificate returns the newest kube-apiserver-to-kubelet-signer +// certificate present in the kubelet-ca.crt bundle. We extract the certificate so we can use its +// expiry date in our metrics/alerting. It's a very important certificate and its expiry will cause +// nodes using it to cease communicating with the cluster. +func (ctrl *Controller) getNewestAPIToKubeletSignerCertificate(statusIgnConfig *ign3types.Config) (*x509.Certificate, error) { + // Retrieve the file data from ignition + kubeletBundle, err := ctrlcommon.GetIgnitionFileDataByPath(statusIgnConfig, kubeletCAFilePath) + if err != nil { + return nil, err + } + + // Parse that bundle into its component certificates + containedCertificates, err := ctrlcommon.GetCertificatesFromPEMBundle(kubeletBundle) + if err != nil { + return nil, err + } + + // We have other problems if this is empty, but it's possible + if len(containedCertificates) == 0 { + return nil, fmt.Errorf("No certificates found in bundle") + } + + // The *original* signer has a different name, the rotated ones have longer names + // The suffix changes with the timstamp on rotation, which is why I'm using prefix here not exact match + newestCertificate := ctrlcommon.GetLongestValidCertificate(containedCertificates, kubeAPIToKubeletSignerNamePrefixes) + + // Shouldn't come back with nothing, but just in case we do + if newestCertificate == nil { + return nil, fmt.Errorf("No matching kube-apiserver-to-kubelet-signer certificates found in bundle") + } + + return newestCertificate, nil +} diff --git a/pkg/controller/node/node_controller_test.go b/pkg/controller/node/node_controller_test.go index edde211f7a..a364fb8283 100644 --- a/pkg/controller/node/node_controller_test.go +++ b/pkg/controller/node/node_controller_test.go @@ -3,7 +3,6 @@ package node import ( "encoding/json" "fmt" - "github.com/openshift/machine-config-operator/pkg/constants" "reflect" "testing" "time" @@ -22,17 +21,20 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + ign3types "github.com/coreos/ignition/v2/config/v3_2/types" apicfgv1 "github.com/openshift/api/config/v1" configv1 "github.com/openshift/api/config/v1" fakeconfigv1client "github.com/openshift/client-go/config/clientset/versioned/fake" configv1informer "github.com/openshift/client-go/config/informers/externalversions" mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "github.com/openshift/machine-config-operator/pkg/constants" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" daemonconsts "github.com/openshift/machine-config-operator/pkg/daemon/constants" "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned/fake" informers "github.com/openshift/machine-config-operator/pkg/generated/informers/externalversions" "github.com/openshift/machine-config-operator/pkg/version" "github.com/openshift/machine-config-operator/test/helpers" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" utilrand "k8s.io/apimachinery/pkg/util/rand" ) @@ -50,6 +52,7 @@ type fixture struct { schedulerClient *fakeconfigv1client.Clientset ccLister []*mcfgv1.ControllerConfig + mcLister []*mcfgv1.MachineConfig mcpLister []*mcfgv1.MachineConfigPool nodeLister []*corev1.Node @@ -78,7 +81,7 @@ func (f *fixture) newController() *Controller { i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc()) k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc()) ci := configv1informer.NewSharedInformerFactory(f.schedulerClient, noResyncPeriodFunc()) - c := New(i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().MachineConfigPools(), k8sI.Core().V1().Nodes(), + c := New(i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().MachineConfigs(), i.Machineconfiguration().V1().MachineConfigPools(), k8sI.Core().V1().Nodes(), ci.Config().V1().Schedulers(), f.kubeclient, f.client) c.ccListerSynced = alwaysReady @@ -213,6 +216,8 @@ func filterInformerActions(actions []core.Action) []core.Action { if len(action.GetNamespace()) == 0 && (action.Matches("list", "machineconfigpools") || action.Matches("watch", "machineconfigpools") || + action.Matches("list", "machineconfigs") || + action.Matches("watch", "machineconfigs") || action.Matches("list", "controllerconfigs") || action.Matches("watch", "controllerconfigs") || action.Matches("list", "nodes") || @@ -934,6 +939,55 @@ func TestPaused(t *testing.T) { f.run(getKey(mcp, t)) } +func TestAlertOnPausedKubeletCA(t *testing.T) { + f := newFixture(t) + mcp := helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v1") + + mcfiles := []ign3types.File{ + helpers.NewIgnFile("/etc/kubernetes/kubelet-ca.crt", TestKubeletCABundle), + helpers.NewIgnFile("/etc/kubernetes/kubelet-ca.crt", "newcertificates"), + } + + mcs := []*mcfgv1.MachineConfig{ + helpers.NewMachineConfig("rendered-worker-1", map[string]string{"node-role/worker": ""}, "dummy://", []ign3types.File{mcfiles[0]}), + helpers.NewMachineConfig("rendered-worker-2", map[string]string{"node-role/worker": ""}, "dummy://1", []ign3types.File{mcfiles[1]}), + } + mcp.Spec.Configuration.Name = "rendered-worker-2" + mcp.Status.Configuration.Name = "rendered-worker-1" + + mcp.Spec.MaxUnavailable = intStrPtr(intstr.FromInt(1)) + mcp.Spec.Paused = true + nodes := []*corev1.Node{ + newNodeWithLabel("node-0", "v1", "v1", map[string]string{"node-role/worker": ""}), + newNodeWithLabel("node-1", "v0", "v0", map[string]string{"node-role/worker": ""}), + } + + f.mcpLister = append(f.mcpLister, mcp) + f.objects = append(f.objects, mcp) + f.nodeLister = append(f.nodeLister, nodes...) + for idx := range nodes { + f.kubeobjects = append(f.kubeobjects, nodes[idx]) + } + f.mcLister = append(f.mcLister, mcs...) + for idx := range mcs { + f.objects = append(f.objects, mcs[idx]) + } + + expStatus := calculateStatus(mcp, nodes) + expMcp := mcp.DeepCopy() + expMcp.Status = expStatus + f.expectUpdateMachineConfigPoolStatus(expMcp) + f.run(getKey(mcp, t)) + + metric := testutil.ToFloat64(ctrlcommon.MachineConfigControllerPausedPoolKubeletCA.WithLabelValues("worker")) + + // The metric should be set the expiry date of the *newest* kube-apiserver-to-kubelet-signer in UTC + // which is 1670379908 -- the unix equivalent of Dec 7 02:25:08 2022 GMT + if metric != 1670379908 { + t.Errorf("Expected metric to be 1670379908 (Dec 7 02:25:08 2022 GMT), metric was %f", metric) + } +} + func TestShouldUpdateStatusOnlyUpdated(t *testing.T) { f := newFixture(t) cc := newControllerConfig(ctrlcommon.ControllerConfigName, configv1.TopologyMode("")) @@ -1073,3 +1127,152 @@ func filterLastTransitionTime(obj runtime.Object) runtime.Object { } return o } + +// TestKubeletCABundle is a fake kubelet CA bundle consisting of 4 certificates +// openshift-kube-apiserver-operator_kube-apiserver-to-kubelet-signer@0 Not After : Dec 5 02:25:08 2022 GMT +// openshift-kube-apiserver-operator_kube-apiserver-to-kubelet-signer@1 Not After : Dec 5 02:25:08 2022 GMT +// openshift-kube-apiserver-operator_kube-apiserver-to-kubelet-signer@2 Not After : Dec 7 02:25:08 2022 GMT +// kubelet-api-to-kubelet-signer Not After : Dec 10 02:25:08 2021 GMT +// Originally this was generated as part of the test, but the crypto was expensive +var TestKubeletCABundle = `-----BEGIN CERTIFICATE----- +MIIGMzCCBBugAwIBAgICB+UwDQYJKoZIhvcNAQELBQAwgaoxDTALBgNVBAYTBFRl +c3QxDTALBgNVBAgTBFRlc3QxDTALBgNVBAcTBFRlc3QxDTALBgNVBAkTBFRlc3Qx +DjAMBgNVBBETBTEyMzQ1MQ0wCwYDVQQKEwRUZXN0MU0wSwYDVQQDDERvcGVuc2hp +ZnQta3ViZS1hcGlzZXJ2ZXItb3BlcmF0b3Jfa3ViZS1hcGlzZXJ2ZXItdG8ta3Vi +ZWxldC1zaWduZXJAMDAeFw0yMTEyMTAwMjI1MDhaFw0yMjEyMDUwMjI1MDhaMIGq +MQ0wCwYDVQQGEwRUZXN0MQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQHEwRUZXN0MQ0w +CwYDVQQJEwRUZXN0MQ4wDAYDVQQREwUxMjM0NTENMAsGA1UEChMEVGVzdDFNMEsG +A1UEAwxEb3BlbnNoaWZ0LWt1YmUtYXBpc2VydmVyLW9wZXJhdG9yX2t1YmUtYXBp +c2VydmVyLXRvLWt1YmVsZXQtc2lnbmVyQDAwggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQC3OOVMPzWJQG5hfAtjluXBVrWgLEeVIfIoiU4IvMmCbLpFxJmp +2Eyb9U1l2yekesReB618HolxgXX0QJZKzlD67X/sKvF1kXUUP5l1E4mq5rgOHyEE +q8sY9EtQQbhY1RhROltoh3tMhlrQlL1O+1mGGorb4anGfDXr171GNatVVdaX29U5 +0AqE+3PWrgL1+oHze+t3DeEV/k02IQPGoiU28L8x5lSzgsrKzJSwaPJ0qx89JKaR +8UrAIqHIyKr3RzO8IYAYaevIDFYrnKK45yN5eBMV6y14VJZigXp0Nv4oXZO2x77R +j/nzgwKhT8FRx+hpv17DRsYMHy5OYOuG+hXFh21rs9ojxRoBwDPtZIKTwKRqeqht +TC5vA5ZFpClTzSYNsF3gtVcvnguv29xEpI06x99SXf/Ih9HoN0bvlDSpCqA2v9/g +FDTNfBLnp7PO9JOjZ8dkE0/SZGLfbElwHNBrBdoUbVa8xEJagv34/RKHKV63GKqv +nyF2BgWEyXDFJUkFjXliB84AtNpoEiqC+oufD2Odwmzt623fPvUvFqUAgjjUCdZJ +RSv/vcljYtmqF5pS5AzhE48IOfHtcgC0EBFHGgxonnpTDNrZe2iet85o/5ldxF5h +XC+V9eAdk1Snsl3A6igs/ur52TmYmi0r8mv3m5hb14uA1VHsZs4WYYNA/wIDAQAB +o2EwXzAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUF +BwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIAWuRGt3DIOqa0gJBEQcxu6 +L2WRMA0GCSqGSIb3DQEBCwUAA4ICAQAAFEbqHKVadL1pdtoWFCSOxJvji8V8wOXS +MF+KcBd7M5rEMMV9ZUh87QwBZaQf/WTlAUYsoVTQp/mM7oFzdze9w2RyXHO96Fea +NxaWA88z8xo/5Q9ikIm5WNHeR7CH5CtV5IxFUMYSHgpfM1uhiHmJNyNsku9Wg8hd +ynTCnB5XJuwwxSijHBfFTzJotDDkpH3CtTbdmC+lCF3l8+R9tcXMhE+kee/f8o1U +EdrmqH1hzsfVUgrCyijz3LVfW5u/JzZ+9jfOqla5bhyrQu9WXP6yXypyFwjvTF5f +ggp2+olabtOrhvIxciyL5roPTTYc8x3x1+vQAnuI5L5pQ+h7br9QFq26g0hV5sGs +VVeU/NspAzWDM7eGTduWBK/TlQAdB9ra7tF7zfhxcry3ZzM45HeJI00oelf1oTN6 +421ru8l+zTq6F8Uj3tzEO5dEvDKnUX7AHekaZOTLLa+l5ovHypKtEHAoFzubdWGC +mF2cYyTd6apgDyFpbDQ5+bQ0alq9dedYH/nkW4OUmdcJFgAPqOfEGX36sqxLetr+ +Lflsydbq5Ogr0nfNvoqUewxcA5I6B1Bj/Crg1skdm/bsdgcbvzKzvfCgwR6EwmZj +Uggj87DQKGQM1suWYwxDoU52N71msAuSCthCmSQkW24rgX6CsyKh1Vyr98MdN2VA +2ypMtC5cPw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIGMzCCBBugAwIBAgICB+UwDQYJKoZIhvcNAQELBQAwgaoxDTALBgNVBAYTBFRl +c3QxDTALBgNVBAgTBFRlc3QxDTALBgNVBAcTBFRlc3QxDTALBgNVBAkTBFRlc3Qx +DjAMBgNVBBETBTEyMzQ1MQ0wCwYDVQQKEwRUZXN0MU0wSwYDVQQDDERvcGVuc2hp +ZnQta3ViZS1hcGlzZXJ2ZXItb3BlcmF0b3Jfa3ViZS1hcGlzZXJ2ZXItdG8ta3Vi +ZWxldC1zaWduZXJAMTAeFw0yMTEyMTAwMjI1MDlaFw0yMjEyMDYwMjI1MDhaMIGq +MQ0wCwYDVQQGEwRUZXN0MQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQHEwRUZXN0MQ0w +CwYDVQQJEwRUZXN0MQ4wDAYDVQQREwUxMjM0NTENMAsGA1UEChMEVGVzdDFNMEsG +A1UEAwxEb3BlbnNoaWZ0LWt1YmUtYXBpc2VydmVyLW9wZXJhdG9yX2t1YmUtYXBp +c2VydmVyLXRvLWt1YmVsZXQtc2lnbmVyQDEwggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQDK7XtbFwv3mUVqbzPM6PLLvjB1b0+9kBoJSKcnVwNfEMYbCjnL +6pXgjW9EeVJZ2JtpaqKiFqLbhmREzRkUaIhaLY/xi2wNVQ0ai5YDjFjvNiLcljRe +2DmUHErxI+RD0LLkFwq7FyU0JXYhjT+dxprcwoZTo3ztIV12j2Nmbxko/PI7us8/ +yjc7Sh+7gtrlpMlRyhuo2g96FnrXSe8jdbfzafY8zM4JZfYDGFbm/YatGtjADACt +q1anGRiE/6WgIm0dNyjXCcwcNBfY81qZBunX7VxXkLGvddIkBBY0G79M4KWH6sm2 +Xx6YTOQernL5U4X0M9CJCiVqjp1QzgoVRHIdI+Zu8K6GRb9/+KKhY+jpZZlpXIpW +lrtMlbphcc07vuDz3QmnMC1RDgqO8LV1aHXYQv2SqEpwgr5/bu2bEr8cx9j0QX4Y +kU1stasozw6HA6Od9JHO3NtpScez7YPkZtmlJCDB7aoSsi3O/julFY56MiOaPF7h +znaTk6iXdvDByphVRLlrqomuDgiu1ErTso6EsyuRoMryrecDhNcrKJIiD+8s0nJK +9QcmycqJvvl14OyV4SL6JA9dhsd0klcyC0MYECGnCRGVkmdrMrZTijGKWZrEGoiz +tFwobFNE88yiDOfTmgLfpAk9S2YuIo0T5Qt27od2OtnJRerzDWJAw8iSgQIDAQAB +o2EwXzAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUF +BwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFGZA1qHJC2uTLh97iLpMy45u +3TmtMA0GCSqGSIb3DQEBCwUAA4ICAQA+SbnXE4CZhL3S4C5arIYqpOU58XpDa/Di +Xl+yXgcF82YIg+KBCXBjDWd7Pt2/KgiBOENHevUVWStyOhHHzCYE9n3nbkYX1KKj +kjR5F7jeR6C3hLuZtOZtvsZSo/v/NlecD6VwdvkAGQQPAKrwWUwpX9MOadufhIxP +T7QQkcZYeDiV4L3GK6QlD+76osnNIrkKHCuS+iKY6ty7c3BcPpj6hNxSRcpkvS4k +z3lX+Cb8HjnRKiZvP5tAmPzCy0ZBq+i+l3Z7yaa6S4AAeQ7hDkQhe/CZXpWG4jRY +fphDJrnsDoIzwW8mZbkc0HeYAIBcr9gf9PXA5v5UGPjKNmLtw0/Mkz0ZoXZ9+ca3 +++S0D7ZgrqHIQ/4TivWC1p8ublHKjaHzJzoDl7cetRKPzp6dCGsNzjQ2Y1WGxaKK ++/KtRGNdQf5PgyE+g+lOndu9ERH80F2TUAQ98e7ZtEnS9+CXofuzn55mhZ1Lczaj +T4zMlf/xJ6dNO8ez8A1rPYlmZw8HC4b5nGfWvz93BZ7B8rWp4TB5oCYYLP+SFL3Y +lNmu4FB15bL+hi5tt41qLatp8KC5yEn8d/o8pz6XtqKtYXaBriiumnqivX/pMCqg +eRhhpMIsg7qOSmAk31ej9Ickk6jcrCQpbMi3pOMx47ditFIVdZSEJmYiT2UyhXpP +gFn9d+CPeQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIGMzCCBBugAwIBAgICB+UwDQYJKoZIhvcNAQELBQAwgaoxDTALBgNVBAYTBFRl +c3QxDTALBgNVBAgTBFRlc3QxDTALBgNVBAcTBFRlc3QxDTALBgNVBAkTBFRlc3Qx +DjAMBgNVBBETBTEyMzQ1MQ0wCwYDVQQKEwRUZXN0MU0wSwYDVQQDDERvcGVuc2hp +ZnQta3ViZS1hcGlzZXJ2ZXItb3BlcmF0b3Jfa3ViZS1hcGlzZXJ2ZXItdG8ta3Vi +ZWxldC1zaWduZXJAMjAeFw0yMTEyMTAwMjI1MTJaFw0yMjEyMDcwMjI1MDhaMIGq +MQ0wCwYDVQQGEwRUZXN0MQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQHEwRUZXN0MQ0w +CwYDVQQJEwRUZXN0MQ4wDAYDVQQREwUxMjM0NTENMAsGA1UEChMEVGVzdDFNMEsG +A1UEAwxEb3BlbnNoaWZ0LWt1YmUtYXBpc2VydmVyLW9wZXJhdG9yX2t1YmUtYXBp +c2VydmVyLXRvLWt1YmVsZXQtc2lnbmVyQDIwggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQDdUXJ8FcsG+AgCVfz3dAg4ljTlB5wn3SG+SyAlMyCBqz+40i9G +I98bcVklDaeirCy7oar2rbfAN9a3I4BEo8t4v1CjwEj/hc8iC2bYK4B6y1N1eD6H +/yvAXHFdFh+NyCQeAhYBnIZ2EejIWP5R30t6XoxyIY0mPaX07ycH3Y29DvZ291fZ +OjsHUAXkhOcXzaxufMYnqbb5fOifoCTAy/bQu/LBaWGOKjaNHsh0mOdIFNL23VwL +pUTp8g/mds+R2jkQuos6WFqN0rx8vmmemAU8kdi8Zmb3uYztEzJDqts8Q/mzX+2A +mEdlNU33KcCQ8yWp2T2d9FIKCpG1uzf0VYE+uxWMe+n8hp1sw2Lw2TnQRkAhcd2c +fQRXUnScTvgBYm1Q0L0uq5y0pBKeNpP5TIXqC04ULM0QT07+1+z47jF0K/eREk2z +AU+6vFCh6/CTJWM7FwDmrLux2RtgbGnzdsRfNJfx7Ugdg7f+t3R0DScZmvPxc+jZ +YIV2pneWVIuvHy1uMgQA92Zj0IX9wTAhxBtepZWiqiEfsU4fZDhyqkA+g1iHvM1V +qlmYkArAvUMJ2fM4Iu5ccj/NlsjjkzJYs27PXv0ckxtb6lB+H5RXqvR8SuoNFx+d +oOVPeAoZvhCtKBSP9wDMr7/PgOupqrHxsTNV5ZCfb7Si2JkYPIfTe74xEQIDAQAB +o2EwXzAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUF +BwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFAmODfFQeMs2ymCnikNvrt1Z +gN5bMA0GCSqGSIb3DQEBCwUAA4ICAQDD7iVJPSk5kV6OkFaN4UU6BKgfPNPA/zcZ +LB/Qv1kGYuAfoQLwRhpql+V9BsPj2GcgWpSrxXD0OOLcqWEsv99EaDHBTS7aOGkd +5idczW8RfiJ/gffI+ybBu+vUqwVyvsrXojfpUPBsVcz22DnMWAvx51QsqRuvYesx +Nfva8L9xYqscZoIwGAA/GaI+OEUK7TS+M5rlwYw2J9Wcv9Y0XCYg+FK9AFqIy1EO +zEmNHwuyUnAs+6HXNDqQQbRHho/iLCinrI+j4uQ86vhUidXEoOpLRwqxkqhDjkur +t475+C0xZklIYCotrfkoWBS7+iWAtOemV2V/utLqjZpCPJghG5PynaO7Pk5Uy8IJ +ycboVzeKJvuR2pcc4l2BIr7mQuyM15pkibQYbBaX88m+CIvmGpEjzWKJpyVjlYzn +D/FPNiAQCtKVlEaxD8FD8s+KB2CCX79P3FumGPnoCyMakuFuW6D641VmB8SiUcBY +xKxNmPBYL7OXeNaPdcm95alyN6jvppnNoiXjTgKPeuRqlto8F/rhMtH9YPidJlSe +DYAk70fyN817YHCG01MngwDKsa5yXvACSspForQy9iAdKiFT8q0DjGeW9q0T6nyv +eJAf6alq4VWHsp05n2XCg53oTNUFda4Hlhsn1IUSAqAqNdU+9ry09UOL8R79QGM1 +atKsq3hqxg== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIF5TCCA82gAwIBAgICB+UwDQYJKoZIhvcNAQELBQAwgYMxDTALBgNVBAYTBFRl +c3QxDTALBgNVBAgTBFRlc3QxDTALBgNVBAcTBFRlc3QxDTALBgNVBAkTBFRlc3Qx +DjAMBgNVBBETBTEyMzQ1MQ0wCwYDVQQKEwRUZXN0MSYwJAYDVQQDEx1rdWJlbGV0 +LWFwaS10by1rdWJlbGV0LXNpZ25lcjAeFw0yMTEyMTAwMjI1MTNaFw0yMTEyMTAw +MjI1MDhaMIGDMQ0wCwYDVQQGEwRUZXN0MQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQH +EwRUZXN0MQ0wCwYDVQQJEwRUZXN0MQ4wDAYDVQQREwUxMjM0NTENMAsGA1UEChME +VGVzdDEmMCQGA1UEAxMda3ViZWxldC1hcGktdG8ta3ViZWxldC1zaWduZXIwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDCDJ18sbSClLUBSZO96FRMPaKZ +bGVfWijjcgThJyJVCX88RYxN/u17+VuIOOJ6A9+bNBpC+/Prmz7ydrOEqjhi/alE +XiThptpV8ChKWZmdxcL4ucAIf+TwUqtRj+RDL3QY/oYFSl+5uwTEGq6g1p+C0+iG +5QtkoaDp/dvncNwMrm78iLugDAJOw3A+p5O54rNAegFixiLMJYg0Mgrv7gkO/KQc +wgyt6VUoHS2j2TnJGvBUkRW4dd0ce85fXayrOh2TegWFfUZrDf/7hdXDtbdTtXwp +FptzgUO7+k0ovPLQUKrBuFmVPX380iZ2VTrbeTAI6vO2a/pem6qBZQMUURp1NyQO +8gJCB2Vjxnpd1MnOaaKdTrDwM8y1SnzQ8tJgK6Pyrfu1NfjCAtvf5srOkjRxB/oe +475iYR91G8LMGnzFx7aHIP7dB5ysDMihfqhvJlO/AZTxzfPlu/woYTpN8jcdLOWg +GpUmACeE7KUQoSsmdMMul+3Q6ELPKgbuCIMrfwTwWrAS/cb4nHVqSRzd0iSjzWpJ +5OK+VPje/ZzwZmAqvU1OfT/4houUbGoCVRxQPxXfTWkdIWZyVxHqY4IMp74iLEAK +//+hTjlIkj+jwSZ9HKveF89xLiPFV1N0jHDSOpzcexENsXOSJlRj/RfpCyo+xpNh +nyvqZdJ8l15zr9WCGwIDAQABo2EwXzAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYw +FAYIKwYBBQUHAwIGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE +FJ/DBOKTbBrZDlS0x+CzTqaPX6orMA0GCSqGSIb3DQEBCwUAA4ICAQAKPXd9eth3 +13OD3PGrDT/V43ywkKtU5PcCnHN6C9XDccBTNJ1vLvosPmI0UbmLvjQ9YV0rQfAX +1qneAYJtOG5KahMXOKgvEN5iXF51fgu4TZQBbXH6dAc8CL6n7l68J1HXR4RB9n+3 +2er9Qg81F3lgtoAUKCm9eOS0TWNu6mgAtd5AlTPjJDM72ZEJo7SPJljfAtLT6gmT +4uY8QFeMCTzbcA+Knvtbv7pnj6jbx4YXD7Ugrn7XCNApK1xOclqKM2kL57fPTmPz +lNgVWeYnVtUj79AaHOx+w7J4omNFN1oMQpPErTnVRm9Ps5NcJI/L9vSbtdDEpNQL +9DHG42JSri6GxTx+94/ZFjgsZP8La+J485Cw/QESeJ8jqIy/PJq31s6LfGV9hQov +uY2aKjSPU0NvDgBJYspVP44Ea0RVbQRMOwhx9MePgZ17pVG9ukmrfJ4JS/q0Wkw0 +9GDhWSsCFDjPa3XFBvwTZeNW34KN+rCqvlEY6d1JCphRgl0ZUGYCNXWOZm6GAM8v +H/dVKL9eYIUIrYbnrloT1Z4phEVzDlk654Tp46DQdeU2QCm7DZPP8VlPUWGidTz4 +jQ8njmN6dBChMIRR2dBeOP/sbrdsa6RZ8R/1ZVHx5Pot9JypihuuOnKgNjKRl/L+ +GjL/XaQj7QkXkVNKrd28YEpsslen5EBjwg== +-----END CERTIFICATE-----` diff --git a/test/e2e-bootstrap/bootstrap_test.go b/test/e2e-bootstrap/bootstrap_test.go index 4481b9cd57..4bc8292ec4 100644 --- a/test/e2e-bootstrap/bootstrap_test.go +++ b/test/e2e-bootstrap/bootstrap_test.go @@ -352,6 +352,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle // The node controller consumes data written by the above node.New( ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(), + ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(), ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(), ctx.KubeInformerFactory.Core().V1().Nodes(), ctx.ConfigInformerFactory.Config().V1().Schedulers(), diff --git a/test/e2e/mcc_test.go b/test/e2e/mcc_test.go new file mode 100644 index 0000000000..91a08b3c97 --- /dev/null +++ b/test/e2e/mcc_test.go @@ -0,0 +1,111 @@ +package e2e_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/openshift/machine-config-operator/test/framework" + "github.com/openshift/machine-config-operator/test/helpers" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Test case to make sure the MCC alerts properly when pools are +// paused and a certificate rotation happens +func TestMCCPausedKubeletCAAlert(t *testing.T) { + var testPool = "master" + + cs := framework.NewClientSet("") + + // Get the machine config pool + mcp, err := cs.MachineConfigPools().Get(context.TODO(), testPool, metav1.GetOptions{}) + require.Nil(t, err) + + // Update our copy of the pool + newMcp := mcp.DeepCopy() + newMcp.Spec.Paused = true + + t.Logf("Pausing pool") + + // Update the pool to be paused + _, err = cs.MachineConfigPools().Update(context.TODO(), newMcp, metav1.UpdateOptions{}) + require.Nil(t, err) + t.Logf("Paused") + + // Rotate the certificates + t.Logf("Patching certificate") + err = helpers.ForceKubeApiserverCertificateRotation(cs) + require.Nil(t, err) + t.Logf("Patched") + + // Verify the pool is paused and the config is pending but not rolling out + t.Logf("Waiting for rendered config to get stuck behind pause") + err = helpers.WaitForPausedConfig(t, cs, testPool) + require.Nil(t, err) + t.Logf("Certificate stuck behind paused (as expected)") + + // Retrieve the token from the prometheus secret, we need it to auth to the oauth proxy + t.Logf("Getting monitoring token") + token, err := helpers.GetMonitoringToken(t, cs) + require.Nil(t, err) + + // Get the service details so we can ge the IP and port to check for the metrics + t.Logf("Getting metrics listener service details") + svc, err := cs.Services("openshift-machine-config-operator").Get(context.TODO(), "machine-config-controller", metav1.GetOptions{}) + require.Nil(t, err) + + // Extract the IP and port and build the URL + requestTarget := svc.Spec.ClusterIP + requestPort := svc.Spec.Ports[0].Port + url := fmt.Sprintf("https://%s:%d/metrics", requestTarget, requestPort) + + // Get a node to execute on (we really just need cluster network access, it doesn't matter what pod) + checkNode, err := helpers.GetNodesByRole(cs, testPool) + require.Nil(t, err) + + // Run our curl command inside the pod to check metrics + out := helpers.ExecCmdOnNode(t, cs, checkNode[0], []string{"curl", "-s", "-k", "-H", "Authorization: Bearer " + string(token), url}...) + + // The /metrics output will contain the metric if it works + if !strings.Contains(out, `machine_config_controller_paused_pool_kubelet_ca{pool="`+testPool+`"} 1`) { + t.Errorf("Metric should have been set after configuration was paused, but it was NOT") + } else { + t.Log("Metric successfully set") + } + + // Get the pool again so we can update it back to unpaosed + mcp2, err := cs.MachineConfigPools().Get(context.TODO(), testPool, metav1.GetOptions{}) + require.Nil(t, err) + + // Set it back to unpaused + newMcp2 := mcp2.DeepCopy() + newMcp2.Spec.Paused = false + + t.Logf("Unpausing pool\n") + // Perform the update + _, err = cs.MachineConfigPools().Update(context.TODO(), newMcp2, metav1.UpdateOptions{}) + require.Nil(t, err) + t.Logf("Waiting for config to sync after unpause...") + + // Wait for the pools to settle again + err = helpers.WaitForPoolCompleteAny(t, cs, testPool) + require.Nil(t, err) + + t.Logf("Unpaused + Synced") + + t.Logf("Checking for metric value...") + + // Get a node again so we can check if the metric shut off + checkNode, err = helpers.GetNodesByRole(cs, testPool) + require.Nil(t, err) + + out = helpers.ExecCmdOnNode(t, cs, checkNode[0], []string{"curl", "-s", "-k", "-H", "Authorization: Bearer " + string(token), url}...) + // The /metrics output will contain the metric if it works + if !strings.Contains(out, `machine_config_controller_paused_pool_kubelet_ca{pool="master"} 0`) { + t.Errorf("Metric should be zero after pool unpaused, but it was NOT") + } else { + t.Logf("Metric has correctly been reset after pool unpaused") + } +} diff --git a/test/helpers/utils.go b/test/helpers/utils.go index bb20e7b0c2..eb0caed3d7 100644 --- a/test/helpers/utils.go +++ b/test/helpers/utils.go @@ -8,6 +8,7 @@ import ( "math/rand" "os" "os/exec" + "strings" "testing" "time" @@ -24,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/util/retry" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" ) @@ -222,6 +224,74 @@ func WaitForNodeConfigChange(t *testing.T, cs *framework.ClientSet, node corev1. return nil } +// WaitForPoolComplete polls a pool until it has completed any update +func WaitForPoolCompleteAny(t *testing.T, cs *framework.ClientSet, pool string) error { + if err := wait.PollImmediate(2*time.Second, 2*time.Minute, func() (bool, error) { + mcp, err := cs.MachineConfigPools().Get(context.TODO(), pool, metav1.GetOptions{}) + if err != nil { + return false, err + } + // wait until the cluter is back to normal (for the next test at least) + if mcfgv1.IsMachineConfigPoolConditionTrue(mcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdated) { + return true, nil + } + return false, nil + }); err != nil { + t.Errorf("Machine config pool did not complete an update: %v", err) + } + return nil +} + +// WaitForPausedConfig waits for configuration to be pending in a paused pool +func WaitForPausedConfig(t *testing.T, cs *framework.ClientSet, pool string) error { + if err := wait.PollImmediate(2*time.Second, 2*time.Minute, func() (bool, error) { + mcp, err := cs.MachineConfigPools().Get(context.TODO(), pool, metav1.GetOptions{}) + if err != nil { + return false, err + } + // paused == not updated, not updating, not degraded + if mcfgv1.IsMachineConfigPoolConditionFalse(mcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdated) && + mcfgv1.IsMachineConfigPoolConditionFalse(mcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdating) && + mcfgv1.IsMachineConfigPoolConditionFalse(mcp.Status.Conditions, mcfgv1.MachineConfigPoolDegraded) { + return true, nil + } + return false, nil + }); err != nil { + t.Errorf("Machine config pool never entered the state where configuration was waiting behind pause: %v", err) + } + return nil +} + +// GetMonitoringToken retrieves the token from the openshift-monitoring secrets in the prometheus-k8s namespace. +// It is equivalent to "oc sa get-token prometheus-k8s -n openshift-monitoring" +func GetMonitoringToken(t *testing.T, cs *framework.ClientSet) (string, error) { + sa, err := cs.ServiceAccounts("openshift-monitoring").Get(context.TODO(), "prometheus-k8s", metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("Failed to retrieve service account: %v", err) + } + for _, secret := range sa.Secrets { + if strings.HasPrefix(secret.Name, "prometheus-k8s-token") { + sec, err := cs.Secrets("openshift-monitoring").Get(context.TODO(), secret.Name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("Failed to retrieve monitoring secret: %v", err) + } + if token, ok := sec.Data["token"]; ok { + return string(token), nil + } + } + } + return "", fmt.Errorf("No token found in openshift-monitoring secrets") +} + +// ForceKubeApiserverCertificateRotation sets the kube-apiserver-to-kubelet-signer's not-after date to nil, which causes the +// apiserver to rotate it +func ForceKubeApiserverCertificateRotation(cs *framework.ClientSet) error { + // Take note that the slash had to be encoded as ~1 because it's a reference: https://www.rfc-editor.org/rfc/rfc6901#section-3 + certPatch := fmt.Sprintf(`[{"op":"replace","path":"/metadata/annotations/auth.openshift.io~1certificate-not-after","value": null }]`) + _, err := cs.Secrets("openshift-kube-apiserver-operator").Patch(context.TODO(), "kube-apiserver-to-kubelet-signer", types.JSONPatchType, []byte(certPatch), metav1.PatchOptions{}) + return err +} + // LabelRandomNodeFromPool gets all nodes in pool and chooses one at random to label func LabelRandomNodeFromPool(t *testing.T, cs *framework.ClientSet, pool, label string) func() { nodes, err := GetNodesByRole(cs, pool) diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/lint.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/lint.go new file mode 100644 index 0000000000..7681877a89 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/lint.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil/promlint" +) + +// CollectAndLint registers the provided Collector with a newly created pedantic +// Registry. It then calls GatherAndLint with that Registry and with the +// provided metricNames. +func CollectAndLint(c prometheus.Collector, metricNames ...string) ([]promlint.Problem, error) { + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(c); err != nil { + return nil, fmt.Errorf("registering collector failed: %s", err) + } + return GatherAndLint(reg, metricNames...) +} + +// GatherAndLint gathers all metrics from the provided Gatherer and checks them +// with the linter in the promlint package. If any metricNames are provided, +// only metrics with those names are checked. +func GatherAndLint(g prometheus.Gatherer, metricNames ...string) ([]promlint.Problem, error) { + got, err := g.Gather() + if err != nil { + return nil, fmt.Errorf("gathering metrics failed: %s", err) + } + if metricNames != nil { + got = filterMetrics(got, metricNames) + } + return promlint.NewWithMetricFamilies(got).Lint() +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/testutil.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/testutil.go new file mode 100644 index 0000000000..9af60ce1d2 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/testutil.go @@ -0,0 +1,230 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testutil provides helpers to test code using the prometheus package +// of client_golang. +// +// While writing unit tests to verify correct instrumentation of your code, it's +// a common mistake to mostly test the instrumentation library instead of your +// own code. Rather than verifying that a prometheus.Counter's value has changed +// as expected or that it shows up in the exposition after registration, it is +// in general more robust and more faithful to the concept of unit tests to use +// mock implementations of the prometheus.Counter and prometheus.Registerer +// interfaces that simply assert that the Add or Register methods have been +// called with the expected arguments. However, this might be overkill in simple +// scenarios. The ToFloat64 function is provided for simple inspection of a +// single-value metric, but it has to be used with caution. +// +// End-to-end tests to verify all or larger parts of the metrics exposition can +// be implemented with the CollectAndCompare or GatherAndCompare functions. The +// most appropriate use is not so much testing instrumentation of your code, but +// testing custom prometheus.Collector implementations and in particular whole +// exporters, i.e. programs that retrieve telemetry data from a 3rd party source +// and convert it into Prometheus metrics. +// +// In a similar pattern, CollectAndLint and GatherAndLint can be used to detect +// metrics that have issues with their name, type, or metadata without being +// necessarily invalid, e.g. a counter with a name missing the “_total” suffix. +package testutil + +import ( + "bytes" + "fmt" + "io" + + "github.com/prometheus/common/expfmt" + + dto "github.com/prometheus/client_model/go" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/internal" +) + +// ToFloat64 collects all Metrics from the provided Collector. It expects that +// this results in exactly one Metric being collected, which must be a Gauge, +// Counter, or Untyped. In all other cases, ToFloat64 panics. ToFloat64 returns +// the value of the collected Metric. +// +// The Collector provided is typically a simple instance of Gauge or Counter, or +// – less commonly – a GaugeVec or CounterVec with exactly one element. But any +// Collector fulfilling the prerequisites described above will do. +// +// Use this function with caution. It is computationally very expensive and thus +// not suited at all to read values from Metrics in regular code. This is really +// only for testing purposes, and even for testing, other approaches are often +// more appropriate (see this package's documentation). +// +// A clear anti-pattern would be to use a metric type from the prometheus +// package to track values that are also needed for something else than the +// exposition of Prometheus metrics. For example, you would like to track the +// number of items in a queue because your code should reject queuing further +// items if a certain limit is reached. It is tempting to track the number of +// items in a prometheus.Gauge, as it is then easily available as a metric for +// exposition, too. However, then you would need to call ToFloat64 in your +// regular code, potentially quite often. The recommended way is to track the +// number of items conventionally (in the way you would have done it without +// considering Prometheus metrics) and then expose the number with a +// prometheus.GaugeFunc. +func ToFloat64(c prometheus.Collector) float64 { + var ( + m prometheus.Metric + mCount int + mChan = make(chan prometheus.Metric) + done = make(chan struct{}) + ) + + go func() { + for m = range mChan { + mCount++ + } + close(done) + }() + + c.Collect(mChan) + close(mChan) + <-done + + if mCount != 1 { + panic(fmt.Errorf("collected %d metrics instead of exactly 1", mCount)) + } + + pb := &dto.Metric{} + m.Write(pb) + if pb.Gauge != nil { + return pb.Gauge.GetValue() + } + if pb.Counter != nil { + return pb.Counter.GetValue() + } + if pb.Untyped != nil { + return pb.Untyped.GetValue() + } + panic(fmt.Errorf("collected a non-gauge/counter/untyped metric: %s", pb)) +} + +// CollectAndCount registers the provided Collector with a newly created +// pedantic Registry. It then calls GatherAndCount with that Registry and with +// the provided metricNames. In the unlikely case that the registration or the +// gathering fails, this function panics. (This is inconsistent with the other +// CollectAnd… functions in this package and has historical reasons. Changing +// the function signature would be a breaking change and will therefore only +// happen with the next major version bump.) +func CollectAndCount(c prometheus.Collector, metricNames ...string) int { + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(c); err != nil { + panic(fmt.Errorf("registering collector failed: %s", err)) + } + result, err := GatherAndCount(reg, metricNames...) + if err != nil { + panic(err) + } + return result +} + +// GatherAndCount gathers all metrics from the provided Gatherer and counts +// them. It returns the number of metric children in all gathered metric +// families together. If any metricNames are provided, only metrics with those +// names are counted. +func GatherAndCount(g prometheus.Gatherer, metricNames ...string) (int, error) { + got, err := g.Gather() + if err != nil { + return 0, fmt.Errorf("gathering metrics failed: %s", err) + } + if metricNames != nil { + got = filterMetrics(got, metricNames) + } + + result := 0 + for _, mf := range got { + result += len(mf.GetMetric()) + } + return result, nil +} + +// CollectAndCompare registers the provided Collector with a newly created +// pedantic Registry. It then calls GatherAndCompare with that Registry and with +// the provided metricNames. +func CollectAndCompare(c prometheus.Collector, expected io.Reader, metricNames ...string) error { + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(c); err != nil { + return fmt.Errorf("registering collector failed: %s", err) + } + return GatherAndCompare(reg, expected, metricNames...) +} + +// GatherAndCompare gathers all metrics from the provided Gatherer and compares +// it to an expected output read from the provided Reader in the Prometheus text +// exposition format. If any metricNames are provided, only metrics with those +// names are compared. +func GatherAndCompare(g prometheus.Gatherer, expected io.Reader, metricNames ...string) error { + got, err := g.Gather() + if err != nil { + return fmt.Errorf("gathering metrics failed: %s", err) + } + if metricNames != nil { + got = filterMetrics(got, metricNames) + } + var tp expfmt.TextParser + wantRaw, err := tp.TextToMetricFamilies(expected) + if err != nil { + return fmt.Errorf("parsing expected metrics failed: %s", err) + } + want := internal.NormalizeMetricFamilies(wantRaw) + + return compare(got, want) +} + +// compare encodes both provided slices of metric families into the text format, +// compares their string message, and returns an error if they do not match. +// The error contains the encoded text of both the desired and the actual +// result. +func compare(got, want []*dto.MetricFamily) error { + var gotBuf, wantBuf bytes.Buffer + enc := expfmt.NewEncoder(&gotBuf, expfmt.FmtText) + for _, mf := range got { + if err := enc.Encode(mf); err != nil { + return fmt.Errorf("encoding gathered metrics failed: %s", err) + } + } + enc = expfmt.NewEncoder(&wantBuf, expfmt.FmtText) + for _, mf := range want { + if err := enc.Encode(mf); err != nil { + return fmt.Errorf("encoding expected metrics failed: %s", err) + } + } + + if wantBuf.String() != gotBuf.String() { + return fmt.Errorf(` +metric output does not match expectation; want: + +%s +got: + +%s`, wantBuf.String(), gotBuf.String()) + + } + return nil +} + +func filterMetrics(metrics []*dto.MetricFamily, names []string) []*dto.MetricFamily { + var filtered []*dto.MetricFamily + for _, m := range metrics { + for _, name := range names { + if m.GetName() == name { + filtered = append(filtered, m) + break + } + } + } + return filtered +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 12e6a4d445..4619e39c23 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -875,6 +875,7 @@ github.com/polyfloyd/go-errorlint/errorlint github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/internal github.com/prometheus/client_golang/prometheus/promhttp +github.com/prometheus/client_golang/prometheus/testutil github.com/prometheus/client_golang/prometheus/testutil/promlint # github.com/prometheus/client_model v0.2.0 ## explicit; go 1.9