Add etcd metrics, Prometheus scrapes, and Grafana dash

* Use etcd v3.3 --listen-metrics-urls to expose only metrics
data via http://0.0.0.0:2381 on controllers
* Hold off on allowing workers firewall access. Move Prometheus
to a controller node for a while.
* Add Prometheus config to scrape etcd members. On Typhoon clusters,
etcd runs across controllers (i.e. 3 controllers == 3 etcd).

Author: dghubble

addons/prometheus/config.yaml

@@ -112,6 +112,26 @@ data:
         target_label: __metrics_path__
         replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
 
+    # Scrap etcd metrics from controllers 
+    - job_name: 'etcd'
+      kubernetes_sd_configs:
+      - role: node
+      scheme: http
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+      relabel_configs:
+        - source_labels: [__meta_kubernetes_node_label_node_role_kubernetes_io_controller]
+          action: keep
+          regex: 'true'
+        - action: labelmap
+          regex: __meta_kubernetes_node_label_(.+)
+        - source_labels: [__meta_kubernetes_node_name]
+          action: replace
+          target_label: __address__
+          replacement: '${1}:2381'
+    
     # Scrape config for service endpoints.
     #
     # The relabeling allows the actual service scrape endpoint to be configured

addons/prometheus/deployment.yaml

@@ -15,6 +15,12 @@ spec:
         name: prometheus
         phase: prod
     spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
       serviceAccountName: prometheus
       containers:
       - name: prometheus

aws/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -13,6 +13,7 @@ systemd:
             Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
             Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
             Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
             Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
             Environment="ETCD_STRICT_RECONFIG_CHECK=true"
             Environment="ETCD_SSL_DIR=/etc/ssl/etcd"

aws/container-linux/kubernetes/security.tf

@@ -81,6 +81,16 @@ resource "aws_security_group_rule" "controller-node-exporter" {
   source_security_group_id = "${aws_security_group.worker.id}"
 }
 
+resource "aws_security_group_rule" "controller-node-exporter-self" {
+  security_group_id = "${aws_security_group.controller.id}"
+
+  type      = "ingress"
+  protocol  = "tcp"
+  from_port = 9100
+  to_port   = 9100
+  self      = true
+}
+
 resource "aws_security_group_rule" "controller-kubelet-self" {
   security_group_id = "${aws_security_group.controller.id}"
 
@@ -256,6 +266,16 @@ resource "aws_security_group_rule" "worker-flannel-self" {
 resource "aws_security_group_rule" "worker-node-exporter" {
   security_group_id = "${aws_security_group.worker.id}"
 
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 9100
+  to_port                  = 9100
+  source_security_group_id = "${aws_security_group.controller.id}"
+}
+
+resource "aws_security_group_rule" "worker-node-exporter-self" {
+  security_group_id = "${aws_security_group.worker.id}"
+
   type      = "ingress"
   protocol  = "tcp"
   from_port = 9100

bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -13,6 +13,7 @@ systemd:
             Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380"
             Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
             Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
             Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
             Environment="ETCD_STRICT_RECONFIG_CHECK=true"
             Environment="ETCD_SSL_DIR=/etc/ssl/etcd"

digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -13,6 +13,7 @@ systemd:
             Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
             Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
             Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
             Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
             Environment="ETCD_STRICT_RECONFIG_CHECK=true"
             Environment="ETCD_SSL_DIR=/etc/ssl/etcd"

google-cloud/container-linux/kubernetes/controllers/cl/controller.yaml.tmpl

@@ -13,6 +13,7 @@ systemd:
             Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
             Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
             Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
             Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
             Environment="ETCD_STRICT_RECONFIG_CHECK=true"
             Environment="ETCD_SSL_DIR=/etc/ssl/etcd"

google-cloud/container-linux/kubernetes/network.tf

@@ -93,7 +93,7 @@ resource "google_compute_firewall" "internal-flannel" {
   target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 }
 
-# Allow prometheus (workload) to scrape node-exporter daemonset
+# Allow Prometheus to scrape node-exporter daemonset
 resource "google_compute_firewall" "internal-node-exporter" {
   name    = "${var.cluster_name}-internal-node-exporter"
   network = "${google_compute_network.network.name}"
@@ -103,7 +103,7 @@ resource "google_compute_firewall" "internal-node-exporter" {
     ports    = [9100]
   }
 
-  source_tags = ["${var.cluster_name}-worker"]
+  source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
   target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 }