From a5eac7a48ffdac7decd1a8aca400c77c46c735a8 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Tue, 19 Jan 2021 10:17:19 -0500 Subject: [PATCH] add sigv4 install script, fix bug Also fixes bug where the deployment scrape config was included with host filtering enabled, where the inverse should've been true. --- CHANGELOG.md | 8 +- production/kubernetes/agent-sigv4.yaml | 357 ++++++++++++++++++ production/kubernetes/agent.yaml | 23 -- production/kubernetes/build/build.sh | 1 + .../build/templates/base-sigv4/main.jsonnet | 22 ++ .../build/templates/base-sigv4/spec.json | 11 + production/kubernetes/install-sigv4.sh | 61 +++ .../tanka/grafana-agent/config.libsonnet | 4 +- 8 files changed, 461 insertions(+), 26 deletions(-) create mode 100644 production/kubernetes/agent-sigv4.yaml create mode 100644 production/kubernetes/build/templates/base-sigv4/main.jsonnet create mode 100644 production/kubernetes/build/templates/base-sigv4/spec.json create mode 100644 production/kubernetes/install-sigv4.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index bd514ce14bde..7f9affbad506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,14 @@ can be found at [#317](https://github.com/grafana/agent/issues/317). ARMv6 docker builds are also now available. (@rfratto) +- [ENHANCEMENT] A sigv4 install script for Prometheus has been added. (@rfratto) + - [BUGFIX] The K8s API server scrape job will use the API server Service name - when resolving IP addresses for Prometheus service discovery using the "Endpoints" role. (@hjet) + when resolving IP addresses for Prometheus service discovery using the + "Endpoints" role. (@hjet) + +- [BUGFIX] The K8s manifests will no longer include the `default/kubernetes` job + twice in both the DaemonSet and the Deployment. (@rfratto) # v0.10.0 (2021-01-13) diff --git a/production/kubernetes/agent-sigv4.yaml b/production/kubernetes/agent-sigv4.yaml new file mode 100644 index 000000000000..86fe49f31811 --- /dev/null +++ b/production/kubernetes/agent-sigv4.yaml @@ -0,0 +1,357 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-agent +--- +apiVersion: v1 +data: + agent.yml: | + prometheus: + configs: + - host_filter: true + name: agent + remote_write: + - sigv4: + enabled: true + url: ${REMOTE_WRITE_URL} + scrape_configs: + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: "false" + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: keep + regex: .*-metrics + source_labels: + - __meta_kubernetes_pod_container_port_name + - action: replace + regex: (https?) + replacement: $1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + replacement: $1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(\:\d+)?;(\d+) + replacement: $1:$3 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: drop + regex: "" + source_labels: + - __meta_kubernetes_pod_label_name + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_label_name + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container + - action: replace + separator: ':' + source_labels: + - __meta_kubernetes_pod_name + - __meta_kubernetes_pod_container_name + - __meta_kubernetes_pod_container_port_name + target_label: instance + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: drop + regex: Succeeded|Failed + source_labels: + - __meta_kubernetes_pod_phase + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: false + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: default/kube-state-metrics + kubernetes_sd_configs: + - namespaces: + names: + - default + role: pod + relabel_configs: + - action: keep + regex: kube-state-metrics + source_labels: + - __meta_kubernetes_pod_label_name + - action: replace + separator: ':' + source_labels: + - __meta_kubernetes_pod_name + - __meta_kubernetes_pod_container_name + - __meta_kubernetes_pod_container_port_name + target_label: instance + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: false + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: default/node-exporter + kubernetes_sd_configs: + - namespaces: + names: + - default + role: pod + relabel_configs: + - action: keep + regex: node-exporter + source_labels: + - __meta_kubernetes_pod_label_name + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: false + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kube-system/kubelet + kubernetes_sd_configs: + - role: node + relabel_configs: + - replacement: kubernetes.default.svc.cluster.local:443 + target_label: __address__ + - replacement: https + target_label: __scheme__ + - regex: (.+) + replacement: /api/v1/nodes/${1}/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: false + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kube-system/cadvisor + kubernetes_sd_configs: + - role: node + metric_relabel_configs: + - action: drop + regex: container_([a-z_]+); + source_labels: + - __name__ + - image + - action: drop + regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + source_labels: + - __name__ + relabel_configs: + - replacement: kubernetes.default.svc.cluster.local:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: false + global: + scrape_interval: 15s + wal_directory: /var/lib/agent/data + server: + log_level: info +kind: ConfigMap +metadata: + name: grafana-agent +--- +apiVersion: v1 +data: + agent.yml: | + prometheus: + configs: + - host_filter: false + name: agent + remote_write: + - sigv4: + enabled: true + url: ${REMOTE_WRITE_URL} + scrape_configs: + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: default/kubernetes + kubernetes_sd_configs: + - role: endpoints + metric_relabel_configs: + - action: drop + regex: apiserver_admission_controller_admission_latencies_seconds_.* + source_labels: + - __name__ + - action: drop + regex: apiserver_admission_step_admission_latencies_seconds_.* + source_labels: + - __name__ + relabel_configs: + - action: keep + regex: apiserver + source_labels: + - __meta_kubernetes_service_label_component + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: false + server_name: kubernetes + global: + scrape_interval: 15s + wal_directory: /var/lib/agent/data + server: + log_level: info +kind: ConfigMap +metadata: + name: grafana-agent-deployment +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: grafana-agent +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: + - get + - list + - watch +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: grafana-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: grafana-agent +subjects: +- kind: ServiceAccount + name: grafana-agent + namespace: default +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: grafana-agent +spec: + minReadySeconds: 10 + selector: + matchLabels: + name: grafana-agent + template: + metadata: + labels: + name: grafana-agent + spec: + containers: + - args: + - -config.file=/etc/agent/agent.yml + - -prometheus.wal-directory=/tmp/agent/data + command: + - /bin/agent + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: grafana/agent:v0.10.0 + imagePullPolicy: IfNotPresent + name: agent + ports: + - containerPort: 80 + name: http-metrics + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /etc/agent + name: grafana-agent + serviceAccount: grafana-agent + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - configMap: + name: grafana-agent + name: grafana-agent + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana-agent-deployment +spec: + minReadySeconds: 10 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: grafana-agent-deployment + template: + metadata: + labels: + name: grafana-agent-deployment + spec: + containers: + - args: + - -config.file=/etc/agent/agent.yml + - -prometheus.wal-directory=/tmp/agent/data + command: + - /bin/agent + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: grafana/agent:v0.10.0 + imagePullPolicy: IfNotPresent + name: agent + ports: + - containerPort: 80 + name: http-metrics + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /etc/agent + name: grafana-agent-deployment + serviceAccount: grafana-agent + volumes: + - configMap: + name: grafana-agent-deployment + name: grafana-agent-deployment diff --git a/production/kubernetes/agent.yaml b/production/kubernetes/agent.yaml index dcf350a15ae3..53007006cf5b 100644 --- a/production/kubernetes/agent.yaml +++ b/production/kubernetes/agent.yaml @@ -176,29 +176,6 @@ data: tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: false - - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - job_name: default/kubernetes - kubernetes_sd_configs: - - role: endpoints - metric_relabel_configs: - - action: drop - regex: apiserver_admission_controller_admission_latencies_seconds_.* - source_labels: - - __name__ - - action: drop - regex: apiserver_admission_step_admission_latencies_seconds_.* - source_labels: - - __name__ - relabel_configs: - - action: keep - regex: apiserver - source_labels: - - __meta_kubernetes_service_label_component - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: false - server_name: kubernetes global: scrape_interval: 15s wal_directory: /var/lib/agent/data diff --git a/production/kubernetes/build/build.sh b/production/kubernetes/build/build.sh index d71272e27c90..6ec66b517500 100644 --- a/production/kubernetes/build/build.sh +++ b/production/kubernetes/build/build.sh @@ -7,6 +7,7 @@ pushd ${DIRNAME} # Make sure dependencies are up to date jb install tk show --dangerous-allow-redirect ./templates/base > ${PWD}/../agent.yaml +tk show --dangerous-allow-redirect ./templates/base-sigv4 > ${PWD}/../agent-sigv4.yaml tk show --dangerous-allow-redirect ./templates/bare > ${PWD}/../agent-bare.yaml tk show --dangerous-allow-redirect ./templates/loki > ${PWD}/../agent-loki.yaml tk show --dangerous-allow-redirect ./templates/tempo > ${PWD}/../agent-tempo.yaml diff --git a/production/kubernetes/build/templates/base-sigv4/main.jsonnet b/production/kubernetes/build/templates/base-sigv4/main.jsonnet new file mode 100644 index 000000000000..fa5e494ddbb5 --- /dev/null +++ b/production/kubernetes/build/templates/base-sigv4/main.jsonnet @@ -0,0 +1,22 @@ +local agent = import 'grafana-agent/grafana-agent.libsonnet'; + +agent { + _images+:: { + agent: (import 'version.libsonnet'), + }, + + _config+:: { + namespace: 'default', + agent_remote_write: [{ + url: '${REMOTE_WRITE_URL}', + sigv4: { + enabled: true, + }, + }], + + // Since the config map isn't managed by Tanka, we don't want to + // add the configmap's hash as an annotation for the Kubernetes + // YAML manifest. + agent_config_hash_annotation: false, + }, +} diff --git a/production/kubernetes/build/templates/base-sigv4/spec.json b/production/kubernetes/build/templates/base-sigv4/spec.json new file mode 100644 index 000000000000..d97d07154983 --- /dev/null +++ b/production/kubernetes/build/templates/base-sigv4/spec.json @@ -0,0 +1,11 @@ +{ + "apiVersion": "tanka.dev/v1alpha1", + "kind": "Environment", + "metadata": { + "name": "template" + }, + "spec": { + "apiServer": "", + "namespace": "" + } +} diff --git a/production/kubernetes/install-sigv4.sh b/production/kubernetes/install-sigv4.sh new file mode 100644 index 000000000000..7f2f248370d9 --- /dev/null +++ b/production/kubernetes/install-sigv4.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# +# install.sh is a really basic installer for the agent. It uses the existing +# Kubernetes YAML example and envsubst to fill in the details for remote write +# URL, username, and password. +# +# There are three ways to provide the inputs for installation: +# +# 1. Environment variables (REMOTE_WRITE_URL) +# +# 2. Flags (-l for remote write URL) +# +# 3. stdin from prompts +# +# Flags override environment variables, and stdin is used as a fallback if a +# value wasn't given from a flag or environment variable. An empty username +# or password is acceptable, as it disables basic auth. However, a remote write +# URL must always be provided. +# + +check_installed() { + if ! type $1 >/dev/null 2>&1; then + echo "error: $1 not installed" >&2 + exit 1 + fi +} + +check_installed curl +check_installed envsubst + +MANIFEST_BRANCH=v0.10.0 +MANIFEST_URL=${MANIFEST_URL:-https://raw.githubusercontent.com/grafana/agent/${MANIFEST_BRANCH}/production/kubernetes/agent-sigv4.yaml} + +while getopts "l:u:p:" opt; do + case "$opt" in + l) + REMOTE_WRITE_URL=$OPTARG + ;; + ?) + echo "usage: $(basename $0) [-l remote write url] [-u remote write username] [-p remote write password]" >&2 + exit 1 + ;; + esac +done + +if [ -z "${REMOTE_WRITE_URL}" ]; then + read -sp 'Enter your remote write URL: ' REMOTE_WRITE_URL + printf $'\n' >&2 + + # We require a remote write URL for the agent; we don't do this same check for + # the username and password as the remote write system may not have basic + # auth enabled. + if [ -z "${REMOTE_WRITE_URL}" ]; then + echo "error: REMOTE_WRITE_URL must be provided by flag, env, or stdin" >&2 + exit 1 + fi +fi + +export REMOTE_WRITE_URL +curl -fsSL $MANIFEST_URL | envsubst diff --git a/production/tanka/grafana-agent/config.libsonnet b/production/tanka/grafana-agent/config.libsonnet index c49907078c06..36dcea7235f2 100644 --- a/production/tanka/grafana-agent/config.libsonnet +++ b/production/tanka/grafana-agent/config.libsonnet @@ -70,9 +70,9 @@ scrape_configs: if $._config.agent_host_filter then - $._config.kubernetes_scrape_configs + $._config.deployment_scrape_configs + $._config.kubernetes_scrape_configs else - $._config.kubernetes_scrape_configs, + $._config.kubernetes_scrape_configs + $._config.deployment_scrape_configs, remote_write: $._config.agent_remote_write, }], },