diff --git a/.github/workflows/markdown.links.config.json b/.github/workflows/markdown.links.config.json index 4087ae8d9dc9..1c42ecb1d4bf 100644 --- a/.github/workflows/markdown.links.config.json +++ b/.github/workflows/markdown.links.config.json @@ -2,6 +2,9 @@ "ignorePatterns": [ { "pattern": "^http://prometheus.k3d.localhost:30080" + }, + { + "pattern": "^http://prometheus.k3d.localhost:50080" } ] } diff --git a/cmd/grafana-agent-crow/README.md b/cmd/grafana-agent-crow/README.md new file mode 100644 index 000000000000..b4040d83bc67 --- /dev/null +++ b/cmd/grafana-agent-crow/README.md @@ -0,0 +1,5 @@ +# Crow + +Crow is a tool similar to Tempo Vulture and Loki Canary that is used to smoke test Grafana Agent. Crow works by generating metrics, then validating them against Prometheus. Crow uses two endpoints; the traditional `/metrics` and then `/validate` that generates the results of Crow checking for successful samples. + +Note: The `/validate` endpoint should only be checked by the Grafana Agent instance that is configured to remote_write. \ No newline at end of file diff --git a/cmd/grafana-agent-crow/main.go b/cmd/grafana-agent-crow/main.go index 2a3655e8a8bf..773c380631e4 100644 --- a/cmd/grafana-agent-crow/main.go +++ b/cmd/grafana-agent-crow/main.go @@ -70,7 +70,7 @@ func main() { EnableOpenMetrics: true, })) - // Register crow's metrics to /metrics and /valiate respectively. + // Register crow's metrics to /metrics and /validate respectively. s.Registerer.MustRegister(c.StateMetrics()) validator.MustRegister(c.TestMetrics()) diff --git a/example/k3d/README.md b/example/k3d/README.md index 4f4b6d84574c..593d137507c7 100644 --- a/example/k3d/README.md +++ b/example/k3d/README.md @@ -1,9 +1,11 @@ -# `k3d` Example +# `k3d` Examples + +## Agent Environment The `k3d` example uses `k3d` and `tanka` to produce a Kubernetes environment -that +that implements a full Grafana Agent environment for testing. -## Requirements +### Requirements - A Unix-y command line (macOS or Linux will do). - Kubectl @@ -11,7 +13,7 @@ that - [Tanka >= v0.9.2](https://github.com/grafana/tanka) - [k3d >= v4.0.0](https://github.com/rancher/k3d) -## Getting Started +### Getting Started Build latest agent images with `make agent-image agentctl-image` in the project root directory if there are local changes to test. @@ -32,3 +34,60 @@ tk apply ./environment # Delete the k3d cluster when you're done with it k3d cluster delete agent-k3d ``` + +## Smoke Test Environment + +The smoke test environment is used to validate samples end to end. + +### Running + +Smoke Test environment is invoked via `/scripts/smoke_test.bash` + +This tool will spin up cluster of Grafana Agent, Cortex, Avalanche and [Crow](../../cmd/grafana-agent-crow/README.md) instances. The tool will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing. + +### What to look for? + +These alerts are viewable [here](http://prometheus.k3d.localhost:50080/alerts). + +Prometheus alerts are triggered: +- If any Crow instances are not running or Crow samples are not being propagated correctly. +- If any Grafana Agents are not running or Grafana Agent limits are outside their norm. + +NOTE: The alerts might be in pending until the system settles down. + +![](./assets/pending_alert.png) + +An alert firing will look similar to the below. + +![](./assets/alert_firing.png) + +If at the end of the test any issues are found they will look similar to the below. + +![](./assets/console_failure.png) + +### How to trigger an alert? + +Changing the avalanche setting for label_count to 1000, located [here](./lib/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit. + +![](./assets/trigger_change.png) + +### Architecture + +By default, a k3d cluster will be created running the following instances + +- agent-single - single instance +- agent-cluster - 3 Grafana Agents in clustered configuration +- crow-cluster - serves the agent cluster +- crow-single - serves the single agent +- cortex +- avalanche - selection of avalanche instances serving traffic + +Crow instance will check to see if the metrics that were scraped shows up in the prometheus endpoint and then will emit metrics on the success of those metrics. This success/failure result will trigger an alert if it is incorrect. + +### Flow + +![](./assets/order.png) + +### Avalanche + +Avalanche is used to add some additional load on the system and general testing. \ No newline at end of file diff --git a/example/k3d/assets/alert_firing.png b/example/k3d/assets/alert_firing.png new file mode 100644 index 000000000000..f134cf8f2b44 Binary files /dev/null and b/example/k3d/assets/alert_firing.png differ diff --git a/example/k3d/assets/alerts.png b/example/k3d/assets/alerts.png new file mode 100644 index 000000000000..d76f62cb0bd3 Binary files /dev/null and b/example/k3d/assets/alerts.png differ diff --git a/example/k3d/assets/console_failure.png b/example/k3d/assets/console_failure.png new file mode 100644 index 000000000000..bf817cbeaacd Binary files /dev/null and b/example/k3d/assets/console_failure.png differ diff --git a/example/k3d/assets/order.png b/example/k3d/assets/order.png new file mode 100644 index 000000000000..d6ca3cd60957 Binary files /dev/null and b/example/k3d/assets/order.png differ diff --git a/example/k3d/assets/order.uml b/example/k3d/assets/order.uml new file mode 100644 index 000000000000..d62485821163 --- /dev/null +++ b/example/k3d/assets/order.uml @@ -0,0 +1,10 @@ +@startuml +Crow -> Crow: Crow Generates Metrics +Agent -> Crow: Agent scrapes /metrics endpoint +Agent -> Prometheus: Agent sends results to prometheus +Crow -> Prometheus: Crow checks that metrics were written to prometheus +Crow -> Crow: Crow updates success/failure metrics and serves those via /validate +Agent -> Crow: Agent scrapes /validate +Agent -> Prometheus: Agent sends validate metrics +Prometheus -> Prometheus: Prometheus checks alerts +@enduml \ No newline at end of file diff --git a/example/k3d/assets/pending_alert.png b/example/k3d/assets/pending_alert.png new file mode 100644 index 000000000000..a8cd3fc3b5ae Binary files /dev/null and b/example/k3d/assets/pending_alert.png differ diff --git a/example/k3d/assets/trigger_change.png b/example/k3d/assets/trigger_change.png new file mode 100644 index 000000000000..42ce8db2eecf Binary files /dev/null and b/example/k3d/assets/trigger_change.png differ diff --git a/example/k3d/lib/avalanche/main.libsonnet b/example/k3d/lib/avalanche/main.libsonnet new file mode 100644 index 000000000000..6887337ce2a9 --- /dev/null +++ b/example/k3d/lib/avalanche/main.libsonnet @@ -0,0 +1,49 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; + +local configMap = k.core.v1.configMap; +local container = k.core.v1.container; +local containerPort = k.core.v1.containerPort; +local deployment = k.apps.v1.deployment; +local pvc = k.core.v1.persistentVolumeClaim; +local service = k.core.v1.service; +local volumeMount = k.core.v1.volumeMount; +local volume = k.core.v1.volume; + +{ + new(name='avalanche', replicas=1, namespace='', config={}):: { + local this = self, + + _config+:: { + image: 'quay.io/freshtracks.io/avalanche:latest', + + metric_count: 500, + label_count: 10, + series_count: 10, + metricname_length: 5, + labelname_length: 5, + value_interval: 30, + series_interval: 30, + metric_interval: 120, + } + config, + + container:: + container.new(name, this._config.image) + + container.withPorts([ + containerPort.newNamed(name='http', containerPort=9001), + ]) + + container.withArgsMixin([ + '--metric-count=%d' % this._config.metric_count, + '--label-count=%d' % this._config.label_count, + '--series-count=%d' % this._config.series_count, + '--metricname-length=%d' % this._config.metricname_length, + '--labelname-length=%d' % this._config.labelname_length, + '--value-interval=%d' % this._config.value_interval, + '--series-interval=%d' % this._config.series_interval, + '--metric-interval=%d' % this._config.metric_interval, + ]), + + deployment: + deployment.new(name, replicas, [self.container]) + + deployment.mixin.metadata.withNamespace(namespace), + }, +} diff --git a/example/k3d/lib/crow/main.libsonnet b/example/k3d/lib/crow/main.libsonnet new file mode 100644 index 000000000000..29268714461c --- /dev/null +++ b/example/k3d/lib/crow/main.libsonnet @@ -0,0 +1,32 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; + +local configMap = k.core.v1.configMap; +local container = k.core.v1.container; +local containerPort = k.core.v1.containerPort; +local deployment = k.apps.v1.deployment; +local pvc = k.core.v1.persistentVolumeClaim; +local service = k.core.v1.service; +local volumeMount = k.core.v1.volumeMount; +local volume = k.core.v1.volume; + +{ + new(name='crow', namespace='', config={}):: { + local this = self, + + _config+:: { + image: 'grafana/agent-crow:latest', + args: {}, + } + config, + + container:: + container.new(name, this._config.image) + + container.withPorts([ + containerPort.newNamed(name='http-metrics', containerPort=80), + ]) + + container.withArgsMixin(k.util.mapToFlags(this._config.args)), + + deployment: + deployment.new(name, 1, [self.container]) + + deployment.mixin.metadata.withNamespace(namespace), + }, +} diff --git a/example/k3d/lib/default/mixins.libsonnet b/example/k3d/lib/default/mixins.libsonnet index afb721e211a7..4da82b233071 100644 --- a/example/k3d/lib/default/mixins.libsonnet +++ b/example/k3d/lib/default/mixins.libsonnet @@ -13,4 +13,18 @@ local fix = { fix + cortex_mixin + agent_debugging_mixin + -agent_mixin +agent_mixin { + _config+: { + // We run a single-node cortex so replace the job names to all + // be the monolith. + job_names+: { + ingester: 'cortex', + distributor: 'cortex', + querier: 'cortex', + query_frontend: 'cortex', + table_manager: 'cortex', + store_gateway: 'cortex', + gateway: 'cortex', + }, + }, +} diff --git a/example/k3d/lib/grafana/config.libsonnet b/example/k3d/lib/grafana/config.libsonnet index 0451a34faf82..79f7d36743d9 100644 --- a/example/k3d/lib/grafana/config.libsonnet +++ b/example/k3d/lib/grafana/config.libsonnet @@ -1,6 +1,6 @@ { _images: { - grafana: 'grafana/grafana:7.0.1', + grafana: 'grafana/grafana:8.0.3', }, _config: { diff --git a/example/k3d/lib/grafana/main.libsonnet b/example/k3d/lib/grafana/main.libsonnet index 5750c6dd36b2..3c2d5f9014ac 100644 --- a/example/k3d/lib/grafana/main.libsonnet +++ b/example/k3d/lib/grafana/main.libsonnet @@ -5,6 +5,7 @@ local configMap = k.core.v1.configMap; local container = k.core.v1.container; local containerPort = k.core.v1.containerPort; local deployment = k.apps.v1.deployment; +local service = k.core.v1.service; { new(dashboards={}, datasources=[], namespace='default'):: { @@ -20,6 +21,7 @@ local deployment = k.apps.v1.deployment; grafana_cm: configMap.new('grafana-config') + + configMap.mixin.metadata.withNamespace(namespace) + configMap.withData({ 'grafana.ini': std.manifestIni(_config.grafana_ini), }), @@ -29,6 +31,7 @@ local deployment = k.apps.v1.deployment; then {} else configMap.new('dashboards') + + configMap.mixin.metadata.withNamespace(namespace) + configMap.withDataMixin({ [name]: std.toString( $.dashboards[name] @@ -40,6 +43,7 @@ local deployment = k.apps.v1.deployment; grafana_dashboard_cms: { ['dashboard-%d' % shard]: configMap.new('dashboards-%d' % shard) + + configMap.mixin.metadata.withNamespace(namespace) + configMap.withDataMixin({ [name]: std.toString( _dashboards[name] @@ -53,12 +57,14 @@ local deployment = k.apps.v1.deployment; grafana_datasource_cm: configMap.new('grafana-datasources') + + configMap.mixin.metadata.withNamespace(namespace) + configMap.withDataMixin(std.foldl(function(acc, obj) acc { ['%s.yml' % obj.datasources[0].name]: k.util.manifestYaml(obj), }, self._datasources, {})), grafana_dashboard_provisioning_cm: configMap.new('grafana-dashboard-provisioning') + + configMap.mixin.metadata.withNamespace(namespace) + configMap.withData({ 'dashboards.yml': k.util.manifestYaml({ apiVersion: 1, @@ -91,6 +97,7 @@ local deployment = k.apps.v1.deployment; deployment: deployment.new('grafana', 1, [self.container]) + + deployment.mixin.metadata.withNamespace(namespace) + deployment.mixin.spec.template.spec.securityContext.withRunAsUser(0) + k.util.configMapVolumeMount(self.grafana_cm, '/etc/grafana-config') + k.util.configMapVolumeMount(self.grafana_datasource_cm, '%(provisioning_dir)s/datasources' % _config) + @@ -111,7 +118,8 @@ local deployment = k.apps.v1.deployment; k.util.podPriority('critical'), service: - k.util.serviceFor(self.deployment), + k.util.serviceFor(self.deployment) + + service.mixin.metadata.withNamespace(namespace), }, // withDashboards sets the list of dashboards. Dashboards is an object where the diff --git a/example/k3d/lib/kube-state-metrics/main.libsonnet b/example/k3d/lib/kube-state-metrics/main.libsonnet new file mode 100644 index 000000000000..48788c8384f9 --- /dev/null +++ b/example/k3d/lib/kube-state-metrics/main.libsonnet @@ -0,0 +1,135 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; + +local container = k.core.v1.container; +local containerPort = k.core.v1.containerPort; +local deployment = k.apps.v1.deployment; +local service = k.core.v1.service; +local serviceAccount = k.core.v1.serviceAccount; +local policyRule = k.rbac.v1.policyRule; + +{ + new(namespace=''):: { + local k = (import 'ksonnet-util/kausal.libsonnet') { _config+:: { namespace: namespace } }, + + container:: + container.new('kube-state-metrics', 'k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.1.0') + + container.withPorts([ + containerPort.newNamed(name='http-metrics', containerPort=8080), + containerPort.newNamed(name='self-metrics', containerPort=8081), + ]) + + container.withArgs([ + '--port=8080', + '--telemetry-host=0.0.0.0', + '--telemetry-port=8081', + ]), + + rbac: + k.util.rbac('kube-state-metrics', [ + policyRule.withApiGroups(['']) + + policyRule.withResources([ + 'configmaps', + 'secrets', + 'nodes', + 'pods', + 'services', + 'resourcequotas', + 'replicationcontrollers', + 'limitranges', + 'persistentvolumeclaims', + 'persistentvolumes', + 'namespaces', + 'endpoints', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['extensions']) + + policyRule.withResources([ + 'daemonsets', + 'deployments', + 'replicasets', + 'ingresses', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['apps']) + + policyRule.withResources([ + 'daemonsets', + 'deployments', + 'replicasets', + 'statefulsets', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['batch']) + + policyRule.withResources([ + 'cronjobs', + 'jobs', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['autoscaling']) + + policyRule.withResources([ + 'horizontalpodautoscalers', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['authorization.k8s.io']) + + policyRule.withResources(['subjectaccessreviews']) + + policyRule.withVerbs(['create']), + + policyRule.withApiGroups(['ingresses']) + + policyRule.withResources(['ingress']) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['policy']) + + policyRule.withResources(['poddisruptionbudgets']) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['certificates.k8s.io']) + + policyRule.withResources(['certificatesigningrequests']) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['storage.k8s.io']) + + policyRule.withResources([ + 'storageclasses', + 'volumeattachments', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['admissionregistration.k8s.io']) + + policyRule.withResources([ + 'mutatingwebhookconfigurations', + 'validatingwebhookconfigurations', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['networking.k8s.io']) + + policyRule.withResources([ + 'networkpolicies', + 'ingresses', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.withApiGroups(['coordination.k8s.io']) + + policyRule.withResources(['leases']) + + policyRule.withVerbs(['list', 'watch']), + ]) { + service_account+: + serviceAccount.mixin.metadata.withNamespace(namespace), + }, + + deployment: + deployment.new('kube-state-metrics', 1, [self.container]) + + deployment.mixin.metadata.withNamespace(namespace) + + deployment.mixin.spec.template.metadata.withAnnotationsMixin({ 'prometheus.io.scrape': 'false' }) + + deployment.mixin.spec.template.spec.withServiceAccount('kube-state-metrics') + + deployment.mixin.spec.template.spec.securityContext.withRunAsUser(65534) + + deployment.mixin.spec.template.spec.securityContext.withRunAsGroup(65534) + + deployment.mixin.spec.template.spec.securityContext.withFsGroup(0) + + k.util.podPriority('critical'), + + service: + k.util.serviceFor(self.deployment) + + service.mixin.metadata.withNamespace(namespace), + }, +} diff --git a/example/k3d/lib/node-exporter/main.libsonnet b/example/k3d/lib/node-exporter/main.libsonnet new file mode 100644 index 000000000000..5f57f9926928 --- /dev/null +++ b/example/k3d/lib/node-exporter/main.libsonnet @@ -0,0 +1,35 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; + +local container = k.core.v1.container; +local containerPort = k.core.v1.containerPort; +local daemonSet = k.apps.v1.daemonSet; +local service = k.core.v1.service; + +{ + new(namespace=''):: { + container:: + container.new('node-exporter', 'quay.io/prometheus/node-exporter:v1.1.2') + + container.withPorts([ + containerPort.newNamed(name='http-metrics', containerPort=9100), + ]) + + container.withArgsMixin([ + '--path.rootfs=/host/root', + '--path.procfs=/host/proc', + '--path.sysfs=/host/sys', + '--collector.netdev.device-exclude=^veth.+$', + ]) + + container.mixin.securityContext.withPrivileged(true) + + container.mixin.securityContext.withRunAsUser(0), + + daemonSet: + daemonSet.new('node-exporter', [self.container]) + + daemonSet.mixin.metadata.withNamespace(namespace) + + daemonSet.mixin.spec.template.metadata.withAnnotationsMixin({ 'prometheus.io.scrape': 'false' }) + + daemonSet.mixin.spec.template.spec.withHostPid(true) + + daemonSet.mixin.spec.template.spec.withHostNetwork(true) + + k.util.hostVolumeMount('proc', '/proc', '/host/proc') + + k.util.hostVolumeMount('sys', '/sys', '/host/sys') + + k.util.hostVolumeMount('root', '/', '/host/root') + + k.util.podPriority('critical'), + }, +} diff --git a/example/k3d/lib/prometheus/main.libsonnet b/example/k3d/lib/prometheus/main.libsonnet new file mode 100644 index 000000000000..74c9cf6b98b1 --- /dev/null +++ b/example/k3d/lib/prometheus/main.libsonnet @@ -0,0 +1,90 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; + +local configMap = k.core.v1.configMap; +local container = k.core.v1.container; +local containerPort = k.core.v1.containerPort; +local deployment = k.apps.v1.deployment; +local statefulSet = k.apps.v1.statefulSet; +local pvc = k.core.v1.persistentVolumeClaim; +local service = k.core.v1.service; +local volumeMount = k.core.v1.volumeMount; +local volume = k.core.v1.volume; +local serviceAccount = k.core.v1.serviceAccount; +local policyRule = k.rbac.v1.policyRule; + +{ + new(namespace=''):: { + local k = (import 'ksonnet-util/kausal.libsonnet') { _config+:: { namespace: namespace } }, + local this = self, + + _images:: { + prom: 'prom/prometheus:v2.28.0', + }, + + _config:: { + rule_files: ['/etc/prometheus/rules.yaml'], + }, + _rules:: {}, + + rbac: + k.util.rbac('prometheus', [ + policyRule.withApiGroups(['']) + + policyRule.withResources(['nodes', 'nodes/proxy', 'services', 'endpoints', 'pods']) + + policyRule.withVerbs(['get', 'list', 'watch']), + + policyRule.withNonResourceUrls('/metrics') + + policyRule.withVerbs(['get']), + ]) { + service_account+: + serviceAccount.mixin.metadata.withNamespace(namespace), + }, + + configMap: + configMap.new('prometheus') + + configMap.mixin.metadata.withNamespace(namespace) + + configMap.withData({ + 'prometheus.yaml': k.util.manifestYaml(this._config), + 'rules.yaml': k.util.manifestYaml(this._rules), + }), + + container:: + container.new('prometheus', this._images.prom) + + container.withPorts([ + containerPort.newNamed(name='http-metrics', containerPort=9090), + ]) + + container.withVolumeMountsMixin( + volumeMount.new('prometheus-data', '/data'), + ) + + container.withArgsMixin([ + '--config.file=/etc/prometheus/prometheus.yaml', + '--storage.tsdb.path=/data', + ]), + + pvc:: + { apiVersion: 'v1', kind: 'PersistentVolumeClaim' } + + pvc.new() + + pvc.mixin.metadata.withName('prometheus-data') + + pvc.mixin.metadata.withNamespace(namespace) + + pvc.mixin.spec.withAccessModes('ReadWriteOnce') + + pvc.mixin.spec.resources.withRequests({ storage: '10Gi' }), + + statefulSet: + statefulSet.new( + name='prometheus', + replicas=1, + containers=[this.container], + volumeClaims=[this.pvc] + ) + + statefulSet.mixin.spec.withServiceName('prometheus') + + k.util.configMapVolumeMount(this.configMap, '/etc/prometheus') + + statefulSet.mixin.spec.template.spec.withServiceAccountName('prometheus') + + statefulSet.mixin.metadata.withNamespace(namespace), + + service: + k.util.serviceFor(this.statefulSet) + + service.mixin.metadata.withNamespace(namespace), + }, + + withConfigMixin(config={}):: { _config+:: config }, + withRulesMixin(rules={}):: { _rules+:: rules }, +} diff --git a/example/k3d/scripts/smoke-test.bash b/example/k3d/scripts/smoke-test.bash new file mode 100755 index 000000000000..0d37d693c664 --- /dev/null +++ b/example/k3d/scripts/smoke-test.bash @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# +# Usage: +# smoke-test.bash [-i] [-d] [-t ] +# +# Dependencies: +# k3d >=3.0 +# Tanka +# jq +# +# smoke-test.bash performs smoke tests that can be used to validate a release. +# It only validates correctness, and does not attempt to do significant load +# testing or performance benchmarks. +# +# It works by deploying a k3d cluster with two pairs of Agent deployments: one +# using the scraping service, and one using host filtering. Each Agent deployment +# monitors a unique set of correctness tools (Loki Canary, Tempo Vulture, +# Cortex text-exporter). These tools expose correctness metrics, generated by +# by querying the backends that the Grafana Agents are configured to send +# telemetry data to. +# +# Grafana and Prometheus are deployed to the cluster and are responsible for +# internal monitoring. Prometheus is configured with a set of alert rules that serve +# as test cases for the smoke tests; alerts generated within the span of the testing +# period are treated as failures. +# +# After the smoke-test duration period (defaulting to 3h), alerts will +# be checked, and the script will end. +# +# The k3d cluster is kept alive after the test for analysis. To clean up assets created +# by the script, re-run the script with the -d flag. + +set -euo pipefail + +# Constants +ROOT=$(git rev-parse --show-toplevel) +K3D_CLUSTER_NAME="agent-smoke-test" +MUTATION_FREQUENCY="5m" +CHAOS_FREQUENCY="30m" + +# Variables + +# Which function will be called +ENTRYPOINT="run" +TEST_DURATION="3h" +IMPORT_IMAGES="" + +while getopts "dt:ih" opt; do + case $opt in + d) ENTRYPOINT="cleanup" ;; + t) TEST_DURATION=$OPTARG ;; + i) IMPORT_IMAGES="yes" ;; + h) + echo "Usage: $0 [-i] [-d] [-t ]" + exit 0 + ;; + *) + echo "Usage: $0 [-i] [-d] [-t ]" + exit 1 + ;; + esac +done + +# Run runs the smoke test for $TEST_DURATION. +run() { + echo "--- Creating k3d cluster $K3D_CLUSTER_NAME" + k3d cluster create $K3D_CLUSTER_NAME \ + --port 50080:80@loadbalancer \ + --api-port 50443 \ + --kubeconfig-update-default=true \ + --kubeconfig-switch-context=true \ + --wait >/dev/null + + # Give the cluster a little bit of time to settle before + # applying the environment + echo "--- Waiting for cluster to warm up" + sleep 10 + + if [[ ! -z "$IMPORT_IMAGES" ]]; then + echo "--- Importing local images" + + k3d image import -c $K3D_CLUSTER_NAME \ + grafana/agent:latest \ + grafana/agentctl:latest \ + grafana/agent-crow:latest + fi + + tk apply $ROOT/example/k3d/smoke --dangerous-auto-approve + + echo "--- Spawning background tasks" + mutation_loop & + MUTATION_PID=$! + chaos_loop & + CHAOS_PID=$! + + # Immediately create a job to sync configs so our two Agent deployments + # are synced up as closely as possible. + kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \ + create job --from=cronjob/grafana-agent-syncer \ + grafana-agent-syncer-startup + + echo "Your environment is now running for the next $TEST_DURATION." + echo "Grafana URL: http://grafana.k3d.localhost:50080" + echo "Prometheus URL: http://prometheus.k3d.localhost:50080" + sleep $TEST_DURATION + + echo "--- Stopping background tasks" + kill $CHAOS_PID $MUTATION_PID + + echo "Smoke tests complete!" + echo "Grafana URL: http://grafana.k3d.localhost:50080" + echo "Prometheus URL: http://prometheus.k3d.localhost:50080" + echo "" + echo "Getting results..." + + get_results +} + +# mutation_loop changes the number of replicas of Avalanche servers +# randomly between 0-10 replicas (inclusive) every $MUTATION_FREQUENCY. +mutation_loop() { + while true; do + sleep $MUTATION_FREQUENCY + + NEW_REPLICAS=$(($RANDOM % 11)) + echo "--- Scaling Avalanche to $NEW_REPLICAS replicas" + kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \ + scale --replicas=$NEW_REPLICAS deployment/avalanche + done +} + +# chaos loop deletes pods every $CHAOS_FREQUENCY. +chaos_loop() { + while true; do + sleep $CHAOS_FREQUENCY + + echo "--- Force deleting pod/grafana-agent-0" + kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \ + delete --grace-period=0 --force pod/grafana-agent-0 + + REPLICA=$(($RANDOM % 3)) + echo "--- Force deleting pod/agent-cluster-$REPLICA" + kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \ + delete --grace-period=0 --force pod/grafana-agent-cluster-$REPLICA + done +} + +get_results() { + NUM_ALERTS=$(curl -s -G \ + -H "Host: prometheus.k3d.localhost" \ + -d "query=count_over_time(ALERTS{alertstate=\"firing\"}[$TEST_DURATION])" \ + 'http://localhost:50080/api/v1/query' \ + | jq '.data.result | length' \ + ) + + if test $NUM_ALERTS -ne 0; then + echo "FAIL: $NUM_ALERTS alerts found over the last $TEST_DURATION." + echo "More information: http://prometheus.k3d.localhost:50080/graph?g0.expr=count_over_time(ALERTS{alertstate%3D%22firing%22}[$TEST_DURATION])" + + exit 1 + else + echo "PASS: 0 alerts found over the last $TEST_DURATION. You're good to go!" + + exit 0 + fi +} + +cleanup() { + echo "--- Deleting k3d cluster $K3D_CLUSTER_NAME" + k3d cluster delete $K3D_CLUSTER_NAME >/dev/null +} + +$ENTRYPOINT diff --git a/example/k3d/smoke/main.jsonnet b/example/k3d/smoke/main.jsonnet new file mode 100644 index 000000000000..57f6e4f26347 --- /dev/null +++ b/example/k3d/smoke/main.jsonnet @@ -0,0 +1,187 @@ +local monitoring = import './monitoring/main.jsonnet'; +local avalanche = import 'avalanche/main.libsonnet'; +local cortex = import 'cortex/main.libsonnet'; +local crow = import 'crow/main.libsonnet'; +local etcd = import 'etcd/main.libsonnet'; +local gragent = import 'grafana-agent/v2/main.libsonnet'; +local k = import 'ksonnet-util/kausal.libsonnet'; + +local namespace = k.core.v1.namespace; +local pvc = k.core.v1.persistentVolumeClaim; +local volumeMount = k.core.v1.volumeMount; + +local images = { + agent: 'grafana/agent:latest', + agentctl: 'grafana/agentctl:latest', +}; + +local new_crow(name, selector) = + crow.new(name, namespace='smoke', config={ + args: { + 'crow.prometheus-addr': 'http://cortex/api/prom', + 'crow.extra-selectors': selector, + }, + }); + +local smoke = { + ns: namespace.new('smoke'), + + cortex: cortex.new('smoke'), + + // Needed to run agent cluster + etcd: etcd.new('smoke'), + + avalanche: avalanche.new(replicas=3, namespace='smoke', config={ + // We're going to be running a lot of these and we're not trying to test + // for load, so reduce the cardinality and churn rate. + metric_count: 50, + series_interval: 300, + metric_interval: 600, + }), + + crows: [ + new_crow('crow-single', 'cluster="grafana-agent"'), + new_crow('crow-cluster', 'cluster="grafana-agent-cluster"'), + ], + + local metric_instances(crow_name) = [{ + name: 'crow', + remote_write: [{ url: 'http://cortex/api/prom/push' }], + scrape_configs: [ + { + job_name: 'crow', + metrics_path: '/validate', + + kubernetes_sd_configs: [{ role: 'pod' }], + tls_config: { + ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + }, + bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', + + relabel_configs: [{ + source_labels: ['__meta_kubernetes_namespace'], + regex: 'smoke', + action: 'keep', + }, { + source_labels: ['__meta_kubernetes_pod_container_name'], + regex: crow_name, + action: 'keep', + }], + }, + ], + }, { + name: 'avalanche', + remote_write: [{ url: 'http://cortex/api/prom/push' }], + scrape_configs: [ + { + job_name: 'avalanche', + kubernetes_sd_configs: [{ role: 'pod' }], + tls_config: { + ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + }, + bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', + + relabel_configs: [{ + source_labels: ['__meta_kubernetes_namespace'], + regex: 'smoke', + action: 'keep', + }, { + source_labels: ['__meta_kubernetes_pod_container_name'], + regex: 'avalanche', + action: 'keep', + }], + }, + ], + }], + + normal_agent: + gragent.new(name='grafana-agent', namespace='smoke') + + gragent.withImagesMixin(images) + + gragent.withStatefulSetController( + replicas=1, + volumeClaims=[ + pvc.new() + + pvc.mixin.metadata.withName('agent-wal') + + pvc.mixin.metadata.withNamespace('smoke') + + pvc.mixin.spec.withAccessModes('ReadWriteOnce') + + pvc.mixin.spec.resources.withRequests({ storage: '5Gi' }), + ], + ) + + gragent.withVolumeMountsMixin([volumeMount.new('agent-wal', '/var/lib/agent')]) + + gragent.withAgentConfig({ + server: { log_level: 'debug' }, + + prometheus: { + global: { + scrape_interval: '15s', + external_labels: { + cluster: 'grafana-agent', + }, + }, + wal_directory: '/var/lib/agent/data', + configs: metric_instances('crow-single'), + }, + }), + + cluster_agent: + gragent.new(name='grafana-agent-cluster', namespace='smoke') + + gragent.withImagesMixin(images) + + gragent.withStatefulSetController( + replicas=3, + volumeClaims=[ + pvc.new() + + pvc.mixin.metadata.withName('agent-cluster-wal') + + pvc.mixin.metadata.withNamespace('smoke') + + pvc.mixin.spec.withAccessModes('ReadWriteOnce') + + pvc.mixin.spec.resources.withRequests({ storage: '5Gi' }), + ], + ) + + gragent.withVolumeMountsMixin([volumeMount.new('agent-cluster-wal', '/var/lib/agent')]) + + gragent.withAgentConfig({ + server: { log_level: 'debug' }, + + prometheus: { + global: { + scrape_interval: '15s', + external_labels: { + cluster: 'grafana-agent-cluster', + }, + }, + wal_directory: '/var/lib/agent/data', + + scraping_service: { + enabled: true, + dangerous_allow_reading_files: true, + kvstore: { + store: 'etcd', + etcd: { endpoints: ['etcd:2379'] }, + }, + lifecycler: { + ring: { + kvstore: { + store: 'etcd', + etcd: { endpoints: ['etcd:2379'] }, + }, + }, + }, + }, + }, + }), + + // Spawn a syncer so our cluster gets the same scrape jobs as our + // normal agent. + sycner: gragent.newSyncer( + name='grafana-agent-syncer', + namespace='smoke', + config={ + image: images.agentctl, + api: 'http://grafana-agent-cluster.smoke.svc.cluster.local', + configs: metric_instances('crow-cluster'), + } + ), +}; + +{ + monitoring: monitoring, + smoke: smoke, +} diff --git a/example/k3d/smoke/monitoring/main.jsonnet b/example/k3d/smoke/monitoring/main.jsonnet new file mode 100644 index 000000000000..fc226923f448 --- /dev/null +++ b/example/k3d/smoke/monitoring/main.jsonnet @@ -0,0 +1,60 @@ +local grafana_mixins = import 'default/mixins.libsonnet'; +local datasource = import 'grafana/datasource.libsonnet'; +local grafana = import 'grafana/main.libsonnet'; +local k = import 'ksonnet-util/kausal.libsonnet'; +local kube_state_metrics = import 'kube-state-metrics/main.libsonnet'; +local node_exporter = import 'node-exporter/main.libsonnet'; +local prometheus = import 'prometheus/main.libsonnet'; + +local namespace = k.core.v1.namespace; +local ingress = k.networking.v1beta1.ingress; +local rule = k.networking.v1beta1.ingressRule; +local path = k.networking.v1beta1.httpIngressPath; + +local prometheus_monitoring = import './prometheus_monitoring.libsonnet'; + +{ + ns: namespace.new('monitoring'), + + grafana: + grafana.new(namespace='monitoring') + + grafana.withDashboards(grafana_mixins.grafanaDashboards) + + grafana.withDataSources([ + datasource.new('Prometheus', 'http://prometheus.monitoring.svc.cluster.local:9090', default='true'), + datasource.new('Cortex', 'http://cortex.smoke.svc.cluster.local/api/prom'), + ]), + + prometheus: + prometheus.new(namespace='monitoring') + + prometheus.withConfigMixin(prometheus_monitoring.config) + + prometheus.withRulesMixin(prometheus_monitoring.rules), + + node_exporter: node_exporter.new(namespace='monitoring'), + kube_state_metrics: kube_state_metrics.new(namespace='monitoring'), + + ingresses: { + prometheus: + ingress.new('prometheus') + + ingress.mixin.metadata.withNamespace('monitoring') + + ingress.mixin.spec.withRules([ + rule.withHost('prometheus.k3d.localhost') + + rule.http.withPaths([ + path.withPath('/') + + path.backend.withServiceName('prometheus') + + path.backend.withServicePort(9090), + ]), + ]), + + grafana: + ingress.new('grafana') + + ingress.mixin.metadata.withNamespace('monitoring') + + ingress.mixin.spec.withRules([ + rule.withHost('grafana.k3d.localhost') + + rule.http.withPaths([ + path.withPath('/') + + path.backend.withServiceName('grafana') + + path.backend.withServicePort(80), + ]), + ]), + }, +} diff --git a/example/k3d/smoke/monitoring/prometheus_monitoring.libsonnet b/example/k3d/smoke/monitoring/prometheus_monitoring.libsonnet new file mode 100644 index 000000000000..a53dc0894747 --- /dev/null +++ b/example/k3d/smoke/monitoring/prometheus_monitoring.libsonnet @@ -0,0 +1,152 @@ +local agent_prometheus = import 'grafana-agent/v1/lib/prometheus.libsonnet'; + +{ + config: { + global: { + scrape_interval: '15s', + }, + scrape_configs: agent_prometheus.scrapeInstanceKubernetes.scrape_configs, + }, + + rules: { + groups: [ + { + name: 'GrafanaAgentChecks', + rules: [ + // Basic sanity checks: ensure that Agents exist, are up, + // and haven't been flapping. + { + alert: 'GrafanaAgentMissing', + expr: ||| + absent(up{ namespace="smoke", pod="grafana-agent-0" }) == 1 or + absent(up{ namespace="smoke", pod="grafana-agent-cluster-0" }) == 1 or + absent(up{ namespace="smoke", pod="grafana-agent-cluster-1" }) == 1 or + absent(up{ namespace="smoke", pod="grafana-agent-cluster-2" }) == 1 + |||, + 'for': '5m', + annotations: { + summary: '{{ $labels.pod }} is not running.', + }, + }, + { + alert: 'GrafanaAgentDown', + expr: ||| + up{ + namespace="smoke", + pod=~"grafana-agent-(0|cluster-0|cluster-1|cluster-2)", + } == 0 + |||, + 'for': '5m', + annotations: { + summary: '{{ $labels.job }} is down', + }, + }, + { + alert: 'GrafanaAgentFlapping', + expr: ||| + avg_over_time(up{ + namespace="smoke", + pod=~"grafana-agent-(0|cluster-0|cluster-1|cluster-2)", + }[5m]) < 1 + |||, + 'for': '15m', + annotations: { + summary: '{{ $labels.job }} is flapping', + }, + }, + + // Checks that the CPU usage doesn't go too high. This was generated + // from main where the CPU usage hovered around 2-3% per pod. + // + // TODO: something less guessworky here. + { + alert: 'GrafanaAgentCPUHigh', + expr: ||| + rate(container_cpu_usage_seconds_total{namespace="smoke", pod=~"grafana-agent-.*"}[1m]) > 0.05 + |||, + 'for': '5m', + annotations: { + summary: '{{ $labels.pod }} is using more than 5% CPU over the last 5 minutes', + }, + }, + + // We assume roughly ~8KB per series. Check that each deployment + // doesn't go too far above this. + // + // We aggregate the memory of the scraping service together since an individual + // node with a really small number of active series will throw this metric off. + { + alert: 'GrafanaAgentMemHigh', + expr: ||| + sum without (pod, instance) (go_memstats_heap_inuse_bytes{job=~"smoke/grafana-agent.*"}) / + sum without (pod, instance, instance_group_name) (agent_wal_storage_active_series{job=~"smoke/grafana-agent.*"}) / 1e3 > 10 + |||, + 'for': '5m', + annotations: { + summary: '{{ $labels.job }} has used more than 10KB per series for more than 5 minutes', + }, + }, + ], + }, + { + name: 'CrowChecks', + rules: [ + { + alert: 'CrowMissing', + expr: ||| + absent(up{container="crow-single"}) == 1 or + absent(up{container="crow-cluster"}) == 1 + |||, + 'for': '5m', + annotations: { + summary: '{{ $labels.container }} is not running.', + }, + }, + { + alert: 'CrowDown', + expr: ||| + up{job=~"smoke/crow-.*"} == 0 + |||, + 'for': '5m', + annotations: { + summary: 'Crow {{ $labels.job }} is down.', + }, + }, + { + alert: 'CrowFlapping', + expr: ||| + avg_over_time(up{job=~"smoke/crow-.*"}[5m]) < 1 + |||, + 'for': '15m', + annotations: { + summary: 'Crow {{ $labels.job }} is flapping.', + }, + }, + { + alert: 'CrowNotScraped', + expr: ||| + rate(crow_test_samples_total[1m]) == 0 + |||, + 'for': '5m', + annotations: { + summary: 'Crow {{ $labels.job }} is not being scraped.', + }, + }, + { + alert: 'CrowFailures', + expr: ||| + ( + rate(crow_test_sample_results_total{result="success"}[1m]) + / ignoring(result) rate(crow_test_samples_total[1m]) + ) < 1 + |||, + 'for': '5m', + annotations: { + summary: 'Crow {{ $labels.job }} has had failures for at least 5m', + }, + }, + ], + }, + ], + }, +} diff --git a/example/k3d/smoke/spec.json b/example/k3d/smoke/spec.json new file mode 100644 index 000000000000..8c90817a7a99 --- /dev/null +++ b/example/k3d/smoke/spec.json @@ -0,0 +1,11 @@ +{ + "apiVersion": "tanka.dev/v1alpha1", + "kind": "Environment", + "metadata": { + "name": "default" + }, + "spec": { + "apiServer": "https://0.0.0.0:50443", + "namespace": "default" + } +} diff --git a/pkg/crow/crow.go b/pkg/crow/crow.go index 0cfff6f8c25a..cd14558a78ab 100644 --- a/pkg/crow/crow.go +++ b/pkg/crow/crow.go @@ -86,7 +86,7 @@ var DefaultConfig = Config{ ValueEpsilon: 0.0001, } -// Crow is a collectness checker that validates scraped metrics reach a +// Crow is a correctness checker that validates scraped metrics reach a // Prometheus-compatible server with the same values and roughly the same // timestamp. // @@ -96,12 +96,12 @@ var DefaultConfig = Config{ // 2. State metrics, exposing state of the Crow checker itself. // // These two metrics should be exposed via different endpoints, and only state -// metrics are safe to be manually collecetd from. +// metrics are safe to be manually collected from. // // Collecting from the set of test metrics generates a validation job, where // Crow will query the Prometheus API to ensure the metrics that were scraped // were written with (approximately) the same timestamp as the scrape time -// and with (approximately) the same floatnig point values exposed in the +// and with (approximately) the same floating point values exposed in the // scrape. // // If a set of test metrics were not found and retries have been exhausted, diff --git a/pkg/crow/samples.go b/pkg/crow/samples.go index 58751d501155..066b21008af4 100644 --- a/pkg/crow/samples.go +++ b/pkg/crow/samples.go @@ -13,7 +13,7 @@ type sample struct { Labels prometheus.Labels Value float64 - // How many times this sample has attempted to be valdated. Starts at 0. + // How many times this sample has attempted to be validated. Starts at 0. ValidationAttempt int } diff --git a/production/grafana-agent-mixin/debugging.libsonnet b/production/grafana-agent-mixin/debugging.libsonnet index 52c12f5a0367..cc671f344e46 100644 --- a/production/grafana-agent-mixin/debugging.libsonnet +++ b/production/grafana-agent-mixin/debugging.libsonnet @@ -1,10 +1,10 @@ -local g = import 'grafana-builder/grafana.libsonnet'; local utils = import './utils.libsonnet'; +local g = import 'grafana-builder/grafana.libsonnet'; { grafanaDashboards+:: { 'agent-operational.json': - utils.injectUtils(g.dashboard('Agent Operational')) + utils.injectUtils(g.dashboard('Agent Operational')) .addMultiTemplate('cluster', 'agent_build_info', 'cluster') .addMultiTemplate('namespace', 'agent_build_info', 'namespace') .addMultiTemplate('container', 'agent_build_info', 'container') @@ -21,7 +21,6 @@ local utils = import './utils.libsonnet'; .addPanel( g.panel('Go Heap') + { yaxes: g.yaxes('decbytes') } + - { stack: 'true' } + g.queryPanel( 'go_memstats_heap_inuse_bytes{cluster=~"$cluster", namespace=~"$namespace", container=~"$container", pod=~"$pod"}', '{{pod}}', @@ -78,7 +77,6 @@ local utils = import './utils.libsonnet'; .addPanel( g.panel('Bytes/Series/Pod') + { yaxes: g.yaxes('decbytes') } + - { stack: 'true' } + g.queryPanel( ||| (sum by (pod) (avg_over_time(go_memstats_heap_inuse_bytes{cluster=~"$cluster", namespace=~"$namespace", container=~"$container", pod=~"$pod"}[1m]))) @@ -91,7 +89,6 @@ local utils = import './utils.libsonnet'; .addPanel( g.panel('Bytes/Series') + { yaxes: g.yaxes('decbytes') } + - { stack: 'true' } + g.queryPanel( ||| (sum by (container) (avg_over_time(go_memstats_heap_inuse_bytes{cluster=~"$cluster", namespace=~"$namespace", container=~"$container", pod=~"$pod"}[1m]))) @@ -103,7 +100,6 @@ local utils = import './utils.libsonnet'; ) .addPanel( g.panel('Series/Pod') + - { stack: 'true' } + g.queryPanel( 'sum by (pod) (agent_wal_storage_active_series{cluster=~"$cluster", namespace=~"$namespace", container=~"$container", pod=~"$pod"})', '{{pod}}', @@ -111,7 +107,6 @@ local utils = import './utils.libsonnet'; ) .addPanel( g.panel('Series/Config') + - { stack: 'true' } + g.queryPanel( 'sum by (instance_group_name) (agent_wal_storage_active_series{cluster=~"$cluster", namespace=~"$namespace", container=~"$container", pod=~"$pod"})', '{{instance_group_name}}', @@ -119,7 +114,6 @@ local utils = import './utils.libsonnet'; ) .addPanel( g.panel('Series') + - { stack: 'true' } + g.queryPanel( 'sum by (container) (agent_wal_storage_active_series{cluster=~"$cluster", namespace=~"$namespace", container=~"$container", pod=~"$pod"})', '{{container}}', diff --git a/production/tanka/grafana-agent/config.libsonnet b/production/tanka/grafana-agent/config.libsonnet index 84497843a108..f19bd723a432 100644 --- a/production/tanka/grafana-agent/config.libsonnet +++ b/production/tanka/grafana-agent/config.libsonnet @@ -1,3 +1,5 @@ +local k8s_v2 = import './v2/internal/helpers/k8s.libsonnet'; + { _images+:: { agent: 'grafana/agent:latest', @@ -91,316 +93,24 @@ }, - // + local all_scrape_configs = k8s_v2.metrics({ + scrape_api_server_endpoints: $._config.scrape_api_server_endpoints, + insecure_skip_verify: $._config.prometheus_insecure_skip_verify, + kubernetes_api_server_address: $._config.prometheus_kubernetes_api_server_address, + ksm_namespace: $._config.namespace, + node_exporter_namespace: $._config.namespace, + }), + // We have two optional extension points for scrape config. One for the // statefulset that holds all the agents attached to a node // (kubernetes_scrape_configs) and One for the single replica deployment // that is used to scrape jobs that don't work with host filtering mode // (deployment_scrape_configs) the later is only used when host_filter = // true. - deployment_scrape_configs: [ - { - job_name: 'default/kubernetes', - kubernetes_sd_configs: [{ - role: - if $._config.scrape_api_server_endpoints - then 'endpoints' - else 'service', - }], - scheme: 'https', - - tls_config: { - ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', - insecure_skip_verify: $._config.prometheus_insecure_skip_verify, - server_name: 'kubernetes', - }, - bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', - relabel_configs: [{ - source_labels: ['__meta_kubernetes_service_label_component'], - regex: 'apiserver', - action: 'keep', - }], - - // Keep limited set of metrics to reduce default usage, drop all others - metric_relabel_configs: [ - { - source_labels: ['__name__'], - regex: 'workqueue_queue_duration_seconds_bucket|process_cpu_seconds_total|process_resident_memory_bytes|workqueue_depth|rest_client_request_duration_seconds_bucket|workqueue_adds_total|up|rest_client_requests_total|apiserver_request_total|go_goroutines', - action: 'keep', - }, - ], - - }, - ], - kubernetes_scrape_configs: [ - { - job_name: 'kubernetes-pods', - kubernetes_sd_configs: [{ - role: 'pod', - }], - - // You can specify the following annotations (on pods): - // prometheus.io/scrape: false - don't scrape this pod - // prometheus.io/scheme: https - use https for scraping - // prometheus.io/port - scrape this port - // prometheus.io/path - scrape this path - // prometheus.io/param- - send ?parameter=value with the scrape - relabel_configs: [ - // Drop anything annotated with prometheus.io/scrape=false - { - source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_scrape'], - action: 'drop', - regex: 'false', - }, - - // Drop any endpoint whose pod port name does not end with metrics - { - source_labels: ['__meta_kubernetes_pod_container_port_name'], - action: 'keep', - regex: '.*-metrics', - }, - - // Allow pods to override the scrape scheme with prometheus.io/scheme=https - { - source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_scheme'], - action: 'replace', - target_label: '__scheme__', - regex: '(https?)', - replacement: '$1', - }, - - // Allow service to override the scrape path with prometheus.io/path=/other_metrics_path - { - source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_path'], - action: 'replace', - target_label: '__metrics_path__', - regex: '(.+)', - replacement: '$1', - }, - - // Allow services to override the scrape port with prometheus.io/port=1234 - { - source_labels: ['__address__', '__meta_kubernetes_pod_annotation_prometheus_io_port'], - action: 'replace', - target_label: '__address__', - regex: '(.+?)(\\:\\d+)?;(\\d+)', - replacement: '$1:$3', - }, - - // Drop pods without a name label - { - source_labels: ['__meta_kubernetes_pod_label_name'], - action: 'drop', - regex: '', - }, - - // Rename jobs to be / - { - source_labels: ['__meta_kubernetes_namespace', '__meta_kubernetes_pod_label_name'], - action: 'replace', - separator: '/', - target_label: 'job', - replacement: '$1', - }, - - // But also include the namespace as a separate label for routing alerts - { - source_labels: ['__meta_kubernetes_namespace'], - action: 'replace', - target_label: 'namespace', - }, - { - source_labels: ['__meta_kubernetes_pod_name'], - action: 'replace', - target_label: 'pod', // Not 'pod_name', which disappeared in K8s 1.16. - }, - { - source_labels: ['__meta_kubernetes_pod_container_name'], - action: 'replace', - target_label: 'container', // Not 'container_name', which disappeared in K8s 1.16. - }, - - // Rename instances to the concatenation of pod:container:port. - // All three components are needed to guarantee a unique instance label. - { - source_labels: [ - '__meta_kubernetes_pod_name', - '__meta_kubernetes_pod_container_name', - '__meta_kubernetes_pod_container_port_name', - ], - action: 'replace', - separator: ':', - target_label: 'instance', - }, - - // Map prometheus.io/param-=value fields to __param_=value - { - regex: '__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)', - action: 'labelmap', - replacement: '__param_$1', - }, - - // Drop pods with phase Succeeded or Failed - { - source_labels: ['__meta_kubernetes_pod_phase'], - action: 'drop', - regex: 'Succeeded|Failed', - }, - ], - }, - - // A separate scrape config for kube-state-metrics which doesn't add a - // namespace label and instead takes the namespace label from the - // exported timeseries. This prevents the exported namespace label from - // being renamed to exported_namespace and allows us to route alerts - // based on namespace. - { - job_name: '%s/kube-state-metrics' % $._config.namespace, - kubernetes_sd_configs: [{ - role: 'pod', - namespaces: { - names: [$._config.namespace], - }, - }], - - relabel_configs: [ - // Drop anything whose service is not kube-state-metrics - { - source_labels: ['__meta_kubernetes_pod_label_name'], - regex: 'kube-state-metrics', - action: 'keep', - }, - - // Rename instances to the concatenation of pod:container:port. - // In the specific case of KSM, we could leave out the container - // name and still have a unique instance label, but we leave it - // in here for consistency with the normal pod scraping. - { - source_labels: [ - '__meta_kubernetes_pod_name', - '__meta_kubernetes_pod_container_name', - '__meta_kubernetes_pod_container_port_name', - ], - action: 'replace', - separator: ':', - target_label: 'instance', - }, - ], - }, - - // A separate scrape config for node-exporter which maps the nodename - // onto the instance label. - { - job_name: '%s/node-exporter' % $._config.namespace, - kubernetes_sd_configs: [{ - role: 'pod', - namespaces: { - names: [$._config.namespace], - }, - }], - - relabel_configs: [ - // Drop anything whose name is not node-exporter. - { - source_labels: ['__meta_kubernetes_pod_label_name'], - regex: 'node-exporter', - action: 'keep', - }, - - // Rename instances to be the node name. - { - source_labels: ['__meta_kubernetes_pod_node_name'], - action: 'replace', - target_label: 'instance', - }, - - // But also include the namespace as a separate label, for - // routing alerts. - { - source_labels: ['__meta_kubernetes_namespace'], - action: 'replace', - target_label: 'namespace', - }, - ], - }, - - // This scrape config gathers all kubelet metrics. - { - job_name: 'kube-system/kubelet', - kubernetes_sd_configs: [{ - role: 'node', - }], - - tls_config: { - ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', - insecure_skip_verify: $._config.prometheus_insecure_skip_verify, - }, - bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', - - relabel_configs: [ - { - target_label: '__address__', - replacement: $._config.prometheus_kubernetes_api_server_address, - }, - { - target_label: '__scheme__', - replacement: 'https', - }, - { - source_labels: ['__meta_kubernetes_node_name'], - regex: '(.+)', - target_label: '__metrics_path__', - replacement: '/api/v1/nodes/$1/proxy/metrics', - }, - ], - }, - - // As of k8s 1.7.3, cAdvisor metrics are available via kubelet using - // the /metrics/cadvisor path. - { - job_name: 'kube-system/cadvisor', - kubernetes_sd_configs: [{ - role: 'node', - }], - scheme: 'https', - - tls_config: { - ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', - insecure_skip_verify: $._config.prometheus_insecure_skip_verify, - }, - bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', - - relabel_configs: [ - { - target_label: '__address__', - replacement: $._config.prometheus_kubernetes_api_server_address, - }, - { - source_labels: ['__meta_kubernetes_node_name'], - regex: '(.+)', - target_label: '__metrics_path__', - replacement: '/api/v1/nodes/$1/proxy/metrics/cadvisor', - }, - ], - - metric_relabel_configs: [ - // Drop container_* metrics with no image. - { - source_labels: ['__name__', 'image'], - regex: 'container_([a-z_]+);', - action: 'drop', - }, - - // Drop a bunch of metrics which are disabled but still sent, - // see https://github.com/google/cadvisor/issues/1925. - { - source_labels: ['__name__'], - regex: 'container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)', - action: 'drop', - }, - ], - }, - ], + deployment_scrape_configs: + std.filter(function(job) job.job_name == 'default/kubernetes', all_scrape_configs), + kubernetes_scrape_configs: + std.filter(function(job) job.job_name != 'default/kubernetes', all_scrape_configs), agent_remote_write: [], }, diff --git a/production/tanka/grafana-agent/v1/README.md b/production/tanka/grafana-agent/v1/README.md index fedaf8ea321f..81b82390de11 100644 --- a/production/tanka/grafana-agent/v1/README.md +++ b/production/tanka/grafana-agent/v1/README.md @@ -1,6 +1,6 @@ # Tanka Configs -**STATUS**: Work in progress, use of these configs is not recommended for production. +**STATUS**: Abandoned. Use v0 (parent directory) or v2 instead. This directory contains the Tanka configs that we use to deploy the Grafana Agent. It is marked as `v1` and is incompatible with the `v0` configs diff --git a/production/tanka/grafana-agent/v1/internal/kubernetes_instance.libsonnet b/production/tanka/grafana-agent/v1/internal/kubernetes_instance.libsonnet index 2b206001bf14..a4749c9797fc 100644 --- a/production/tanka/grafana-agent/v1/internal/kubernetes_instance.libsonnet +++ b/production/tanka/grafana-agent/v1/internal/kubernetes_instance.libsonnet @@ -1,10 +1,4 @@ -local k8s_tls_config(config) = { - tls_config: { - ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', - insecure_skip_verify: config.insecure_skip_verify, - }, - bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', -}; +local k8s_v2 = import '../../v2/internal/helpers/k8s.libsonnet'; { kubernetesScrapeInstanceConfig:: { @@ -17,289 +11,17 @@ local k8s_tls_config(config) = { }, newKubernetesScrapeInstance(config, namespace='default'):: { - name: 'kubernetes', - scrape_configs: [ - k8s_tls_config(config) { - job_name: 'default/kubernetes', - kubernetes_sd_configs: [{ - role: if config.scrape_api_server_endpoints then 'endpoints' else 'service', - }], - scheme: 'https', - tls_config+: { - server_name: 'kubernetes', - }, - - relabel_configs: [{ - source_labels: ['__meta_kubernetes_service_label_component'], - regex: 'apiserver', - action: 'keep', - }], - - // Keep limited set of metrics to reduce default usage, drop all others - metric_relabel_configs: [ - { - source_labels: ['__name__'], - regex: 'workqueue_queue_duration_seconds_bucket|process_cpu_seconds_total|process_resident_memory_bytes|workqueue_depth|rest_client_request_duration_seconds_bucket|workqueue_adds_total|up|rest_client_requests_total|apiserver_request_total|go_goroutines', - action: 'keep', - }, - ], - }, - - { - job_name: 'kubernetes-pods', - kubernetes_sd_configs: [{ - role: 'pod', - }], - - // You can specify the following annotations (on pods): - // prometheus.io/scrape: false - don't scrape this pod - // prometheus.io/scheme: https - use https for scraping - // prometheus.io/port - scrape this port - // prometheus.io/path - scrape this path - // prometheus.io/param- - send ?parameter=value with the scrape - relabel_configs: [ - // Drop anything annotated with prometheus.io/scrape=false - { - source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_scrape'], - action: 'drop', - regex: 'false', - }, - - // Drop any endpoint whose pod port name does not end with metrics - { - source_labels: ['__meta_kubernetes_pod_container_port_name'], - action: 'keep', - regex: '.*-metrics', - }, - - // Allow pods to override the scrape scheme with prometheus.io/scheme=https - { - source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_scheme'], - action: 'replace', - target_label: '__scheme__', - regex: '(https?)', - replacement: '$1', - }, - - // Allow service to override the scrape path with prometheus.io/path=/other_metrics_path - { - source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_path'], - action: 'replace', - target_label: '__metrics_path__', - regex: '(.+)', - replacement: '$1', - }, - - // Allow services to override the scrape port with prometheus.io/port=1234 - { - source_labels: ['__address__', '__meta_kubernetes_pod_annotation_prometheus_io_port'], - action: 'replace', - target_label: '__address__', - regex: '(.+?)(\\:\\d+)?;(\\d+)', - replacement: '$1:$3', - }, - - // Drop pods without a name label - { - source_labels: ['__meta_kubernetes_pod_label_name'], - action: 'drop', - regex: '', - }, - - // Rename jobs to be / - { - source_labels: ['__meta_kubernetes_namespace', '__meta_kubernetes_pod_label_name'], - action: 'replace', - separator: '/', - target_label: 'job', - replacement: '$1', - }, - - // But also include the namespace as a separate label for routing alerts - { - source_labels: ['__meta_kubernetes_namespace'], - action: 'replace', - target_label: 'namespace', - }, - { - source_labels: ['__meta_kubernetes_pod_name'], - action: 'replace', - target_label: 'pod', // Not 'pod_name', which disappeared in K8s 1.16. - }, - { - source_labels: ['__meta_kubernetes_pod_container_name'], - action: 'replace', - target_label: 'container', // Not 'container_name', which disappeared in K8s 1.16. - }, - - // Rename instances to the concatenation of pod:container:port. - // All three components are needed to guarantee a unique instance label. - { - source_labels: [ - '__meta_kubernetes_pod_name', - '__meta_kubernetes_pod_container_name', - '__meta_kubernetes_pod_container_port_name', - ], - action: 'replace', - separator: ':', - target_label: 'instance', - }, - - // Map prometheus.io/param-=value fields to __param_=value - { - regex: '__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)', - action: 'labelmap', - replacement: '__param_$1', - }, + local _config = $.kubernetesScrapeInstanceConfig + config, - // Drop pods with phase Succeeded or Failed - { - source_labels: ['__meta_kubernetes_pod_phase'], - action: 'drop', - regex: 'Succeeded|Failed', - }, - ], - }, - - // A separate scrape config for kube-state-metrics which doesn't add a - // namespace label and instead takes the namespace label from the exported - // timeseries. This prevents the exported namespace label from being - // renamed to exported_namesapce and allows us to route alerts based on - // namespace. - { - job_name: '%s/kube-state-metrics' % namespace, - kubernetes_sd_configs: [{ - role: 'pod', - namespaces: { - names: [namespace], - }, - }], - - relabel_configs: [ - // Drop anything whose service is not kube-state-metrics - { - source_labels: ['__meta_kubernetes_pod_label_name'], - regex: 'kube-state-metrics', - action: 'keep', - }, - - // Rename instances to the concatenation of pod:container:port. - // In the specific case of KSM, we could leave out the container - // name and still have a unique instance label, but we leave it - // in here for consistency with the normal pod scraping. - { - source_labels: [ - '__meta_kubernetes_pod_name', - '__meta_kubernetes_pod_container_name', - '__meta_kubernetes_pod_container_port_name', - ], - action: 'replace', - separator: ':', - target_label: 'instance', - }, - ], - }, - - // A separate scrape config for node-exporter which maps the node name - // onto the instance label. - { - job_name: '%s/node-exporter' % namespace, - kubernetes_sd_configs: [{ - role: 'pod', - namespaces: { - names: [namespace], - }, - }], - - relabel_configs: [ - // Drop anything whose name is not node-exporter. - { - source_labels: ['__meta_kubernetes_pod_label_name'], - regex: 'node-exporter', - action: 'keep', - }, - - // Rename instances to be the node name. - { - source_labels: ['__meta_kubernetes_pod_node_name'], - action: 'replace', - target_label: 'instance', - }, - - // But also include the namespace as a separate label, for - // routing alerts. - { - source_labels: ['__meta_kubernetes_namespace'], - action: 'replace', - target_label: 'namespace', - }, - ], - }, - - // This scrape config gathers all kubelet metrics. - k8s_tls_config(config) { - job_name: 'kube-system/kubelet', - kubernetes_sd_configs: [{ - role: 'node', - }], - - relabel_configs: [ - { - target_label: '__address__', - replacement: config.kubernetes_api_server_address, - }, - { - target_label: '__scheme__', - replacement: 'https', - }, - { - source_labels: ['__meta_kubernetes_node_name'], - regex: '(.+)', - target_label: '__metrics_path__', - replacement: '/api/v1/nodes/${1}/proxy/metrics', - }, - ], - }, - - // As of k8s 1.7.3, cAdvisor metrics are available via kubelet using the - // /metrics/cadvisor path. - k8s_tls_config(config) { - job_name: 'kube-system/cadvisor', - kubernetes_sd_configs: [{ - role: 'node', - }], - scheme: 'https', - - relabel_configs: [ - { - target_label: '__address__', - replacement: config.kubernetes_api_server_address, - }, - { - source_labels: ['__meta_kubernetes_node_name'], - regex: '(.+)', - target_label: '__metrics_path__', - replacement: '/api/v1/nodes/${1}/proxy/metrics/cadvisor', - }, - ], - - metric_relabel_configs: [ - // Drop container_* metrics with no image. - { - source_labels: ['__name__', 'image'], - regex: 'container_([a-z_]+);', - action: 'drop', - }, - - // Drop a bunch of metrics which are disabled but still sent, - // see https://github.com/google/cadvisor/issues/1925. - { - source_labels: ['__name__'], - regex: 'container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)', - action: 'drop', - }, - ], - }, - ], + name: 'kubernetes', + scrape_configs: k8s_v2.metrics({ + scrape_api_server_endpoints: _config.scrape_api_server_endpoints, + insecure_skip_verify: _config.insecure_skip_verify, + cluster_dns_tld: _config.cluster_dns_tld, + cluster_dns_suffix: _config.cluster_dns_suffix, + kubernetes_api_server_address: _config.kubernetes_api_server_address, + ksm_namespace: namespace, + node_exporter_namespace: namespace, + }), }, } diff --git a/production/tanka/grafana-agent/v1/internal/kubernetes_logs.libsonnet b/production/tanka/grafana-agent/v1/internal/kubernetes_logs.libsonnet index 0038bb651f66..8ef2d4f40200 100644 --- a/production/tanka/grafana-agent/v1/internal/kubernetes_logs.libsonnet +++ b/production/tanka/grafana-agent/v1/internal/kubernetes_logs.libsonnet @@ -1,185 +1,7 @@ -local gen_scrape_config(job_name, pod_uid) = { - job_name: job_name, - pipeline_stages: [{ - docker: {}, - }], - kubernetes_sd_configs: [{ - role: 'pod', - }], - - relabel_configs: self.prelabel_config + [ - // Only scrape local pods; Promtail will drop targets with a __host__ label - // that does not match the current host name. - { - source_labels: ['__meta_kubernetes_pod_node_name'], - target_label: '__host__', - }, - - // Drop pods without a __service__ label. - { - source_labels: ['__service__'], - action: 'drop', - regex: '', - }, - - // Include all the other labels on the pod. - // Perform this mapping before applying additional label replacement rules - // to prevent a supplied label from overwriting any of the following labels. - { - action: 'labelmap', - regex: '__meta_kubernetes_pod_label_(.+)', - }, - - // Rename jobs to be /. - { - source_labels: ['__meta_kubernetes_namespace', '__service__'], - action: 'replace', - separator: '/', - target_label: 'job', - replacement: '$1', - }, - - // But also include the namespace, pod, container as separate - // labels. They uniquely identify a container. They are also - // identical to the target labels configured in Prometheus - // (but note that Loki does not use an instance label). - { - source_labels: ['__meta_kubernetes_namespace'], - action: 'replace', - target_label: 'namespace', - }, - { - source_labels: ['__meta_kubernetes_pod_name'], - action: 'replace', - target_label: 'pod', // Not 'pod_name', which disappeared in K8s 1.16. - }, - { - source_labels: ['__meta_kubernetes_pod_container_name'], - action: 'replace', - target_label: 'container', // Not 'container_name', which disappeared in K8s 1.16. - }, - - // Kubernetes puts logs under subdirectories keyed pod UID and container_name. - { - source_labels: [pod_uid, '__meta_kubernetes_pod_container_name'], - target_label: '__path__', - separator: '/', - replacement: '/var/log/pods/*$1/*.log', - }, - ], -}; +local k8s_v2 = import '../../v2/internal/helpers/k8s.libsonnet'; { newKubernetesLogsCollector():: { - scrape_configs: [ - // Scrape config to scrape any pods with a 'name' label. - gen_scrape_config('kubernetes-pods-name', '__meta_kubernetes_pod_uid') { - prelabel_config:: [ - // Use name label as __service__. - { - source_labels: ['__meta_kubernetes_pod_label_name'], - target_label: '__service__', - }, - ], - }, - - // Scrape config to scrape any pods with an 'app' label. - gen_scrape_config('kubernetes-pods-app', '__meta_kubernetes_pod_uid') { - prelabel_config:: [ - // Drop pods with a 'name' label. They will have already been added by - // the scrape_config that matches on the 'name' label - { - source_labels: ['__meta_kubernetes_pod_label_name'], - action: 'drop', - regex: '.+', - }, - - // Use app label as the __service__. - { - source_labels: ['__meta_kubernetes_pod_label_app'], - target_label: '__service__', - }, - ], - }, - - // Scrape config to scrape any pods with a direct controller (eg - // StatefulSets). - gen_scrape_config('kubernetes-pods-direct-controllers', '__meta_kubernetes_pod_uid') { - prelabel_config:: [ - // Drop pods with a 'name' or 'app' label. They will have already been added by - // the scrape_config that matches above. - { - source_labels: ['__meta_kubernetes_pod_label_name', '__meta_kubernetes_pod_label_app'], - separator: '', - action: 'drop', - regex: '.+', - }, - - // Drop pods with an indirect controller. eg Deployments create replicaSets - // which then create pods. - { - source_labels: ['__meta_kubernetes_pod_controller_name'], - action: 'drop', - regex: '[0-9a-z-.]+-[0-9a-f]{8,10}', - }, - - // Use controller name as __service__. - { - source_labels: ['__meta_kubernetes_pod_controller_name'], - target_label: '__service__', - }, - ], - }, - - // Scrape config to scrape any pods with an indirect controller (eg - // Deployments). - gen_scrape_config('kubernetes-pods-indirect-controller', '__meta_kubernetes_pod_uid') { - prelabel_config:: [ - // Drop pods with a 'name' or 'app' label. They will have already been added by - // the scrape_config that matches above. - { - source_labels: ['__meta_kubernetes_pod_label_name', '__meta_kubernetes_pod_label_app'], - separator: '', - action: 'drop', - regex: '.+', - }, - - // Drop pods not from an indirect controller. eg StatefulSets, DaemonSets - { - source_labels: ['__meta_kubernetes_pod_controller_name'], - regex: '[0-9a-z-.]+-[0-9a-f]{8,10}', - action: 'keep', - }, - - // Put the indirect controller name into a temp label. - { - source_labels: ['__meta_kubernetes_pod_controller_name'], - action: 'replace', - regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}', - target_label: '__service__', - }, - ], - }, - - // Scrape config to scrape any control plane static pods (e.g. kube-apiserver - // etcd, kube-controller-manager & kube-scheduler) - gen_scrape_config('kubernetes-pods-static', '__meta_kubernetes_pod_annotation_kubernetes_io_config_mirror') { - prelabel_config:: [ - // Ignore pods that aren't mirror pods - { - action: 'drop', - source_labels: ['__meta_kubernetes_pod_annotation_kubernetes_io_config_mirror'], - regex: '', - }, - - // Static control plane pods usually have a component label that identifies them - { - action: 'replace', - source_labels: ['__meta_kubernetes_pod_label_component'], - target_label: '__service__', - }, - ], - }, - ], + scrape_configs: k8s_v2.logs(), }, } diff --git a/production/tanka/grafana-agent/v2/README.md b/production/tanka/grafana-agent/v2/README.md new file mode 100644 index 000000000000..41cc84fc5246 --- /dev/null +++ b/production/tanka/grafana-agent/v2/README.md @@ -0,0 +1,80 @@ +# Tanka Configs + +**STATUS**: Work in progress, use of these configs is not recommended for production. + +This directory contains the Tanka configs that we use to deploy the Grafana +Agent. It is marked as `v2` and is incompatible previous versions of the library +located in other directories. + +This library is currently a work in progress and backwards-incompatible changes +may occur. Once the library is considered complete, no further backwards +incompatible changes will be made. + +## Capabilities + +This library is significantly simplified over the `v0` and `v1` counterparts. +Since there are many ways to combine the various functionalities of the Grafana +Agent, the `v2` library aims to stay out of your way and provide optional composible +helpers that may be useful for some people. + +Users of the library will pick a controller for their deployment. They are +expected to know what feature are compatible with which controller: + +| Controller | Metrics | Logs | Traces | Integrations | +| ---------------- | ------------------- | --------- | ------ | ------------ | +| DaemonSet | If host filtering | Yes | Yes | No | +| Deployment | Yes | No | No | Yes | +| StatefulSet | Yes | No | No | Yes | + +Creating an incompatible deployment will cause runtime issues when running the +Agent (for example, if configuring Logs with a StatefulSet, you will only get +logs from the node the pods are running on). + +To get full coverage of features, you must create multiple deployments of the +library. You may wish to combine a StatefulSet for metrics and integrations, a +Deployment for Traces, and a DaemonSet for logs. + +## API + +## Generate Agent Deployment + +- `new(name='grafana-agent', namespace='')`: Create a new Agent without a + controller. +- `withDeploymentController(replicas=1)`: Attach a Deployment as the Agent + controller. Number of replicas may optionally be given. +- `withDaemonSetController()`: Attach a DaemonSet as the Agent controller. +- `withStatefulSetController(replicas=1, volumeClaims=[])`: Attach a StatefulSet + as the Agent controller. Number of replicas and a set of volume claim + templates may be given. + +## Generate Scraping Service Syncer + +The Scraping Service Syncer is used to sync metrics instance configs against the +scraping service config management API. + +- `newSyncer(name='grafana-agent-sycner', namespace='', config={})` + +## General + +- `withAgentConfig(config)`: Provide a custom Agent config. +- `withImagesMixin(images)`: Use custom images instead of the defaults. +- `withConfigHash(include=true)`: Whether to include a config hash annotation. +- `withPortsMixin(ports=[])`: Mixin ports from `k.core.v1.containerPort` against + the container and service. +- `withVolumesMixin(volumes=[])`: Volume to attach to the pod. +- `withVolumeMountsMixin(mounts=[])`: Volume mounts to attach to the container. + +## Helpers + +- `newKubernetesMetrics(config)`: Creates a set of metrics scrape_configs for + collecting metrics from Kubernetes pods. +- `newKubernetesLogs(config)`: Creates a set of logs scrape_configs for + collecting logs from Kubernetes pods. +- `newKubernetesTraces(config)`: Creates a set of traces scrape_configs for + associating spans with metadata from discovered Kubernetes pods. +- `withLogVolumeMounts()`: Adds volume mounts to the controller for collecting + logs. +- `withLogPermissions()`: Runs the container as privileged and as the root user + so logs can be collected properly. + + diff --git a/production/tanka/grafana-agent/v2/internal/base.libsonnet b/production/tanka/grafana-agent/v2/internal/base.libsonnet new file mode 100644 index 000000000000..79fac7537c4a --- /dev/null +++ b/production/tanka/grafana-agent/v2/internal/base.libsonnet @@ -0,0 +1,48 @@ +function(name='grafana-agent', namespace='') { + local k = (import 'ksonnet-util/kausal.libsonnet') { _config+:: { namespace: namespace } }, + + local container = k.core.v1.container, + local configMap = k.core.v1.configMap, + local containerPort = k.core.v1.containerPort, + local policyRule = k.rbac.v1.policyRule, + local serviceAccount = k.core.v1.serviceAccount, + + local this = self, + + _images:: { + agent: 'grafana/agent:v0.16.1', + agentctl: 'grafana/agentctl:v0.16.1', + }, + _config:: { + name: name, + namespace: namespace, + config_hash: true, + agent_config: '', + }, + + rbac: k.util.rbac(name, [ + policyRule.withApiGroups(['']) + + policyRule.withResources(['nodes', 'nodes/proxy', 'services', 'endpoints', 'pods']) + + policyRule.withVerbs(['get', 'list', 'watch']), + + policyRule.withNonResourceUrls('/metrics') + + policyRule.withVerbs(['get']), + ]) { + service_account+: serviceAccount.mixin.metadata.withNamespace(namespace), + }, + + configMap: + configMap.new(name) + + configMap.mixin.metadata.withNamespace(namespace) + + configMap.withData({ + 'agent.yaml': k.util.manifestYaml(this._config.agent_config), + }), + + container:: + container.new(name, this._images.agent) + + container.withPorts(containerPort.new('http-metrics', 80)) + + container.withCommand('/bin/agent') + + container.withArgsMixin(k.util.mapToFlags({ + 'config.file': '/etc/agent/agent.yaml', + })), +} diff --git a/production/tanka/grafana-agent/v2/internal/controllers/daemonset.libsonnet b/production/tanka/grafana-agent/v2/internal/controllers/daemonset.libsonnet new file mode 100644 index 000000000000..c4afe3c725e7 --- /dev/null +++ b/production/tanka/grafana-agent/v2/internal/controllers/daemonset.libsonnet @@ -0,0 +1,22 @@ +function() { + local this = self, + local _config = this._config, + local name = _config.name, + local namespace = _config.namespace, + + local k = (import 'ksonnet-util/kausal.libsonnet') { _config+:: this._config }, + local daemonSet = k.apps.v1.daemonSet, + + controller: + daemonSet.new(name, [this.container]) + + daemonSet.mixin.metadata.withNamespace(namespace) + + daemonSet.mixin.spec.template.spec.withServiceAccount(name) + + ( + if _config.config_hash + then daemonSet.mixin.spec.template.metadata.withAnnotationsMixin({ + config_hash: std.md5(std.toString(_config.agent_config)), + }) + else {} + ) + + k.util.configVolumeMount(name, '/etc/agent'), +} diff --git a/production/tanka/grafana-agent/v2/internal/controllers/deployment.libsonnet b/production/tanka/grafana-agent/v2/internal/controllers/deployment.libsonnet new file mode 100644 index 000000000000..14abca1d415b --- /dev/null +++ b/production/tanka/grafana-agent/v2/internal/controllers/deployment.libsonnet @@ -0,0 +1,23 @@ +function(replicas=1) { + local this = self, + local _config = this._config, + local name = _config.name, + local namespace = _config.namespace, + + local k = (import 'ksonnet-util/kausal.libsonnet') { _config+:: this._config } }, + + local deployment = k.apps.v1.deployment, + + controller: + deployment.new(name, replicas, [this.container]) + + deployment.mixin.metadata.withNamespace(namespace) + + deployment.mixin.spec.template.spec.withServiceAccount(name) + + ( + if _config.config_hash + then deployment.mixin.spec.template.metadata.withAnnotationsMixin({ + config_hash: std.md5(std.toString(_config.agent_config)), + }) + else {} + ) + + k.util.configVolumeMount(name, '/etc/agent'), +} diff --git a/production/tanka/grafana-agent/v2/internal/controllers/statefulset.libsonnet b/production/tanka/grafana-agent/v2/internal/controllers/statefulset.libsonnet new file mode 100644 index 000000000000..f77179271505 --- /dev/null +++ b/production/tanka/grafana-agent/v2/internal/controllers/statefulset.libsonnet @@ -0,0 +1,29 @@ +function(replicas=1, volumeClaims=[]) { + local this = self, + local _config = this._config, + local name = _config.name, + local namespace = _config.namespace, + + local k = (import 'ksonnet-util/kausal.libsonnet') { _config+:: this._config }, + + local statefulSet = k.apps.v1.statefulSet, + local service = k.core.v1.service, + + controller: + statefulSet.new(name, replicas, [this.container], volumeClaims) + + statefulSet.mixin.metadata.withNamespace(namespace) + + statefulSet.mixin.spec.withServiceName(name) + + statefulSet.mixin.spec.template.spec.withServiceAccount(name) + + ( + if _config.config_hash + then statefulSet.mixin.spec.template.metadata.withAnnotationsMixin({ + config_hash: std.md5(std.toString(_config.agent_config)), + }) + else {} + ) + + k.util.configVolumeMount(name, '/etc/agent'), + + service: + k.util.serviceFor(this.controller) + + service.mixin.metadata.withNamespace(namespace), +} diff --git a/production/tanka/grafana-agent/v2/internal/helpers/k8s.libsonnet b/production/tanka/grafana-agent/v2/internal/helpers/k8s.libsonnet new file mode 100644 index 000000000000..d3e63c119f8e --- /dev/null +++ b/production/tanka/grafana-agent/v2/internal/helpers/k8s.libsonnet @@ -0,0 +1,515 @@ +local k8s_tls_config(config) = { + tls_config: { + ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + insecure_skip_verify: config.insecure_skip_verify, + }, + bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', +}; + +local gen_scrape_config(job_name, pod_uid) = { + job_name: job_name, + pipeline_stages: [{ + docker: {}, + }], + kubernetes_sd_configs: [{ + role: 'pod', + }], + + relabel_configs: self.prelabel_config + [ + // Only scrape local pods; Promtail will drop targets with a __host__ label + // that does not match the current host name. + { + source_labels: ['__meta_kubernetes_pod_node_name'], + target_label: '__host__', + }, + + // Drop pods without a __service__ label. + { + source_labels: ['__service__'], + action: 'drop', + regex: '', + }, + + // Include all the other labels on the pod. + // Perform this mapping before applying additional label replacement rules + // to prevent a supplied label from overwriting any of the following labels. + { + action: 'labelmap', + regex: '__meta_kubernetes_pod_label_(.+)', + }, + + // Rename jobs to be /. + { + source_labels: ['__meta_kubernetes_namespace', '__service__'], + action: 'replace', + separator: '/', + target_label: 'job', + replacement: '$1', + }, + + // But also include the namespace, pod, container as separate + // labels. They uniquely identify a container. They are also + // identical to the target labels configured in Prometheus + // (but note that Loki does not use an instance label). + { + source_labels: ['__meta_kubernetes_namespace'], + action: 'replace', + target_label: 'namespace', + }, + { + source_labels: ['__meta_kubernetes_pod_name'], + action: 'replace', + target_label: 'pod', // Not 'pod_name', which disappeared in K8s 1.16. + }, + { + source_labels: ['__meta_kubernetes_pod_container_name'], + action: 'replace', + target_label: 'container', // Not 'container_name', which disappeared in K8s 1.16. + }, + + // Kubernetes puts logs under subdirectories keyed pod UID and container_name. + { + source_labels: [pod_uid, '__meta_kubernetes_pod_container_name'], + target_label: '__path__', + separator: '/', + replacement: '/var/log/pods/*$1/*.log', + }, + ], +}; + +{ + metrics(config):: + local _config = { + scrape_api_server_endpoints: false, + insecure_skip_verify: false, + + cluster_dns_tld: 'local', + cluster_dns_suffix: 'cluster.' + self.cluster_dns_tld, + kubernetes_api_server_address: 'kubernetes.default.svc.%(cluster_dns_suffix)s:443' % self, + + ksm_namespace: 'kube-system', + node_exporter_namespace: 'kube-system', + } + config; + + [ + k8s_tls_config(_config) { + job_name: 'default/kubernetes', + kubernetes_sd_configs: [{ + role: if _config.scrape_api_server_endpoints then 'endpoints' else 'service', + }], + scheme: 'https', + tls_config+: { + server_name: 'kubernetes', + }, + + relabel_configs: [{ + source_labels: ['__meta_kubernetes_service_label_component'], + regex: 'apiserver', + action: 'keep', + }], + + // Keep limited set of metrics to reduce default usage, drop all others + metric_relabel_configs: [ + { + source_labels: ['__name__'], + regex: 'workqueue_queue_duration_seconds_bucket|process_cpu_seconds_total|process_resident_memory_bytes|workqueue_depth|rest_client_request_duration_seconds_bucket|workqueue_adds_total|up|rest_client_requests_total|apiserver_request_total|go_goroutines', + action: 'keep', + }, + ], + }, + + { + job_name: 'kubernetes-pods', + kubernetes_sd_configs: [{ + role: 'pod', + }], + + // You can specify the following annotations (on pods): + // prometheus.io/scrape: false - don't scrape this pod + // prometheus.io/scheme: https - use https for scraping + // prometheus.io/port - scrape this port + // prometheus.io/path - scrape this path + // prometheus.io/param- - send ?parameter=value with the scrape + relabel_configs: [ + // Drop anything annotated with prometheus.io/scrape=false + { + source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_scrape'], + action: 'drop', + regex: 'false', + }, + + // Drop any endpoint whose pod port name does not end with metrics + { + source_labels: ['__meta_kubernetes_pod_container_port_name'], + action: 'keep', + regex: '.*-metrics', + }, + + // Allow pods to override the scrape scheme with prometheus.io/scheme=https + { + source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_scheme'], + action: 'replace', + target_label: '__scheme__', + regex: '(https?)', + replacement: '$1', + }, + + // Allow service to override the scrape path with prometheus.io/path=/other_metrics_path + { + source_labels: ['__meta_kubernetes_pod_annotation_prometheus_io_path'], + action: 'replace', + target_label: '__metrics_path__', + regex: '(.+)', + replacement: '$1', + }, + + // Allow services to override the scrape port with prometheus.io/port=1234 + { + source_labels: ['__address__', '__meta_kubernetes_pod_annotation_prometheus_io_port'], + action: 'replace', + target_label: '__address__', + regex: '(.+?)(\\:\\d+)?;(\\d+)', + replacement: '$1:$3', + }, + + // Drop pods without a name label + { + source_labels: ['__meta_kubernetes_pod_label_name'], + action: 'drop', + regex: '', + }, + + // Rename jobs to be / + { + source_labels: ['__meta_kubernetes_namespace', '__meta_kubernetes_pod_label_name'], + action: 'replace', + separator: '/', + target_label: 'job', + replacement: '$1', + }, + + // But also include the namespace as a separate label for routing alerts + { + source_labels: ['__meta_kubernetes_namespace'], + action: 'replace', + target_label: 'namespace', + }, + { + source_labels: ['__meta_kubernetes_pod_name'], + action: 'replace', + target_label: 'pod', // Not 'pod_name', which disappeared in K8s 1.16. + }, + { + source_labels: ['__meta_kubernetes_pod_container_name'], + action: 'replace', + target_label: 'container', // Not 'container_name', which disappeared in K8s 1.16. + }, + + // Rename instances to the concatenation of pod:container:port. + // All three components are needed to guarantee a unique instance label. + { + source_labels: [ + '__meta_kubernetes_pod_name', + '__meta_kubernetes_pod_container_name', + '__meta_kubernetes_pod_container_port_name', + ], + action: 'replace', + separator: ':', + target_label: 'instance', + }, + + // Map prometheus.io/param-=value fields to __param_=value + { + regex: '__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)', + action: 'labelmap', + replacement: '__param_$1', + }, + + // Drop pods with phase Succeeded or Failed + { + source_labels: ['__meta_kubernetes_pod_phase'], + action: 'drop', + regex: 'Succeeded|Failed', + }, + ], + }, + + // A separate scrape config for kube-state-metrics which doesn't add a + // namespace label and instead takes the namespace label from the exported + // timeseries. This prevents the exported namespace label from being + // renamed to exported_namesapce and allows us to route alerts based on + // namespace. + { + job_name: '%s/kube-state-metrics' % _config.ksm_namespace, + kubernetes_sd_configs: [{ + role: 'pod', + namespaces: { + names: [_config.ksm_namespace], + }, + }], + + relabel_configs: [ + // Drop anything whose service is not kube-state-metrics + { + source_labels: ['__meta_kubernetes_pod_label_name'], + regex: 'kube-state-metrics', + action: 'keep', + }, + + // Rename instances to the concatenation of pod:container:port. + // In the specific case of KSM, we could leave out the container + // name and still have a unique instance label, but we leave it + // in here for consistency with the normal pod scraping. + { + source_labels: [ + '__meta_kubernetes_pod_name', + '__meta_kubernetes_pod_container_name', + '__meta_kubernetes_pod_container_port_name', + ], + action: 'replace', + separator: ':', + target_label: 'instance', + }, + ], + }, + + // A separate scrape config for node-exporter which maps the node name + // onto the instance label. + { + job_name: '%s/node-exporter' % _config.node_exporter_namespace, + kubernetes_sd_configs: [{ + role: 'pod', + namespaces: { + names: [_config.node_exporter_namespace], + }, + }], + + relabel_configs: [ + // Drop anything whose name is not node-exporter. + { + source_labels: ['__meta_kubernetes_pod_label_name'], + regex: 'node-exporter', + action: 'keep', + }, + + // Rename instances to be the node name. + { + source_labels: ['__meta_kubernetes_pod_node_name'], + action: 'replace', + target_label: 'instance', + }, + + // But also include the namespace as a separate label, for + // routing alerts. + { + source_labels: ['__meta_kubernetes_namespace'], + action: 'replace', + target_label: 'namespace', + }, + ], + }, + + // This scrape config gathers all kubelet metrics. + k8s_tls_config(_config) { + job_name: 'kube-system/kubelet', + kubernetes_sd_configs: [{ role: 'node' }], + + relabel_configs: [ + { + target_label: '__address__', + replacement: _config.kubernetes_api_server_address, + }, + { + target_label: '__scheme__', + replacement: 'https', + }, + { + source_labels: ['__meta_kubernetes_node_name'], + regex: '(.+)', + target_label: '__metrics_path__', + replacement: '/api/v1/nodes/${1}/proxy/metrics', + }, + ], + }, + + // As of k8s 1.7.3, cAdvisor metrics are available via kubelet using the + // /metrics/cadvisor path. + k8s_tls_config(_config) { + job_name: 'kube-system/cadvisor', + kubernetes_sd_configs: [{ + role: 'node', + }], + scheme: 'https', + + relabel_configs: [ + { + target_label: '__address__', + replacement: _config.kubernetes_api_server_address, + }, + { + source_labels: ['__meta_kubernetes_node_name'], + regex: '(.+)', + target_label: '__metrics_path__', + replacement: '/api/v1/nodes/${1}/proxy/metrics/cadvisor', + }, + ], + + metric_relabel_configs: [ + // Drop container_* metrics with no image. + { + source_labels: ['__name__', 'image'], + regex: 'container_([a-z_]+);', + action: 'drop', + }, + + // Drop a bunch of metrics which are disabled but still sent, + // see https://github.com/google/cadvisor/issues/1925. + { + source_labels: ['__name__'], + regex: 'container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)', + action: 'drop', + }, + ], + }, + ], + + logs(config={}):: [ + // Scrape config to scrape any pods with a 'name' label. + gen_scrape_config('kubernetes-pods-name', '__meta_kubernetes_pod_uid') { + prelabel_config:: [ + // Use name label as __service__. + { + source_labels: ['__meta_kubernetes_pod_label_name'], + target_label: '__service__', + }, + ], + }, + + // Scrape config to scrape any pods with an 'app' label. + gen_scrape_config('kubernetes-pods-app', '__meta_kubernetes_pod_uid') { + prelabel_config:: [ + // Drop pods with a 'name' label. They will have already been added by + // the scrape_config that matches on the 'name' label + { + source_labels: ['__meta_kubernetes_pod_label_name'], + action: 'drop', + regex: '.+', + }, + + // Use app label as the __service__. + { + source_labels: ['__meta_kubernetes_pod_label_app'], + target_label: '__service__', + }, + ], + }, + + // Scrape config to scrape any pods with a direct controller (eg + // StatefulSets). + gen_scrape_config('kubernetes-pods-direct-controllers', '__meta_kubernetes_pod_uid') { + prelabel_config:: [ + // Drop pods with a 'name' or 'app' label. They will have already been added by + // the scrape_config that matches above. + { + source_labels: ['__meta_kubernetes_pod_label_name', '__meta_kubernetes_pod_label_app'], + separator: '', + action: 'drop', + regex: '.+', + }, + + // Drop pods with an indirect controller. eg Deployments create replicaSets + // which then create pods. + { + source_labels: ['__meta_kubernetes_pod_controller_name'], + action: 'drop', + regex: '[0-9a-z-.]+-[0-9a-f]{8,10}', + }, + + // Use controller name as __service__. + { + source_labels: ['__meta_kubernetes_pod_controller_name'], + target_label: '__service__', + }, + ], + }, + + // Scrape config to scrape any pods with an indirect controller (eg + // Deployments). + gen_scrape_config('kubernetes-pods-indirect-controller', '__meta_kubernetes_pod_uid') { + prelabel_config:: [ + // Drop pods with a 'name' or 'app' label. They will have already been added by + // the scrape_config that matches above. + { + source_labels: ['__meta_kubernetes_pod_label_name', '__meta_kubernetes_pod_label_app'], + separator: '', + action: 'drop', + regex: '.+', + }, + + // Drop pods not from an indirect controller. eg StatefulSets, DaemonSets + { + source_labels: ['__meta_kubernetes_pod_controller_name'], + regex: '[0-9a-z-.]+-[0-9a-f]{8,10}', + action: 'keep', + }, + + // Put the indirect controller name into a temp label. + { + source_labels: ['__meta_kubernetes_pod_controller_name'], + action: 'replace', + regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}', + target_label: '__service__', + }, + ], + }, + + // Scrape config to scrape any control plane static pods (e.g. kube-apiserver + // etcd, kube-controller-manager & kube-scheduler) + gen_scrape_config('kubernetes-pods-static', '__meta_kubernetes_pod_annotation_kubernetes_io_config_mirror') { + prelabel_config:: [ + // Ignore pods that aren't mirror pods + { + action: 'drop', + source_labels: ['__meta_kubernetes_pod_annotation_kubernetes_io_config_mirror'], + regex: '', + }, + + // Static control plane pods usually have a component label that identifies them + { + action: 'replace', + source_labels: ['__meta_kubernetes_pod_label_component'], + target_label: '__service__', + }, + ], + }, + ], + + traces(conifg={}):: [ + { + bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', + job_name: 'kubernetes-pods', + kubernetes_sd_configs: [{ role: 'pod' }], + relabel_configs: [ + { + action: 'replace', + source_labels: ['__meta_kubernetes_namespace'], + target_label: 'namespace', + }, + { + action: 'replace', + source_labels: ['__meta_kubernetes_pod_name'], + target_label: 'pod', + }, + { + action: 'replace', + source_labels: ['__meta_kubernetes_pod_container_name'], + target_label: 'container', + }, + ], + tls_config: { + ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + insecure_skip_verify: false, + }, + }, + ], +} diff --git a/production/tanka/grafana-agent/v2/internal/helpers/logs.libsonnet b/production/tanka/grafana-agent/v2/internal/helpers/logs.libsonnet new file mode 100644 index 000000000000..2b2fc4eecaa3 --- /dev/null +++ b/production/tanka/grafana-agent/v2/internal/helpers/logs.libsonnet @@ -0,0 +1,21 @@ +local k = import 'ksonnet-util/kausal.libsonnet', +local container = k.core.v1.container, + +{ + volumeMounts(config={}):: { + controller+: + // For reading docker containers. /var/log is used for the positions file + // and shouldn't be set to readonly. + k.util.hostVolumeMount('varlog', '/var/log', '/var/log') + + k.util.hostVolumeMount('varlibdockercontainers', '/var/lib/docker/containers', '/var/lib/docker/containers', readOnly=true) + + + // For reading journald + k.util.hostVolumeMount('etcmachineid', '/etc/machine-id', '/etc/machine-id', readOnly=true), + }, + + permissions(config={}):: { + container+:: + container.mixin.securityContext.withPrivileged(true) + + container.mixin.securityContext.withRunAsUser(0), + } +} diff --git a/production/tanka/grafana-agent/v2/internal/syncer.libsonnet b/production/tanka/grafana-agent/v2/internal/syncer.libsonnet new file mode 100644 index 000000000000..360ba96dcd94 --- /dev/null +++ b/production/tanka/grafana-agent/v2/internal/syncer.libsonnet @@ -0,0 +1,62 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; + +local cronJob = k.batch.v1beta1.cronJob; +local configMap = k.core.v1.configMap; +local container = k.core.v1.container; +local deployment = k.apps.v1.deployment; +local volumeMount = k.core.v1.volumeMount; +local volume = k.core.v1.volume; + +function( + name='grafana-agent-syncer', + namespace='', + config={}, +) { + local _config = { + api: error 'api must be set', + image: 'grafana/agentctl:v0.16.1', + schedule: '*/5 * * * *', + configs: [], + } + config, + + local this = self, + local _configs = std.foldl( + function(agg, cfg) + // Sanitize the name and remove / so every file goes into the same + // folder. + local name = std.strReplace(cfg.name, '/', '_'); + + agg { ['%s.yml' % name]: k.util.manifestYaml(cfg) }, + _config.configs, + {}, + ), + + configMap: + configMap.new(name) + + configMap.mixin.metadata.withNamespace(namespace) + + configMap.withData(_configs), + + container:: + container.new(name, _config.image) + + container.withArgsMixin([ + 'config-sync', + '--addr=%s' % _config.api, + '/etc/configs', + ]) + + container.withVolumeMounts(volumeMount.new(name, '/etc/configs')), + + job: + cronJob.new(name, _config.schedule, this.container) + + cronJob.mixin.metadata.withNamespace(namespace) + + cronJob.mixin.spec.withSuccessfulJobsHistoryLimit(1) + + cronJob.mixin.spec.withFailedJobsHistoryLimit(3) + + cronJob.mixin.spec.jobTemplate.spec.template.spec.withRestartPolicy('OnFailure') + + cronJob.mixin.spec.jobTemplate.spec.template.spec.withActiveDeadlineSeconds(600) + + cronJob.mixin.spec.jobTemplate.spec.withTtlSecondsAfterFinished(120) + + cronJob.mixin.spec.jobTemplate.spec.template.spec.withVolumes([ + volume.fromConfigMap( + name=name, + configMapName=this.configMap.metadata.name, + ), + ]), +} diff --git a/production/tanka/grafana-agent/v2/main.libsonnet b/production/tanka/grafana-agent/v2/main.libsonnet new file mode 100644 index 000000000000..aa5629bdb6ab --- /dev/null +++ b/production/tanka/grafana-agent/v2/main.libsonnet @@ -0,0 +1,41 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; +local container = k.core.v1.container; + +{ + new(name='grafana-agent', namespace=''):: + (import './internal/base.libsonnet')(name, namespace), + + // Controllers + withDeploymentController(replicas=1):: + (import './internal/controllers/deployment.libsonnet')(replicas), + withDaemonSetController():: + (import './internal/controllers/daemonset.libsonnet')(), + withStatefulSetController(replicas=1, volumeClaims=[]):: + (import './internal/controllers/statefulset.libsonnet')(replicas, volumeClaims), + + // Syncer + newSyncer(name='grafana-agent-syncer', namespace='', config={}):: + (import './internal/syncer.libsonnet')(name, namespace, config), + + // General + withAgentConfig(config):: { _config+: { agent_config: config } }, + withImagesMixin(images):: { _images+: images }, + withConfigHash(include=true):: { _config+: { config_hash: include } }, + withPortsMixin(ports=[]):: { container+:: container.withPortsMixin(ports) }, + withVolumeMountsMixin(mounts=[]):: { container+:: container.withVolumeMountsMixin(mounts) }, + withVolumesMixin(volumes=[]):: { + controller+: self.controller.mixin.spec.template.spec.withVolumesMixin(volumes), + }, + + // Helpers + newKubernetesMetrics(config):: + (import './internal/helpers/k8s.libsonnet').metrics(config), + newKubernetesLogs(config):: + (import './internal/helpers/k8s.libsonnet').logs(config), + newKubernetesTraces(config):: + (import './internal/helpers/k8s.libsonnet').traces(config), + withLogVolumeMounts(config):: + (import './internal/helpers/logs.libsonnet').volumeMounts(config), + withLogPermissions(config):: + (import './internal/helpers/logs.libsonnet').permissions(config), +}