Automated smoke tests for metrics

.github/workflows/markdown.links.config.json

@@ -2,6 +2,9 @@
   "ignorePatterns": [
     {
       "pattern": "^http://prometheus.k3d.localhost:30080"
+    },
+    {
+      "pattern": "^http://prometheus.k3d.localhost:50080"
     }
   ]
 }

cmd/grafana-agent-crow/README.md

@@ -0,0 +1,5 @@
+# Crow
+
+Crow is a tool similar to Tempo Vulture and Loki Canary that is used to smoke test Grafana Agent. Crow works by generating metrics, then validating them against Prometheus. Crow uses two endpoints; the traditional `/metrics` and then `/validate` that generates the results of Crow checking for successful samples.
+
+Note: The `/validate` endpoint should only be checked by the Grafana Agent instance that is configured to remote_write. 

cmd/grafana-agent-crow/main.go

@@ -70,7 +70,7 @@ func main() {
 		EnableOpenMetrics: true,
 	}))
 
-	// Register crow's metrics to /metrics and /valiate respectively.
+	// Register crow's metrics to /metrics and /validate respectively.
 	s.Registerer.MustRegister(c.StateMetrics())
 	validator.MustRegister(c.TestMetrics())
 

example/k3d/README.md

@@ -1,17 +1,19 @@
-# `k3d` Example
+# `k3d` Examples
+
+## Agent Environment
 
 The `k3d` example uses `k3d` and `tanka` to produce a Kubernetes environment
-that
+that implements a full Grafana Agent environment for testing.
 
-## Requirements
+### Requirements
 
 - A Unix-y command line (macOS or Linux will do).
 - Kubectl
 - Docker
 - [Tanka >= v0.9.2](https://github.com/grafana/tanka)
 - [k3d >= v4.0.0](https://github.com/rancher/k3d)
 
-## Getting Started
+### Getting Started
 
 Build latest agent images with `make agent-image agentctl-image` in the project root directory if there are local changes to test.
 
@@ -32,3 +34,60 @@ tk apply ./environment
 # Delete the k3d cluster when you're done with it
 k3d cluster delete agent-k3d
 ```
+
+## Smoke Test Environment
+
+The smoke test environment is used to validate samples end to end. 
+
+### Running
+
+Smoke Test environment is invoked via `/scripts/smoke_test.bash`
+
+This tool will spin up cluster of Grafana Agent, Cortex, Avalanche and [Crow](../../cmd/grafana-agent-crow/README.md) instances. The tool will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing. 
+
+### What to look for?
+
+These alerts are viewable [here](http://prometheus.k3d.localhost:50080/alerts). 
+
+Prometheus alerts are triggered:
+- If any Crow instances are not running or Crow samples are not being propagated correctly.
+- If any Grafana Agents are not running or Grafana Agent limits are outside their norm.
+
+NOTE: The alerts might be in pending until the system settles down.
+
+![](./assets/pending_alert.png)
+
+An alert firing will look similar to the below.
+
+![](./assets/alert_firing.png)
+
+If at the end of the test any issues are found they will look similar to the below.
+
+![](./assets/console_failure.png)
+
+### How to trigger an alert?
+
+Changing the avalanche setting for label_count to 1000, located [here](./lib/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit.
+
+![](./assets/trigger_change.png)
+
+### Architecture
+
+By default, a k3d cluster will be created running the following instances
+
+- agent-single - single instance
+- agent-cluster - 3 Grafana Agents in clustered configuration
+- crow-cluster - serves the agent cluster
+- crow-single - serves the single agent
+- cortex
+- avalanche - selection of avalanche instances serving traffic
+
+Crow instance will check to see if the metrics that were scraped shows up in the prometheus endpoint and then will emit metrics on the success of those metrics. This success/failure result will trigger an alert if it is incorrect.
+
+### Flow
+
+![](./assets/order.png)
+
+### Avalanche
+
+Avalanche is used to add some additional load on the system and general testing. 

example/k3d/assets/order.uml

@@ -0,0 +1,10 @@
+@startuml
+Crow -> Crow: Crow Generates Metrics
+Agent -> Crow: Agent scrapes /metrics endpoint
+Agent -> Prometheus: Agent sends results to prometheus
+Crow -> Prometheus: Crow checks that metrics were written to prometheus
+Crow -> Crow: Crow updates success/failure metrics and serves those via /validate
+Agent -> Crow: Agent scrapes /validate
+Agent -> Prometheus: Agent sends validate metrics
+Prometheus -> Prometheus: Prometheus checks alerts
+@enduml

example/k3d/lib/avalanche/main.libsonnet

@@ -0,0 +1,49 @@
+local k = import 'ksonnet-util/kausal.libsonnet';
+
+local configMap = k.core.v1.configMap;
+local container = k.core.v1.container;
+local containerPort = k.core.v1.containerPort;
+local deployment = k.apps.v1.deployment;
+local pvc = k.core.v1.persistentVolumeClaim;
+local service = k.core.v1.service;
+local volumeMount = k.core.v1.volumeMount;
+local volume = k.core.v1.volume;
+
+{
+  new(name='avalanche', replicas=1, namespace='', config={}):: {
+    local this = self,
+
+    _config+:: {
+      image: 'quay.io/freshtracks.io/avalanche:latest',
+
+      metric_count: 500,
+      label_count: 10,
+      series_count: 10,
+      metricname_length: 5,
+      labelname_length: 5,
+      value_interval: 30,
+      series_interval: 30,
+      metric_interval: 120,
+    } + config,
+
+    container::
+      container.new(name, this._config.image) +
+      container.withPorts([
+        containerPort.newNamed(name='http', containerPort=9001),
+      ]) +
+      container.withArgsMixin([
+        '--metric-count=%d' % this._config.metric_count,
+        '--label-count=%d' % this._config.label_count,
+        '--series-count=%d' % this._config.series_count,
+        '--metricname-length=%d' % this._config.metricname_length,
+        '--labelname-length=%d' % this._config.labelname_length,
+        '--value-interval=%d' % this._config.value_interval,
+        '--series-interval=%d' % this._config.series_interval,
+        '--metric-interval=%d' % this._config.metric_interval,
+      ]),
+
+    deployment:
+      deployment.new(name, replicas, [self.container]) +
+      deployment.mixin.metadata.withNamespace(namespace),
+  },
+}

example/k3d/lib/crow/main.libsonnet

@@ -0,0 +1,32 @@
+local k = import 'ksonnet-util/kausal.libsonnet';
+
+local configMap = k.core.v1.configMap;
+local container = k.core.v1.container;
+local containerPort = k.core.v1.containerPort;
+local deployment = k.apps.v1.deployment;
+local pvc = k.core.v1.persistentVolumeClaim;
+local service = k.core.v1.service;
+local volumeMount = k.core.v1.volumeMount;
+local volume = k.core.v1.volume;
+
+{
+  new(name='crow', namespace='', config={}):: {
+    local this = self,
+
+    _config+:: {
+      image: 'grafana/agent-crow:latest',
+      args: {},
+    } + config,
+
+    container::
+      container.new(name, this._config.image) +
+      container.withPorts([
+        containerPort.newNamed(name='http-metrics', containerPort=80),
+      ]) +
+      container.withArgsMixin(k.util.mapToFlags(this._config.args)),
+
+    deployment:
+      deployment.new(name, 1, [self.container]) +
+      deployment.mixin.metadata.withNamespace(namespace),
+  },
+}

example/k3d/lib/default/mixins.libsonnet

@@ -13,4 +13,18 @@ local fix = {
 fix +
 cortex_mixin +
 agent_debugging_mixin +
-agent_mixin
+agent_mixin {
+  _config+: {
+    // We run a single-node cortex so replace the job names to all
+    // be the monolith.
+    job_names+: {
+      ingester: 'cortex',
+      distributor: 'cortex',
+      querier: 'cortex',
+      query_frontend: 'cortex',
+      table_manager: 'cortex',
+      store_gateway: 'cortex',
+      gateway: 'cortex',
+    },
+  },
+}

example/k3d/lib/grafana/config.libsonnet

@@ -1,6 +1,6 @@
 {
   _images: {
-    grafana: 'grafana/grafana:7.0.1',
+    grafana: 'grafana/grafana:8.0.3',
   },
 
   _config: {

example/k3d/lib/grafana/main.libsonnet

@@ -5,6 +5,7 @@ local configMap = k.core.v1.configMap;
 local container = k.core.v1.container;
 local containerPort = k.core.v1.containerPort;
 local deployment = k.apps.v1.deployment;
+local service = k.core.v1.service;
 
 {
   new(dashboards={}, datasources=[], namespace='default'):: {
@@ -20,6 +21,7 @@ local deployment = k.apps.v1.deployment;
 
     grafana_cm:
       configMap.new('grafana-config') +
+      configMap.mixin.metadata.withNamespace(namespace) +
       configMap.withData({
         'grafana.ini': std.manifestIni(_config.grafana_ini),
       }),
@@ -29,6 +31,7 @@ local deployment = k.apps.v1.deployment;
       then {}
       else
         configMap.new('dashboards') +
+        configMap.mixin.metadata.withNamespace(namespace) +
         configMap.withDataMixin({
           [name]: std.toString(
             $.dashboards[name]
@@ -40,6 +43,7 @@ local deployment = k.apps.v1.deployment;
     grafana_dashboard_cms: {
       ['dashboard-%d' % shard]:
         configMap.new('dashboards-%d' % shard) +
+        configMap.mixin.metadata.withNamespace(namespace) +
         configMap.withDataMixin({
           [name]: std.toString(
             _dashboards[name]
@@ -53,12 +57,14 @@ local deployment = k.apps.v1.deployment;
 
     grafana_datasource_cm:
       configMap.new('grafana-datasources') +
+      configMap.mixin.metadata.withNamespace(namespace) +
       configMap.withDataMixin(std.foldl(function(acc, obj) acc {
         ['%s.yml' % obj.datasources[0].name]: k.util.manifestYaml(obj),
       }, self._datasources, {})),
 
     grafana_dashboard_provisioning_cm:
       configMap.new('grafana-dashboard-provisioning') +
+      configMap.mixin.metadata.withNamespace(namespace) +
       configMap.withData({
         'dashboards.yml': k.util.manifestYaml({
           apiVersion: 1,
@@ -91,6 +97,7 @@ local deployment = k.apps.v1.deployment;
 
     deployment:
       deployment.new('grafana', 1, [self.container]) +
+      deployment.mixin.metadata.withNamespace(namespace) +
       deployment.mixin.spec.template.spec.securityContext.withRunAsUser(0) +
       k.util.configMapVolumeMount(self.grafana_cm, '/etc/grafana-config') +
       k.util.configMapVolumeMount(self.grafana_datasource_cm, '%(provisioning_dir)s/datasources' % _config) +
@@ -111,7 +118,8 @@ local deployment = k.apps.v1.deployment;
       k.util.podPriority('critical'),
 
     service:
-      k.util.serviceFor(self.deployment),
+      k.util.serviceFor(self.deployment) +
+      service.mixin.metadata.withNamespace(namespace),
   },
 
   // withDashboards sets the list of dashboards. Dashboards is an object where the