Add initial documentation for crow.

.github/workflows/markdown.links.config.json

@@ -2,6 +2,9 @@
   "ignorePatterns": [
     {
       "pattern": "^http://prometheus.k3d.localhost:30080"
+    },
+    {
+      "pattern": "^http://prometheus.k3d.localhost:50080"
     }
   ]
 }

cmd/grafana-agent-crow/README.md

@@ -0,0 +1,5 @@
+# Crow
+
+Crow is a tool similar to Tempo Vulture and Loki Canary that is used to smoke test Grafana Agent. Crow works by generating metrics, then validating them against Prometheus. Crow uses two endpoints; the traditional `/metrics` and then `/validate` that generates the results of Crow checking for successful samples.
+
+Note: The `/validate` endpoint should only be checked by the Grafana Agent instance that is configured to remote_write. 

cmd/grafana-agent-crow/main.go

@@ -70,7 +70,7 @@ func main() {
 		EnableOpenMetrics: true,
 	}))
 
-	// Register crow's metrics to /metrics and /valiate respectively.
+	// Register crow's metrics to /metrics and /validate respectively.
 	s.Registerer.MustRegister(c.StateMetrics())
 	validator.MustRegister(c.TestMetrics())
 

example/k3d/README.md

@@ -1,17 +1,19 @@
-# `k3d` Example
+# `k3d` Examples
+
+## Agent Environment
 
 The `k3d` example uses `k3d` and `tanka` to produce a Kubernetes environment
-that
+that implements a full Grafana Agent environment for testing.
 
-## Requirements
+### Requirements
 
 - A Unix-y command line (macOS or Linux will do).
 - Kubectl
 - Docker
 - [Tanka >= v0.9.2](https://github.com/grafana/tanka)
 - [k3d >= v4.0.0](https://github.com/rancher/k3d)
 
-## Getting Started
+### Getting Started
 
 Build latest agent images with `make agent-image agentctl-image` in the project root directory if there are local changes to test.
 
@@ -32,3 +34,60 @@ tk apply ./environment
 # Delete the k3d cluster when you're done with it
 k3d cluster delete agent-k3d
 ```
+
+## Smoke Test Environment
+
+The smoke test environment is used to validate samples end to end. 
+
+### Running
+
+Smoke Test environment is invoked via `/scripts/smoke_test.bash`
+
+This tool will spin up cluster of Grafana Agent, Cortex, Avalanche and [Crow](../../cmd/grafana-agent-crow/README.md) instances. The tool will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing. 
+
+### What to look for?
+
+These alerts are viewable [here](http://prometheus.k3d.localhost:50080/alerts). 
+
+Prometheus alerts are triggered:
+- If any Crow instances are not running or Crow samples are not being propagated correctly.
+- If any Grafana Agents are not running or Grafana Agent limits are outside their norm.
+
+NOTE: The alerts might be in pending until the system settles down.
+
+![](./assets/pending_alert.png)
+
+An alert firing will look similar to the below.
+
+![](./assets/alert_firing.png)
+
+If at the end of the test any issues are found they will look similar to the below.
+
+![](./assets/console_failure.png)
+
+### How to trigger an alert?
+
+Changing the avalanche setting for label_count to 1000, located [here](./lib/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit.
+
+![](./assets/trigger_change.png)
+
+### Architecture
+
+By default, a k3d cluster will be created running the following instances
+
+- agent-single - single instance
+- agent-cluster - 3 Grafana Agents in clustered configuration
+- crow-cluster - serves the agent cluster
+- crow-single - serves the single agent
+- cortex
+- avalanche - selection of avalanche instances serving traffic
+
+Crow instance will check to see if the metrics that were scraped shows up in the prometheus endpoint and then will emit metrics on the success of those metrics. This success/failure result will trigger an alert if it is incorrect.
+
+### Flow
+
+![](./assets/order.png)
+
+### Avalanche
+
+Avalanche is used to add some additional load on the system and general testing. 

example/k3d/assets/order.uml

@@ -0,0 +1,10 @@
+@startuml
+Crow -> Crow: Crow Generates Metrics
+Agent -> Crow: Agent scrapes /metrics endpoint
+Agent -> Prometheus: Agent sends results to prometheus
+Crow -> Prometheus: Crow checks that metrics were written to prometheus
+Crow -> Crow: Crow updates success/failure metrics and serves those via /validate
+Agent -> Crow: Agent scrapes /validate
+Agent -> Prometheus: Agent sends validate metrics
+Prometheus -> Prometheus: Prometheus checks alerts
+@enduml

example/k3d/scripts/smoke-test.bash

@@ -9,7 +9,7 @@
 #   jq
 #
 # smoke-test.bash performs smoke tests that can be used to validate a release.
-# It only validates correctness, and does not attempt to do significiant load
+# It only validates correctness, and does not attempt to do significant load
 # testing or performance benchmarks.
 #
 # It works by deploying a k3d cluster with two pairs of Agent deployments: one
@@ -24,7 +24,7 @@
 # as test cases for the smoke tests; alerts generated within the span of the testing
 # period are treated as failures.
 #
-# After the smoke-test duration period (defualting to 3h), alerts will
+# After the smoke-test duration period (defaulting to 3h), alerts will
 # be checked, and the script will end.
 #
 # The k3d cluster is kept alive after the test for analysis. To clean up assets created

pkg/crow/crow.go

@@ -86,7 +86,7 @@ var DefaultConfig = Config{
 	ValueEpsilon:      0.0001,
 }
 
-// Crow is a collectness checker that validates scraped metrics reach a
+// Crow is a correctness checker that validates scraped metrics reach a
 // Prometheus-compatible server with the same values and roughly the same
 // timestamp.
 //
@@ -96,12 +96,12 @@ var DefaultConfig = Config{
 // 2. State metrics, exposing state of the Crow checker itself.
 //
 // These two metrics should be exposed via different endpoints, and only state
-// metrics are safe to be manually collecetd from.
+// metrics are safe to be manually collected from.
 //
 // Collecting from the set of test metrics generates a validation job, where
 // Crow will query the Prometheus API to ensure the metrics that were scraped
 // were written with (approximately) the same timestamp as the scrape time
-// and with (approximately) the same floatnig point values exposed in the
+// and with (approximately) the same floating point values exposed in the
 // scrape.
 //
 // If a set of test metrics were not found and retries have been exhausted,

pkg/crow/samples.go

@@ -13,7 +13,7 @@ type sample struct {
 	Labels     prometheus.Labels
 	Value      float64
 
-	// How many times this sample has attempted to be valdated. Starts at 0.
+	// How many times this sample has attempted to be validated. Starts at 0.
 	ValidationAttempt int
 }
 

production/tanka/grafana-agent/v2/internal/helpers/k8s.libsonnet

@@ -6,7 +6,7 @@ local k8s_tls_config(config) = {
   bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token',
 };
 
-local gen_logs_scrape_config(job_name, pod_uid) = {
+local gen_scrape_config(job_name, pod_uid) = {
   job_name: job_name,
   pipeline_stages: [{
     docker: {},