smoke framework refactor

.drone/drone.yml

@@ -53,7 +53,7 @@ steps:
       - tar -C /usr/local -xzf go1.17.6.linux-amd64.tar.gz
       - rm go1.17.6.linux-amd64.tar.gz
       - export PATH=$PATH:/usr/local/go/bin
-      - make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator cmd/grafana-agent-crow/grafana-agent-crow
+      - make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator tools/crow/grafana-agent-crow tools/smoke/grafana-agent-smoke
       - make K8S_USE_DOCKER_NETWORK=1 DRONE=true BUILD_IN_CONTAINER=false test
 depends_on:
   - Lint
@@ -113,7 +113,11 @@ steps:
         from_secret: DOCKER_LOGIN
       DOCKER_PASSWORD:
         from_secret: DOCKER_PASSWORD
+      GCR_CREDS:
+        from_secret: gcr_admin
     commands:
+      - mkdir -p $HOME/.docker
+      - printenv GCR_CREDS > $HOME/.docker/config.json
       - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
       - apk update && apk add make bash wget git qemu
       - wget -q https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64
@@ -128,6 +132,7 @@ steps:
       - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agentctl-image
       - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-operator-image
       - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true grafana-agent-crow-image
+      - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-smoke-image
       - docker buildx rm multiarch
 
 depends_on:
@@ -243,6 +248,14 @@ get:
   path: secret/data/common/gcr
   name: .dockerconfigjson
 
+---
+kind: secret
+name: gcr_admin
+
+get:
+  name: .dockerconfigjson
+  path: infra/data/ci/gcr-admin
+
 ---
 kind: secret
 name: gh_token
@@ -252,6 +265,6 @@ get:
   name: pat
 ---
 kind: signature
-hmac: 87a9a23731884dac8e0b5c3ee0fa2b5cef9d7fa490fcad649f7783f6ea9d2d32
+hmac: f59c8b25058782d021a2d6bdc1fc59a049f3b352fb41de6a14c28fd05f578073
 
 ...

Makefile

@@ -19,6 +19,9 @@ IMAGE_BRANCH_TAG = latest
 endif
 DRONE ?= false
 
+# INTERNAL_REGISTRY used for pushing images to grafana internal registry
+INTERNAL_REGISTRY ?= us.gcr.io/kubernetes-dev
+
 # TARGETPLATFORM is specifically called from `docker buildx --platform`, this is mainly used when pushing docker image manifests, normal generally means NON DRONE builds
 TARGETPLATFORM ?=normal
 
@@ -180,7 +183,8 @@ all: protos agent agentctl
 agent: cmd/agent/agent
 agentctl: cmd/agentctl/agentctl
 agent-operator: cmd/agent-operator/agent-operator
-grafana-agent-crow: cmd/grafana-agent-crow/grafana-agent-crow
+agent-smoke: tools/smoke/grafana-agent-smoke
+grafana-agent-crow: tools/crow/grafana-agent-crow
 
 # In general DRONE variable should overwrite any other options, if DRONE is not set then fallback to normal behavior
 
@@ -196,24 +200,33 @@ cmd/agent-operator/agent-operator: cmd/agent-operator/main.go
 	$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
-cmd/grafana-agent-crow/grafana-agent-crow: cmd/grafana-agent-crow/main.go
+tools/crow/grafana-agent-crow: tools/crow/main.go
+	$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
+	$(NETGO_CHECK)
+
+tools/smoke/grafana-agent-smoke: tools/smoke/main.go
 	$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
+
+
 agent-image:
 	$(docker-build)  -t $(IMAGE_PREFIX)/agent:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent:$(IMAGE_TAG) -f cmd/agent/$(DOCKERFILE) .
 agentctl-image:
 	$(docker-build)  -t $(IMAGE_PREFIX)/agentctl:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agentctl:$(IMAGE_TAG) -f cmd/agentctl/$(DOCKERFILE) .
 agent-operator-image:
 	$(docker-build)  -t $(IMAGE_PREFIX)/agent-operator:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent-operator:$(IMAGE_TAG) -f cmd/agent-operator/$(DOCKERFILE) .
 grafana-agent-crow-image:
-	$(docker-build)  -t $(IMAGE_PREFIX)/agent-crow:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent-crow:$(IMAGE_TAG) -f cmd/grafana-agent-crow/$(DOCKERFILE) .
+	$(docker-build)  -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-crow:$(IMAGE_BRANCH_TAG) -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-crow:$(IMAGE_TAG) -f tools/crow/$(DOCKERFILE) .
+agent-smoke-image:
+	$(docker-build)  -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-smoke:$(IMAGE_BRANCH_TAG) -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-smoke:$(IMAGE_TAG) -f tools/smoke/$(DOCKERFILE) .
 
 install:
 	CGO_ENABLED=1 go install $(CGO_FLAGS) ./cmd/agent
 	CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agentctl
 	CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agent-operator
-	CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/grafana-agent-crow
+	CGO_ENABLED=0 go install $(GO_FLAGS) ./tools/crow
+	CGO_ENABLED=0 go install $(GO_FLAGS) ./tools/smoke
 
 #######################
 # Development targets #
@@ -306,7 +319,6 @@ endif
 dist/agent-freebsd-amd64: seego
 	$(call SetBuildVarsConditional,freebsd);  $(seego) build $(CGO_FLAGS) -o $@ ./cmd/agent
 
-
 #######################
 # BEGIN AGENTCTL DIST #
 #######################

example/k3d/README.md

@@ -46,7 +46,13 @@ The smoke test environment is used to validate samples end to end.
 
 Smoke Test environment is invoked via `/scripts/smoke-test.bash`
 
-This tool will spin up cluster of Grafana Agent, Cortex, Avalanche and [Crow](../../cmd/grafana-agent-crow/README.md) instances. The tool will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing.
+This tool will spin up cluster of Grafana Agent, Cortex, Avalanche, Smoke and [Crow](../../tools/crow/README.md) instances. The Smoke deployment will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing.
+
+For users who do not have access to the `us.gcr.io/kubernetes-dev` container registry, you will need to build the Smoke and Crow images locally before running the smoke test.
+
+```
+make grafana-agent-crow-image agent-smoke-image
+```
 
 ### What to look for?
 
@@ -70,7 +76,7 @@ If at the end of the test any issues are found they will look similar to the bel
 
 ### How to trigger an alert?
 
-Changing the avalanche setting for label_count to 1000, located [here](./lib/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit.
+Changing the avalanche setting for label_count to 1000, located [here](../../production/tanka/grafana-agent/smoke/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit.
 
 ![](./assets/trigger_change.png)
 
@@ -84,6 +90,7 @@ By default, a k3d cluster will be created running the following instances
 - crow-single - serves the single agent
 - cortex
 - avalanche - selection of avalanche instances serving traffic
+- smoke - scales avalanche replicas and introduces chaos by deleting agent pods during testing
 
 Crow instance will check to see if the metrics that were scraped shows up in the prometheus endpoint and then will emit metrics on the success of those metrics. This success/failure result will trigger an alert if it is incorrect.
 

example/k3d/environment/main.jsonnet

@@ -1,6 +1,6 @@
 local collector = import 'collector/main.libsonnet';
 local default = import 'default/main.libsonnet';
-local etcd = import 'etcd/main.libsonnet';
+local etcd = import 'grafana-agent/smoke/etcd/main.libsonnet';
 local agent_cluster = import 'grafana-agent/scraping-svc/main.libsonnet';
 local k = import 'ksonnet-util/kausal.libsonnet';
 local load_generator = import 'load-generator/main.libsonnet';

example/k3d/jsonnetfile.lock.json

@@ -18,8 +18,8 @@
           "subdir": "grafonnet"
         }
       },
-      "version": "0d4f64154ee2024820396aa66841a64d851302b0",
-      "sum": "1DvaFCCiwjI9624YYhLPdwiEPDvLsrCOES0aBxr7wrM="
+      "version": "3626fc4dc2326931c530861ac5bebe39444f6cbf",
+      "sum": "gF8foHByYcB25jcUOBqP6jxk0OPifQMjPvKY0HaCk6w="
     },
     {
       "source": {

example/k3d/scripts/smoke-test.bash

@@ -35,14 +35,13 @@ set -euo pipefail
 # Constants
 ROOT=$(git rev-parse --show-toplevel)
 K3D_CLUSTER_NAME="agent-smoke-test"
-MUTATION_FREQUENCY="5m"
-CHAOS_FREQUENCY="30m"
+SKIP_CREATE=""
 
 # Variables
 
 # Which function will be called
 ENTRYPOINT="run"
-TEST_DURATION="3h"
+TEST_DURATION="10800"
 IMPORT_IMAGES=""
 
 while getopts "dt:ish" opt; do
@@ -85,18 +84,13 @@ run() {
     k3d image import -c $K3D_CLUSTER_NAME \
       grafana/agent:main \
       grafana/agentctl:main \
-      grafana/agent-crow:main
+      us.gcr.io/kubernetes-dev/grafana/agent-crow:main \
+      us.gcr.io/kubernetes-dev/grafana/agent-smoke:main
   fi
 
   (cd $ROOT/example/k3d && jb install)
   tk apply $ROOT/example/k3d/smoke --dangerous-auto-approve
 
-  echo "--- Spawning background tasks"
-  mutation_loop &
-  MUTATION_PID=$!
-  chaos_loop &
-  CHAOS_PID=$!
-
   # Immediately create a job to sync configs so our two Agent deployments
   # are synced up as closely as possible.
   kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \
@@ -106,10 +100,11 @@ run() {
   echo "Your environment is now running for the next $TEST_DURATION."
   echo "Grafana URL: http://grafana.k3d.localhost:50080"
   echo "Prometheus URL: http://prometheus.k3d.localhost:50080"
+  echo "Check smoke test logs: "
+  echo "  kubectl logs --namespace=smoke -f deployment/smoke-test"
   sleep $TEST_DURATION
 
-  echo "--- Stopping background tasks"
-  kill $CHAOS_PID $MUTATION_PID
+  kubectl scale -n smoke --replicas=0 deployment/smoke-test
 
   echo "Smoke tests complete!"
   echo "Grafana URL: http://grafana.k3d.localhost:50080"
@@ -120,43 +115,13 @@ run() {
   get_results
 }
 
-# mutation_loop changes the number of replicas of Avalanche servers
-# randomly between 0-10 replicas (inclusive) every $MUTATION_FREQUENCY.
-mutation_loop() {
-  while true; do
-    sleep $MUTATION_FREQUENCY
-
-    NEW_REPLICAS=$(($RANDOM % 11))
-    echo "--- Scaling Avalanche to $NEW_REPLICAS replicas"
-    kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \
-      scale --replicas=$NEW_REPLICAS deployment/avalanche
-  done
-}
-
-# chaos loop deletes pods every $CHAOS_FREQUENCY.
-chaos_loop() {
-  while true; do
-    sleep $CHAOS_FREQUENCY
-
-    echo "--- Force deleting pod/grafana-agent-0"
-    kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \
-      delete --grace-period=0 --force pod/grafana-agent-0
-
-    REPLICA=$(($RANDOM % 3))
-    echo "--- Force deleting pod/agent-cluster-$REPLICA"
-    kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \
-      delete --grace-period=0 --force pod/grafana-agent-cluster-$REPLICA
-  done
-}
-
 get_results() {
   NUM_ALERTS=$(curl -s -G \
     -H "Host: prometheus.k3d.localhost"    \
     -d "query=count_over_time(ALERTS{alertstate=\"firing\"}[$TEST_DURATION])" \
     'http://localhost:50080/api/v1/query'  \
       | jq '.data.result | length'         \
   )
-
   if test $NUM_ALERTS -ne 0; then
     echo "FAIL: $NUM_ALERTS alerts found over the last $TEST_DURATION."
     echo "More information: http://prometheus.k3d.localhost:50080/graph?g0.expr=count_over_time(ALERTS{alertstate%3D%22firing%22}[$TEST_DURATION])"

example/k3d/smoke/main.jsonnet

@@ -1,8 +1,9 @@
 local monitoring = import './monitoring/main.jsonnet';
-local avalanche = import 'avalanche/main.libsonnet';
+local avalanche = import 'grafana-agent/smoke/avalanche/main.libsonnet';
 local cortex = import 'cortex/main.libsonnet';
-local crow = import 'crow/main.libsonnet';
-local etcd = import 'etcd/main.libsonnet';
+local crow = import 'grafana-agent/smoke/crow/main.libsonnet';
+local smoke = import 'grafana-agent/smoke/main.libsonnet';
+local etcd = import 'grafana-agent/smoke/etcd/main.libsonnet';
 local gragent = import 'grafana-agent/v2/main.libsonnet';
 local k = import 'ksonnet-util/kausal.libsonnet';
 
@@ -23,6 +24,13 @@ local new_crow(name, selector) =
     },
   });
 
+local new_smoke(name) = smoke.new(name, namespace='smoke', config={
+    mutationFrequency: '5m',
+    chaosFrequency: '30m',
+});
+
+
+
 local smoke = {
   ns: namespace.new('smoke'),
 
@@ -39,6 +47,8 @@ local smoke = {
     metric_interval: 600,
   }),
 
+  smoke_test: new_smoke('smoke-test'),
+
   crows: [
     new_crow('crow-single', 'cluster="grafana-agent"'),
     new_crow('crow-cluster', 'cluster="grafana-agent-cluster"'),

example/k3d/smoke/monitoring/main.jsonnet

@@ -11,7 +11,8 @@ local ingress = k.networking.v1beta1.ingress;
 local rule = k.networking.v1beta1.ingressRule;
 local path = k.networking.v1beta1.httpIngressPath;
 
-local prometheus_monitoring = import './prometheus_monitoring.libsonnet';
+
+local prometheus_monitoring = import 'grafana-agent/smoke/prometheus_monitoring.libsonnet';
 
 {
   ns: namespace.new('monitoring'),

pkg/util/logger.go

@@ -29,6 +29,17 @@ func NewLogger(cfg *server.Config) *Logger {
 	return newLogger(cfg, defaultLogger)
 }
 
+// NewLoggerFromLevel creates a new logger from logging.Level and logging.Format.
+func NewLoggerFromLevel(lvl logging.Level, fmt logging.Format) *Logger {
+	logger, err := makeDefaultLogger(lvl, fmt)
+	if err != nil {
+		panic(err)
+	}
+	return &Logger{
+		l: logger,
+	}
+}
+
 func newLogger(cfg *server.Config, ctor func(*server.Config) (log.Logger, error)) *Logger {
 	l := Logger{makeLogger: ctor}
 	if err := l.ApplyConfig(cfg); err != nil {
@@ -52,9 +63,13 @@ func (l *Logger) ApplyConfig(cfg *server.Config) error {
 }
 
 func defaultLogger(cfg *server.Config) (log.Logger, error) {
+	return makeDefaultLogger(cfg.LogLevel, cfg.LogFormat)
+}
+
+func makeDefaultLogger(lvl logging.Level, fmt logging.Format) (log.Logger, error) {
 	var l log.Logger
 
-	l, err := cortex_log.NewPrometheusLogger(cfg.LogLevel, cfg.LogFormat)
+	l, err := cortex_log.NewPrometheusLogger(lvl, fmt)
 	if err != nil {
 		return nil, err
 	}

example/k3d/lib/crow/main.libsonnet → production/tanka/grafana-agent/smoke/crow/main.libsonnet

@@ -14,8 +14,9 @@ local volume = k.core.v1.volume;
     local this = self,
 
     _config+:: {
-      image: 'grafana/agent-crow:latest',
+      image: 'us.gcr.io/kubernetes-dev/grafana/agent-crow:main',
       args: {},
+      pull_secret: '',
     } + config,
 
     container::
@@ -27,6 +28,7 @@ local volume = k.core.v1.volume;
 
     deployment:
       deployment.new(name, 1, [self.container]) +
-      deployment.mixin.metadata.withNamespace(namespace),
+      deployment.mixin.metadata.withNamespace(namespace) +
+      deployment.spec.template.spec.withImagePullSecrets({ name: this._config.pull_secret }),
   },
 }