Agent smoke test

.drone/drone.yml

@@ -53,7 +53,7 @@ steps:
       - tar -C /usr/local -xzf go1.17.6.linux-amd64.tar.gz
       - rm go1.17.6.linux-amd64.tar.gz
       - export PATH=$PATH:/usr/local/go/bin
-      - make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator cmd/grafana-agent-crow/grafana-agent-crow
+      - make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator tools/crow/grafana-agent-crow tools/smoke/grafana-agent-smoke
       - make K8S_USE_DOCKER_NETWORK=1 DRONE=true BUILD_IN_CONTAINER=false test
 depends_on:
   - Lint
@@ -113,7 +113,11 @@ steps:
         from_secret: DOCKER_LOGIN
       DOCKER_PASSWORD:
         from_secret: DOCKER_PASSWORD
+      GCR_CREDS:
+        from_secret: gcr_admin
     commands:
+      - mkdir -p $HOME/.docker
+      - printenv GCR_CREDS > $HOME/.docker/config.json
       - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
       - apk update && apk add make bash wget git qemu
       - wget -q https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64
@@ -128,6 +132,7 @@ steps:
       - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agentctl-image
       - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-operator-image
       - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true grafana-agent-crow-image
+      - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-smoke-image
       - docker buildx rm multiarch
 
 depends_on:
@@ -243,6 +248,14 @@ get:
   path: secret/data/common/gcr
   name: .dockerconfigjson
 
+---
+kind: secret
+name: gcr_admin
+
+get:
+  name: .dockerconfigjson
+  path: infra/data/ci/gcr-admin
+
 ---
 kind: secret
 name: gh_token
@@ -252,6 +265,6 @@ get:
   name: pat
 ---
 kind: signature
-hmac: 87a9a23731884dac8e0b5c3ee0fa2b5cef9d7fa490fcad649f7783f6ea9d2d32
+hmac: f59c8b25058782d021a2d6bdc1fc59a049f3b352fb41de6a14c28fd05f578073
 
 ...

Makefile

@@ -19,6 +19,9 @@ IMAGE_BRANCH_TAG = latest
 endif
 DRONE ?= false
 
+# INTERNAL_REGISTRY used for pushing images to grafana internal registry
+INTERNAL_REGISTRY ?= us.gcr.io/kubernetes-dev
+
 # TARGETPLATFORM is specifically called from `docker buildx --platform`, this is mainly used when pushing docker image manifests, normal generally means NON DRONE builds
 TARGETPLATFORM ?=normal
 
@@ -180,7 +183,8 @@ all: protos agent agentctl
 agent: cmd/agent/agent
 agentctl: cmd/agentctl/agentctl
 agent-operator: cmd/agent-operator/agent-operator
-grafana-agent-crow: cmd/grafana-agent-crow/grafana-agent-crow
+agent-smoke: tools/smoke/grafana-agent-smoke
+grafana-agent-crow: tools/crow/grafana-agent-crow
 
 # In general DRONE variable should overwrite any other options, if DRONE is not set then fallback to normal behavior
 
@@ -196,24 +200,33 @@ cmd/agent-operator/agent-operator: cmd/agent-operator/main.go
 	$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
-cmd/grafana-agent-crow/grafana-agent-crow: cmd/grafana-agent-crow/main.go
+tools/crow/grafana-agent-crow: tools/crow/main.go
+	$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
+	$(NETGO_CHECK)
+
+tools/smoke/grafana-agent-smoke: tools/smoke/main.go
 	$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
+
+
 agent-image:
 	$(docker-build)  -t $(IMAGE_PREFIX)/agent:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent:$(IMAGE_TAG) -f cmd/agent/$(DOCKERFILE) .
 agentctl-image:
 	$(docker-build)  -t $(IMAGE_PREFIX)/agentctl:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agentctl:$(IMAGE_TAG) -f cmd/agentctl/$(DOCKERFILE) .
 agent-operator-image:
 	$(docker-build)  -t $(IMAGE_PREFIX)/agent-operator:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent-operator:$(IMAGE_TAG) -f cmd/agent-operator/$(DOCKERFILE) .
 grafana-agent-crow-image:
-	$(docker-build)  -t $(IMAGE_PREFIX)/agent-crow:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent-crow:$(IMAGE_TAG) -f cmd/grafana-agent-crow/$(DOCKERFILE) .
+	$(docker-build)  -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-crow:$(IMAGE_BRANCH_TAG) -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-crow:$(IMAGE_TAG) -f tools/crow/$(DOCKERFILE) .
+agent-smoke-image:
+	$(docker-build)  -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-smoke:$(IMAGE_BRANCH_TAG) -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-smoke:$(IMAGE_TAG) -f tools/smoke/$(DOCKERFILE) .
 
 install:
 	CGO_ENABLED=1 go install $(CGO_FLAGS) ./cmd/agent
 	CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agentctl
 	CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agent-operator
-	CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/grafana-agent-crow
+	CGO_ENABLED=0 go install $(GO_FLAGS) ./tools/crow
+	CGO_ENABLED=0 go install $(GO_FLAGS) ./tools/smoke
 
 #######################
 # Development targets #
@@ -306,7 +319,6 @@ endif
 dist/agent-freebsd-amd64: seego
 	$(call SetBuildVarsConditional,freebsd);  $(seego) build $(CGO_FLAGS) -o $@ ./cmd/agent
 
-
 #######################
 # BEGIN AGENTCTL DIST #
 #######################

example/k3d/README.md

@@ -46,7 +46,7 @@ The smoke test environment is used to validate samples end to end.
 
 Smoke Test environment is invoked via `/scripts/smoke-test.bash`
 
-This tool will spin up cluster of Grafana Agent, Cortex, Avalanche and [Crow](../../cmd/grafana-agent-crow/README.md) instances. The tool will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing.
+This tool will spin up cluster of Grafana Agent, Cortex, Avalanche and [Crow](../../tools/crow/README.md) instances. The tool will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing.
 
 ### What to look for?
 
@@ -70,7 +70,7 @@ If at the end of the test any issues are found they will look similar to the bel
 
 ### How to trigger an alert?
 
-Changing the avalanche setting for label_count to 1000, located [here](./lib/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit.
+Changing the avalanche setting for label_count to 1000, located [here](../../production/tanka/grafana-agent/smoke/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit.
 
 ![](./assets/trigger_change.png)
 

example/k3d/environment/main.jsonnet

@@ -1,6 +1,6 @@
 local collector = import 'collector/main.libsonnet';
 local default = import 'default/main.libsonnet';
-local etcd = import 'etcd/main.libsonnet';
+local etcd = import 'grafana-agent/smoke/etcd/main.libsonnet';
 local agent_cluster = import 'grafana-agent/scraping-svc/main.libsonnet';
 local k = import 'ksonnet-util/kausal.libsonnet';
 local load_generator = import 'load-generator/main.libsonnet';

example/k3d/jsonnetfile.lock.json

@@ -18,8 +18,8 @@
           "subdir": "grafonnet"
         }
       },
-      "version": "0d4f64154ee2024820396aa66841a64d851302b0",
-      "sum": "1DvaFCCiwjI9624YYhLPdwiEPDvLsrCOES0aBxr7wrM="
+      "version": "3626fc4dc2326931c530861ac5bebe39444f6cbf",
+      "sum": "gF8foHByYcB25jcUOBqP6jxk0OPifQMjPvKY0HaCk6w="
     },
     {
       "source": {

example/k3d/scripts/smoke-test.bash

@@ -35,14 +35,13 @@ set -euo pipefail
 # Constants
 ROOT=$(git rev-parse --show-toplevel)
 K3D_CLUSTER_NAME="agent-smoke-test"
-MUTATION_FREQUENCY="5m"
-CHAOS_FREQUENCY="30m"
+SKIP_CREATE=""
 
 # Variables
 
 # Which function will be called
 ENTRYPOINT="run"
-TEST_DURATION="3h"
+TEST_DURATION="10800"
 IMPORT_IMAGES=""
 
 while getopts "dt:ish" opt; do
@@ -85,18 +84,13 @@ run() {
     k3d image import -c $K3D_CLUSTER_NAME \
       grafana/agent:main \
       grafana/agentctl:main \
-      grafana/agent-crow:main
+      us.gcr.io/kubernetes-dev/grafana/agent-crow:main \
+      us.gcr.io/kubernetes-dev/grafana/agent-smoke:main
   fi
 
   (cd $ROOT/example/k3d && jb install)
   tk apply $ROOT/example/k3d/smoke --dangerous-auto-approve
 
-  echo "--- Spawning background tasks"
-  mutation_loop &
-  MUTATION_PID=$!
-  chaos_loop &
-  CHAOS_PID=$!
-
   # Immediately create a job to sync configs so our two Agent deployments
   # are synced up as closely as possible.
   kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \
@@ -106,10 +100,11 @@ run() {
   echo "Your environment is now running for the next $TEST_DURATION."
   echo "Grafana URL: http://grafana.k3d.localhost:50080"
   echo "Prometheus URL: http://prometheus.k3d.localhost:50080"
+  echo "Check smoke test logs: "
+  echo "  kubectl logs --namespace=smoke -f deployment/smoke-test"
   sleep $TEST_DURATION
 
-  echo "--- Stopping background tasks"
-  kill $CHAOS_PID $MUTATION_PID
+  kubectl scale -n smoke --replicas=0 deployment/smoke-test
 
   echo "Smoke tests complete!"
   echo "Grafana URL: http://grafana.k3d.localhost:50080"
@@ -120,43 +115,13 @@ run() {
   get_results
 }
 
-# mutation_loop changes the number of replicas of Avalanche servers
-# randomly between 0-10 replicas (inclusive) every $MUTATION_FREQUENCY.
-mutation_loop() {
-  while true; do
-    sleep $MUTATION_FREQUENCY
-
-    NEW_REPLICAS=$(($RANDOM % 11))
-    echo "--- Scaling Avalanche to $NEW_REPLICAS replicas"
-    kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \
-      scale --replicas=$NEW_REPLICAS deployment/avalanche
-  done
-}
-
-# chaos loop deletes pods every $CHAOS_FREQUENCY.
-chaos_loop() {
-  while true; do
-    sleep $CHAOS_FREQUENCY
-
-    echo "--- Force deleting pod/grafana-agent-0"
-    kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \
-      delete --grace-period=0 --force pod/grafana-agent-0
-
-    REPLICA=$(($RANDOM % 3))
-    echo "--- Force deleting pod/agent-cluster-$REPLICA"
-    kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \
-      delete --grace-period=0 --force pod/grafana-agent-cluster-$REPLICA
-  done
-}
-
 get_results() {
   NUM_ALERTS=$(curl -s -G \
     -H "Host: prometheus.k3d.localhost"    \
     -d "query=count_over_time(ALERTS{alertstate=\"firing\"}[$TEST_DURATION])" \
     'http://localhost:50080/api/v1/query'  \
       | jq '.data.result | length'         \
   )
-
   if test $NUM_ALERTS -ne 0; then
     echo "FAIL: $NUM_ALERTS alerts found over the last $TEST_DURATION."
     echo "More information: http://prometheus.k3d.localhost:50080/graph?g0.expr=count_over_time(ALERTS{alertstate%3D%22firing%22}[$TEST_DURATION])"

example/k3d/smoke/main.jsonnet

@@ -1,8 +1,9 @@
 local monitoring = import './monitoring/main.jsonnet';
-local avalanche = import 'avalanche/main.libsonnet';
+local avalanche = import 'grafana-agent/smoke/avalanche/main.libsonnet';
 local cortex = import 'cortex/main.libsonnet';
-local crow = import 'crow/main.libsonnet';
-local etcd = import 'etcd/main.libsonnet';
+local crow = import 'grafana-agent/smoke/crow/main.libsonnet';
+local smoke = import 'grafana-agent/smoke/main.libsonnet';
+local etcd = import 'grafana-agent/smoke/etcd/main.libsonnet';
 local gragent = import 'grafana-agent/v2/main.libsonnet';
 local k = import 'ksonnet-util/kausal.libsonnet';
 
@@ -13,6 +14,7 @@ local volumeMount = k.core.v1.volumeMount;
 local images = {
   agent: 'grafana/agent:main',
   agentctl: 'grafana/agentctl:main',
+  agentsmoke: 'us.gcr.io/kubernetes-dev/grafana/agent-smoke:main',
 };
 
 local new_crow(name, selector) =
@@ -23,6 +25,9 @@ local new_crow(name, selector) =
     },
   });
 
+local new_smoke(name) = smoke.new(name, namespace='smoke',
+    mutationFrequency='5m', chaosFrequency='30m', image=images.agentsmoke);
+
 local smoke = {
   ns: namespace.new('smoke'),
 
@@ -39,6 +44,8 @@ local smoke = {
     metric_interval: 600,
   }),
 
+  smoke_test: new_smoke('smoke-test'),
+
   crows: [
     new_crow('crow-single', 'cluster="grafana-agent"'),
     new_crow('crow-cluster', 'cluster="grafana-agent-cluster"'),

example/k3d/smoke/monitoring/main.jsonnet

@@ -11,7 +11,8 @@ local ingress = k.networking.v1beta1.ingress;
 local rule = k.networking.v1beta1.ingressRule;
 local path = k.networking.v1beta1.httpIngressPath;
 
-local prometheus_monitoring = import './prometheus_monitoring.libsonnet';
+
+local prometheus_monitoring = import 'grafana-agent/smoke/prometheus_monitoring.libsonnet';
 
 {
   ns: namespace.new('monitoring'),

pkg/util/logger.go

@@ -29,6 +29,17 @@ func NewLogger(cfg *server.Config) *Logger {
 	return newLogger(cfg, defaultLogger)
 }
 
+// NewLoggerFromLevel creates a new logger from logging.Level and logging.Format.
+func NewLoggerFromLevel(lvl logging.Level, fmt logging.Format) *Logger {
+	logger, err := makeDefaultLogger(lvl, fmt)
+	if err != nil {
+		panic(err)
+	}
+	return &Logger{
+		l: logger,
+	}
+}
+
 func newLogger(cfg *server.Config, ctor func(*server.Config) (log.Logger, error)) *Logger {
 	l := Logger{makeLogger: ctor}
 	if err := l.ApplyConfig(cfg); err != nil {
@@ -52,9 +63,13 @@ func (l *Logger) ApplyConfig(cfg *server.Config) error {
 }
 
 func defaultLogger(cfg *server.Config) (log.Logger, error) {
+	return makeDefaultLogger(cfg.LogLevel, cfg.LogFormat)
+}
+
+func makeDefaultLogger(lvl logging.Level, fmt logging.Format) (log.Logger, error) {
 	var l log.Logger
 
-	l, err := cortex_log.NewPrometheusLogger(cfg.LogLevel, cfg.LogFormat)
+	l, err := cortex_log.NewPrometheusLogger(lvl, fmt)
 	if err != nil {
 		return nil, err
 	}

example/k3d/lib/crow/main.libsonnet → production/tanka/grafana-agent/smoke/crow/main.libsonnet

@@ -14,7 +14,7 @@ local volume = k.core.v1.volume;
     local this = self,
 
     _config+:: {
-      image: 'grafana/agent-crow:latest',
+      image: 'us.gcr.io/kubernetes-dev/grafana/agent-crow:main',
       args: {},
     } + config,
 

production/tanka/grafana-agent/smoke/internal/smoke.libsonnet

@@ -0,0 +1,37 @@
+local k = import 'ksonnet-util/kausal.libsonnet';
+
+local policyRule = k.rbac.v1.policyRule;
+local serviceAccount = k.core.v1.serviceAccount;
+local container = k.core.v1.container;
+local deployment = k.apps.v1.deployment;
+
+{
+    newSmoke(name='grafana-agent-smoke', namespace='default', mutationFrequency='5m', chaosFrequency='30m', image):: {
+        local k = (import 'ksonnet-util/kausal.libsonnet') { _config+:: { namespace: namespace } },
+
+        rbac:
+            k.util.rbac(name, [
+                policyRule.withApiGroups(['apps']) +
+                policyRule.withResources(['deployments/scale']) +
+                policyRule.withVerbs(['get', 'update']),
+            ]) {
+                service_account+:
+                  serviceAccount.mixin.metadata.withNamespace(namespace),
+            },
+
+        container::
+          container.new('agent-smoke', image) +
+          container.withCommand('/bin/grafana-agent-smoke') +
+          container.withArgsMixin(k.util.mapToFlags({
+            'log.level': 'debug',
+            'namespace': namespace,
+            'mutation-frequency': mutationFrequency,
+            'chaos-frequency': chaosFrequency,
+          })),
+
+        agentsmoke:
+          deployment.new(name, 1, [self.container]) +
+          deployment.mixin.metadata.withNamespace(namespace) +
+          deployment.mixin.spec.template.spec.withServiceAccount(name),
+    },
+}

production/tanka/grafana-agent/smoke/main.libsonnet

@@ -0,0 +1,14 @@
+local smoke = import './internal/smoke.libsonnet';
+
+{
+    _images:: {
+        agentsmoke: 'us.gcr.io/kubernetes-dev/grafana/agent-smoke:latest',
+    },
+
+    new(name='grafana-agent-smoke', namespace='grafana-agent-smoke', mutationFrequency='5m', chaosFrequency='30m', image=self._images.agentsmoke):: {
+        smoke:
+            smoke.newSmoke(name, namespace, mutationFrequency, chaosFrequency, image)
+    },
+
+    monitoring: (import './prometheus_monitoring.libsonnet'),
+}

tools/crow/.gitignore

@@ -0,0 +1 @@
+grafana-agent-crow