diff --git a/.drone/drone.yml b/.drone/drone.yml index f9c7a9f5d9c5..6118bd3de15a 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -53,7 +53,7 @@ steps: - tar -C /usr/local -xzf go1.17.6.linux-amd64.tar.gz - rm go1.17.6.linux-amd64.tar.gz - export PATH=$PATH:/usr/local/go/bin - - make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator cmd/grafana-agent-crow/grafana-agent-crow + - make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator tools/crow/grafana-agent-crow tools/smoke/grafana-agent-smoke - make K8S_USE_DOCKER_NETWORK=1 DRONE=true BUILD_IN_CONTAINER=false test depends_on: - Lint @@ -113,7 +113,11 @@ steps: from_secret: DOCKER_LOGIN DOCKER_PASSWORD: from_secret: DOCKER_PASSWORD + GCR_CREDS: + from_secret: gcr_admin commands: + - mkdir -p $HOME/.docker + - printenv GCR_CREDS > $HOME/.docker/config.json - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - apk update && apk add make bash wget git qemu - wget -q https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 @@ -128,6 +132,7 @@ steps: - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agentctl-image - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-operator-image - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true grafana-agent-crow-image + - make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-smoke-image - docker buildx rm multiarch depends_on: @@ -243,6 +248,14 @@ get: path: secret/data/common/gcr name: .dockerconfigjson +--- +kind: secret +name: gcr_admin + +get: + name: .dockerconfigjson + path: infra/data/ci/gcr-admin + --- kind: secret name: gh_token @@ -252,6 +265,6 @@ get: name: pat --- kind: signature -hmac: 87a9a23731884dac8e0b5c3ee0fa2b5cef9d7fa490fcad649f7783f6ea9d2d32 +hmac: f59c8b25058782d021a2d6bdc1fc59a049f3b352fb41de6a14c28fd05f578073 ... diff --git a/Makefile b/Makefile index 8454824a028b..fbe0c4902aa9 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,9 @@ IMAGE_BRANCH_TAG = latest endif DRONE ?= false +# INTERNAL_REGISTRY used for pushing images to grafana internal registry +INTERNAL_REGISTRY ?= us.gcr.io/kubernetes-dev + # TARGETPLATFORM is specifically called from `docker buildx --platform`, this is mainly used when pushing docker image manifests, normal generally means NON DRONE builds TARGETPLATFORM ?=normal @@ -180,7 +183,8 @@ all: protos agent agentctl agent: cmd/agent/agent agentctl: cmd/agentctl/agentctl agent-operator: cmd/agent-operator/agent-operator -grafana-agent-crow: cmd/grafana-agent-crow/grafana-agent-crow +agent-smoke: tools/smoke/grafana-agent-smoke +grafana-agent-crow: tools/crow/grafana-agent-crow # In general DRONE variable should overwrite any other options, if DRONE is not set then fallback to normal behavior @@ -196,10 +200,16 @@ cmd/agent-operator/agent-operator: cmd/agent-operator/main.go $(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D) $(NETGO_CHECK) -cmd/grafana-agent-crow/grafana-agent-crow: cmd/grafana-agent-crow/main.go +tools/crow/grafana-agent-crow: tools/crow/main.go + $(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + +tools/smoke/grafana-agent-smoke: tools/smoke/main.go $(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D) $(NETGO_CHECK) + + agent-image: $(docker-build) -t $(IMAGE_PREFIX)/agent:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent:$(IMAGE_TAG) -f cmd/agent/$(DOCKERFILE) . agentctl-image: @@ -207,13 +217,16 @@ agentctl-image: agent-operator-image: $(docker-build) -t $(IMAGE_PREFIX)/agent-operator:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent-operator:$(IMAGE_TAG) -f cmd/agent-operator/$(DOCKERFILE) . grafana-agent-crow-image: - $(docker-build) -t $(IMAGE_PREFIX)/agent-crow:$(IMAGE_BRANCH_TAG) -t $(IMAGE_PREFIX)/agent-crow:$(IMAGE_TAG) -f cmd/grafana-agent-crow/$(DOCKERFILE) . + $(docker-build) -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-crow:$(IMAGE_BRANCH_TAG) -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-crow:$(IMAGE_TAG) -f tools/crow/$(DOCKERFILE) . +agent-smoke-image: + $(docker-build) -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-smoke:$(IMAGE_BRANCH_TAG) -t $(INTERNAL_REGISTRY)/$(IMAGE_PREFIX)/agent-smoke:$(IMAGE_TAG) -f tools/smoke/$(DOCKERFILE) . install: CGO_ENABLED=1 go install $(CGO_FLAGS) ./cmd/agent CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agentctl CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agent-operator - CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/grafana-agent-crow + CGO_ENABLED=0 go install $(GO_FLAGS) ./tools/crow + CGO_ENABLED=0 go install $(GO_FLAGS) ./tools/smoke ####################### # Development targets # @@ -306,7 +319,6 @@ endif dist/agent-freebsd-amd64: seego $(call SetBuildVarsConditional,freebsd); $(seego) build $(CGO_FLAGS) -o $@ ./cmd/agent - ####################### # BEGIN AGENTCTL DIST # ####################### diff --git a/example/k3d/README.md b/example/k3d/README.md index e46fb3ffa279..715a10553c5e 100644 --- a/example/k3d/README.md +++ b/example/k3d/README.md @@ -46,7 +46,13 @@ The smoke test environment is used to validate samples end to end. Smoke Test environment is invoked via `/scripts/smoke-test.bash` -This tool will spin up cluster of Grafana Agent, Cortex, Avalanche and [Crow](../../cmd/grafana-agent-crow/README.md) instances. The tool will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing. +This tool will spin up cluster of Grafana Agent, Cortex, Avalanche, Smoke and [Crow](../../tools/crow/README.md) instances. The Smoke deployment will then periodically kill instances and check for any failed alerts. At the end of the duration (default 3h) it will end the testing. + +For users who do not have access to the `us.gcr.io/kubernetes-dev` container registry, you will need to build the Smoke and Crow images locally before running the smoke test. + +``` +make grafana-agent-crow-image agent-smoke-image +``` ### What to look for? @@ -70,7 +76,7 @@ If at the end of the test any issues are found they will look similar to the bel ### How to trigger an alert? -Changing the avalanche setting for label_count to 1000, located [here](./lib/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit. +Changing the avalanche setting for label_count to 1000, located [here](../../production/tanka/grafana-agent/smoke/avalanche/main.libsonnet). This will ensure the [GrafanaAgentMemHigh](http://prometheus.k3d.localhost:50080/graph?g0.expr=ALERTS%7Balertname%3D%22GrafanaAgentMemHigh%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0.g0.range_input=1h.) alert exceeds the limit. ![](./assets/trigger_change.png) @@ -84,6 +90,7 @@ By default, a k3d cluster will be created running the following instances - crow-single - serves the single agent - cortex - avalanche - selection of avalanche instances serving traffic +- smoke - scales avalanche replicas and introduces chaos by deleting agent pods during testing Crow instance will check to see if the metrics that were scraped shows up in the prometheus endpoint and then will emit metrics on the success of those metrics. This success/failure result will trigger an alert if it is incorrect. diff --git a/example/k3d/environment/main.jsonnet b/example/k3d/environment/main.jsonnet index e5766fbe46a2..53658d08ae4a 100644 --- a/example/k3d/environment/main.jsonnet +++ b/example/k3d/environment/main.jsonnet @@ -1,6 +1,6 @@ local collector = import 'collector/main.libsonnet'; local default = import 'default/main.libsonnet'; -local etcd = import 'etcd/main.libsonnet'; +local etcd = import 'grafana-agent/smoke/etcd/main.libsonnet'; local agent_cluster = import 'grafana-agent/scraping-svc/main.libsonnet'; local k = import 'ksonnet-util/kausal.libsonnet'; local load_generator = import 'load-generator/main.libsonnet'; diff --git a/example/k3d/jsonnetfile.lock.json b/example/k3d/jsonnetfile.lock.json index 2537becf63bd..0cd970b024a2 100644 --- a/example/k3d/jsonnetfile.lock.json +++ b/example/k3d/jsonnetfile.lock.json @@ -18,8 +18,8 @@ "subdir": "grafonnet" } }, - "version": "0d4f64154ee2024820396aa66841a64d851302b0", - "sum": "1DvaFCCiwjI9624YYhLPdwiEPDvLsrCOES0aBxr7wrM=" + "version": "3626fc4dc2326931c530861ac5bebe39444f6cbf", + "sum": "gF8foHByYcB25jcUOBqP6jxk0OPifQMjPvKY0HaCk6w=" }, { "source": { diff --git a/example/k3d/scripts/smoke-test.bash b/example/k3d/scripts/smoke-test.bash index 36059107d3ce..a0d45a4b023e 100755 --- a/example/k3d/scripts/smoke-test.bash +++ b/example/k3d/scripts/smoke-test.bash @@ -35,14 +35,13 @@ set -euo pipefail # Constants ROOT=$(git rev-parse --show-toplevel) K3D_CLUSTER_NAME="agent-smoke-test" -MUTATION_FREQUENCY="5m" -CHAOS_FREQUENCY="30m" +SKIP_CREATE="" # Variables # Which function will be called ENTRYPOINT="run" -TEST_DURATION="3h" +TEST_DURATION="10800" IMPORT_IMAGES="" while getopts "dt:ish" opt; do @@ -85,18 +84,13 @@ run() { k3d image import -c $K3D_CLUSTER_NAME \ grafana/agent:main \ grafana/agentctl:main \ - grafana/agent-crow:main + us.gcr.io/kubernetes-dev/grafana/agent-crow:main \ + us.gcr.io/kubernetes-dev/grafana/agent-smoke:main fi (cd $ROOT/example/k3d && jb install) tk apply $ROOT/example/k3d/smoke --dangerous-auto-approve - echo "--- Spawning background tasks" - mutation_loop & - MUTATION_PID=$! - chaos_loop & - CHAOS_PID=$! - # Immediately create a job to sync configs so our two Agent deployments # are synced up as closely as possible. kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \ @@ -106,10 +100,11 @@ run() { echo "Your environment is now running for the next $TEST_DURATION." echo "Grafana URL: http://grafana.k3d.localhost:50080" echo "Prometheus URL: http://prometheus.k3d.localhost:50080" + echo "Check smoke test logs: " + echo " kubectl logs --namespace=smoke -f deployment/smoke-test" sleep $TEST_DURATION - echo "--- Stopping background tasks" - kill $CHAOS_PID $MUTATION_PID + kubectl scale -n smoke --replicas=0 deployment/smoke-test echo "Smoke tests complete!" echo "Grafana URL: http://grafana.k3d.localhost:50080" @@ -120,35 +115,6 @@ run() { get_results } -# mutation_loop changes the number of replicas of Avalanche servers -# randomly between 0-10 replicas (inclusive) every $MUTATION_FREQUENCY. -mutation_loop() { - while true; do - sleep $MUTATION_FREQUENCY - - NEW_REPLICAS=$(($RANDOM % 11)) - echo "--- Scaling Avalanche to $NEW_REPLICAS replicas" - kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \ - scale --replicas=$NEW_REPLICAS deployment/avalanche - done -} - -# chaos loop deletes pods every $CHAOS_FREQUENCY. -chaos_loop() { - while true; do - sleep $CHAOS_FREQUENCY - - echo "--- Force deleting pod/grafana-agent-0" - kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \ - delete --grace-period=0 --force pod/grafana-agent-0 - - REPLICA=$(($RANDOM % 3)) - echo "--- Force deleting pod/agent-cluster-$REPLICA" - kubectl --context=k3d-$K3D_CLUSTER_NAME --namespace=smoke \ - delete --grace-period=0 --force pod/grafana-agent-cluster-$REPLICA - done -} - get_results() { NUM_ALERTS=$(curl -s -G \ -H "Host: prometheus.k3d.localhost" \ @@ -156,7 +122,6 @@ get_results() { 'http://localhost:50080/api/v1/query' \ | jq '.data.result | length' \ ) - if test $NUM_ALERTS -ne 0; then echo "FAIL: $NUM_ALERTS alerts found over the last $TEST_DURATION." echo "More information: http://prometheus.k3d.localhost:50080/graph?g0.expr=count_over_time(ALERTS{alertstate%3D%22firing%22}[$TEST_DURATION])" diff --git a/example/k3d/smoke/main.jsonnet b/example/k3d/smoke/main.jsonnet index df4dcf8f4dc2..22bca8630eaa 100644 --- a/example/k3d/smoke/main.jsonnet +++ b/example/k3d/smoke/main.jsonnet @@ -1,8 +1,9 @@ local monitoring = import './monitoring/main.jsonnet'; -local avalanche = import 'avalanche/main.libsonnet'; +local avalanche = import 'grafana-agent/smoke/avalanche/main.libsonnet'; local cortex = import 'cortex/main.libsonnet'; -local crow = import 'crow/main.libsonnet'; -local etcd = import 'etcd/main.libsonnet'; +local crow = import 'grafana-agent/smoke/crow/main.libsonnet'; +local smoke = import 'grafana-agent/smoke/main.libsonnet'; +local etcd = import 'grafana-agent/smoke/etcd/main.libsonnet'; local gragent = import 'grafana-agent/v2/main.libsonnet'; local k = import 'ksonnet-util/kausal.libsonnet'; @@ -23,6 +24,13 @@ local new_crow(name, selector) = }, }); +local new_smoke(name) = smoke.new(name, namespace='smoke', config={ + mutationFrequency: '5m', + chaosFrequency: '30m', +}); + + + local smoke = { ns: namespace.new('smoke'), @@ -39,6 +47,8 @@ local smoke = { metric_interval: 600, }), + smoke_test: new_smoke('smoke-test'), + crows: [ new_crow('crow-single', 'cluster="grafana-agent"'), new_crow('crow-cluster', 'cluster="grafana-agent-cluster"'), diff --git a/example/k3d/smoke/monitoring/main.jsonnet b/example/k3d/smoke/monitoring/main.jsonnet index fc226923f448..06e93f4a5f5c 100644 --- a/example/k3d/smoke/monitoring/main.jsonnet +++ b/example/k3d/smoke/monitoring/main.jsonnet @@ -11,7 +11,8 @@ local ingress = k.networking.v1beta1.ingress; local rule = k.networking.v1beta1.ingressRule; local path = k.networking.v1beta1.httpIngressPath; -local prometheus_monitoring = import './prometheus_monitoring.libsonnet'; + +local prometheus_monitoring = import 'grafana-agent/smoke/prometheus_monitoring.libsonnet'; { ns: namespace.new('monitoring'), diff --git a/pkg/util/logger.go b/pkg/util/logger.go index bc627c3a0b26..5e90090d718c 100644 --- a/pkg/util/logger.go +++ b/pkg/util/logger.go @@ -29,6 +29,17 @@ func NewLogger(cfg *server.Config) *Logger { return newLogger(cfg, defaultLogger) } +// NewLoggerFromLevel creates a new logger from logging.Level and logging.Format. +func NewLoggerFromLevel(lvl logging.Level, fmt logging.Format) *Logger { + logger, err := makeDefaultLogger(lvl, fmt) + if err != nil { + panic(err) + } + return &Logger{ + l: logger, + } +} + func newLogger(cfg *server.Config, ctor func(*server.Config) (log.Logger, error)) *Logger { l := Logger{makeLogger: ctor} if err := l.ApplyConfig(cfg); err != nil { @@ -52,9 +63,13 @@ func (l *Logger) ApplyConfig(cfg *server.Config) error { } func defaultLogger(cfg *server.Config) (log.Logger, error) { + return makeDefaultLogger(cfg.LogLevel, cfg.LogFormat) +} + +func makeDefaultLogger(lvl logging.Level, fmt logging.Format) (log.Logger, error) { var l log.Logger - l, err := cortex_log.NewPrometheusLogger(cfg.LogLevel, cfg.LogFormat) + l, err := cortex_log.NewPrometheusLogger(lvl, fmt) if err != nil { return nil, err } diff --git a/example/k3d/lib/avalanche/main.libsonnet b/production/tanka/grafana-agent/smoke/avalanche/main.libsonnet similarity index 100% rename from example/k3d/lib/avalanche/main.libsonnet rename to production/tanka/grafana-agent/smoke/avalanche/main.libsonnet diff --git a/example/k3d/lib/crow/main.libsonnet b/production/tanka/grafana-agent/smoke/crow/main.libsonnet similarity index 77% rename from example/k3d/lib/crow/main.libsonnet rename to production/tanka/grafana-agent/smoke/crow/main.libsonnet index 29268714461c..10dbf3d0fa46 100644 --- a/example/k3d/lib/crow/main.libsonnet +++ b/production/tanka/grafana-agent/smoke/crow/main.libsonnet @@ -14,8 +14,9 @@ local volume = k.core.v1.volume; local this = self, _config+:: { - image: 'grafana/agent-crow:latest', + image: 'us.gcr.io/kubernetes-dev/grafana/agent-crow:main', args: {}, + pull_secret: '', } + config, container:: @@ -27,6 +28,7 @@ local volume = k.core.v1.volume; deployment: deployment.new(name, 1, [self.container]) + - deployment.mixin.metadata.withNamespace(namespace), + deployment.mixin.metadata.withNamespace(namespace) + + deployment.spec.template.spec.withImagePullSecrets({ name: this._config.pull_secret }), }, } diff --git a/example/k3d/lib/etcd/main.libsonnet b/production/tanka/grafana-agent/smoke/etcd/main.libsonnet similarity index 100% rename from example/k3d/lib/etcd/main.libsonnet rename to production/tanka/grafana-agent/smoke/etcd/main.libsonnet diff --git a/production/tanka/grafana-agent/smoke/main.libsonnet b/production/tanka/grafana-agent/smoke/main.libsonnet new file mode 100644 index 000000000000..a8648ad6d346 --- /dev/null +++ b/production/tanka/grafana-agent/smoke/main.libsonnet @@ -0,0 +1,55 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; +local policyRule = k.rbac.v1.policyRule; +local serviceAccount = k.core.v1.serviceAccount; +local container = k.core.v1.container; +local deployment = k.apps.v1.deployment; + +{ + new(name='grafana-agent-smoke', namespace='default', config={}):: { + local k = (import 'ksonnet-util/kausal.libsonnet') { _config+:: { namespace: namespace } }, + + local this = self, + + _images:: { + agentsmoke: 'us.gcr.io/kubernetes-dev/grafana/agent-smoke:main', + }, + + _config:: { + mutationFrequency: '5m', + chaosFrequency: '30m', + image: this._images.agentsmoke, + pull_secret: '', + } + config, + + rbac: + k.util.rbac(name, [ + policyRule.withApiGroups(['apps']) + + policyRule.withResources(['deployments/scale']) + + policyRule.withVerbs(['get', 'update']), + policyRule.withApiGroups(['']) + + policyRule.withResources(['pods']) + + policyRule.withVerbs(['list', 'delete']), + ]) { + service_account+: + serviceAccount.mixin.metadata.withNamespace(namespace), + }, + + container:: + container.new('agent-smoke', this._config.image) + + container.withCommand('/bin/grafana-agent-smoke') + + container.withArgsMixin(k.util.mapToFlags({ + 'log.level': 'debug', + 'namespace': namespace, + 'mutation-frequency': this._config.mutationFrequency, + 'chaos-frequency': this._config.chaosFrequency, + })), + + agentsmoke_deployment: + deployment.new(name, 1, [self.container]) + + deployment.mixin.metadata.withNamespace(namespace) + + deployment.mixin.spec.template.spec.withServiceAccount(name) + + deployment.spec.template.spec.withImagePullSecrets({ name: this._config.pull_secret }), + }, + + monitoring: (import './prometheus_monitoring.libsonnet'), +} diff --git a/example/k3d/smoke/monitoring/prometheus_monitoring.libsonnet b/production/tanka/grafana-agent/smoke/prometheus_monitoring.libsonnet similarity index 100% rename from example/k3d/smoke/monitoring/prometheus_monitoring.libsonnet rename to production/tanka/grafana-agent/smoke/prometheus_monitoring.libsonnet diff --git a/tools/crow/.gitignore b/tools/crow/.gitignore new file mode 100644 index 000000000000..4abc650571dc --- /dev/null +++ b/tools/crow/.gitignore @@ -0,0 +1 @@ +grafana-agent-crow diff --git a/cmd/grafana-agent-crow/Dockerfile b/tools/crow/Dockerfile similarity index 75% rename from cmd/grafana-agent-crow/Dockerfile rename to tools/crow/Dockerfile index ed7e4ba31451..4ccc1703d66a 100644 --- a/cmd/grafana-agent-crow/Dockerfile +++ b/tools/crow/Dockerfile @@ -7,5 +7,5 @@ ARG IMAGE_TAG RUN make clean && make IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false grafana-agent-crow FROM debian:buster-slim -COPY --from=build /src/agent/cmd/grafana-agent-crow/grafana-agent-crow /bin/grafana-agent-crow +COPY --from=build /src/agent/tools/crow/grafana-agent-crow /bin/grafana-agent-crow ENTRYPOINT ["/bin/grafana-agent-crow"] diff --git a/cmd/grafana-agent-crow/Dockerfile.buildx b/tools/crow/Dockerfile.buildx similarity index 90% rename from cmd/grafana-agent-crow/Dockerfile.buildx rename to tools/crow/Dockerfile.buildx index fa7bf16fc092..79d861ae07a3 100644 --- a/cmd/grafana-agent-crow/Dockerfile.buildx +++ b/tools/crow/Dockerfile.buildx @@ -20,5 +20,5 @@ ARG IMAGE_TAG RUN make clean && IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false DRONE=true make grafana-agent-crow FROM debian:buster-slim -COPY --from=build /src/agent/cmd/grafana-agent-crow/grafana-agent-crow /bin/grafana-agent-crow +COPY --from=build /src/agent/tools/crow/grafana-agent-crow /bin/grafana-agent-crow ENTRYPOINT ["/bin/grafana-agent-crow"] diff --git a/cmd/grafana-agent-crow/README.md b/tools/crow/README.md similarity index 100% rename from cmd/grafana-agent-crow/README.md rename to tools/crow/README.md diff --git a/cmd/grafana-agent-crow/main.go b/tools/crow/main.go similarity index 100% rename from cmd/grafana-agent-crow/main.go rename to tools/crow/main.go diff --git a/tools/smoke/.gitignore b/tools/smoke/.gitignore new file mode 100644 index 000000000000..a85290032261 --- /dev/null +++ b/tools/smoke/.gitignore @@ -0,0 +1 @@ +grafana-agent-smoke diff --git a/tools/smoke/Dockerfile b/tools/smoke/Dockerfile new file mode 100644 index 000000000000..82facd92bce7 --- /dev/null +++ b/tools/smoke/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.17.6-buster as build +COPY . /src/agent +WORKDIR /src/agent +ARG RELEASE_BUILD=true +ARG IMAGE_TAG + +# Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression +RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list +RUN apt-get update && apt-get install -t buster-backports -qy libsystemd-dev + +RUN make clean && make IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false agent-smoke + +FROM debian:buster-slim + +# Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression +RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list +RUN apt-get update && apt-get install -t buster-backports -qy libsystemd-dev && \ + apt-get install -qy tzdata ca-certificates && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY --from=build /src/agent/tools/smoke/grafana-agent-smoke /bin/grafana-agent-smoke + +ENTRYPOINT ["/bin/grafana-agent-smoke"] diff --git a/tools/smoke/Dockerfile.buildx b/tools/smoke/Dockerfile.buildx new file mode 100644 index 000000000000..702a7d926d1f --- /dev/null +++ b/tools/smoke/Dockerfile.buildx @@ -0,0 +1,32 @@ +FROM --platform=$BUILDPLATFORM rfratto/seego:latest as build +ARG TARGETPLATFORM +ARG BUILDPLATFORM + +# Use custom Go version instead of one prepacked in seego +ENV GOLANG_VERSION 1.17.6 +ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz +ENV GOLANG_DOWNLOAD_SHA256 231654bbf2dab3d86c1619ce799e77b03d96f9b50770297c8f4dff8836fc8ca2 +RUN rm -rf /usr/local/go \ + && curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ + && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ + && tar -C /usr/local -xzf golang.tar.gz \ + && rm golang.tar.gz + +COPY . /src/agent +WORKDIR /src/agent +ARG RELEASE_BUILD=true +ARG IMAGE_TAG + +RUN make clean && IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false DRONE=true make agent-smoke + +FROM debian:buster-slim + +# Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression +RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list +RUN apt-get update && apt-get install -t buster-backports -qy libsystemd-dev && \ + apt-get install -qy tzdata ca-certificates libsystemd-dev && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY --from=build /src/agent/tools/smoke/grafana-agent-smoke /bin/grafana-agent-smoke + +ENTRYPOINT ["/bin/grafana-agent-smoke"] diff --git a/tools/smoke/internal/smoke.go b/tools/smoke/internal/smoke.go new file mode 100644 index 000000000000..f7a10b310711 --- /dev/null +++ b/tools/smoke/internal/smoke.go @@ -0,0 +1,126 @@ +package smoke + +import ( + "context" + "flag" + "time" + + "github.com/go-kit/log" + "github.com/oklog/run" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// Smoke is the top level object for a smoke test. +type Smoke struct { + logger log.Logger + tasks []repeatingTask +} + +// Config struct to pass configuration to the Smoke constructor. +type Config struct { + Kubeconfig string + Namespace string + ChaosFrequency time.Duration + MutationFrequency time.Duration +} + +// RegisterFlags registers flags for the config to the given FlagSet. +func (c *Config) RegisterFlags(f *flag.FlagSet) { + f.StringVar(&c.Namespace, "namespace", DefaultConfig.Namespace, "namespace smoke test should run in") + f.StringVar(&c.Kubeconfig, "kubeconfig", DefaultConfig.Kubeconfig, "absolute path to the kubeconfig file") + f.DurationVar(&c.ChaosFrequency, "chaos-frequency", DefaultConfig.ChaosFrequency, "chaos frequency duration") + f.DurationVar(&c.MutationFrequency, "mutation-frequency", DefaultConfig.MutationFrequency, "mutation frequency duration") +} + +// DefaultConfig holds defaults for Smoke settings. +var DefaultConfig = Config{ + Kubeconfig: "", + Namespace: "default", + ChaosFrequency: 30 * time.Minute, + MutationFrequency: 5 * time.Minute, +} + +// New is the constructor for a Smoke object. +func New(logger log.Logger, cfg Config) (*Smoke, error) { + s := &Smoke{ + logger: logger, + } + if s.logger == nil { + s.logger = log.NewNopLogger() + } + + // use the current context in kubeconfig. this falls back to in-cluster config if kubeconfig is empty + config, err := clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig) + if err != nil { + return nil, err + } + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + // add default tasks + s.tasks = append(s.tasks, + repeatingTask{ + Task: &deletePodTask{ + logger: log.With(s.logger, "task", "delete_pod", "pod", "grafana-agent-0"), + clientset: clientset, + namespace: cfg.Namespace, + pod: "grafana-agent-0", + }, + frequency: cfg.ChaosFrequency, + }, + repeatingTask{ + Task: &deletePodBySelectorTask{ + logger: log.With(s.logger, "task", "delete_pod_by_selector"), + clientset: clientset, + namespace: cfg.Namespace, + selector: "name=grafana-agent-cluster", + }, + frequency: cfg.ChaosFrequency, + }, + repeatingTask{ + Task: &scaleDeploymentTask{ + logger: log.With(s.logger, "task", "scale_deployment", "deployment", "avalanche"), + clientset: clientset, + namespace: cfg.Namespace, + deployment: "avalanche", + maxReplicas: 11, + minReplicas: 2, + }, + frequency: cfg.MutationFrequency, + }) + + return s, nil +} + +// Run starts the smoke test and runs the tasks concurrently. +func (s *Smoke) Run(ctx context.Context) error { + var g run.Group + ctx, cancel := context.WithCancel(ctx) + defer cancel() + taskFn := func(t repeatingTask) func() error { + return func() error { + tick := time.NewTicker(t.frequency) + defer tick.Stop() + for { + select { + case <-tick.C: + if err := t.Run(ctx); err != nil { + return err + } + case <-ctx.Done(): + return nil + } + } + } + } + for _, task := range s.tasks { + g.Add(taskFn(task), func(err error) { + cancel() + }) + } + return g.Run() +} diff --git a/tools/smoke/internal/tasks.go b/tools/smoke/internal/tasks.go new file mode 100644 index 000000000000..3b887d577f77 --- /dev/null +++ b/tools/smoke/internal/tasks.go @@ -0,0 +1,101 @@ +package smoke + +import ( + "context" + "math/rand" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/pointer" +) + +// The Task interface represents some unit of work performed concurrently. +type Task interface { + Run(context.Context) error +} + +type repeatingTask struct { + Task + frequency time.Duration +} + +type deletePodTask struct { + logger log.Logger + clientset kubernetes.Interface + namespace string + pod string +} + +func (t *deletePodTask) Run(ctx context.Context) error { + level.Debug(t.logger).Log("msg", "deleting pod") + if err := t.clientset.CoreV1().Pods(t.namespace).Delete(ctx, t.pod, metav1.DeleteOptions{ + GracePeriodSeconds: pointer.Int64(0), + }); err != nil { + level.Error(t.logger).Log("msg", "failed to delete pod", "err", err) + } + return nil +} + +type scaleDeploymentTask struct { + logger log.Logger + clientset kubernetes.Interface + namespace string + deployment string + maxReplicas int + minReplicas int +} + +func (t *scaleDeploymentTask) Run(ctx context.Context) error { + newReplicas := rand.Intn(t.maxReplicas-t.minReplicas) + t.minReplicas + level.Debug(t.logger).Log("msg", "scaling replicas", "replicas", newReplicas) + + scale, err := t.clientset.AppsV1().Deployments(t.namespace). + GetScale(ctx, t.deployment, metav1.GetOptions{}) + if err != nil { + level.Error(t.logger).Log("msg", "failed to get autoscalingv1.Scale object", "err", err) + return nil + } + + sc := *scale + sc.Spec.Replicas = int32(newReplicas) + _, err = t.clientset.AppsV1().Deployments(t.namespace). + UpdateScale(ctx, t.deployment, &sc, metav1.UpdateOptions{}) + if err != nil { + level.Error(t.logger).Log("msg", "failed to scale deployment", "err", err) + } + return nil +} + +type deletePodBySelectorTask struct { + logger log.Logger + clientset kubernetes.Interface + namespace string + selector string +} + +func (t *deletePodBySelectorTask) Run(ctx context.Context) error { + list, err := t.clientset.CoreV1().Pods(t.namespace).List(ctx, metav1.ListOptions{ + LabelSelector: t.selector, + }) + if err != nil { + level.Error(t.logger).Log("msg", "failed to list pods", "err", err) + return nil + } + + l := len(list.Items) + if l > 0 { + i := rand.Intn(l) + pod := list.Items[i].Name + level.Debug(t.logger).Log("msg", "deleting pod", "pod", pod) + if err := t.clientset.CoreV1().Pods(t.namespace).Delete(ctx, pod, metav1.DeleteOptions{ + GracePeriodSeconds: pointer.Int64(0), + }); err != nil { + level.Error(t.logger).Log("msg", "failed to delete pod", "pod", pod, "err", err) + } + } + + return nil +} diff --git a/tools/smoke/internal/tasks_test.go b/tools/smoke/internal/tasks_test.go new file mode 100644 index 000000000000..76fb1cbd2789 --- /dev/null +++ b/tools/smoke/internal/tasks_test.go @@ -0,0 +1,144 @@ +package smoke + +import ( + "context" + "testing" + + "github.com/go-kit/log" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +// Note: these tests are mostly worthless at this point +// but would allow easy debugging of tasks as they become more +// complex. Using https://pkg.go.dev/k8s.io/client-go/testing#ObjectTracker +// to mock responses from the fake client is also possible. + +func Test_deletePodBySelectorTask_Run(t1 *testing.T) { + type fields struct { + logger log.Logger + clientset kubernetes.Interface + namespace string + selector string + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "deletePodBySelectorTask", + fields: fields{ + logger: log.NewNopLogger(), + clientset: fake.NewSimpleClientset(), + namespace: "foo", + selector: "foo=bar", + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := &deletePodBySelectorTask{ + logger: tt.fields.logger, + clientset: tt.fields.clientset, + namespace: tt.fields.namespace, + selector: tt.fields.selector, + } + if err := t.Run(tt.args.ctx); (err != nil) != tt.wantErr { + t1.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_deletePodTask_Run(t1 *testing.T) { + type fields struct { + logger log.Logger + clientset kubernetes.Interface + namespace string + pod string + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "deletePodTask", + fields: fields{ + logger: log.NewNopLogger(), + clientset: fake.NewSimpleClientset(), + namespace: "foo", + pod: "bar", + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := &deletePodTask{ + logger: tt.fields.logger, + clientset: tt.fields.clientset, + namespace: tt.fields.namespace, + pod: tt.fields.pod, + } + if err := t.Run(tt.args.ctx); (err != nil) != tt.wantErr { + t1.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_scaleDeploymentTask_Run(t1 *testing.T) { + type fields struct { + logger log.Logger + clientset kubernetes.Interface + namespace string + deployment string + maxReplicas int + minReplicas int + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "scaleDeploymentTask", + fields: fields{ + logger: log.NewNopLogger(), + clientset: fake.NewSimpleClientset(), + namespace: "foo", + deployment: "bar", + maxReplicas: 11, + minReplicas: 2, + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := &scaleDeploymentTask{ + logger: tt.fields.logger, + clientset: tt.fields.clientset, + namespace: tt.fields.namespace, + deployment: tt.fields.deployment, + maxReplicas: tt.fields.maxReplicas, + minReplicas: tt.fields.minReplicas, + } + if err := t.Run(tt.args.ctx); (err != nil) != tt.wantErr { + t1.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/tools/smoke/main.go b/tools/smoke/main.go new file mode 100644 index 000000000000..568979501b89 --- /dev/null +++ b/tools/smoke/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "flag" + "os" + "time" + + "github.com/go-kit/log/level" + "github.com/grafana/agent/pkg/util" + smoke "github.com/grafana/agent/tools/smoke/internal" + "github.com/weaveworks/common/logging" +) + +func main() { + var ( + cfg smoke.Config + logLevel logging.Level + logFormat logging.Format + withTimeout time.Duration + ) + + cfg.RegisterFlags(flag.CommandLine) + logLevel.RegisterFlags(flag.CommandLine) + logFormat.RegisterFlags(flag.CommandLine) + flag.DurationVar(&withTimeout, "duration", time.Duration(0), "test duration") + flag.Parse() + + logger := util.NewLoggerFromLevel(logLevel, logFormat) + + ctx := context.Background() + if withTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, withTimeout) + defer cancel() + level.Debug(logger).Log("msg", "running with duration", "duration", withTimeout.String()) + } + + level.Info(logger).Log("msg", "starting smoke test") + smokeTest, err := smoke.New(logger, cfg) + if err != nil { + level.Error(logger).Log("msg", "error constructing smoke test", "err", err) + os.Exit(1) + } + if err := smokeTest.Run(ctx); err != nil { + level.Error(logger).Log("msg", "smoke test run failure", "err", err) + os.Exit(1) + } +}