Use K6 operator to execute test cases (#5)

.github/workflows/performance-tests.yaml

@@ -14,28 +14,21 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-go@v4
-        with:
-          go-version: "1.20"
-
       - uses: oNaiPs/secrets-to-env-action@v1
         with:
           secrets: ${{ toJSON(secrets) }}
 
-      - name: Build k6 runner
-        run: make generate-k6
-
       - name: Get cluster context
         run: make get-cluster-context
 
       - name: Deploy dependencies
         run: make deploy
 
-      - name: Login in Grafana Cloud
-        run: make login-k6
-
       - name: Execute performance tests
         run: make execute-k6
+        # TODO: We need to edit this before accepting PRs because this will fail with forks
+        env:
+          REPO_BRANCH: ${{ github.head_ref || github.ref_name }}
 
       - name: Cleanup dependencies
         run: make undeploy

Makefile

@@ -16,8 +16,10 @@ KEDA_VERSION ?= main
 GRAFANA_PROMETHEUS_URL_PUSH ?= $(TF_GRAFANA_PROMETHEUS_URL)/api/prom/push
 GRAFANA_PROMETHEUS_URL_QUERY ?= $(TF_GRAFANA_PROMETHEUS_URL)/api/prom
 PROMETHEUS_NAMESPACE ?= prometheus-performance
-
-K6_ENVS ?= PROMETHEUS_URL="$(GRAFANA_PROMETHEUS_URL_QUERY)" PROMETHEUS_USER="$(TF_GRAFANA_PROMETHEUS_USER)" PROMETHEUS_PASSWORD="$(TF_GRAFANA_PROMETHEUS_PASSWORD)"
+K6_OPERATOR_NAMESPACE = k6-operator-system
+REPO_URL ?= https://github.com/kedacore/keda-performance.git
+REPO_BRANCH ?= main
+TEST_CONFIG ?= config.json
 
 ##################################################
 # Kubernetes context                             #
@@ -36,9 +38,9 @@ get-cluster-context: az-login ## Get Azure cluster context.
 # Deployments                                    #
 ##################################################
 
-deploy: deploy-keda deploy-prometheus
+deploy: deploy-keda deploy-k6-operator deploy-prometheus
 
-undeploy: clean-up-testing-namespaces undeploy-prometheus undeploy-keda	
+undeploy: clean-up-testing-namespaces undeploy-prometheus undeploy-k6-operator undeploy-keda	
 
 deploy-keda:
 	mkdir -p deps
@@ -66,40 +68,48 @@ undeploy-prometheus:
 	helm uninstall prometheus -n $(PROMETHEUS_NAMESPACE)
 	kubectl delete ns $(PROMETHEUS_NAMESPACE)
 
+# we have to replace this with the helm chart when they release it
+# https://github.com/grafana/k6-operator/pull/98
+deploy-k6-operator:
+	mkdir -p deps
+	git clone https://github.com/grafana/k6-operator deps/k6-operator --depth 1	
+	make -C deps/k6-operator deploy
+	kubectl create secret generic k6-operator-cloud-token --from-literal="token=$(TF_GRAFANA_TOKEN)" -n $(K6_OPERATOR_NAMESPACE)
+	kubectl label secret k6-operator-cloud-token -n $(K6_OPERATOR_NAMESPACE) "k6cloud=token"
+	kubectl annotate secret k6-operator-cloud-token -n $(K6_OPERATOR_NAMESPACE) "kubernetes.io/service-account.name=k6-operator-controller"
+
+undeploy-k6-operator:
+	make -C deps/k6-operator delete
+	rm -rf deps/k6-operator
+
 clean-up-testing-namespaces:
 	kubectl delete ns -l type=e2e
 
 ##################################################
 # Grafana k6                                     #
 ##################################################
 
-generate-k6:
-	go install go.k6.io/xk6/cmd/xk6@latest
-	xk6 build \
-		--output ./k6 \
-		--with github.com/szkiba/xk6-yaml@latest \
-		--with github.com/grafana/xk6-kubernetes@latest \
-		--with github.com/grafana/xk6-disruptor@latest \
-		--with github.com/JorTurFer/xk6-input-prometheus
-
-login-k6:
-	@./k6 login cloud --token $(TF_GRAFANA_TOKEN)
-
-execute-k6: # execute-k6-scaledobjects-cases
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=1000 TARGET_METRICS=1 ./k6 run --out cloud tests/test-scaledobject.js
-
-execute-k6-scaledobjects-cases: # disabled temporally to reduce execution times + grafana VUs
-	# Increasing ScaledObject count (10, 100, 1000 metrics in total)
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=10 TARGET_METRICS=1 ./k6 run --out cloud tests/test-scaledobject.js
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=100 TARGET_METRICS=1 ./k6 run --out cloud tests/test-scaledobject.js
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=1000 TARGET_METRICS=1 ./k6 run --out cloud tests/test-scaledobject.js
-
-	# Increasing metrics per ScaledObject (10, 100, 1000 metrics in total)
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=1 TARGET_METRICS=10 ./k6 run --out cloud tests/test-scaledobject.js
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=10 TARGET_METRICS=10 ./k6 run --out cloud tests/test-scaledobject.js
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=10 TARGET_METRICS=100 ./k6 run --out cloud tests/test-scaledobject.js
-
-	# Switching between ScaledObject count and metrics per ScaledObject (1000 metrics in total)
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=20 TARGET_METRICS=50 ./k6 run --out cloud tests/test-scaledobject.js
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=100 TARGET_METRICS=10 ./k6 run --out cloud tests/test-scaledobject.js
-	@$(K6_ENVS) TARGET_SCALABLEDOBJECTS=500 TARGET_METRICS=2 ./k6 run --out cloud tests/test-scaledobject.js
+execute-k6: execute-k6-scaledobjects
+
+execute-k6-scaledobjects:
+	@for file in $(shell find ./configs/scaledobjects -maxdepth 1 -not -type d); do \
+		make execute-k6-scaled-object-case TEST_CONFIG="$${file}" ; \
+	done
+
+execute-k6-scaled-object-case:
+	@helm install k6-test \
+		chart	\
+		-n $(K6_OPERATOR_NAMESPACE) \
+		--create-namespace \
+		--set test.file=tests/test-scaledobject.js \
+		--set test.config=$(TEST_CONFIG) \
+		--set repo.url=$(REPO_URL) \
+		--set repo.branch=$(REPO_BRANCH) \
+		--set test.extraConfig.PROMETHEUS_URL=$(GRAFANA_PROMETHEUS_URL_QUERY) \
+		--set test.extraConfig.PROMETHEUS_USER=$(TF_GRAFANA_PROMETHEUS_USER) \
+		--set test.extraConfig.PROMETHEUS_PASSWORD=$(TF_GRAFANA_PROMETHEUS_PASSWORD) \
+		--set test.extraArgs="--out cloud"
+
+	./hack/wait-test-case.sh $(K6_OPERATOR_NAMESPACE)
+
+	helm uninstall k6-test -n $(K6_OPERATOR_NAMESPACE)

chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

chart/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: chart
+description: A Helm chart for Kubernetes
+type: application
+version: 0.1.0
+appVersion: "1.16.0"

chart/templates/cm.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: extra-cm
+data:
+{{- range $key, $value := .Values.test.extraConfig }}
+  {{ upper $key }}: {{ $value | quote }} 
+{{- end -}}

chart/templates/rbac.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: k6-test-role
+subjects:
+  - kind: ServiceAccount
+    name: k6-runner
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin

chart/templates/sa.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: k6-runner

chart/templates/test.yaml

@@ -0,0 +1,36 @@
+apiVersion: k6.io/v1alpha1
+kind: K6
+metadata:
+  name: k6-case
+spec:
+  parallelism: {{ .Values.test.parallelism }}
+  script:
+    localFile: /repo/{{ .Values.test.file }}
+  arguments: --config /repo/{{ .Values.test.config }} {{ .Values.test.extraArgs }}
+  runner:
+    image: ghcr.io/kedacore/keda-k6-runner:9e4410f
+    serviceAccountName: k6-runner
+    volumes:
+      - name: git-repo
+        emptyDir: {}
+    volumeMounts:
+      - name: git-repo
+        mountPath: /repo
+    initContainers:
+    - name: repo-puller
+      image: bitnami/git
+      command:
+        - "git" 
+        - "clone"
+        - "-b"
+        - "{{ .Values.repo.branch }}"
+        - "{{ .Values.repo.url }}"
+        - "/repo"
+        - "--depth" 
+        - "1"
+      volumeMounts:
+        - name: git-repo
+          mountPath: /repo
+    envFrom:
+      - configMapRef:
+          name: extra-cm

chart/values.yaml

@@ -0,0 +1,9 @@
+test:
+  parallelism: 1
+  file: ""
+  config: ""
+  extraConfig: {}
+  extraArgs: ""
+repo:
+  url: ""
+  branch: main

configs/scaledobjects/10so1m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "10-ScaleObjects-1-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 10,
+      "metricsPerScaledobject": 1
+    }
+  }
+}

configs/scaledobjects/1so10m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "1-ScaleObjects-10-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 1,
+      "metricsPerScaledobject": 10
+    }
+  }
+}

configs/scaledobjects/disabled/1000so1m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "1000-ScaleObjects-1-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 1000,
+      "metricsPerScaledobject": 1
+    }
+  }
+}

configs/scaledobjects/disabled/100so10m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "100-ScaleObjects-10-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 100,
+      "metricsPerScaledobject": 10
+    }
+  }
+}

configs/scaledobjects/disabled/100so1m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "100-ScaleObjects-1-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 100,
+      "metricsPerScaledobject": 1
+    }
+  }
+}

configs/scaledobjects/disabled/10so100m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "10-ScaleObjects-100-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 10,
+      "metricsPerScaledobject": 100
+    }
+  }
+}

configs/scaledobjects/disabled/10so10m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "10-ScaleObjects-10-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 10,
+      "metricsPerScaledobject": 10
+    }
+  }
+}

configs/scaledobjects/disabled/20so50m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "20-ScaleObjects-50-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 20,
+      "metricsPerScaledobject": 50
+    }
+  }
+}

configs/scaledobjects/disabled/500so2m.json

@@ -0,0 +1,12 @@
+{
+  "ext": {
+    "loadimpact": {
+      "projectID": 3645343,
+      "name": "500-ScaleObjects-2-Metrics"
+    },
+    "keda": {
+      "scaledobjects": 500,
+      "metricsPerScaledobject": 2
+    }
+  }
+}