Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update prometheus_metrics_test.go #4437

Merged
merged 6 commits into from
Apr 25, 2023
Merged

Update prometheus_metrics_test.go #4437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3540bdf
Update prometheus_metrics_test.go
Parthiba-Hazra Apr 5, 2023
ebda240
Merge branch 'kedacore:main' into test-prometheus
Parthiba-Hazra Apr 6, 2023
d1ac303
Merge branch 'main' into test-prometheus
Parthiba-Hazra Apr 14, 2023
9bc66d9
Update prometheus_metrics_test.go
Parthiba-Hazra Apr 14, 2023
40f40d5
Update CHANGELOG.md
Parthiba-Hazra Apr 25, 2023
0a4411e
Merge branch 'main' into test-prometheus
Parthiba-Hazra Apr 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
- **AWS SQS Scaler**: Respect `scaleOnInFlight` value ([#4276](https://github.com/kedacore/keda/issue/4276))
- **Azure Pipelines**: Fix for disallowing `$top` on query when using `meta.parentID` method ([#4397])
- **Azure Pipelines**: Respect all required demands ([#4404](https://github.com/kedacore/keda/issues/4404))
- **Prometheus Metrics**: Create e2e tests for all exposed Prometheus metrics ([#4127](https://github.com/kedacore/keda/issues/4127))

### Deprecations

Expand Down
148 changes: 148 additions & 0 deletions tests/sequential/prometheus_metrics/prometheus_metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"strings"
"testing"
"time"

promModel "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
Expand All @@ -29,6 +30,7 @@ var (
deploymentName = fmt.Sprintf("%s-deployment", testName)
monitoredDeploymentName = fmt.Sprintf("%s-monitored", testName)
scaledObjectName = fmt.Sprintf("%s-so", testName)
wrongScaledObjectName = fmt.Sprintf("%s-wrong", testName)
cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName)
clientName = fmt.Sprintf("%s-client", testName)
kedaOperatorPrometheusURL = "http://keda-operator.keda.svc.cluster.local:8080/metrics"
Expand All @@ -41,6 +43,7 @@ type templateData struct {
TestNamespace string
DeploymentName string
ScaledObjectName string
WrongScaledObjectName string
CronScaledJobName string
MonitoredDeploymentName string
ClientName string
Expand Down Expand Up @@ -114,6 +117,29 @@ spec:
value: '1'
`

wrongScaledObjectTemplate = `
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{.WrongScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
scaleTargetRef:
name: {{.DeploymentName}}
pollingInterval: 2
idleReplicaCount: 0
minReplicaCount: 1
maxReplicaCount: 2
cooldownPeriod: 10
triggers:
- type: prometheus
metadata:
serverAddress: http://keda-prometheus.keda.svc.cluster.local:8080
metricName: keda_scaler_errors_total
threshold: '1'
query: 'keda_scaler_errors_total{namespace="{{.TestNamespace}}",scaledObject="{{.WrongScaledObjectName}}"}'
`

cronScaledJobTemplate = `
apiVersion: keda.sh/v1alpha1
kind: ScaledJob
Expand Down Expand Up @@ -230,6 +256,9 @@ func TestPrometheusMetrics(t *testing.T) {
testScalerMetricValue(t)
testScalerMetricLatency(t)
testScalerActiveMetric(t)
testScaledObjectErrors(t, data)
testScalerErrors(t, data)
testScalerErrorsTotal(t, data)
testMetricsServerScalerMetricValue(t)
testOperatorMetrics(t, kc, data)
testWebhookMetrics(t, data)
Expand All @@ -244,6 +273,7 @@ func getTemplateData() (templateData, []Template) {
TestNamespace: testNamespace,
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
WrongScaledObjectName: wrongScaledObjectName,
MonitoredDeploymentName: monitoredDeploymentName,
ClientName: clientName,
CronScaledJobName: cronScaledJobName,
Expand Down Expand Up @@ -292,6 +322,124 @@ func testScalerMetricValue(t *testing.T) {
}
}

func testScaledObjectErrors(t *testing.T, data templateData) {
t.Log("--- testing scaled object errors ---")

KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)

family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaled_object_errors"]; ok {
errCounterVal1 := getErrorMetricsValue(val)

// wait for 2 seconds as pollinginterval is 2
time.Sleep(2 * time.Second)

family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaled_object_errors"]; ok {
errCounterVal2 := getErrorMetricsValue(val)
assert.NotEqual(t, errCounterVal2, float64(0))
assert.GreaterOrEqual(t, errCounterVal2, errCounterVal1)
} else {
t.Errorf("metric not available")
}
} else {
t.Errorf("metric not available")
}

KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
}

func testScalerErrors(t *testing.T, data templateData) {
t.Log("--- testing scaler errors ---")

KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)

family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaler_errors"]; ok {
errCounterVal1 := getErrorMetricsValue(val)

// wait for 2 seconds as pollinginterval is 2
time.Sleep(2 * time.Second)

Comment on lines +365 to +366
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we need to increase this time to ensure that the metric is exposed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should I increase it from 2 to 4?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or even bigger xD, we need to ensure that the test sin't flaky. I mean, in this case, waiting 2,4,10 seconds doesn't change the test because it's checking > 0, never mind if 1 or 100. If you think that 10 is safer, go with 10 :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually its confusing, cause in my local cluster it's worked well. At least it should pass after 4 seconds, I will change it to 4.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should I create a new PR on this?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, please 🙏

family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaler_errors"]; ok {
errCounterVal2 := getErrorMetricsValue(val)
assert.NotEqual(t, errCounterVal2, float64(0))
assert.GreaterOrEqual(t, errCounterVal2, errCounterVal1)
} else {
t.Errorf("metric not available")
}
} else {
t.Errorf("metric not available")
}

KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
}

func testScalerErrorsTotal(t *testing.T, data templateData) {
t.Log("--- testing scaler errors total ---")

KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)

family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaler_errors_total"]; ok {
errCounterVal1 := getErrorMetricsValue(val)

// wait for 2 seconds as pollinginterval is 2
time.Sleep(2 * time.Second)

family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaler_errors_total"]; ok {
errCounterVal2 := getErrorMetricsValue(val)
assert.NotEqual(t, errCounterVal2, float64(0))
assert.GreaterOrEqual(t, errCounterVal2, errCounterVal1)
} else {
t.Errorf("metric not available")
}
} else {
t.Errorf("metric not available")
}

KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
}

func getErrorMetricsValue(val *promModel.MetricFamily) float64 {
switch val.GetName() {
case "keda_scaler_errors_total":
metrics := val.GetMetric()
for _, metric := range metrics {
return metric.GetCounter().GetValue()
}
case "keda_scaled_object_errors":
metrics := val.GetMetric()
for _, metric := range metrics {
labels := metric.GetLabel()
for _, label := range labels {
if *label.Name == "scaledObject" && *label.Value == wrongScaledObjectName {
return *metric.Counter.Value
}
}
}
case "keda_scaler_errors":
metrics := val.GetMetric()
for _, metric := range metrics {
labels := metric.GetLabel()
for _, label := range labels {
if *label.Name == "scaler" && *label.Value == "prometheusScaler" {
return *metric.Counter.Value
}
}
}
}
return 0
}

func testScalerMetricLatency(t *testing.T) {
t.Log("--- testing scaler metric latency ---")

Expand Down