Skip to content

Commit

Permalink
Paused ScaledObject continues after removing pause annotation (#4734)
Browse files Browse the repository at this point in the history
  • Loading branch information
JorTurFer authored Jun 23, 2023
1 parent 107947c commit 04ca96e
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 50 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio

### Fixes

- TODO ([#XXX](https://github.com/kedacore/keda/issue/XXX))
- **General**: Paused ScaledObject continues working after removing the annotation ([#4733](https://github.com/kedacore/keda/issues/4733))

### Deprecations

Expand Down
20 changes: 12 additions & 8 deletions controllers/keda/util/predicate.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,19 @@ func (PausedReplicasPredicate) Update(e event.UpdateEvent) bool {

newAnnotations := e.ObjectNew.GetAnnotations()
oldAnnotations := e.ObjectOld.GetAnnotations()
if newAnnotations != nil && oldAnnotations != nil {
if newVal, ok1 := newAnnotations[PausedReplicasAnnotation]; ok1 {
if oldVal, ok2 := oldAnnotations[PausedReplicasAnnotation]; ok2 {
return newVal != oldVal
}
return true
}

newPausedValue := ""
oldPausedValue := ""

if newAnnotations != nil {
newPausedValue = newAnnotations[PausedReplicasAnnotation]
}
return false

if oldAnnotations != nil {
oldPausedValue = oldAnnotations[PausedReplicasAnnotation]
}

return newPausedValue != oldPausedValue
}

type ScaleObjectReadyConditionPredicate struct {
Expand Down
64 changes: 23 additions & 41 deletions tests/internals/pause_scaledobject/pause_scaledobject_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ type templateData struct {
DeploymentName string
ScaledObjectName string
MonitoredDeploymentName string
PausedReplicaCount int
CooldownPeriod int
}

const (
Expand Down Expand Up @@ -99,29 +97,7 @@ spec:
pollingInterval: 5
minReplicaCount: 0
maxReplicaCount: 1
cooldownPeriod: {{.CooldownPeriod}}
triggers:
- type: kubernetes-workload
metadata:
podSelector: 'app={{.MonitoredDeploymentName}}'
value: '1'
`

scaledObjectAnnotatedTemplate = `
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{.ScaledObjectName}}
namespace: {{.TestNamespace}}
annotations:
autoscaling.keda.sh/paused-replicas: "{{.PausedReplicaCount}}"
spec:
scaleTargetRef:
name: {{.DeploymentName}}
pollingInterval: 5
minReplicaCount: 0
maxReplicaCount: 1
cooldownPeriod: {{.CooldownPeriod}}
cooldownPeriod: 5
triggers:
- type: kubernetes-workload
metadata:
Expand All @@ -146,9 +122,9 @@ func TestScaler(t *testing.T) {

// test scaling
testPauseAt0(t, kc)
testScaleOut(t, kc, data)
testPauseAtN(t, kc, data, 5)
testScaleIn(t, kc, data)
testScaleOut(t, kc)
testPauseAtN(t, kc, 5)
testScaleIn(t, kc)

// cleanup
DeleteKubernetesResources(t, testNamespace, data, templates)
Expand All @@ -160,42 +136,49 @@ func getTemplateData() (templateData, []Template) {
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
MonitoredDeploymentName: monitoredDeploymentName,
PausedReplicaCount: 0,
CooldownPeriod: 10,
}, []Template{
{Name: "deploymentTemplate", Config: deploymentTemplate},
{Name: "monitoredDeploymentTemplate", Config: monitoredDeploymentTemplate},
{Name: "scaledObjectAnnotatedTemplate", Config: scaledObjectAnnotatedTemplate},
{Name: "scaledObjectAnnotatedTemplate", Config: scaledObjectTemplate},
}
}

func upsertScaledObjectAnnotation(t assert.TestingT, value int) {
_, err := ExecuteCommand(fmt.Sprintf("kubectl annotate scaledobject/%s -n %s autoscaling.keda.sh/paused-replicas=%d --overwrite", scaledObjectName, testNamespace, value))
assert.NoErrorf(t, err, "cannot execute command - %s", err)
}

func removeScaledObjectAnnotation(t assert.TestingT) {
_, err := ExecuteCommand(fmt.Sprintf("kubectl annotate scaledobject/%s -n %s autoscaling.keda.sh/paused-replicas- --overwrite", scaledObjectName, testNamespace))
assert.NoErrorf(t, err, "cannot execute command - %s", err)
}

func testPauseAt0(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing pausing at 0 ---")

upsertScaledObjectAnnotation(t, 0)
KubernetesScaleDeployment(t, kc, monitoredDeploymentName, 2, testNamespace)
assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, monitoredDeploymentName, testNamespace, 2, 60, testScaleOutWaitMin),
"monitoredDeploymentName replica count should be 2 after %d minute(s)", testScaleOutWaitMin)

AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, 0, 60)
}

func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) {
func testScaleOut(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing scale out ---")
data.CooldownPeriod = 15
KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)

removeScaledObjectAnnotation(t)
assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, monitoredDeploymentName, testNamespace, 2, 60, testScaleOutWaitMin),
"monitoredDeploymentName replica count should be 2 after %d minute(s)", testScaleOutWaitMin)

assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, testScaleOutWaitMin),
"replica count should be 1 after %d minute(s)", testScaleOutWaitMin)
}

func testPauseAtN(t *testing.T, kc *kubernetes.Clientset, data templateData, n int) {
func testPauseAtN(t *testing.T, kc *kubernetes.Clientset, n int) {
t.Log("--- testing pausing at N ---")
data.PausedReplicaCount = n
data.CooldownPeriod = 10
KubectlApplyWithTemplate(t, data, "scaledObjectAnnotatedTemplate", scaledObjectAnnotatedTemplate)

upsertScaledObjectAnnotation(t, n)
KubernetesScaleDeployment(t, kc, monitoredDeploymentName, 0, testNamespace)

assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, monitoredDeploymentName, testNamespace, 0, 60, testPauseAtNWaitMin),
Expand All @@ -205,11 +188,10 @@ func testPauseAtN(t *testing.T, kc *kubernetes.Clientset, data templateData, n i
"replica count should be %d after %d minute(s)", n, testPauseAtNWaitMin)
}

func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) {
func testScaleIn(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing scale in ---")
data.CooldownPeriod = 15
KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)

removeScaledObjectAnnotation(t)
assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, testScaleInWaitMin),
"replica count should be 0 after %d minutes", testScaleInWaitMin)
}

0 comments on commit 04ca96e

Please sign in to comment.