Skip to content

Commit

Permalink
Implement conditional remediation
Browse files Browse the repository at this point in the history 
Fixes #41
  • Loading branch information
@seaneagan
seaneagan committed Aug 20, 2020
1 parent 3a7b447 commit 2668e4d
Show file tree
Hide file tree
Showing 29 changed files with 1,462 additions and 205 deletions.
331 changes: 331 additions & 0 deletions .github/workflows/e2e.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,336 @@ jobs:
if [ "$RESULT" != "$EXPECTED" ]; then
echo -e "$RESULT\n\ndoes not equal\n\n$EXPECTED"
fi
- name: Run install fail test
run: |
test_name=install-fail
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="False" and .Ready=="False"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed and not uninstalled.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
- name: Run test fail test
run: |
test_name=test-fail
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Tested=="False" and .Ready=="False"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed and not uninstalled.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
- name: Run ignore test fail test
run: |
test_name=test-fail-ignore
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Tested=="False" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed and not uninstalled.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
- name: Run test fail with remedition test
run: |
test_name=test-fail-remediate
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Tested=="False" and .Ready=="False" and .Uninstalled=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Ensure release does not exist (was uninstalled).
HISTORY=$(helm -n helm-system history $test_name; exit 0)
if [ "$HISTORY" != 'Error: release: not found' ]; then
echo -e "Unexpected release history: $HISTORY"
fi
- name: Run test fail with retry test
run: |
test_name=test-fail-retry
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.installFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Tested == "False" and .Ready=="False" )' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
- name: Run upgrade fail test
run: |
test_name=upgrade-fail
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and and .Ready=="False"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was upgraded and not rolled back.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 2 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
- name: Run upgrade test fail test
run: |
test_name=upgrade-test-fail
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="True" and .Tested=="False" and .Ready=="False"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was upgraded and not rolled back.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 2 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
- name: Run upgrade test fail with remediation test
run: |
test_name=upgrade-test-fail-remediate
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="True" and .Tested=="False" and .Ready=="False" and .RolledBack=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was upgraded and then rolled back.
HISTORY=$(helm -n helm-system history -o json $test_name)
REVISION_COUNT=$(echo "$HISTORY" | jq 'length')
if [ "$REVISION_COUNT" != 3 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
LAST_REVISION_DESCRIPTION=$(echo "$HISTORY" | jq 'last | .description')
if [ "$LAST_REVISION_DESCRIPTION" != "Rollback to 2" ]; then
echo -e "Unexpected last revision description: $LAST_REVISION_DESCRIPTION"
fi
- name: Run upgrade test fail retry test
run: |
test_name=upgrade-test-fail-retry
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Upgraded=="True" and .Tested=="False" and .Ready=="False" and .RolledBack=="True" )' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was upgraded rolled back twice.
HISTORY=$(helm -n helm-system history -o json $test_name)
REVISION_COUNT=$(echo "$HISTORY" | jq 'length')
if [ "$REVISION_COUNT" != 5 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
LAST_REVISION_DESCRIPTION=$(echo "$HISTORY" | jq 'last | .description')
if [ "$LAST_REVISION_DESCRIPTION" != "Rollback to 4" ]; then
echo -e "Unexpected last revision description: $LAST_REVISION_DESCRIPTION"
fi
- name: Run upgrade test fail with uninstall remediation strategy test
run: |
test_name=upgrade-test-fail-remediate-uninstall
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Validate release was installed.
REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
if [ "$REVISION_COUNT" != 1 ]; then
echo -e "Unexpected revision count: $REVISION_COUNT"
fi
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="True" and .Tested=="False" and .Ready=="False" and .Uninstalled=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
if [[ ${count} -eq 24 ]]; then
echo ' No more retries left!'
exit 1
fi
done
echo ' done'
# Ensure release does not exist (was uninstalled).
HISTORY=$(helm -n helm-system history $test_name; exit 0)
if [ "$HISTORY" != 'Error: release: not found' ]; then
echo -e "Unexpected release history: $HISTORY"
fi
- name: Logs
run: |
kubectl -n helm-system logs deploy/source-controller
Expand All @@ -90,5 +420,6 @@ jobs:
kubectl -n helm-system get helmcharts -oyaml
kubectl -n helm-system get helmreleases -oyaml
kubectl -n helm-system get all
helm ls -n helm-system
kubectl -n helm-system logs deploy/source-controller
kubectl -n helm-system logs deploy/helm-controller
5 changes: 4 additions & 1 deletion api/v2alpha1/condition_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ const (
// UpgradeFailedReason represents the fact that the Helm upgrade for the release failed.
UpgradeFailedReason string = "UpgradeFailed"

// TestFailedReason represents the fact that the Helm test for the release failed.
// TestSucceededReason represents the fact that the Helm test for the release succeeded.
TestSucceededReason string = "TestSucceeded"

// TestFailedReason represents the fact that the Helm test for the release failed.
Expand All @@ -110,6 +110,9 @@ const (
// InitFailedReason represents the fact that the initialization of the Helm configuration failed.
InitFailedReason string = "InitFailed"

// GetLastReleaseFailedReason represents the fact that observing the last release failed.
GetLastReleaseFailedReason string = "GetLastReleaseFailed"

// ProgressingReason represents the fact that the reconciliation for the resource is underway.
ProgressingReason string = "Progressing"

Expand Down
Loading

0 comments on commit 2668e4d

Please sign in to comment.