add test on different cluster versions

.travis.yml

@@ -11,15 +11,16 @@ go:
   - tip
 
 before_install:
-  - openssl aes-256-cbc -K $ENCRYPTED_IR_RSA_PUB_KEY -iv $ENCRYPTED_IR_RSA_PUB_IV -in tests/id_rsa.pub.enc -out tests/id_rsa.pub -d
-  - openssl aes-256-cbc -K $ENCRYPTED_IR_RSA_PRV_KEY -iv $ENCRYPTED_IR_RSA_PRV_IV -in tests/id_rsa.enc     -out tests/id_rsa     -d
   - go get -u github.com/golang/dep/cmd/dep
   - go get -u golang.org/x/lint/golint
   - go get -u honnef.co/go/tools/...
   - go get -u mvdan.cc/unparam
   - go get -u github.com/client9/misspell/cmd/misspell
   - dep ensure
 
+after_script:
+  - make hard-cleanup
+
 script:
   - go test -v ./...
 
@@ -38,9 +39,7 @@ jobs:
     - stage: e2e test
       script:
         - go build
-        - make info
-        - make context
-        - make ssh-key
-        - make cluster
+        - make preparare
+        - make test-all
         - make cleanup
       go: 1.11.x

Makefile

@@ -1,42 +1,66 @@
 HETZNER_KUBE=${TRAVIS_BUILD_DIR}/hetzner-kube
 
-SSH_KEY_PATH_PRIVATE=${TRAVIS_BUILD_DIR}/tests/id_rsa
-SSH_KEY_PATH_PUBLIC=${SSH_KEY_PATH_PRIVATE}.pub
+SSH_KEY_FOLDER=${TRAVIS_BUILD_DIR}/tests/keys
 DATACENTER=fsn1-dc8
-SERVER_MASTER_COUNT=1
-SERVER_WORKER_COUNT=1
 
 SSH_KEY_NAME=testing-ssh-key-${TRAVIS_JOB_NUMBER}
 CONTEXT_NAME=testing-context-${TRAVIS_JOB_NUMBER}
 CLUSTER_NAME=testing-cluster-${TRAVIS_JOB_NUMBER}
 
-info:
+preparare:
+	mkdir -p ${SSH_KEY_FOLDER}
+	ssh-keygen -t rsa -b 4096 -P "" -f ${SSH_KEY_FOLDER}/id_rsa
+
+test-all: test-info test-context test-ssh-key test-clusters
+
+test-info:
 	${HETZNER_KUBE} help
 	${HETZNER_KUBE} version
 
-context:
+test-context:
 	${HETZNER_KUBE} context add ${CONTEXT_NAME} --token ${HETZNER_API_KEY}
 	${HETZNER_KUBE} context list
 	${HETZNER_KUBE} context use ${CONTEXT_NAME}
 	${HETZNER_KUBE} context current
+	# We move it on clanup since they are used to test cluster operations
 	# ${HETZNER_KUBE} context delete ${CONTEXT_NAME}
 
-ssh-key:
-	mkdir ${TRAVIS_BUILD_DIR}/tests
-	ssh-keygen -b 2048 -t rsa -f ${SSH_KEY_PATH_PRIVATE} -q -N ""
-	${HETZNER_KUBE} ssh-key add --name ${SSH_KEY_NAME} --private-key-path ${SSH_KEY_PATH_PRIVATE} --public-key-path ${SSH_KEY_PATH_PUBLIC}
+test-ssh-key:
+	${HETZNER_KUBE} ssh-key add --name ${SSH_KEY_NAME} --private-key-path ${SSH_KEY_FOLDER}/id_rsa --public-key-path ${SSH_KEY_FOLDER}/id_rsa.pub
 	${HETZNER_KUBE} ssh-key list
+	# We move it on clanup since they are used to test cluster operations
 	# ${HETZNER_KUBE} ssh-key delete --name ${SSH_KEY_NAME}
 
-cluster:
-	${HETZNER_KUBE} cluster create --worker-count ${SERVER_WORKER_COUNT} --datacenters ${DATACENTER} --ssh-key ${SSH_KEY_NAME} --name ${CLUSTER_NAME}
+test-clusters: test-cluster-without-ha test-cluster-with-ha-level3 test-cluster-with-ha-level4
+
+test-cluster-without-ha:
+	${HETZNER_KUBE} cluster create --worker-count 1 --datacenters ${DATACENTER} --ssh-key ${SSH_KEY_NAME} --name ${CLUSTER_NAME}-without-ha
+	${HETZNER_KUBE} cluster list
+	${HETZNER_KUBE} cluster master-ip ${CLUSTER_NAME}-without-ha
+	${HETZNER_KUBE} cluster kubeconfig -f ${CLUSTER_NAME}-without-ha
+	${HETZNER_KUBE} cluster add-worker --nodes 1 --datacenters ${DATACENTER} --name ${CLUSTER_NAME}-without-ha
+	${HETZNER_KUBE} cluster delete ${CLUSTER_NAME}-without-ha
+
+test-cluster-with-ha-level3:
+	${HETZNER_KUBE} cluster create --worker-count 1 --master-count 3 --ha-enabled --datacenters ${DATACENTER} --ssh-key ${SSH_KEY_NAME} --name ${CLUSTER_NAME}-with-ha-level3
+	${HETZNER_KUBE} cluster list
+	${HETZNER_KUBE} cluster master-ip ${CLUSTER_NAME}-with-ha-level3
+	${HETZNER_KUBE} cluster kubeconfig -f ${CLUSTER_NAME}-with-ha-level3
+	${HETZNER_KUBE} cluster add-worker --nodes 1 --datacenters ${DATACENTER} --name ${CLUSTER_NAME}-with-ha-level3
+	${HETZNER_KUBE} cluster delete ${CLUSTER_NAME}-with-ha-level3
+
+test-cluster-with-ha-level4:
+	${HETZNER_KUBE} cluster create --worker-count 1 --master-count 3 --etcd-count 3 --ha-enabled --isolated-etcd --datacenters ${DATACENTER} --ssh-key ${SSH_KEY_NAME} --name ${CLUSTER_NAME}-with-ha-level4
 	${HETZNER_KUBE} cluster list
-	${HETZNER_KUBE} cluster master-ip ${CLUSTER_NAME}
-	${HETZNER_KUBE} cluster kubeconfig ${CLUSTER_NAME}
-	${HETZNER_KUBE} cluster add-worker --nodes 1 --datacenters ${DATACENTER} --name ${CLUSTER_NAME}
-	# ${HETZNER_KUBE} cluster delete ${CLUSTER_NAME}
+	${HETZNER_KUBE} cluster master-ip ${CLUSTER_NAME}-with-ha-level4
+	${HETZNER_KUBE} cluster kubeconfig -f ${CLUSTER_NAME}-with-ha-level4
+	${HETZNER_KUBE} cluster add-worker --nodes 1 --datacenters ${DATACENTER} --name ${CLUSTER_NAME}-with-ha-level4
+	${HETZNER_KUBE} cluster delete ${CLUSTER_NAME}-with-ha-level4
 
 cleanup:
-	${HETZNER_KUBE} cluster delete ${CLUSTER_NAME}
 	${HETZNER_KUBE} ssh-key delete --name ${SSH_KEY_NAME}
 	${HETZNER_KUBE} context delete ${CONTEXT_NAME}
+
+hard-cleanup:
+	curl -s --request GET --url https://api.hetzner.cloud/v1/servers --header "Authorization: Bearer ${HETZNER_API_KEY}" | jq '.servers[] | "\(.name) \(.id)"' | grep ${TRAVIS_JOB_NUMBER} | awk 'BEGIN { FS = "\"" } ; { print $$2 }' | awk '{ print $$2 }' | xargs -I '{}' bash -c "curl -s --request DELETE https://api.hetzner.cloud/v1/servers/{} --header \"Authorization: Bearer ${HETZNER_API_KEY}\""
+	curl -s --request GET --url https://api.hetzner.cloud/v1/ssh_keys --header "Authorization: Bearer ${HETZNER_API_KEY}" | jq '.ssh_keys[] | "\(.name) \(.id)"' | grep ${TRAVIS_JOB_NUMBER} | awk 'BEGIN { FS = "\"" } ; { print $$2 }' | awk '{ print $$2 }' | xargs -I '{}' bash -c "curl -s --request DELETE https://api.hetzner.cloud/v1/ssh_keys/{} --header \"Authorization: Bearer ${HETZNER_API_KEY}\""