From c083cc950d90fd1b9e7bb83298700b0e8e5368c6 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Sun, 15 Mar 2020 19:36:39 +0100 Subject: [PATCH 1/3] finalize v1alpha3 --- .dockerignore | 6 +- .golangci.yml | 39 +- CONTRIBUTING.md | 12 +- LICENSE | 4 +- Makefile | 142 +++--- README.md | 40 +- RELEASE.md | 5 +- api/v1alpha3/openstackcluster_types.go | 10 +- api/v1alpha3/openstackmachine_types.go | 10 +- api/v1alpha3/types.go | 16 + api/v1alpha3/zz_generated.deepcopy.go | 8 + clusterctl-settings.json | 5 +- code-of-conduct.md | 8 - ...re.cluster.x-k8s.io_openstackclusters.yaml | 45 +- ...re.cluster.x-k8s.io_openstackmachines.yaml | 28 +- ...er.x-k8s.io_openstackmachinetemplates.yaml | 4 - .../manager_role_aggregation_patch.yaml | 2 +- config/manager/manager.yaml | 2 +- config/manager/manager_auth_proxy_patch.yaml | 3 +- config/manager/manager_image_patch.yaml | 2 +- config/manager/manager_webhook_patch.yaml | 23 - config/manager/webhookcainjection_patch.yaml | 15 - config/rbac/role.yaml | 10 +- controllers/helpers.go | 15 + controllers/openstackcluster_controller.go | 183 ++++---- controllers/openstackmachine_controller.go | 326 ++++++++------ controllers/suite_test.go | 1 + docs/{config.md => configuration.md} | 0 docs/getting-started.md | 112 ----- examples/.gitignore | 1 - examples/README.md | 85 ---- examples/clouds-secrets/.gitignore | 1 - examples/clouds-secrets/kustomization.yaml | 13 - examples/cluster/multi-node/cluster.yaml | 50 --- .../cluster/multi-node/kustomization.yaml | 7 - .../cluster/multi-node/kustomizeconfig.yaml | 6 - examples/cluster/single-node/cluster.yaml | 47 -- .../cluster/single-node/kustomization.yaml | 7 - .../cluster/single-node/kustomizeconfig.yaml | 6 - .../controlplane/multi-node/controlplane.yaml | 93 ---- .../multi-node/kustomization.yaml | 7 - .../multi-node/kustomizeconfig.yaml | 15 - .../single-node/controlplane.yaml | 93 ---- .../single-node/kustomization.yaml | 7 - .../single-node/kustomizeconfig.yaml | 15 - examples/generate.sh | 244 ---------- examples/machinedeployment/kustomization.yaml | 7 - .../machinedeployment/kustomizeconfig.yaml | 11 - .../machinedeployment/machinedeployment.yaml | 87 ---- examples/provider-components/.gitignore | 1 - .../provider-components/kustomization.yaml | 4 - examples/provider-components/namespace.yaml | 6 - go.mod | 2 +- go.sum | 18 + hack/boilerplate/boilerplate.Dockerfile.txt | 14 + hack/boilerplate/boilerplate.Makefile.txt | 14 + hack/boilerplate/boilerplate.go.txt | 15 + hack/boilerplate/boilerplate.py.txt | 16 + hack/boilerplate/boilerplate.sh.txt | 14 + hack/ci/ci-bootstrap.yaml | 90 ++++ hack/ci/e2e-conformance.sh | 424 ++++++++++++++++++ hack/tools/go.sum | 113 ++--- hack/verify-boilerplate.sh | 2 +- .../services/compute/availabilityzone.go | 36 ++ pkg/cloud/services/compute/instance.go | 91 ++-- .../services/loadbalancer/loadbalancer.go | 56 ++- pkg/cloud/services/networking/floatingip.go | 16 + pkg/cloud/services/networking/network.go | 5 +- .../services/networking/securitygroups.go | 13 +- pkg/cloud/services/provider/provider.go | 16 + templates/cluster-template-without-lb.yaml | 199 ++++++++ templates/cluster-template.yaml | 200 +++++++++ templates/env.rc | 122 +++++ templates/kustomization.yaml | 5 + templates/kustomizeversions.yaml | 167 +++++++ 75 files changed, 2071 insertions(+), 1466 deletions(-) delete mode 100644 config/manager/manager_webhook_patch.yaml delete mode 100644 config/manager/webhookcainjection_patch.yaml rename docs/{config.md => configuration.md} (100%) delete mode 100644 docs/getting-started.md delete mode 100644 examples/.gitignore delete mode 100644 examples/README.md delete mode 100644 examples/clouds-secrets/.gitignore delete mode 100644 examples/clouds-secrets/kustomization.yaml delete mode 100644 examples/cluster/multi-node/cluster.yaml delete mode 100644 examples/cluster/multi-node/kustomization.yaml delete mode 100644 examples/cluster/multi-node/kustomizeconfig.yaml delete mode 100644 examples/cluster/single-node/cluster.yaml delete mode 100644 examples/cluster/single-node/kustomization.yaml delete mode 100644 examples/cluster/single-node/kustomizeconfig.yaml delete mode 100644 examples/controlplane/multi-node/controlplane.yaml delete mode 100644 examples/controlplane/multi-node/kustomization.yaml delete mode 100644 examples/controlplane/multi-node/kustomizeconfig.yaml delete mode 100644 examples/controlplane/single-node/controlplane.yaml delete mode 100644 examples/controlplane/single-node/kustomization.yaml delete mode 100644 examples/controlplane/single-node/kustomizeconfig.yaml delete mode 100755 examples/generate.sh delete mode 100644 examples/machinedeployment/kustomization.yaml delete mode 100644 examples/machinedeployment/kustomizeconfig.yaml delete mode 100644 examples/machinedeployment/machinedeployment.yaml delete mode 100644 examples/provider-components/.gitignore delete mode 100644 examples/provider-components/kustomization.yaml delete mode 100644 examples/provider-components/namespace.yaml create mode 100644 hack/boilerplate/boilerplate.Dockerfile.txt create mode 100644 hack/boilerplate/boilerplate.Makefile.txt create mode 100644 hack/boilerplate/boilerplate.go.txt create mode 100644 hack/boilerplate/boilerplate.py.txt create mode 100644 hack/boilerplate/boilerplate.sh.txt create mode 100644 hack/ci/ci-bootstrap.yaml create mode 100755 hack/ci/e2e-conformance.sh create mode 100644 pkg/cloud/services/compute/availabilityzone.go create mode 100644 templates/cluster-template-without-lb.yaml create mode 100644 templates/cluster-template.yaml create mode 100755 templates/env.rc create mode 100644 templates/kustomization.yaml create mode 100644 templates/kustomizeversions.yaml diff --git a/.dockerignore b/.dockerignore index 9bcfb6aa6b..9257eaf784 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,13 @@ .git .github .vscode +_artifacts bin/ config/ hack/ docs/ -examples/ +templates/ +tmp +scripts/ **/.md +tilt-provider.json diff --git a/.golangci.yml b/.golangci.yml index d419ae6587..4dffd49edf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,23 +1,28 @@ linters: - enable: - - golint - - govet - - gofmt - - structcheck - - varcheck - - interfacer - - unconvert - - ineffassign - - goconst - - misspell - - nakedret - - prealloc - - deadcode - disable-all: true + enable-all: true + disable: + - dupl + - funlen + - gochecknoglobals + - gochecknoinits + - lll + - maligned + - godox + - wsl + - whitespace + - gocognit + - gomnd # Run with --fast=false for more extensive checks fast: true -issue: +issues: max-same-issues: 0 max-per-linter: 0 + # List of regexps of issue texts to exclude, empty list by default. + exclude: + - Using the variable on range scope `(tc)|(rt)|(tt)|(test)|(testcase)|(testCase)` in function literal + - "G108: Profiling endpoint is automatically exposed on /debug/pprof" run: - deadline: 4m + timeout: 6m + skip-files: + - "zz_generated.*\\.go$" + - ".*_mock\\.go" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6de71ef3e..a27706035a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,13 +1,3 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Contributing guidelines](#contributing-guidelines) - - [Sign the CLA](#sign-the-cla) - - [Contributing A Patch](#contributing-a-patch) - - - # Contributing guidelines ## Sign the CLA @@ -21,4 +11,4 @@ Kubernetes projects require that you sign a Contributor License Agreement (CLA) 1. If your proposed change is accepted, and you haven't already done so, sign a Contributor License Agreement (see details above). 1. Fork the desired repo, develop and test your code changes. 1. Submit a pull request. -1. All code PR must be labeled with ⚠️ (⚠️, major or breaking changes), ✨ (✨, minor or feature additions), 🐛 (🐛, patch and bugfixes), 📖 (📖, documentation or proposals), or 🏃 (🏃, other). +1. All code PR must be labeled with ⚠️ (:warning:, major or breaking changes), ✨ (:sparkles:, minor or feature additions), 🐛 (:bug:, patch and bugfixes), 📖 (:book:, documentation or proposals), or 🏃 (:running:, other) diff --git a/LICENSE b/LICENSE index 8dada3edaf..261eeb9e9f 100644 --- a/LICENSE +++ b/LICENSE @@ -178,7 +178,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile index 8c980e5be1..cd9ca56750 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ RELEASE_NOTES := $(TOOLS_DIR)/$(RELEASE_NOTES_BIN) GINKGO := $(abspath $(TOOLS_BIN_DIR)/ginkgo) # Define Docker related variables. Releases should modify and double check these vars. -REGISTRY ?= gcr.io/$(shell gcloud config get-value project) +REGISTRY ?= gcr.io/k8s-staging-capi-openstack STAGING_REGISTRY := gcr.io/k8s-staging-capi-openstack PROD_REGISTRY := us.gcr.io/k8s-artifacts-prod/capi-openstack IMAGE_NAME ?= capi-openstack-controller @@ -64,7 +64,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac # Allow overriding the imagePullPolicy -PULL_POLICY ?= Always +PULL_POLICY ?= IfNotPresent # Hosts running SELinux need :z added to volume mounts SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0) @@ -76,9 +76,6 @@ endif # Set build time variables including version details LDFLAGS := $(shell source ./hack/version.sh; version::ldflags) -# Check if binaries exist -HAS_YQ := $(shell command -v yq;) - ## -------------------------------------- ## Help ## -------------------------------------- @@ -95,7 +92,7 @@ help: ## Display this help images: docker-build ## Build all images .PHONY: check -check: modules generate lint-full test +check: modules generate lint-full test verify ## -------------------------------------- ## Testing @@ -104,28 +101,11 @@ check: modules generate lint-full test .PHONY: test test: generate lint ## Run tests $(MAKE) test-go - $(MAKE) test-generate-examples .PHONY: test-go test-go: ## Run golang tests go test -v ./... -.PHONY: test-generate-examples -# See: -# * https://github.com/mikefarah/yq/issues/291 -# * https://github.com/mikefarah/yq/issues/289 -test-generate-examples: $(KUSTOMIZE) $(ENVSUBST) -ifndef HAS_YQ - echo "installing yq" - GO111MODULE=on go get github.com/mikefarah/yq/v2 -endif - # Create a dummy file for test only - mkdir -p tmp/dummy-make-auto-test - echo 'clouds' > tmp/dummy-make-auto-test/dummy-clouds-test.yaml - PATH=$(TOOLS_DIR)/$(BIN_DIR):${PATH} examples/generate.sh -f tmp/dummy-make-auto-test/dummy-clouds-test.yaml openstack tmp/dummy-make-auto-test/_out - # the folder will be generated under same folder of examples - rm -rf tmp/dummy-make-auto-test - ## -------------------------------------- ## Binaries ## -------------------------------------- @@ -217,14 +197,6 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. output:rbac:dir=$(RBAC_ROOT) \ rbac:roleName=manager-role -.PHONY: generate-examples -generate-examples: clean-examples ## Generate examples configurations to run a cluster. -ifndef HAS_YQ - echo "installing yq" - GO111MODULE=on go get github.com/mikefarah/yq/v2 -endif - ./examples/generate.sh -f ${OPENSTACK_CONFIG_FILE} ${CLUSTER_NAME} ./examples/_out single-node - ## -------------------------------------- ## Docker ## -------------------------------------- @@ -318,13 +290,41 @@ release-notes: $(RELEASE_NOTES) ## Development ## -------------------------------------- -# This is used in the get-kubeconfig call below in the create-cluster target. It may be overridden by the -# e2e-conformance.sh script, which is why we need it as a variable here. -CLUSTER_NAME ?= test1 +# Properties for create-cluster +OPENSTACK_CONTROLPLANE_IP ?= "192.168.200.195" +OPENSTACK_FAILURE_DOMAIN ?= "nova" +OPENSTACK_CLOUD ?= "capi-quickstart" +OPENSTACK_CLOUD_CACERT_B64 ?= "Cg==" +OPENSTACK_CLOUD_PROVIDER_CONF_B64 ?= "" +OPENSTACK_CLOUD_YAML_B64 ?= "" +OPENSTACK_EXTERNAL_NETWORK_ID ?= "" +OPENSTACK_DNS_NAMESERVERS ?= "192.168.200.1" +OPENSTACK_IMAGE_NAME ?= "ubuntu-1910-kube-v1.17.3" +OPENSTACK_SSH_AUTHORIZED_KEY ?= "" +OPENSTACK_NODE_MACHINE_FLAVOR ?= "m1.medium" +OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR ?= "m1.medium" +CLUSTER_NAME ?= "capi-quickstart" +OPENSTACK_CLUSTER_TEMPLATE ?= "./templates/cluster-template-without-lb.yaml" +KUBERNETES_VERSION ?= "v1.17.3" +CONTROL_PLANE_MACHINE_COUNT ?= "1" +WORKER_MACHINE_COUNT ?= "3" +LOAD_IMAGE=$(CONTROLLER_IMG)-$(ARCH):dev -# NOTE: do not add 'generate-exmaples' as a prerequisite of this target. It will break e2e conformance testing. .PHONY: create-cluster create-cluster: $(CLUSTERCTL) $(ENVSUBST) ## Create a development Kubernetes cluster on OpenStack in a KIND management cluster. + + # Create clusterctl.yaml to use local OpenStack provider + mkdir -p ./out/infrastructure-openstack/v0.3.0 + echo "providers:" > ./out/clusterctl.yaml + echo "- name: openstack" >> ./out/clusterctl.yaml + echo " url: $(PWD)/out/infrastructure-openstack/v0.3.0/infrastructure-components.yaml" >> ./out/clusterctl.yaml + echo " type: InfrastructureProvider" >> ./out/clusterctl.yaml + + echo "releaseSeries:" > ./out/infrastructure-openstack/v0.3.0/metadata.yaml + echo "- major: 0" >> ./out/infrastructure-openstack/v0.3.0/metadata.yaml + echo " minor: 3" >> ./out/infrastructure-openstack/v0.3.0/metadata.yaml + echo " contract: v1alpha3" >> ./out/infrastructure-openstack/v0.3.0/metadata.yaml + @if [ -z `kind get clusters | grep clusterapi` ]; then \ kind create cluster --name=clusterapi; \ fi @@ -332,35 +332,51 @@ create-cluster: $(CLUSTERCTL) $(ENVSUBST) ## Create a development Kubernetes clu echo "loading ${LOAD_IMAGE} into kind cluster ..." && \ kind --name="clusterapi" load docker-image "${LOAD_IMAGE}"; \ fi - # Install cert manager and wait for availability - kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v0.11.1/cert-manager.yaml - kubectl wait --for=condition=Available --timeout=5m apiservice v1beta1.webhook.cert-manager.io - # Deploy CAPI - kubectl apply -f https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.0/cluster-api-components.yaml + # (Re-)install Core providers + $(CLUSTERCTL) delete --all + $(CLUSTERCTL) init --core cluster-api:v0.3.0 --bootstrap kubeadm:v0.3.0 --control-plane kubeadm:v0.3.0 - # Deploy CAPO - kustomize build config | $(ENVSUBST) | kubectl apply -f - + # (Re-)deploy CAPO provider + MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image + kustomize build config > ./out/infrastructure-openstack/v0.3.0/infrastructure-components.yaml + $(CLUSTERCTL) delete --infrastructure openstack --include-namespace --namespace capo-system || true + kubectl wait --for=delete ns/capo-system || true + $(CLUSTERCTL) init --config ./out/clusterctl.yaml --infrastructure openstack # Wait for CAPI pods - kubectl wait --for=condition=Ready --timeout=5m -n capi-system pod -l cluster.x-k8s.io/provider=cluster-api - kubectl wait --for=condition=Ready --timeout=5m -n capi-kubeadm-bootstrap-system pod -l cluster.x-k8s.io/provider=bootstrap-kubeadm - kubectl wait --for=condition=Ready --timeout=5m -n capi-kubeadm-control-plane-system pod -l cluster.x-k8s.io/provider=control-plane-kubeadm - - # Wait for CAPO pods - kubectl wait --for=condition=Ready --timeout=5m -n capo-system pod -l cluster.x-k8s.io/provider=infrastructure-openstack + kubectl wait --for=condition=Ready --timeout=5m -n capi-system pod --all + kubectl wait --for=condition=Ready --timeout=5m -n capi-webhook-system pod --all + kubectl wait --for=condition=Ready --timeout=5m -n capi-kubeadm-bootstrap-system pod --all + kubectl wait --for=condition=Ready --timeout=5m -n capi-kubeadm-control-plane-system pod --all + kubectl wait --for=condition=Ready --timeout=5m -n capo-system pod --all - # FIXME: - # Create Cluster. - #sleep 10 - #kustomize build templates | $(ENVSUBST) | kubectl apply -f - + # Wait for CAPO CRDs + kubectl wait --for condition=established --timeout=60s crds/openstackmachines.infrastructure.cluster.x-k8s.io + kubectl wait --for condition=established --timeout=60s crds/openstackmachinetemplates.infrastructure.cluster.x-k8s.io + kubectl wait --for condition=established --timeout=60s crds/openstackclusters.infrastructure.cluster.x-k8s.io - # Apply provider-components. - kubectl apply -f examples/_out/provider-components.yaml # Create Cluster. - kubectl apply -f examples/_out/cluster.yaml - # Create control plane machine. - kubectl apply -f examples/_out/controlplane.yaml + kubectl create ns $(CLUSTER_NAME) || true + PULL_POLICY=$(PULL_POLICY) \ + OPENSTACK_CONTROLPLANE_IP=$(OPENSTACK_CONTROLPLANE_IP) \ + OPENSTACK_FAILURE_DOMAIN=$(OPENSTACK_FAILURE_DOMAIN) \ + OPENSTACK_CLOUD=$(OPENSTACK_CLOUD) \ + OPENSTACK_CLOUD_CACERT_B64=$(OPENSTACK_CLOUD_CACERT_B64) \ + OPENSTACK_CLOUD_PROVIDER_CONF_B64=$(OPENSTACK_CLOUD_PROVIDER_CONF_B64) \ + OPENSTACK_CLOUD_YAML_B64=$(OPENSTACK_CLOUD_YAML_B64) \ + OPENSTACK_EXTERNAL_NETWORK_ID=$(OPENSTACK_EXTERNAL_NETWORK_ID) \ + OPENSTACK_DNS_NAMESERVERS=$(OPENSTACK_DNS_NAMESERVERS) \ + OPENSTACK_IMAGE_NAME=$(OPENSTACK_IMAGE_NAME) \ + OPENSTACK_SSH_AUTHORIZED_KEY="$(OPENSTACK_SSH_AUTHORIZED_KEY)" \ + OPENSTACK_NODE_MACHINE_FLAVOR=$(OPENSTACK_NODE_MACHINE_FLAVOR) \ + OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR=$(OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR) \ + $(CLUSTERCTL) config cluster $(CLUSTER_NAME) \ + --from=$(OPENSTACK_CLUSTER_TEMPLATE) \ + --kubernetes-version $(KUBERNETES_VERSION) \ + --control-plane-machine-count=$(CONTROL_PLANE_MACHINE_COUNT) \ + --worker-machine-count=$(WORKER_MACHINE_COUNT) \ + --target-namespace=$(CLUSTER_NAME) | kubectl apply -f - # Wait for the kubeconfig to become available. timeout 300 bash -c "while ! kubectl -n $(CLUSTER_NAME) get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done" @@ -369,10 +385,8 @@ create-cluster: $(CLUSTERCTL) $(ENVSUBST) ## Create a development Kubernetes clu timeout 900 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done" # Deploy calico - kubectl --kubeconfig=./kubeconfig apply -f https://docs.projectcalico.org/manifests/calico.yaml - - # Create a worker node with MachineDeployment. - kubectl apply -f examples/_out/machinedeployment.yaml + curl https://docs.projectcalico.org/manifests/calico.yaml | sed "s/veth_mtu:.*/veth_mtu: \"1400\"/g" | \ + kubectl --kubeconfig=./kubeconfig apply -f - .PHONY: kind-reset kind-reset: ## Destroys the "clusterapi" kind cluster. @@ -401,12 +415,6 @@ clean-temporary: ## Remove all temporary files and folders clean-release: ## Remove the release folder rm -rf $(RELEASE_DIR) -# FIXME: -.PHONY: clean-examples -clean-examples: ## Remove all the temporary files generated in the examples folder - rm -rf examples/_out/ - rm -f examples/provider-components/provider-components-*.yaml - .PHONY: verify verify: verify-boilerplate verify-modules verify-gen diff --git a/README.md b/README.md index 42b75b9f3b..189052b93d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ previous cluster managers such as [kops][kops] and ## Launching a Kubernetes cluster on OpenStack - Check out the [Cluster API Quick Start](https://cluster-api.sigs.k8s.io/user/quick-start.html) to create your first Kubernetes cluster on OpenStack using Cluster API. -- Check out the [getting started guide](./docs/getting-started.md) for launching a cluster on OpenStack using `clusterctl`. ## Features @@ -30,7 +29,7 @@ previous cluster managers such as [kops][kops] and - Support for single and multi-node control plane clusters - Deploy clusters with and without LBaaS available - Support for security groups -- Doesn't use SSH for bootstrapping nodes +- cloud-init based nodes bootstrapping ------ @@ -38,26 +37,35 @@ previous cluster managers such as [kops][kops] and This provider's versions are compatible with the following versions of Cluster API: -||Cluster API v1alpha1 (v0.1)|Cluster API v1alpha2 (v0.2)| -|-|-|-| -|OpenStack Provider v1alpha1 (release-0.1 branch)|✓|| -|OpenStack Provider v1alpha2 (v0.2)||✓| +| abc | def | adfa |adf |asdf | +|---|---|---|---|---| +| sdf | sdfsdf |sfddsf |sdfsd |sdf | +| | | | | | +| | | | | | + +| | Cluster API v1alpha1 (v0.1) | Cluster API v1alpha2 (v0.2) | Cluster API v1alpha2 (v0.3) | +|-------------------------------------------------|---|---|---| +| OpenStack Provider v1alpha1 (release-0.1 branch)| ✓ | | | +| OpenStack Provider v1alpha2 (v0.2) | | ✓ | | +| OpenStack Provider v1alpha3 (v0.3) | | | ✓ | This provider's versions are able to install and manage the following versions of Kubernetes: -||Kubernetes 1.13|Kubernetes 1.14|Kubernetes 1.15| -|-|-|-|-| -|OpenStack Provider v1alpha1 (release-0.1 branch)|✓|✓|✓| -|OpenStack Provider v1alpha2 (v0.2)|||✓| +| | Kubernetes 1.13 | Kubernetes 1.14 | Kubernetes 1.15 | Kubernetes 1.16 | Kubernetes 1.17 | +|-------------------------------------------------|---|---|---|---|---| +|OpenStack Provider v1alpha1 (release-0.1 branch) | ✓ | ✓ | ✓ | | | +|OpenStack Provider v1alpha2 (v0.2) | | | ✓ | | | +|OpenStack Provider v1alpha3 (v0.3) | | | | ✓ | ✓ | This provider's versions are able to install Kubernetes to the following versions of OpenStack: -||OpenStack Pike|OpenStack Queens|OpenStack Rocky|OpenStack Stein| -|-|-|-|-|-| -|OpenStack Provider v1alpha1 (release-0.1 branch)|✓|✓|✓|✓| -|OpenStack Provider v1alpha2 (v0.2)|+|✓|+|✓| +| | OpenStack Pike | OpenStack Queens | OpenStack Rocky | OpenStack Stein | OpenStack Train | +|-------------------------------------------------|---|---|---|---|---| +| OpenStack Provider v1alpha1 (release-0.1 branch)| ✓ | ✓ | ✓ | ✓ | + | +| OpenStack Provider v1alpha2 (v0.2) | + | ✓ | + | ✓ | + | +| OpenStack Provider v1alpha3 (v0.3) | + | + | + | + | ✓ | -Key: +Test status: * `✓` tested * `+` should work, but we weren't able to test it @@ -80,7 +88,7 @@ Reference images can be found in [kubernetes-sigs/image-builder](https://github. ## Documentation -Documentation is in the `/docs` directory +Documentation can be found in the `/docs` directory ## Getting involved and contributing diff --git a/RELEASE.md b/RELEASE.md index 24b5e72b2a..ddd86996f0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -28,8 +28,9 @@ 6. Checkout the tag you've just created and make sure git is in a clean state 7. Run `make release` 8. Attach the files to the drafted release: - 3. `./out/infrastructure-components.yaml` - 4. `./templates/cluster-template.yaml` + 1. `./out/infrastructure-components.yaml` + 2. `./templates/cluster-template.yaml` + 3. `./templates/cluster-template-without-lb.yaml` 9. Perform the [image promotion process](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io#image-promoter). The staging repository is at https://console.cloud.google.com/gcr/images/k8s-staging-capi-openstack/GLOBAL. Be sure to choose the top level `capi-openstack-controller`, which will provide the multi-arch manifest, rather than one for a specific architecture. diff --git a/api/v1alpha3/openstackcluster_types.go b/api/v1alpha3/openstackcluster_types.go index 19808a5951..9ce0d3e248 100644 --- a/api/v1alpha3/openstackcluster_types.go +++ b/api/v1alpha3/openstackcluster_types.go @@ -125,6 +125,9 @@ type OpenStackClusterStatus struct { // It includes Subnets and Router. Network *Network `json:"network,omitempty"` + // FailureDomains represent OpenStack availability zones + FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` + // ControlPlaneSecurityGroups contains all the information about the OpenStack // Security Group that needs to be applied to control plane nodes. // TODO: Maybe instead of two properties, we add a property to the group? @@ -136,9 +139,14 @@ type OpenStackClusterStatus struct { } // +kubebuilder:object:root=true -// +kubebuilder:resource:path=openstackclusters,scope=Namespaced +// +kubebuilder:resource:path=openstackclusters,scope=Namespaced,categories=cluster-api // +kubebuilder:storageversion // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this OpenStackCluster belongs" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Cluster infrastructure is ready for OpenStack instances" +// +kubebuilder:printcolumn:name="Network",type="string",JSONPath=".status.network.id",description="Network the cluster is using" +// +kubebuilder:printcolumn:name="Subnet",type="string",JSONPath=".status.network.subnet.id",description="Subnet the cluster is using" +// +kubebuilder:printcolumn:name="Endpoint",type="string",JSONPath=".status.network.apiServerLoadBalancer.IP",description="API Endpoint",priority=1 // OpenStackCluster is the Schema for the openstackclusters API type OpenStackCluster struct { diff --git a/api/v1alpha3/openstackmachine_types.go b/api/v1alpha3/openstackmachine_types.go index 98e80570e7..27463ed8f2 100644 --- a/api/v1alpha3/openstackmachine_types.go +++ b/api/v1alpha3/openstackmachine_types.go @@ -58,9 +58,6 @@ type OpenStackMachineSpec struct { // The floatingIP should have been created and haven't been associated. FloatingIP string `json:"floatingIP,omitempty"` - // The availability zone from which to launch the server. - AvailabilityZone string `json:"availabilityZone,omitempty"` - // The names of the security groups to assign to the instance SecurityGroups []SecurityGroupParam `json:"securityGroups,omitempty"` @@ -121,9 +118,14 @@ type OpenStackMachineStatus struct { } // +kubebuilder:object:root=true -// +kubebuilder:resource:path=openstackmachines,scope=Namespaced +// +kubebuilder:resource:path=openstackmachines,scope=Namespaced,categories=cluster-api // +kubebuilder:storageversion // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this OpenStackMachine belongs" +// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.instanceState",description="OpenStack instance state" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Machine ready status" +// +kubebuilder:printcolumn:name="InstanceID",type="string",JSONPath=".spec.providerID",description="OpenStack instance ID" +// +kubebuilder:printcolumn:name="Machine",type="string",JSONPath=".metadata.ownerReferences[?(@.kind==\"Machine\")].name",description="Machine object which owns with this OpenStackMachine" // OpenStackMachine is the Schema for the openstackmachines API type OpenStackMachine struct { diff --git a/api/v1alpha3/types.go b/api/v1alpha3/types.go index 252e7f3c5b..94be6b7e3d 100644 --- a/api/v1alpha3/types.go +++ b/api/v1alpha3/types.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package v1alpha3 // OpenStackMachineTemplateResource describes the data needed to create a OpenStackMachine from a template diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index e1810f7772..67f569bc01 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -23,6 +23,7 @@ package v1alpha3 import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + apiv1alpha3 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/errors" ) @@ -291,6 +292,13 @@ func (in *OpenStackClusterStatus) DeepCopyInto(out *OpenStackClusterStatus) { *out = new(Network) (*in).DeepCopyInto(*out) } + if in.FailureDomains != nil { + in, out := &in.FailureDomains, &out.FailureDomains + *out = make(apiv1alpha3.FailureDomains, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } if in.ControlPlaneSecurityGroup != nil { in, out := &in.ControlPlaneSecurityGroup, &out.ControlPlaneSecurityGroup *out = new(SecurityGroup) diff --git a/clusterctl-settings.json b/clusterctl-settings.json index d8f9be2f50..91d0fca798 100644 --- a/clusterctl-settings.json +++ b/clusterctl-settings.json @@ -1,8 +1,7 @@ { - "name": "openstack", + "name": "infrastructure-openstack", "config": { "componentsFile": "infrastructure-components.yaml", - "nextVersion": "v0.2.0", - "type": "InfrastructureProvider" + "nextVersion": "v0.3.0" } } diff --git a/code-of-conduct.md b/code-of-conduct.md index a922d1afae..0d15c00cf3 100644 --- a/code-of-conduct.md +++ b/code-of-conduct.md @@ -1,11 +1,3 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Kubernetes Community Code of Conduct](#kubernetes-community-code-of-conduct) - - - # Kubernetes Community Code of Conduct Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml index c2cde878d9..9fb7873078 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml @@ -10,13 +10,37 @@ metadata: spec: group: infrastructure.cluster.x-k8s.io names: + categories: + - cluster-api kind: OpenStackCluster listKind: OpenStackClusterList plural: openstackclusters singular: openstackcluster scope: Namespaced versions: - - name: v1alpha3 + - additionalPrinterColumns: + - description: Cluster to which this OpenStackCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: Cluster infrastructure is ready for OpenStack instances + jsonPath: .status.ready + name: Ready + type: string + - description: Network the cluster is using + jsonPath: .status.network.id + name: Network + type: string + - description: Subnet the cluster is using + jsonPath: .status.network.subnet.id + name: Subnet + type: string + - description: API Endpoint + jsonPath: .status.network.apiServerLoadBalancer.IP + name: Endpoint + priority: 1 + type: string + name: v1alpha3 schema: openAPIV3Schema: description: OpenStackCluster is the Schema for the openstackclusters API @@ -378,6 +402,25 @@ spec: - name - rules type: object + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: FailureDomains represent OpenStack availability zones + type: object globalSecurityGroup: description: GlobalSecurityGroup contains all the information about the OpenStack Security Group that needs to be applied to all nodes, diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachines.yaml index edef3069b2..27c2073820 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachines.yaml @@ -10,13 +10,36 @@ metadata: spec: group: infrastructure.cluster.x-k8s.io names: + categories: + - cluster-api kind: OpenStackMachine listKind: OpenStackMachineList plural: openstackmachines singular: openstackmachine scope: Namespaced versions: - - name: v1alpha3 + - additionalPrinterColumns: + - description: Cluster to which this OpenStackMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: OpenStack instance state + jsonPath: .status.instanceState + name: State + type: string + - description: Machine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: OpenStack instance ID + jsonPath: .spec.providerID + name: InstanceID + type: string + - description: Machine object which owns with this OpenStackMachine + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + type: string + name: v1alpha3 schema: openAPIV3Schema: description: OpenStackMachine is the Schema for the openstackmachines API @@ -36,9 +59,6 @@ spec: spec: description: OpenStackMachineSpec defines the desired state of OpenStackMachine properties: - availabilityZone: - description: The availability zone from which to launch the server. - type: string cloudName: description: The name of the cloud to use from the clouds secret type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml index 5ad6e0b035..a4adba495b 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml @@ -48,10 +48,6 @@ spec: description: Spec is the specification of the desired behavior of the machine. properties: - availabilityZone: - description: The availability zone from which to launch the - server. - type: string cloudName: description: The name of the cloud to use from the clouds secret diff --git a/config/default/manager_role_aggregation_patch.yaml b/config/default/manager_role_aggregation_patch.yaml index effd90c540..8e335f4930 100644 --- a/config/default/manager_role_aggregation_patch.yaml +++ b/config/default/manager_role_aggregation_patch.yaml @@ -12,4 +12,4 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: capi-aggregated-manager-role + name: capi-system-capi-aggregated-manager-role diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 67f62190e5..8efaac677b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -22,7 +22,7 @@ spec: - args: - --enable-leader-election image: controller:latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: manager ports: - containerPort: 9440 diff --git a/config/manager/manager_auth_proxy_patch.yaml b/config/manager/manager_auth_proxy_patch.yaml index d3994fb918..61cb5e7cb6 100644 --- a/config/manager/manager_auth_proxy_patch.yaml +++ b/config/manager/manager_auth_proxy_patch.yaml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kube-rbac-proxy - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1 args: - "--secure-listen-address=0.0.0.0:8443" - "--upstream=http://127.0.0.1:8080/" @@ -22,3 +22,4 @@ spec: - name: manager args: - "--metrics-addr=127.0.0.1:8080" + - "--enable-leader-election" diff --git a/config/manager/manager_image_patch.yaml b/config/manager/manager_image_patch.yaml index 2a68b85fdb..473c22d67e 100644 --- a/config/manager/manager_image_patch.yaml +++ b/config/manager/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: us.gcr.io/k8s-artifacts-prod/capi-openstack/capi-openstack-controller:v0.2.0 + - image: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller-amd64:dev name: manager diff --git a/config/manager/manager_webhook_patch.yaml b/config/manager/manager_webhook_patch.yaml deleted file mode 100644 index f2f7157b46..0000000000 --- a/config/manager/manager_webhook_patch.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - ports: - - containerPort: 443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: webhook-server-cert diff --git a/config/manager/webhookcainjection_patch.yaml b/config/manager/webhookcainjection_patch.yaml deleted file mode 100644 index 02025d2f7b..0000000000 --- a/config/manager/webhookcainjection_patch.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# This patch add annotation to admission webhook config and -# the variables $(NAMESPACE) and $(CERTIFICATENAME) will be substituted by kustomize. -apiVersion: admissionregistration.k8s.io/v1beta1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - certmanager.k8s.io/inject-ca-from: $(NAMESPACE)/$(CERTIFICATENAME) ---- -apiVersion: admissionregistration.k8s.io/v1beta1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - certmanager.k8s.io/inject-ca-from: $(NAMESPACE)/$(CERTIFICATENAME) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index eb15bd98f8..db8659421c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -10,7 +10,6 @@ rules: - "" resources: - events - - secrets verbs: - create - get @@ -18,6 +17,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch - apiGroups: - cluster.x-k8s.io resources: @@ -30,7 +37,6 @@ rules: - apiGroups: - cluster.x-k8s.io resources: - - clusters - machines - machines/status verbs: diff --git a/controllers/helpers.go b/controllers/helpers.go index 1ede1fd489..68e0f6cd33 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -20,8 +20,23 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" clusterutil "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" ) +// TODO: Move to Cluster API +var pausePredicates = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + return !isPaused(nil, e.MetaNew) + }, + CreateFunc: func(e event.CreateEvent) bool { + return !isPaused(nil, e.Meta) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return !isPaused(nil, e.Meta) + }, +} + // TODO: Fix up Cluster API's clusterutil.IsPaused function func isPaused(cluster *clusterv1.Cluster, v metav1.Object) bool { if cluster == nil { diff --git a/controllers/openstackcluster_controller.go b/controllers/openstackcluster_controller.go index 3238730c58..f7d061299a 100644 --- a/controllers/openstackcluster_controller.go +++ b/controllers/openstackcluster_controller.go @@ -1,4 +1,5 @@ /* +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,6 +19,7 @@ package controllers import ( "context" "fmt" + "github.com/go-logr/logr" "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" @@ -25,6 +27,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/record" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/compute" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/loadbalancer" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/provider" @@ -38,30 +41,24 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -const ( - clusterControllerName = "openstackcluster-controller" -) - // OpenStackClusterReconciler reconciles a OpenStackCluster object type OpenStackClusterReconciler struct { client.Client - Log logr.Logger Recorder record.EventRecorder + Log logr.Logger } // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=openstackclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=openstackclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch -func (r *OpenStackClusterReconciler) Reconcile(request ctrl.Request) (_ ctrl.Result, reterr error) { +func (r *OpenStackClusterReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr error) { ctx := context.TODO() - logger := r.Log.WithName(clusterControllerName). - WithName(fmt.Sprintf("namespace=%s", request.Namespace)). - WithName(fmt.Sprintf("openStackCluster=%s", request.Name)) + log := r.Log.WithValues("namespace", req.Namespace, "openStackCluster", req.Name) // Fetch the OpenStackCluster instance openStackCluster := &infrav1.OpenStackCluster{} - err := r.Get(ctx, request.NamespacedName, openStackCluster) + err := r.Get(ctx, req.NamespacedName, openStackCluster) if err != nil { if apierrors.IsNotFound(err) { return reconcile.Result{}, nil @@ -69,8 +66,6 @@ func (r *OpenStackClusterReconciler) Reconcile(request ctrl.Request) (_ ctrl.Res return reconcile.Result{}, err } - logger = logger.WithName(openStackCluster.APIVersion) - // Fetch the Cluster. cluster, err := util.GetOwnerCluster(ctx, r.Client, openStackCluster.ObjectMeta) if err != nil { @@ -78,21 +73,23 @@ func (r *OpenStackClusterReconciler) Reconcile(request ctrl.Request) (_ ctrl.Res } if isPaused(cluster, openStackCluster) { - logger.Info("OpenStackCluster or linked Cluster is marked as paused. Won't reconcile") + log.Info("OpenStackCluster or linked Cluster is marked as paused. Won't reconcile") return reconcile.Result{}, nil } if cluster == nil { - logger.Info("Cluster Controller has not yet set OwnerRef") + log.Info("Cluster Controller has not yet set OwnerRef") return reconcile.Result{}, nil } - logger = logger.WithName(fmt.Sprintf("cluster=%s", cluster.Name)) + log = log.WithValues("cluster", cluster.Name) patchHelper, err := patch.NewHelper(openStackCluster, r) if err != nil { return ctrl.Result{}, err } + + // Always patch the openStackCluster when exiting this function so we can persist any OpenStackCluster changes. defer func() { if err := patchHelper.Patch(ctx, openStackCluster); err != nil { if reterr == nil { @@ -103,15 +100,66 @@ func (r *OpenStackClusterReconciler) Reconcile(request ctrl.Request) (_ ctrl.Res // Handle deleted clusters if !openStackCluster.DeletionTimestamp.IsZero() { - return r.reconcileClusterDelete(logger, cluster, openStackCluster) + return r.reconcileDelete(log, cluster, openStackCluster) } // Handle non-deleted clusters - return r.reconcileCluster(ctx, logger, patchHelper, cluster, openStackCluster) + return r.reconcileNormal(ctx, log, patchHelper, cluster, openStackCluster) +} + +func (r *OpenStackClusterReconciler) reconcileDelete(log logr.Logger, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) (ctrl.Result, error) { + log.Info("Reconciling Cluster delete") + + clusterName := fmt.Sprintf("%s-%s", cluster.Namespace, cluster.Name) + osProviderClient, clientOpts, err := provider.NewClientFromCluster(r.Client, openStackCluster) + if err != nil { + return reconcile.Result{}, err + } + + networkingService, err := networking.NewService(osProviderClient, clientOpts, log) + if err != nil { + return reconcile.Result{}, err + } + + loadBalancerService, err := loadbalancer.NewService(osProviderClient, clientOpts, log, openStackCluster.Spec.UseOctavia) + if err != nil { + return reconcile.Result{}, err + } + + if openStackCluster.Spec.ManagedAPIServerLoadBalancer { + err = loadBalancerService.DeleteLoadBalancer(clusterName, openStackCluster) + if err != nil { + return reconcile.Result{}, errors.Errorf("failed to delete load balancer: %v", err) + } + } + + // Delete other things + if openStackCluster.Status.GlobalSecurityGroup != nil { + log.Info("Deleting global security group", "name", openStackCluster.Status.GlobalSecurityGroup.Name) + err := networkingService.DeleteSecurityGroups(openStackCluster.Status.GlobalSecurityGroup) + if err != nil { + return reconcile.Result{}, errors.Errorf("failed to delete security group: %v", err) + } + } + + if openStackCluster.Status.ControlPlaneSecurityGroup != nil { + log.Info("Deleting control plane security group", "name", openStackCluster.Status.ControlPlaneSecurityGroup.Name) + err := networkingService.DeleteSecurityGroups(openStackCluster.Status.ControlPlaneSecurityGroup) + if err != nil { + return reconcile.Result{}, errors.Errorf("failed to delete security group: %v", err) + } + } + + // TODO(sbueringer) Delete network/subnet/router/... if created by CAPO + + // Cluster is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(openStackCluster, infrav1.ClusterFinalizer) + + return reconcile.Result{}, nil } -func (r *OpenStackClusterReconciler) reconcileCluster(ctx context.Context, logger logr.Logger, patchHelper *patch.Helper, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) (_ ctrl.Result, reterr error) { - logger.Info("Reconciling Cluster") +func (r *OpenStackClusterReconciler) reconcileNormal(ctx context.Context, log logr.Logger, patchHelper *patch.Helper, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) (ctrl.Result, error) { + log.Info("Reconciling Cluster") clusterName := fmt.Sprintf("%s-%s", cluster.Namespace, cluster.Name) @@ -127,19 +175,24 @@ func (r *OpenStackClusterReconciler) reconcileCluster(ctx context.Context, logge return reconcile.Result{}, err } - networkingService, err := networking.NewService(osProviderClient, clientOpts, logger) + computeService, err := compute.NewService(osProviderClient, clientOpts, log) + if err != nil { + return reconcile.Result{}, err + } + + networkingService, err := networking.NewService(osProviderClient, clientOpts, log) if err != nil { return reconcile.Result{}, err } - loadbalancerService, err := loadbalancer.NewService(osProviderClient, clientOpts, logger, openStackCluster.Spec.UseOctavia) + loadBalancerService, err := loadbalancer.NewService(osProviderClient, clientOpts, log, openStackCluster.Spec.UseOctavia) if err != nil { return reconcile.Result{}, err } - logger.Info("Reconciling network components") + log.Info("Reconciling network components") if openStackCluster.Spec.NodeCIDR == "" { - logger.V(4).Info("No need to reconcile network, searching network and subnet instead") + log.V(4).Info("No need to reconcile network, searching network and subnet instead") netOpts := networks.ListOpts(openStackCluster.Spec.Network) networkList, err := networkingService.GetNetworksByFilter(&netOpts) @@ -182,7 +235,7 @@ func (r *OpenStackClusterReconciler) reconcileCluster(ctx context.Context, logge return reconcile.Result{}, errors.Errorf("failed to reconcile router: %v", err) } if openStackCluster.Spec.ManagedAPIServerLoadBalancer { - err = loadbalancerService.ReconcileLoadBalancer(clusterName, openStackCluster) + err = loadBalancerService.ReconcileLoadBalancer(clusterName, openStackCluster) if err != nil { return reconcile.Result{}, errors.Errorf("failed to reconcile load balancer: %v", err) } @@ -201,14 +254,14 @@ func (r *OpenStackClusterReconciler) reconcileCluster(ctx context.Context, logge Port: int32(openStackCluster.Spec.APIServerLoadBalancerPort), } } else { - controlPlaneMachine, err := r.getControlPlaneMachine() + controlPlaneMachine, err := getControlPlaneMachine(r.Client) if err != nil { - return reconcile.Result{}, errors.Errorf("failed to get control plane machine: %v", err) + return ctrl.Result{}, errors.Errorf("failed to get control plane machine: %v", err) } if controlPlaneMachine != nil { var apiPort int if cluster.Spec.ClusterNetwork.APIServerPort == nil { - logger.Info("No API endpoint given, default to 6443") + log.Info("No API endpoint given, default to 6443") apiPort = 6443 } else { apiPort = int(*cluster.Spec.ClusterNetwork.APIServerPort) @@ -219,89 +272,55 @@ func (r *OpenStackClusterReconciler) reconcileCluster(ctx context.Context, logge Port: int32(apiPort), } } else { - logger.Info("No control plane node found yet, could not write OpenStackCluster.Spec.ControlPlaneEndpoint") + log.Info("No control plane node found yet, could not write OpenStackCluster.Spec.ControlPlaneEndpoint") } } - // No errors, so mark us ready so the Cluster API Cluster Controller can pull it - openStackCluster.Status.Ready = true - - logger.Info("Reconciled Cluster create successfully") - return reconcile.Result{}, nil -} - -func (r *OpenStackClusterReconciler) reconcileClusterDelete(logger logr.Logger, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) (ctrl.Result, error) { - - logger.Info("Reconcile Cluster delete") - clusterName := fmt.Sprintf("%s-%s", cluster.Namespace, cluster.Name) - osProviderClient, clientOpts, err := provider.NewClientFromCluster(r.Client, openStackCluster) - if err != nil { - return reconcile.Result{}, err - } - - networkingService, err := networking.NewService(osProviderClient, clientOpts, logger) - if err != nil { - return reconcile.Result{}, err - } - - loadbalancerService, err := loadbalancer.NewService(osProviderClient, clientOpts, logger, openStackCluster.Spec.UseOctavia) + availabilityZones, err := computeService.GetAvailabilityZones() if err != nil { - return reconcile.Result{}, err + return ctrl.Result{}, err } - - if openStackCluster.Spec.ManagedAPIServerLoadBalancer { - err = loadbalancerService.DeleteLoadBalancer(clusterName, openStackCluster) - if err != nil { - return reconcile.Result{}, errors.Errorf("failed to delete load balancer: %v", err) - } + if openStackCluster.Status.FailureDomains == nil { + openStackCluster.Status.FailureDomains = make(clusterv1.FailureDomains) } - - // Delete other things - if openStackCluster.Status.GlobalSecurityGroup != nil { - logger.Info("Deleting global security group", "name", openStackCluster.Status.GlobalSecurityGroup.Name) - err := networkingService.DeleteSecurityGroups(openStackCluster.Status.GlobalSecurityGroup) - if err != nil { - return reconcile.Result{}, errors.Errorf("failed to delete security group: %v", err) + for _, az := range availabilityZones { + // I'm actually not sure if that's just my local devstack, + // but we probably shouldn't use the "internal" AZ + if az.ZoneName == "internal" { + continue } - } - - if openStackCluster.Status.ControlPlaneSecurityGroup != nil { - logger.Info("Deleting control plane security group", "name", openStackCluster.Status.ControlPlaneSecurityGroup.Name) - err := networkingService.DeleteSecurityGroups(openStackCluster.Status.ControlPlaneSecurityGroup) - if err != nil { - return reconcile.Result{}, errors.Errorf("failed to delete security group: %v", err) + openStackCluster.Status.FailureDomains[az.ZoneName] = clusterv1.FailureDomainSpec{ + ControlPlane: true, } } - // TODO(sbueringer) Delete network/subnet/router/... if created by CAPO - - logger.Info("Reconciled Cluster delete successfully") - // Cluster is deleted so remove the finalizer. - controllerutil.RemoveFinalizer(openStackCluster, infrav1.ClusterFinalizer) - return reconcile.Result{}, nil + openStackCluster.Status.Ready = true + return ctrl.Result{}, nil } func (r *OpenStackClusterReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { return ctrl.NewControllerManagedBy(mgr). WithOptions(options). For(&infrav1.OpenStackCluster{}). + WithEventFilter(pausePredicates). Complete(r) } -func (r *OpenStackClusterReconciler) getControlPlaneMachine() (*infrav1.OpenStackMachine, error) { +func getControlPlaneMachine(client client.Client) (*infrav1.OpenStackMachine, error) { machines := &clusterv1.MachineList{} - if err := r.Client.List(context.TODO(), machines); err != nil { + if err := client.List(context.TODO(), machines); err != nil { return nil, err } openStackMachines := &infrav1.OpenStackMachineList{} - if err := r.Client.List(context.TODO(), openStackMachines); err != nil { + if err := client.List(context.TODO(), openStackMachines); err != nil { return nil, err } var controlPlaneMachine *clusterv1.Machine for _, machine := range machines.Items { - if util.IsControlPlaneMachine(&machine) { - controlPlaneMachine = &machine + m := machine + if util.IsControlPlaneMachine(&m) { + controlPlaneMachine = &m break } } diff --git a/controllers/openstackmachine_controller.go b/controllers/openstackmachine_controller.go index 0e27dfdedb..0d3573bd51 100644 --- a/controllers/openstackmachine_controller.go +++ b/controllers/openstackmachine_controller.go @@ -1,4 +1,5 @@ /* +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,46 +21,42 @@ import ( "encoding/base64" "encoding/json" "fmt" - "k8s.io/client-go/tools/record" "net" "os" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "strconv" "time" + "github.com/go-logr/logr" "github.com/gophercloud/gophercloud" "github.com/gophercloud/utils/openstack/clientconfig" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime/schema" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" "k8s.io/utils/pointer" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha3" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/compute" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/loadbalancer" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/provider" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + capierrors "sigs.k8s.io/cluster-api/errors" + "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "github.com/go-logr/logr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha3" - capierrors "sigs.k8s.io/cluster-api/errors" - "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" ) const ( waitForClusterInfrastructureReadyDuration = 15 * time.Second - machineControllerName = "openstackmachine-controller" TimeoutInstanceCreate = 5 - TimeoutInstanceDelete = 5 RetryIntervalInstanceStatus = 10 * time.Second ) @@ -72,45 +69,41 @@ type OpenStackMachineReconciler struct { // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=openstackmachines,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=openstackmachines/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;machines;machines/status,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=events;secrets,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines;machines/status,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch -func (r *OpenStackMachineReconciler) Reconcile(request ctrl.Request) (_ ctrl.Result, reterr error) { +func (r *OpenStackMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr error) { ctx := context.TODO() - logger := r.Log. - WithName(machineControllerName). - WithName(fmt.Sprintf("namespace=%s", request.Namespace)). - WithName(fmt.Sprintf("openStackMachine=%s", request.Name)) + logger := r.Log.WithValues("namespace", req.Namespace, "openStackMachine", req.Name) // Fetch the OpenStackMachine instance. openStackMachine := &infrav1.OpenStackMachine{} - err := r.Get(ctx, request.NamespacedName, openStackMachine) + err := r.Get(ctx, req.NamespacedName, openStackMachine) if err != nil { if apierrors.IsNotFound(err) { - return reconcile.Result{}, nil + return ctrl.Result{}, nil } - return reconcile.Result{}, err + return ctrl.Result{}, err } - logger = logger.WithName(openStackMachine.APIVersion) - // Fetch the Machine. machine, err := util.GetOwnerMachine(ctx, r.Client, openStackMachine.ObjectMeta) if err != nil { - return reconcile.Result{}, err + return ctrl.Result{}, err } if machine == nil { logger.Info("Machine Controller has not yet set OwnerRef") - return reconcile.Result{}, nil + return ctrl.Result{}, nil } - logger = logger.WithName(fmt.Sprintf("machine=%s", machine.Name)) + logger = logger.WithValues("machine", machine.Name) // Fetch the Cluster. cluster, err := util.GetClusterFromMetadata(ctx, r.Client, machine.ObjectMeta) if err != nil { logger.Info("Machine is missing cluster label or cluster does not exist") - return reconcile.Result{}, nil + return ctrl.Result{}, nil } if isPaused(cluster, openStackMachine) { @@ -118,26 +111,28 @@ func (r *OpenStackMachineReconciler) Reconcile(request ctrl.Request) (_ ctrl.Res return ctrl.Result{}, nil } - logger = logger.WithName(fmt.Sprintf("cluster=%s", cluster.Name)) + logger = logger.WithValues("cluster", cluster.Name) openStackCluster := &infrav1.OpenStackCluster{} - openStackClusterName := types.NamespacedName{ + + openStackClusterName := client.ObjectKey{ Namespace: openStackMachine.Namespace, Name: cluster.Spec.InfrastructureRef.Name, } if err := r.Client.Get(ctx, openStackClusterName, openStackCluster); err != nil { logger.Info("OpenStackCluster is not available yet") - return reconcile.Result{}, nil + return ctrl.Result{}, nil } - logger = logger.WithName(fmt.Sprintf("openStackCluster=%s", openStackCluster.Name)) + logger = logger.WithValues("openStackCluster", openStackCluster.Name) // Initialize the patch helper patchHelper, err := patch.NewHelper(openStackMachine, r) if err != nil { return ctrl.Result{}, err } - // Always attempt to Patch the Machine object and status after each reconciliation. + + // Always patch the openStackMachine when exiting this function so we can persist any AWSMachine changes. defer func() { if err := patchHelper.Patch(ctx, openStackMachine); err != nil { if reterr == nil { @@ -146,42 +141,138 @@ func (r *OpenStackMachineReconciler) Reconcile(request ctrl.Request) (_ ctrl.Res } }() - // Handle deleted clusters + // Handle deleted machines if !openStackMachine.DeletionTimestamp.IsZero() { - return r.reconcileMachineDelete(logger, machine, openStackMachine, cluster, openStackCluster) + return r.reconcileDelete(logger, machine, openStackMachine, cluster, openStackCluster) } // Handle non-deleted clusters - return r.reconcileMachine(ctx, logger, patchHelper, machine, openStackMachine, cluster, openStackCluster) + return r.reconcileNormal(ctx, logger, patchHelper, machine, openStackMachine, cluster, openStackCluster) +} + +func (r *OpenStackMachineReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { + controller, err := ctrl.NewControllerManagedBy(mgr). + WithOptions(options). + For(&infrav1.OpenStackMachine{}). + Watches( + &source.Kind{Type: &clusterv1.Machine{}}, + &handler.EnqueueRequestsFromMapFunc{ + ToRequests: util.MachineToInfrastructureMapFunc(infrav1.GroupVersion.WithKind("OpenStackMachine")), + }, + ). + Watches( + &source.Kind{Type: &infrav1.OpenStackCluster{}}, + &handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.OpenStackClusterToOpenStackMachines)}, + ). + WithEventFilter(pausePredicates). + Build(r) + + if err != nil { + return err + } + + return controller.Watch( + &source.Kind{Type: &clusterv1.Cluster{}}, + &handler.EnqueueRequestsFromMapFunc{ + ToRequests: handler.ToRequestsFunc(r.requeueOpenStackMachinesForUnpausedCluster), + }, + predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldCluster := e.ObjectOld.(*clusterv1.Cluster) + newCluster := e.ObjectNew.(*clusterv1.Cluster) + return oldCluster.Spec.Paused && !newCluster.Spec.Paused + }, + CreateFunc: func(e event.CreateEvent) bool { + cluster := e.Object.(*clusterv1.Cluster) + return !cluster.Spec.Paused + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + }, + ) +} + +func (r *OpenStackMachineReconciler) reconcileDelete(logger logr.Logger, machine *clusterv1.Machine, openStackMachine *infrav1.OpenStackMachine, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) (ctrl.Result, error) { + logger.Info("Handling deleted OpenStackMachine") + + clusterName := fmt.Sprintf("%s-%s", cluster.ObjectMeta.Namespace, cluster.Name) + + osProviderClient, clientOpts, err := provider.NewClientFromMachine(r.Client, openStackMachine) + if err != nil { + return ctrl.Result{}, err + } + + computeService, err := compute.NewService(osProviderClient, clientOpts, logger) + if err != nil { + return ctrl.Result{}, err + } + + loadBalancerService, err := loadbalancer.NewService(osProviderClient, clientOpts, logger, openStackCluster.Spec.UseOctavia) + if err != nil { + return ctrl.Result{}, err + } + if openStackCluster.Spec.ManagedAPIServerLoadBalancer { + err = loadBalancerService.DeleteLoadBalancerMember(clusterName, machine, openStackMachine, openStackCluster) + if err != nil { + return ctrl.Result{}, err + } + } + + instance, err := computeService.InstanceExists(openStackMachine) + if err != nil { + return ctrl.Result{}, err + } + + if instance == nil { + logger.Info("Skipped deleting machine that is already deleted") + controllerutil.RemoveFinalizer(openStackMachine, infrav1.MachineFinalizer) + return ctrl.Result{}, nil + } + + // TODO(sbueringer) wait for instance deleted + err = computeService.InstanceDelete(machine) + if err != nil { + handleUpdateMachineError(logger, openStackMachine, errors.Errorf("error deleting Openstack instance: %v", err)) + return ctrl.Result{}, nil + } + + logger.Info("OpenStack machine deleted successfully") + r.Recorder.Eventf(openStackMachine, corev1.EventTypeNormal, "SuccessfulTerminate", "Terminated instance %q", instance.ID) + + // Instance is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(openStackMachine, infrav1.MachineFinalizer) + + return ctrl.Result{}, nil } -func (r *OpenStackMachineReconciler) reconcileMachine(ctx context.Context, logger logr.Logger, patchHelper *patch.Helper, machine *clusterv1.Machine, openStackMachine *infrav1.OpenStackMachine, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) (_ ctrl.Result, reterr error) { +func (r *OpenStackMachineReconciler) reconcileNormal(ctx context.Context, logger logr.Logger, patchHelper *patch.Helper, machine *clusterv1.Machine, openStackMachine *infrav1.OpenStackMachine, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) (_ ctrl.Result, reterr error) { // If the OpenStackMachine is in an error state, return early. if openStackMachine.Status.FailureReason != nil || openStackMachine.Status.FailureMessage != nil { logger.Info("Error state detected, skipping reconciliation") - return reconcile.Result{}, nil + return ctrl.Result{}, nil } // If the OpenStackMachine doesn't have our finalizer, add it. controllerutil.AddFinalizer(openStackMachine, infrav1.ClusterFinalizer) // Register the finalizer immediately to avoid orphaning OpenStack resources on delete if err := patchHelper.Patch(ctx, openStackMachine); err != nil { - return reconcile.Result{}, err + return ctrl.Result{}, err } if !cluster.Status.InfrastructureReady { logger.Info("Cluster infrastructure is not ready yet, requeuing machine") - return reconcile.Result{RequeueAfter: waitForClusterInfrastructureReadyDuration}, nil + return ctrl.Result{RequeueAfter: waitForClusterInfrastructureReadyDuration}, nil } // Make sure bootstrap data is available and populated. if machine.Spec.Bootstrap.DataSecretName == nil { logger.Info("Waiting for bootstrap data to be available") - return reconcile.Result{RequeueAfter: 10 * time.Second}, nil + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } userData, err := r.getBootstrapData(machine, openStackMachine) if err != nil { - return reconcile.Result{}, err + return ctrl.Result{}, err } logger.Info("Creating Machine") @@ -190,29 +281,29 @@ func (r *OpenStackMachineReconciler) reconcileMachine(ctx context.Context, logge osProviderClient, clientOpts, err := provider.NewClientFromMachine(r.Client, openStackMachine) if err != nil { - return reconcile.Result{}, err + return ctrl.Result{}, err } computeService, err := compute.NewService(osProviderClient, clientOpts, logger) if err != nil { - return reconcile.Result{}, err + return ctrl.Result{}, err } networkingService, err := networking.NewService(osProviderClient, clientOpts, logger) if err != nil { - return reconcile.Result{}, err + return ctrl.Result{}, err } instance, err := r.getOrCreate(computeService, machine, openStackMachine, cluster, openStackCluster, userData) if err != nil { - handleMachineError(logger, openStackMachine, capierrors.UpdateMachineError, errors.Errorf("OpenStack instance cannot be created: %v", err)) - return reconcile.Result{}, err + handleUpdateMachineError(logger, openStackMachine, errors.Errorf("OpenStack instance cannot be created: %v", err)) + return ctrl.Result{}, err } // Set an error message if we couldn't find the instance. if instance == nil { - handleMachineError(logger, openStackMachine, capierrors.UpdateMachineError, errors.New("OpenStack instance cannot be found")) - return reconcile.Result{}, nil + handleUpdateMachineError(logger, openStackMachine, errors.New("OpenStack instance cannot be found")) + return ctrl.Result{}, nil } // TODO(sbueringer) From CAPA: TODO(ncdc): move this validation logic into a validating webhook (for us: create validation logic in webhook) @@ -234,79 +325,28 @@ func (r *OpenStackMachineReconciler) reconcileMachine(ctx context.Context, logge case infrav1.InstanceStateBuilding: logger.Info("Machine instance is BUILDING", "instance-id", instance.ID) default: - handleMachineError(logger, openStackMachine, capierrors.UpdateMachineError, errors.Errorf("OpenStack instance state %q is unexpected", instance.State)) - return reconcile.Result{}, nil + handleUpdateMachineError(logger, openStackMachine, errors.Errorf("OpenStack instance state %q is unexpected", instance.State)) + return ctrl.Result{}, nil } if openStackMachine.Spec.FloatingIP != "" { err = r.reconcileFloatingIP(computeService, networkingService, instance, openStackMachine, openStackCluster) if err != nil { - handleMachineError(logger, openStackMachine, capierrors.UpdateMachineError, errors.Errorf("FloatingIP cannot be reconciled: %v", err)) - return reconcile.Result{}, nil + handleUpdateMachineError(logger, openStackMachine, errors.Errorf("FloatingIP cannot be reconciled: %v", err)) + return ctrl.Result{}, nil } } if openStackCluster.Spec.ManagedAPIServerLoadBalancer { err = r.reconcileLoadBalancerMember(logger, osProviderClient, clientOpts, instance, clusterName, machine, openStackMachine, openStackCluster) if err != nil { - handleMachineError(logger, openStackMachine, capierrors.UpdateMachineError, errors.Errorf("LoadBalancerMember cannot be reconciled: %v", err)) - return reconcile.Result{}, nil + handleUpdateMachineError(logger, openStackMachine, errors.Errorf("LoadBalancerMember cannot be reconciled: %v", err)) + return ctrl.Result{}, nil } } logger.Info("Reconciled Machine create successfully") - return reconcile.Result{}, nil -} - -func (r *OpenStackMachineReconciler) reconcileMachineDelete(logger logr.Logger, machine *clusterv1.Machine, openStackMachine *infrav1.OpenStackMachine, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) (ctrl.Result, error) { - - logger.Info("Deleting Machine") - - clusterName := fmt.Sprintf("%s-%s", cluster.ObjectMeta.Namespace, cluster.Name) - - osProviderClient, clientOpts, err := provider.NewClientFromMachine(r.Client, openStackMachine) - if err != nil { - return reconcile.Result{}, err - } - - computeService, err := compute.NewService(osProviderClient, clientOpts, logger) - if err != nil { - return reconcile.Result{}, err - } - - loadbalancerService, err := loadbalancer.NewService(osProviderClient, clientOpts, logger, openStackCluster.Spec.UseOctavia) - if err != nil { - return reconcile.Result{}, err - } - if openStackCluster.Spec.ManagedAPIServerLoadBalancer { - err = loadbalancerService.DeleteLoadBalancerMember(clusterName, machine, openStackMachine, openStackCluster) - if err != nil { - return reconcile.Result{}, err - } - } - - instance, err := computeService.InstanceExists(openStackMachine) - if err != nil { - return reconcile.Result{}, err - } - - if instance == nil { - logger.Info("Skipped deleting machine that is already deleted") - controllerutil.RemoveFinalizer(openStackMachine, infrav1.MachineFinalizer) - return reconcile.Result{}, nil - } - - // TODO(sbueringer) wait for instance deleted - err = computeService.InstanceDelete(machine) - if err != nil { - handleMachineError(logger, openStackMachine, capierrors.UpdateMachineError, errors.Errorf("error deleting Openstack instance: %v", err)) - return reconcile.Result{}, nil - } - - logger.Info("Reconciled Machine delete successfully") - // Instance is deleted so remove the finalizer. - controllerutil.RemoveFinalizer(openStackMachine, infrav1.MachineFinalizer) - return reconcile.Result{}, nil + return ctrl.Result{}, nil } func (r *OpenStackMachineReconciler) getOrCreate(computeService *compute.Service, machine *clusterv1.Machine, openStackMachine *infrav1.OpenStackMachine, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster, userData string) (*compute.Instance, error) { @@ -322,7 +362,7 @@ func (r *OpenStackMachineReconciler) getOrCreate(computeService *compute.Service return nil, errors.Errorf("error creating Openstack instance: %v", err) } instanceCreateTimeout := getTimeout("CLUSTER_API_OPENSTACK_INSTANCE_CREATE_TIMEOUT", TimeoutInstanceCreate) - instanceCreateTimeout = instanceCreateTimeout * time.Minute + instanceCreateTimeout *= time.Minute // instance in PollImmediate has to overwrites instance of the outer scope to get an updated instance state, // which is then returned at the end of getOrCreate err = util.PollImmediate(RetryIntervalInstanceStatus, instanceCreateTimeout, func() (bool, error) { @@ -340,11 +380,12 @@ func (r *OpenStackMachineReconciler) getOrCreate(computeService *compute.Service return instance, nil } -func handleMachineError(logger logr.Logger, openstackMachine *infrav1.OpenStackMachine, reason capierrors.MachineStatusError, message error) { - openstackMachine.Status.FailureReason = &reason +func handleUpdateMachineError(logger logr.Logger, openstackMachine *infrav1.OpenStackMachine, message error) { + err := capierrors.UpdateMachineError + openstackMachine.Status.FailureReason = &err openstackMachine.Status.FailureMessage = pointer.StringPtr(message.Error()) // TODO remove if this error is logged redundantly - logger.Error(fmt.Errorf(string(reason)), message.Error()) + logger.Error(fmt.Errorf(string(err)), message.Error()) } func getTimeout(name string, timeout int) time.Duration { @@ -357,24 +398,6 @@ func getTimeout(name string, timeout int) time.Duration { return time.Duration(timeout) } -func (r *OpenStackMachineReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { - return ctrl.NewControllerManagedBy(mgr). - WithOptions(options). - For(&infrav1.OpenStackMachine{}).Watches( - &source.Kind{Type: &clusterv1.Machine{}}, - &handler.EnqueueRequestsFromMapFunc{ - ToRequests: util.MachineToInfrastructureMapFunc(schema.GroupVersionKind{ - Group: infrav1.SchemeBuilder.GroupVersion.Group, - Version: infrav1.SchemeBuilder.GroupVersion.Version, - Kind: "OpenStackMachine", - }), - }, - ).Watches( - &source.Kind{Type: &infrav1.OpenStackCluster{}}, - &handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.OpenStackClusterToOpenStackMachines)}, - ).Complete(r) -} - func (r *OpenStackMachineReconciler) reconcileFloatingIP(computeService *compute.Service, networkingService *networking.Service, instance *compute.Instance, openStackMachine *infrav1.OpenStackMachine, openStackCluster *infrav1.OpenStackCluster) error { err := networkingService.CreateFloatingIPIfNecessary(openStackCluster, openStackMachine.Spec.FloatingIP) if err != nil { @@ -490,9 +513,9 @@ func (r *OpenStackMachineReconciler) getBootstrapData(machine *clusterv1.Machine } secret := &corev1.Secret{} - key := types.NamespacedName{Namespace: machine.ObjectMeta.Namespace, Name: *machine.Spec.Bootstrap.DataSecretName} + key := types.NamespacedName{Namespace: machine.Namespace, Name: *machine.Spec.Bootstrap.DataSecretName} if err := r.Client.Get(context.TODO(), key, secret); err != nil { - return "", errors.Wrapf(err, "failed to retrieve bootstrap data secret for Openstack Machine %s/%s", machine.ObjectMeta.Namespace, openStackMachine.Name) + return "", errors.Wrapf(err, "failed to retrieve bootstrap data secret for Openstack Machine %s/%s", machine.Namespace, openStackMachine.Name) } value, ok := secret.Data["value"] @@ -502,3 +525,36 @@ func (r *OpenStackMachineReconciler) getBootstrapData(machine *clusterv1.Machine return base64.StdEncoding.EncodeToString(value), nil } + +func (r *OpenStackMachineReconciler) requeueOpenStackMachinesForUnpausedCluster(o handler.MapObject) []ctrl.Request { + c, ok := o.Object.(*clusterv1.Cluster) + if !ok { + r.Log.Error(errors.Errorf("expected a Cluster but got a %T", o.Object), "failed to get OpenStackMachines for unpaused Cluster") + return nil + } + + // Don't handle deleted clusters + if !c.ObjectMeta.DeletionTimestamp.IsZero() { + return nil + } + + return r.requestsForCluster(c.Namespace, c.Name) +} + +func (r *OpenStackMachineReconciler) requestsForCluster(namespace, name string) []ctrl.Request { + log := r.Log.WithValues("Cluster", name, "Namespace", namespace) + labels := map[string]string{clusterv1.ClusterLabelName: name} + machineList := &clusterv1.MachineList{} + if err := r.Client.List(context.TODO(), machineList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { + log.Error(err, "failed to get owned Machines") + return nil + } + + result := make([]ctrl.Request, 0, len(machineList.Items)) + for _, m := range machineList.Items { + if m.Spec.InfrastructureRef.Name != "" { + result = append(result, ctrl.Request{NamespacedName: client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.InfrastructureRef.Name}}) + } + } + return result +} diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 577215f944..129f069437 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -1,4 +1,5 @@ /* +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/docs/config.md b/docs/configuration.md similarity index 100% rename from docs/config.md rename to docs/configuration.md diff --git a/docs/getting-started.md b/docs/getting-started.md deleted file mode 100644 index 9d40adb1fa..0000000000 --- a/docs/getting-started.md +++ /dev/null @@ -1,112 +0,0 @@ -# Getting started - -## Prerequisites - -1. Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -1. Install `kustomize` `v3.1.0+` (see [kustomize/releases](https://github.com/kubernetes-sigs/kustomize/releases)) -1. Download the latest `v0.2.x` release of `clusterctl` from [cluster-api/releases](https://github.com/kubernetes-sigs/cluster-api/releases) -1. You can use either a VM, container or existing Kubernetes cluster as bootstrap cluster. - - If you want to use VM, install [Minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/), version 0.30.0 or greater. Also install a [driver](https://github.com/kubernetes/minikube/blob/master/docs/drivers.md). For Linux, we recommend `kvm2`. For MacOS, we recommend `VirtualBox`. - - If you want to use a container, install [Kind](https://github.com/kubernetes-sigs/kind#installation-and-usage). - - If you want to use an existing Kubernetes cluster, prepare a kubeconfig which for this cluster. -1. The CAPO provider requires an OS image (available in OpenStack), you have several ways to provide one: - - Build image via [image-builder](https://github.com/kubernetes-sigs/image-builder/tree/master/images/capi) - - Or as a test env, you can deploy a VM through OpenStack, log on to the VM and follow the steps in [install kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/) to install container runtime and download kubectl, kubelet and kubeadm but *don't* execute kubeadm, then capture the image in OpenStack cloud and use it as OS image. - -## Cluster Creation - -1. Generate example YAML files if needed. You can use the `examples/generate.sh` script as documented [here](./../examples/README.md). - -2. Create a cluster: - - If you are using Minikube: - - ```bash - clusterctl create cluster \ - --bootstrap-type minikube --bootstrap-flags kubernetes-version=v1.15.0 \ - -c examples/_out/cluster.yaml \ - -m examples/_out/controlplane.yaml \ - -p examples/_out/provider-components.yaml \ - -a examples/addons.yaml - ``` - - To choose a specific Minikube driver, please use the `--bootstrap-flags vm-driver=xxx` command line parameter. For example to use the `kvm2` driver with clusterctl you would add `--bootstrap-flags vm-driver=kvm2`, for linux, if you haven't installed any driver, you can add `--bootstrap-flags vm-driver=none`. - - - If you are using `Kind`: - - ```bash - clusterctl create cluster \ - --bootstrap-type kind --bootstrap-cluster-cleanup=false \ - -c examples/_out/cluster.yaml \ - -m examples/_out/controlplane.yaml \ - -p examples/_out/provider-components.yaml \ - -a examples/addons.yaml - ``` - - - If you are using an existing Kubernetes cluster: - - ```bash - clusterctl create cluster \ - --bootstrap-cluster-kubeconfig ~/.kube/config \ - -c examples/_out/cluster.yaml \ - -m examples/_out/controlplane.yaml \ - -p examples/_out/provider-components.yaml \ - -a examples/addons.yaml - ``` - -### Interacting with your cluster - -Once you have created a cluster, you can interact with the cluster and machine resources via `kubectl`: - -```bash -export KUBECONFIG=./kubeconfig -kubectl get clusters -kubectl get machines -``` - -### Deploying additional machines - -You can deploy additional machines via the `examples/_out/machinedeployment.yaml`. You probably have to customize the -configuration of these machines. - -## Cluster Deletion - -Use the following command to delete a cluster and all resources it created. - -```bash -clusterctl delete cluster --cluster --bootstrap-type kind --kubeconfig kubeconfig --provider-components examples/_out/provider-components.yaml -``` - -Or you can manually delete all resources that were created as part of -your openstack Cluster API Kubernetes cluster. - -1. Delete all of the worker machines in the cluster. Make sure to wait for the - corresponding machines to be deleted before moving onto the next step. After this - step, only the control plane node(s) will remain. - - ```bash - kubectl --kubeconfig=kubeconfig delete machinedeployment -l cluster.x-k8s.io/cluster-name= - kubectl --kubeconfig=kubeconfig get nodes - ``` - -2. Delete the control plane machines. - ```bash - kubectl --kubeconfig=kubeconfig delete machines -l cluster.x-k8s.io/cluster-name= - ``` - -3. (optional) Delete the load balancer in your OpenStack cloud if one has been created. - -4. Delete the `kubeconfig` file that were created for your cluster. - - ```bash - rm kubeconfig - ``` - -5. Delete the ssh keypair that was created for your machines. - - ```bash - rm -rf $HOME/.ssh/openstack_tmp* - ``` - -## Troubleshooting - -Please refer to [Trouble shooting documentation](./troubleshooting.md) for further info. diff --git a/examples/.gitignore b/examples/.gitignore deleted file mode 100644 index c1d18d8a8b..0000000000 --- a/examples/.gitignore +++ /dev/null @@ -1 +0,0 @@ -_out diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 996ba94302..0000000000 --- a/examples/README.md +++ /dev/null @@ -1,85 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Openstack Example Files](#openstack-example-files) - - [Contents](#contents) - - [Prerequisites](#prerequisites) - - [Generation](#generation) - - [Manual Modification](#manual-modification) - - - -# Openstack Samples - -## Prerequisites - -1. Install `yq` (see [here](https://github.com/mikefarah/yq)). -1. Install `envsubst` (see [here](http://linuxcommandlibrary.com/man/envsubst.html)). -2. Install `kustomize` v3.1.0+ (see [here](https://github.com/kubernetes-sigs/kustomize/releases)). - -## Generation - -For convenience, a script is provided which generates example YAMLs. The generated YAMLs then have to be customized manually. - -1. Run the generation script. - -``` -cd examples -export CLUSTER_NAME= -./generate.sh [options] [single/multi-node] -``` - - `` is a YAML configuration file for Openstack, for more details refer to [OpenStack client configuration files](https://docs.openstack.org/python-openstackclient/latest/configuration/index.html#configuration-files). - - `` is the cloud you are going to use, e.g. multiple clouds might be defined in `clouds.yaml`. - This will be the cloud on which the new workload Kubernetes cluster will be created. For example, assume you have - multiple clouds defined in the `clouds.yaml` like shown below. You have to decide if you want to create your cluster - in `openstack1` or `ds-admin`. - - ``` - clouds: - openstack1: - auth: - auth_url: http://192.168.122.10:35357/ - region_name: RegionOne - ds-admin: - auth: - auth_url: http://192.168.122.10:35357/ - region_name: RegionOne - ``` - In case your OpenStack cluster endpoint is using SSL and the cert is signed by an unknown CA, a specific CA certificate - can be provided via the cacert field. - - `` is where the YAML files will be stored. - - `[single/multi-node]` is for the control plane preference, by default it's `single-node` . - -## Manual Modification - -You **will need** to make changes to the generated files to create a working cluster. -You can find some guidance on what needs to be edited, and how to create some of the -required OpenStack resources in the [Configuration documentation](../docs/config.md). - - -#### Quick notes on clouds.yaml - -We no longer support generating clouds.yaml. You should be able to get a valid clouds.yaml from your openstack cluster. -However, make sure that the following fields are included, and correct. - -- `username` -- `user_domain_name` -- `project_id` -- `region_name` -- `auth_url` -- `password` - -#### Notes on ssh keys for debug purpose. - -When running `generate.sh` the first time, a new ssh keypair is generated and stored in `$HOME/.ssh/openstack_tmp` and -`$HOME/.ssh/openstack_tmp.pub`. The key is for debugging purposes only. You have to create the ssh key manually in OpenStack -before creating the cluster, e.g.: - -```bash -openstack keypair create --public-key ~/.ssh/openstack_tmp.pub cluster-api-provider-openstack -``` diff --git a/examples/clouds-secrets/.gitignore b/examples/clouds-secrets/.gitignore deleted file mode 100644 index 52e5937d23..0000000000 --- a/examples/clouds-secrets/.gitignore +++ /dev/null @@ -1 +0,0 @@ -configs \ No newline at end of file diff --git a/examples/clouds-secrets/kustomization.yaml b/examples/clouds-secrets/kustomization.yaml deleted file mode 100644 index 453e1c00c5..0000000000 --- a/examples/clouds-secrets/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -generatorOptions: - disableNameSuffixHash: true - -secretGenerator: -- name: cloud-config - files: - - clouds.yaml=configs/clouds.yaml - - cacert=configs/cacert - type: Opaque - -namespace: ${CLUSTER_NAME} diff --git a/examples/cluster/multi-node/cluster.yaml b/examples/cluster/multi-node/cluster.yaml deleted file mode 100644 index de325ed118..0000000000 --- a/examples/cluster/multi-node/cluster.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: ${CLUSTER_NAME} -spec: - clusterNetwork: - services: - cidrBlocks: ["10.96.0.0/12"] - pods: - cidrBlocks: ["192.168.0.0/16"] # CIDR block used by Calico. - serviceDomain: "cluster.local" - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: OpenStackCluster - name: ${CLUSTER_NAME} - namespace: ${CLUSTER_NAME} - controlPlaneRef: - kind: KubeadmControlPlane - apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 - name: "${CLUSTER_NAME}-control-plane" - namespace: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: OpenStackCluster -metadata: - name: ${CLUSTER_NAME} - namespace: ${CLUSTER_NAME} -spec: - cloudName: $CLOUD - cloudsSecret: - name: cloud-config - namespace: ${CLUSTER_NAME} - nodeCidr: ${NODE_CIDR} - managedAPIServerLoadBalancer: true - apiServerLoadBalancerFloatingIP: ${CONTROLPLANE_FLOATING_IP} - apiServerLoadBalancerPort: 6443 - apiServerLoadBalancerAdditionalPorts: - - 22 - dnsNameservers: [] - # multi-node control-plane: - # * externalNetworkId is required for: - # * creating routers - # * creating floating ips - # * creating load balancer - externalNetworkId: ${EXTERNAL_NETWORK_ID} - disablePortSecurity: true - disableServerTags: true - useOctavia: true diff --git a/examples/cluster/multi-node/kustomization.yaml b/examples/cluster/multi-node/kustomization.yaml deleted file mode 100644 index d3e20f6823..0000000000 --- a/examples/cluster/multi-node/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: ${CLUSTER_NAME} -resources: -- cluster.yaml -configurations: -- kustomizeconfig.yaml diff --git a/examples/cluster/multi-node/kustomizeconfig.yaml b/examples/cluster/multi-node/kustomizeconfig.yaml deleted file mode 100644 index 008398b422..0000000000 --- a/examples/cluster/multi-node/kustomizeconfig.yaml +++ /dev/null @@ -1,6 +0,0 @@ -namespace: -- kind: Cluster - group: cluster.x-k8s.io - version: v1alpha3 - path: spec/infrastructureRef/namespace - create: true diff --git a/examples/cluster/single-node/cluster.yaml b/examples/cluster/single-node/cluster.yaml deleted file mode 100644 index 07eb279c47..0000000000 --- a/examples/cluster/single-node/cluster.yaml +++ /dev/null @@ -1,47 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: ${CLUSTER_NAME} -spec: - clusterNetwork: - services: - cidrBlocks: ["10.96.0.0/12"] - pods: - cidrBlocks: ["192.168.0.0/16"] # CIDR block used by Calico. - serviceDomain: "cluster.local" - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: OpenStackCluster - name: ${CLUSTER_NAME} - namespace: ${CLUSTER_NAME} - controlPlaneRef: - kind: KubeadmControlPlane - apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 - name: "${CLUSTER_NAME}-control-plane" - namespace: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: OpenStackCluster -metadata: - name: ${CLUSTER_NAME} - namespace: ${CLUSTER_NAME} -spec: - controlPlaneEndpoint: - host: ${CONTROLPLANE_FLOATING_IP} - port: 6443 - cloudName: $CLOUD - cloudsSecret: - name: cloud-config - namespace: ${CLUSTER_NAME} - nodeCidr: ${NODE_CIDR} - dnsNameservers: [] - # multi-node control-plane: - # * externalNetworkId is required for: - # * creating routers - # * creating floating ips - # * creating load balancer - externalNetworkId: ${EXTERNAL_NETWORK_ID} - disablePortSecurity: true - disableServerTags: true diff --git a/examples/cluster/single-node/kustomization.yaml b/examples/cluster/single-node/kustomization.yaml deleted file mode 100644 index d3e20f6823..0000000000 --- a/examples/cluster/single-node/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: ${CLUSTER_NAME} -resources: -- cluster.yaml -configurations: -- kustomizeconfig.yaml diff --git a/examples/cluster/single-node/kustomizeconfig.yaml b/examples/cluster/single-node/kustomizeconfig.yaml deleted file mode 100644 index 008398b422..0000000000 --- a/examples/cluster/single-node/kustomizeconfig.yaml +++ /dev/null @@ -1,6 +0,0 @@ -namespace: -- kind: Cluster - group: cluster.x-k8s.io - version: v1alpha3 - path: spec/infrastructureRef/namespace - create: true diff --git a/examples/controlplane/multi-node/controlplane.yaml b/examples/controlplane/multi-node/controlplane.yaml deleted file mode 100644 index 81e07c9142..0000000000 --- a/examples/controlplane/multi-node/controlplane.yaml +++ /dev/null @@ -1,93 +0,0 @@ -##################################################### -# ${CLUSTER_NAME}-controlplane -##################################################### -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 -metadata: - name: "${CLUSTER_NAME}-control-plane" - namespace: ${CLUSTER_NAME} -spec: - replicas: 3 - infrastructureTemplate: - kind: OpenStackMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - name: "${CLUSTER_NAME}-control-plane" - kubeadmConfigSpec: - initConfiguration: - localAPIEndpoint: - advertiseAddress: '{{ ds.ec2_metadata.local_ipv4 }}' - bindPort: 6443 - nodeRegistration: - name: '{{ local_hostname }}' - kubeletExtraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf - clusterConfiguration: - controlPlaneEndpoint: "${LOADBALANCER_IP}:6443" - imageRepository: k8s.gcr.io - apiServer: - extraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf - extraVolumes: - - name: cloud - hostPath: /etc/kubernetes/cloud.conf - mountPath: /etc/kubernetes/cloud.conf - readOnly: true - controllerManager: - extraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf - extraVolumes: - - name: cloud - hostPath: /etc/kubernetes/cloud.conf - mountPath: /etc/kubernetes/cloud.conf - readOnly: true - - name: cacerts - hostPath: /etc/certs/cacert - mountPath: /etc/certs/cacert - readOnly: true - joinConfiguration: - nodeRegistration: - name: '{{ local_hostname }}' - kubeletExtraArgs: - cloud-config: /etc/kubernetes/cloud.conf - cloud-provider: openstack - files: - - path: /etc/kubernetes/cloud.conf - owner: root - permissions: "0600" - content: |- - # cloud.conf to communicate with OpenStack - $OPENSTACK_CLOUD_PROVIDER_CONF8 - - path: /etc/certs/cacert - owner: root - permissions: "0600" - content: | - $OPENSTACK_CLOUD_CACERT_CONFIG8 - ntp: - servers: [] - users: - - name: capo - sudo: "ALL=(ALL) NOPASSWD:ALL" - sshAuthorizedKeys: - - "$MACHINE_CONTROLLER_SSH_PUBLIC_FILE_CONTENT" - - version: "${KUBERNETES_VERSION}" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: OpenStackMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${CLUSTER_NAME} -spec: - template: - spec: - flavor: m1.medium - image: ${OS_IMAGE_NAME} - availabilityZone: ${AVAILABILITY_ZONE} - floatingIP: ${CONTROLPLANE_FLOATING_IP} - cloudName: $CLOUD - cloudsSecret: - name: cloud-config - namespace: ${CLUSTER_NAME} diff --git a/examples/controlplane/multi-node/kustomization.yaml b/examples/controlplane/multi-node/kustomization.yaml deleted file mode 100644 index 44a9cc2831..0000000000 --- a/examples/controlplane/multi-node/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: ${CLUSTER_NAME} -resources: -- controlplane.yaml -configurations: -- kustomizeconfig.yaml diff --git a/examples/controlplane/multi-node/kustomizeconfig.yaml b/examples/controlplane/multi-node/kustomizeconfig.yaml deleted file mode 100644 index 7e2a7bafec..0000000000 --- a/examples/controlplane/multi-node/kustomizeconfig.yaml +++ /dev/null @@ -1,15 +0,0 @@ -namespace: -- kind: Machine - group: cluster.x-k8s.io - version: v1alpha3 - path: spec/infrastructureRef/namespace - create: true -- kind: Machine - group: cluster.x-k8s.io - version: v1alpha3 - path: spec/bootstrap/configRef/namespace - create: true - -commonLabels: -- path: metadata/labels - create: true diff --git a/examples/controlplane/single-node/controlplane.yaml b/examples/controlplane/single-node/controlplane.yaml deleted file mode 100644 index 046a04ac2e..0000000000 --- a/examples/controlplane/single-node/controlplane.yaml +++ /dev/null @@ -1,93 +0,0 @@ -##################################################### -# ${CLUSTER_NAME}-controlplane -##################################################### -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 -metadata: - name: "${CLUSTER_NAME}-control-plane" - namespace: ${CLUSTER_NAME} -spec: - replicas: 1 - infrastructureTemplate: - kind: OpenStackMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - name: "${CLUSTER_NAME}-control-plane" - kubeadmConfigSpec: - initConfiguration: - localAPIEndpoint: - advertiseAddress: '{{ ds.ec2_metadata.local_ipv4 }}' - bindPort: 6443 - nodeRegistration: - name: '{{ local_hostname }}' - kubeletExtraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf - clusterConfiguration: - controlPlaneEndpoint: "${CONTROLPLANE_FLOATING_IP}:6443" - imageRepository: k8s.gcr.io - apiServer: - extraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf - extraVolumes: - - name: cloud - hostPath: /etc/kubernetes/cloud.conf - mountPath: /etc/kubernetes/cloud.conf - readOnly: true - controllerManager: - extraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf - extraVolumes: - - name: cloud - hostPath: /etc/kubernetes/cloud.conf - mountPath: /etc/kubernetes/cloud.conf - readOnly: true - - name: cacerts - hostPath: /etc/certs/cacert - mountPath: /etc/certs/cacert - readOnly: true - joinConfiguration: - nodeRegistration: - name: '{{ local_hostname }}' - kubeletExtraArgs: - cloud-config: /etc/kubernetes/cloud.conf - cloud-provider: openstack - files: - - path: /etc/kubernetes/cloud.conf - owner: root - permissions: "0600" - content: |- - # cloud.conf to communicate with OpenStack - $OPENSTACK_CLOUD_PROVIDER_CONF8 - - path: /etc/certs/cacert - owner: root - permissions: "0600" - content: | - $OPENSTACK_CLOUD_CACERT_CONFIG8 - ntp: - servers: [] - users: - - name: capo - sudo: "ALL=(ALL) NOPASSWD:ALL" - sshAuthorizedKeys: - - "$MACHINE_CONTROLLER_SSH_PUBLIC_FILE_CONTENT" - - version: "${KUBERNETES_VERSION}" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: OpenStackMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${CLUSTER_NAME} -spec: - template: - spec: - flavor: m1.medium - image: ${OS_IMAGE_NAME} - availabilityZone: ${AVAILABILITY_ZONE} - floatingIP: ${CONTROLPLANE_FLOATING_IP} - cloudName: $CLOUD - cloudsSecret: - name: cloud-config - namespace: ${CLUSTER_NAME} diff --git a/examples/controlplane/single-node/kustomization.yaml b/examples/controlplane/single-node/kustomization.yaml deleted file mode 100644 index 44a9cc2831..0000000000 --- a/examples/controlplane/single-node/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: ${CLUSTER_NAME} -resources: -- controlplane.yaml -configurations: -- kustomizeconfig.yaml diff --git a/examples/controlplane/single-node/kustomizeconfig.yaml b/examples/controlplane/single-node/kustomizeconfig.yaml deleted file mode 100644 index 7e2a7bafec..0000000000 --- a/examples/controlplane/single-node/kustomizeconfig.yaml +++ /dev/null @@ -1,15 +0,0 @@ -namespace: -- kind: Machine - group: cluster.x-k8s.io - version: v1alpha3 - path: spec/infrastructureRef/namespace - create: true -- kind: Machine - group: cluster.x-k8s.io - version: v1alpha3 - path: spec/bootstrap/configRef/namespace - create: true - -commonLabels: -- path: metadata/labels - create: true diff --git a/examples/generate.sh b/examples/generate.sh deleted file mode 100755 index 00db754214..0000000000 --- a/examples/generate.sh +++ /dev/null @@ -1,244 +0,0 @@ -#!/bin/bash -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# Directories. -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Binaries -ENVSUBST=${ENVSUBST:-envsubst} -command -v "${ENVSUBST}" >/dev/null 2>&1 || echo -v "Cannot find ${ENVSUBST} in path." - -# Cluster. -export CLUSTER_NAME="${CLUSTER_NAME:-test-cluster}" -export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.16.0}" - -# Machine settings. -export CONTROL_PLANE_MACHINE_TYPE="${CONTROL_PLANE_MACHINE_TYPE:-m1.medium}" -export NODE_MACHINE_TYPE="${CONTROL_PLANE_MACHINE_TYPE:-m1.medium}" -export SSH_KEY_NAME="${SSH_KEY_NAME:-cluster-api-provider-openstack}" -export CONTROLPLANE="single-node" - -# Overwrite flag. -OVERWRITE=0 - -SCRIPT=$(basename "$0") -while test $# -gt 0; do - case "$1" in - -h|--help) - echo "$SCRIPT - generates input yaml files for Cluster API on OpenStack" - echo " " - echo "$SCRIPT [options] [single-node/multi-node]" - echo " " - echo "options:" - echo "-h, --help show brief help" - echo "-f, --force-overwrite if file to be generated already exists, force script to overwrite it" - exit 0 - ;; - -f) - OVERWRITE=1 - shift - ;; - --force-overwrite) - OVERWRITE=1 - shift - ;; - *) - break - ;; - esac -done - -# Check if clouds.yaml file provided -if [[ -n "${1-}" ]] && [[ $1 != -* ]] && [[ $1 != --* ]];then - CLOUDS_PATH="$1" -else - echo "Error: No clouds.yaml provided" - echo "You must provide a valid clouds.yaml script to generate a cloud.conf" - echo "" - exit 1 -fi - -# Check if os cloud is provided -if [[ -n "${2-}" ]] && [[ $2 != -* ]] && [[ $2 != --* ]]; then - export CLOUD=$2 -else - echo "Error: No cloud specified" - echo "You must specify which cloud you want to use." - echo "" - exit 1 -fi - -# Check if output folder directory is provided -if [[ -n "${3-}" ]] && [[ $3 != -* ]] && [[ $3 != --* ]]; then - OUTPUT_DIR=$(echo $3 | tr '[:upper:]' '[:lower:]') -else - echo "Error: No output folder provided" - echo "You must specify the output folder." - echo "" - exit 1 -fi - -# Check if correct controlplane preference is given -if [[ -n "${4-}" ]] && [[ $4 != -* ]] && [[ $4 != --* ]]; then - if [[ "$4" == "single-node" || "$4" == "multi-node" ]]; then - export CONTROLPLANE=$4 - else - echo "\"${4}\" is not a valid keyword. Use \"single-node\" or \"multi-node\" instead." - echo "" - exit 1 - fi -else - echo "Controlplane preference not given, generating samples for \"single-node\" by default" -fi - -if [[ ${OVERWRITE} -ne 1 ]] && [[ -d "$OUTPUT_DIR" ]]; then - echo "ERR: Folder ${OUTPUT_DIR} already exists. Delete it manually before running this script." - exit 1 -fi - -yq_type=$(file "$(which yq)") -if [[ ${yq_type} == *"Python script"* ]]; then - echo "Wrong version of 'yq' installed, please install the one from https://github.com/mikefarah/yq" - echo "" - exit 1 -fi - -# Outputs. -COMPONENTS_OPENSTACK_GENERATED_FILE=${SOURCE_DIR}/provider-components/provider-components-openstack.yaml -COMPONENTS_OPENSTACK_CLOUDS_SECRETS_GENERATED_FILE=${SOURCE_DIR}/provider-components/provider-components-openstack-clouds-secrets.yaml -CLOUDS_SECRETS_CONFIG_DIR=${SOURCE_DIR}/clouds-secrets/configs -MACHINE_CONTROLLER_SSH_PRIVATE_FILE=${HOME}/.ssh/openstack_tmp - -PROVIDER_COMPONENTS_GENERATED_FILE=${OUTPUT_DIR}/provider-components.yaml -CLUSTER_GENERATED_FILE=${OUTPUT_DIR}/cluster.yaml -CONTROLPLANE_GENERATED_FILE=${OUTPUT_DIR}/controlplane.yaml -MACHINEDEPLOYMENT_GENERATED_FILE=${OUTPUT_DIR}/machinedeployment.yaml - -rm -rf "${OUTPUT_DIR}" -rm -rf "${CLOUDS_SECRETS_CONFIG_DIR}" -mkdir -p "${OUTPUT_DIR}" -mkdir -p "${CLOUDS_SECRETS_CONFIG_DIR}" - -# Check if the ssh key already exists. If not, generate and copy to the .ssh dir. -if [[ ! -f ${MACHINE_CONTROLLER_SSH_PRIVATE_FILE} ]]; then - echo "Generating SSH key files for machine controller." - # This is needed because GetKubeConfig assumes the key in the home .ssh dir. - ssh-keygen -t rsa -f ${MACHINE_CONTROLLER_SSH_PRIVATE_FILE} -N "" -fi -export MACHINE_CONTROLLER_SSH_PUBLIC_FILE_CONTENT -MACHINE_CONTROLLER_SSH_PUBLIC_FILE_CONTENT="\"$(cat ${MACHINE_CONTROLLER_SSH_PRIVATE_FILE}.pub)\"" - -CLOUDS_PATH=${CLOUDS_PATH:-""} -OPENSTACK_CLOUD_CONFIG_PLAIN=$(cat "$CLOUDS_PATH") - -# Just blindly parse the cloud.conf here, overwriting old vars. -AUTH_URL=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.auth.auth_url) -USERNAME=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.auth.username) -PASSWORD=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.auth.password) -REGION=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.region_name) -PROJECT_ID=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.auth.project_id) -DOMAIN_NAME=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.auth.user_domain_name) -if [[ "$DOMAIN_NAME" = "null" ]]; then - DOMAIN_NAME=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.auth.domain_name) -fi -DOMAIN_ID=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.auth.user_domain_id) -if [[ "$DOMAIN_ID" = "null" ]]; then - DOMAIN_ID=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.auth.domain_id) -fi -CACERT_ORIGINAL=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.${CLOUD}.cacert) - -# use only the selected cloud not the whole clouds.yaml -OPENSTACK_CLOUD_CONFIG_SELECTED_CLOUD=$(echo "${OPENSTACK_CLOUD_CONFIG_PLAIN}" | yq r - clouds.${CLOUD} | yq p - clouds.${CLOUD}) - -# Basic cloud.conf, no LB configuration as that data is not known yet. -OPENSTACK_CLOUD_PROVIDER_CONF=" -[Global] -auth-url=$AUTH_URL -username=\"$USERNAME\" -password=\"$PASSWORD\" -tenant-id=\"$PROJECT_ID\" -" - -if [[ "$DOMAIN_NAME" != "null" ]]; then - OPENSTACK_CLOUD_PROVIDER_CONF="$OPENSTACK_CLOUD_PROVIDER_CONF -domain-name=\"${DOMAIN_NAME}\" - " -fi -if [[ "$DOMAIN_ID" != "null" ]]; then - OPENSTACK_CLOUD_PROVIDER_CONF="$OPENSTACK_CLOUD_PROVIDER_CONF -domain-id=\"${DOMAIN_ID}\" - " -fi - -if [[ "$CACERT_ORIGINAL" != "null" ]]; then - OPENSTACK_CLOUD_PROVIDER_CONF="$OPENSTACK_CLOUD_PROVIDER_CONF -ca-file=\"${CACERT_ORIGINAL}\" - " -fi -if [[ "$REGION" != "null" ]]; then - OPENSTACK_CLOUD_PROVIDER_CONF="$OPENSTACK_CLOUD_PROVIDER_CONF -region=\"${REGION}\" - " -fi - -export OPENSTACK_CLOUD_PROVIDER_CONF8 -OPENSTACK_CLOUD_PROVIDER_CONF8=$(echo "${OPENSTACK_CLOUD_PROVIDER_CONF}" | sed -e 's/^/ /') -export OPENSTACK_CLOUD_PROVIDER_CONF10 -OPENSTACK_CLOUD_PROVIDER_CONF10=$(echo "${OPENSTACK_CLOUD_PROVIDER_CONF}" | sed -e 's/^/ /') - -if [[ "$CACERT_ORIGINAL" != "null" ]]; then - export OPENSTACK_CLOUD_CACERT_CONFIG6 - OPENSTACK_CLOUD_CACERT_CONFIG6="$(cat "$CACERT_ORIGINAL" | head -1) -$(cat "$CACERT_ORIGINAL" | tail -n +2 | sed -e 's/^/ /')" - export OPENSTACK_CLOUD_CACERT_CONFIG10 - OPENSTACK_CLOUD_CACERT_CONFIG10="$(cat "$CACERT_ORIGINAL" | head -1) -$(cat "$CACERT_ORIGINAL" | tail -n +2 | sed -e 's/^/ /')" -fi - -echo "${OPENSTACK_CLOUD_CONFIG_SELECTED_CLOUD}" > ${CLOUDS_SECRETS_CONFIG_DIR}/clouds.yaml -if [[ "$CACERT_ORIGINAL" != "null" ]]; then - cat "$CACERT_ORIGINAL" > ${CLOUDS_SECRETS_CONFIG_DIR}/cacert -else - echo "dummy" > ${CLOUDS_SECRETS_CONFIG_DIR}/cacert -fi - -# Generate cluster resources. -kustomize build "${SOURCE_DIR}/cluster/${CONTROLPLANE}" | envsubst > "${CLUSTER_GENERATED_FILE}" -echo "Generated ${CLUSTER_GENERATED_FILE}" - -# Generate controlplane resources. -kustomize build "${SOURCE_DIR}/controlplane/${CONTROLPLANE}" | envsubst > "${CONTROLPLANE_GENERATED_FILE}" -echo "Generated ${CONTROLPLANE_GENERATED_FILE}" - -# Generate machinedeployment resources. -kustomize build "${SOURCE_DIR}/machinedeployment" | envsubst >> "${MACHINEDEPLOYMENT_GENERATED_FILE}" -echo "Generated ${MACHINEDEPLOYMENT_GENERATED_FILE}" - -# Generate OpenStack Infrastructure Provider components file. -kustomize build "${SOURCE_DIR}/../config" | envsubst > "${COMPONENTS_OPENSTACK_GENERATED_FILE}" -echo "Generated ${COMPONENTS_OPENSTACK_GENERATED_FILE}" - -# Generate OpenStack Infrastructure Provider cloud-secrets file. -kustomize build "${SOURCE_DIR}/clouds-secrets" | envsubst > "${COMPONENTS_OPENSTACK_CLOUDS_SECRETS_GENERATED_FILE}" -echo "Generated ${COMPONENTS_OPENSTACK_CLOUDS_SECRETS_GENERATED_FILE}" -echo "WARNING: ${COMPONENTS_OPENSTACK_CLOUDS_SECRETS_GENERATED_FILE} includes OpenStack credentials" - -# Generate a single provider components file. -kustomize build "${SOURCE_DIR}/provider-components" | envsubst > "${PROVIDER_COMPONENTS_GENERATED_FILE}" -echo "Generated ${PROVIDER_COMPONENTS_GENERATED_FILE}" -echo "WARNING: ${PROVIDER_COMPONENTS_GENERATED_FILE} includes OpenStack credentials" diff --git a/examples/machinedeployment/kustomization.yaml b/examples/machinedeployment/kustomization.yaml deleted file mode 100644 index cc68c00a09..0000000000 --- a/examples/machinedeployment/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: ${CLUSTER_NAME} -resources: -- machinedeployment.yaml -configurations: -- kustomizeconfig.yaml diff --git a/examples/machinedeployment/kustomizeconfig.yaml b/examples/machinedeployment/kustomizeconfig.yaml deleted file mode 100644 index e0e4091702..0000000000 --- a/examples/machinedeployment/kustomizeconfig.yaml +++ /dev/null @@ -1,11 +0,0 @@ -namespace: -- kind: MachineDeployment - group: cluster.x-k8s.io - version: v1alpha3 - path: spec/template/spec/infrastructureRef/namespace - create: true -- kind: MachineDeployment - group: cluster.x-k8s.io - version: v1alpha3 - path: spec/template/spec/bootstrap/configRef/namespace - create: true diff --git a/examples/machinedeployment/machinedeployment.yaml b/examples/machinedeployment/machinedeployment.yaml deleted file mode 100644 index 0f04aa6798..0000000000 --- a/examples/machinedeployment/machinedeployment.yaml +++ /dev/null @@ -1,87 +0,0 @@ -##################################################### -# ${CLUSTER_NAME}-md-0 -##################################################### -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${CLUSTER_NAME} - labels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - nodepool: nodepool-0 -spec: - clusterName: "${CLUSTER_NAME}" - replicas: 1 - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - nodepool: nodepool-0 - template: - metadata: - labels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - nodepool: nodepool-0 - spec: - clusterName: "${CLUSTER_NAME}" - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - namespace: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: OpenStackMachineTemplate - name: ${CLUSTER_NAME}-md-0 - namespace: ${CLUSTER_NAME} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: OpenStackMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${CLUSTER_NAME} -spec: - template: - spec: - availabilityZone: ${AVAILABILITY_ZONE} - cloudName: $CLOUD - cloudsSecret: - name: cloud-config - namespace: ${CLUSTER_NAME} - flavor: m1.medium - image: ${OS_IMAGE_NAME} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${CLUSTER_NAME} -spec: - template: - spec: - files: - - content: |- - # cloud.conf to communicate with OpenStack - $OPENSTACK_CLOUD_PROVIDER_CONF10 - owner: root - path: /etc/kubernetes/cloud.conf - permissions: "0600" - - content: | - $OPENSTACK_CLOUD_CACERT_CONFIG10 - owner: root - path: /etc/certs/cacert - permissions: "0600" - joinConfiguration: - nodeRegistration: - name: '{{ local_hostname }}' - kubeletExtraArgs: - cloud-config: /etc/kubernetes/cloud.conf - cloud-provider: openstack - ntp: - servers: [] - users: - - name: capo - sudo: "ALL=(ALL) NOPASSWD:ALL" - sshAuthorizedKeys: - - "$MACHINE_CONTROLLER_SSH_PUBLIC_FILE_CONTENT" diff --git a/examples/provider-components/.gitignore b/examples/provider-components/.gitignore deleted file mode 100644 index 038f4f2176..0000000000 --- a/examples/provider-components/.gitignore +++ /dev/null @@ -1 +0,0 @@ -provider-components-* \ No newline at end of file diff --git a/examples/provider-components/kustomization.yaml b/examples/provider-components/kustomization.yaml deleted file mode 100644 index de12e89975..0000000000 --- a/examples/provider-components/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: -- namespace.yaml -- provider-components-openstack-clouds-secrets.yaml -- provider-components-openstack.yaml diff --git a/examples/provider-components/namespace.yaml b/examples/provider-components/namespace.yaml deleted file mode 100644 index 51a42b59af..0000000000 --- a/examples/provider-components/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - controller-tools.k8s.io: "1.0" - name: ${CLUSTER_NAME} diff --git a/go.mod b/go.mod index 211a46b332..05b1caa5a3 100644 --- a/go.mod +++ b/go.mod @@ -16,5 +16,5 @@ require ( k8s.io/klog v1.0.0 k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab sigs.k8s.io/cluster-api v0.3.0 - sigs.k8s.io/controller-runtime v0.5.0 + sigs.k8s.io/controller-runtime v0.5.1 ) diff --git a/go.sum b/go.sum index 2457549623..0a8ec81899 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,7 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -176,7 +177,9 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= @@ -215,6 +218,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -254,6 +258,7 @@ github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao= github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -271,6 +276,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY= github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -307,6 +313,7 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -357,11 +364,15 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs= github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -370,6 +381,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -379,6 +391,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -415,6 +428,7 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e h1:egKlR8l7Nu9vHGWbcUV8lqR4987UfUbBd7GbhqGzNYU= golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= @@ -483,6 +497,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -552,6 +567,7 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= @@ -609,6 +625,8 @@ sigs.k8s.io/cluster-api v0.3.0 h1:e0t4IJmefMEammok9DsXw29vPu8UAMe0k19zdLFiU2w= sigs.k8s.io/cluster-api v0.3.0/go.mod h1:+i6vt6ot1/UlIAZZ1sCtchkBIJw33m41mu3fclRWQXA= sigs.k8s.io/controller-runtime v0.5.0 h1:CbqIy5fbUX+4E9bpnBFd204YAzRYlM9SWW77BbrcDQo= sigs.k8s.io/controller-runtime v0.5.0/go.mod h1:REiJzC7Y00U+2YkMbT8wxgrsX5USpXKGhb2sCtAXiT8= +sigs.k8s.io/controller-runtime v0.5.1 h1:TNidCfVoU/cs2i+9xoTcL/l7yhl0bDhYXU0NCG6wmiE= +sigs.k8s.io/controller-runtime v0.5.1/go.mod h1:Uojny7gvg55YLQnEGnPzRE3dC4ik2tRlZJgOUCWXAV4= sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802/go.mod h1:HIZ3PWUezpklcjkqpFbnYOqaqsAE1JeCTEwkgvPLXjk= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= diff --git a/hack/boilerplate/boilerplate.Dockerfile.txt b/hack/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 0000000000..384f325abf --- /dev/null +++ b/hack/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,14 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate.Makefile.txt b/hack/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 0000000000..384f325abf --- /dev/null +++ b/hack/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,14 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate.go.txt b/hack/boilerplate/boilerplate.go.txt new file mode 100644 index 0000000000..4b76f1fdd8 --- /dev/null +++ b/hack/boilerplate/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright YEAR The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/hack/boilerplate/boilerplate.py.txt b/hack/boilerplate/boilerplate.py.txt new file mode 100644 index 0000000000..a2e72e5988 --- /dev/null +++ b/hack/boilerplate/boilerplate.py.txt @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate.sh.txt b/hack/boilerplate/boilerplate.sh.txt new file mode 100644 index 0000000000..384f325abf --- /dev/null +++ b/hack/boilerplate/boilerplate.sh.txt @@ -0,0 +1,14 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/ci/ci-bootstrap.yaml b/hack/ci/ci-bootstrap.yaml new file mode 100644 index 0000000000..f8052a898a --- /dev/null +++ b/hack/ci/ci-bootstrap.yaml @@ -0,0 +1,90 @@ +preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh +files: + - path: /tmp/kubeadm-bootstrap.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + GSUTIL=gsutil + if ! command -v ${GSUTIL} > /dev/null; then + curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1 + GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil + # For faster downloads + pip install --no-cache-dir -U crcmod + fi + ${GSUTIL} version + + # This test installs debian packages that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + + LINE_SEPARATOR="*************************************************" + echo "$LINE_SEPARATOR" + + CI_VERSION=${CI_VERSION:-""} + if [[ "${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $CI_DIR + + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + PACKAGE_EXT="deb" + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + + echo "* testing CI version $CI_VERSION" + + # Check for semver + if [[ "${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="gs://kubernetes-release/release/$CI_VERSION/bin/linux/amd64" + + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list + apt-get update + + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep ${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + + for CI_PACKAGE in "${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $CI_PACKAGE ${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $CI_PACKAGE=$PACKAGE_VERSION + done + else + CI_URL="gs://kubernetes-release-dev/ci/$CI_VERSION-bazel/bin/linux/amd64" + + for CI_PACKAGE in "${PACKAGES_TO_TEST[@]}"; do + echo "* downloading package: $CI_URL/$CI_PACKAGE.$PACKAGE_EXT" + ${GSUTIL} cp "$CI_URL/$CI_PACKAGE.$PACKAGE_EXT" "$CI_DIR/$CI_PACKAGE.$PACKAGE_EXT" + ${SUDO} dpkg -i "$CI_DIR/$CI_PACKAGE.$PACKAGE_EXT" || echo "* ignoring expected 'dpkg -i' result" + done + fi + + for CI_CONTAINER in "${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $CI_URL/$CI_CONTAINER.$CONTAINER_EXT" + ${GSUTIL} cp "$CI_URL/$CI_CONTAINER.$CONTAINER_EXT" "$CI_DIR/$CI_CONTAINER.$CONTAINER_EXT" + ${SUDO} ctr -n k8s.io images import "$CI_DIR/$CI_CONTAINER.$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + ${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$CI_CONTAINER-amd64:"${CI_VERSION//+/_}" k8s.gcr.io/$CI_CONTAINER:"${CI_VERSION//+/_}" + ${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$CI_CONTAINER-amd64:"${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$CI_CONTAINER:"${CI_VERSION//+/_}" + done + fi + + echo "* checking binary versions" + + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + + echo "$LINE_SEPARATOR" diff --git a/hack/ci/e2e-conformance.sh b/hack/ci/e2e-conformance.sh new file mode 100755 index 0000000000..a0f99ae49c --- /dev/null +++ b/hack/ci/e2e-conformance.sh @@ -0,0 +1,424 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# hack script for running a cluster-api-provider-openstack e2e + +set -o errexit -o nounset -o pipefail + +OPENSTACK_CLOUD_YAML_FILE=${OPENSTACK_CLOUD_YAML_FILE:-"/tmp/clouds.yaml"} +OPENSTACK_SSH_AUTHORIZED_KEY_PATH=${OPENSTACK_SSH_AUTHORIZED_KEY_PATH:-"/tmp/id_rsa.pub"} +OPENSTACK_SSH_PRIVATE_KEY_PATH=${OPENSTACK_SSH_PRIVATE_KEY_PATH:-"/tmp/id_rsa"} +OPENSTACK_IMAGE_NAME="ubuntu-1910-kube-v1.17.3" +CLUSTER_NAME=${CLUSTER_NAME:-"capi-quickstart"} +KUBERNETES_VERSION=${KUBERNETES_VERSION:-"v1.17.3"} +TIMESTAMP=$(date +"%Y-%m-%dT%H:%M:%SZ") + +ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" + +# dump logs from kind and all the nodes +dump-logs() { + + echo "Dump logs" + mkdir -p "${ARTIFACTS}/logs" + + # log version information + echo "=== versions ===" + echo "kind : $(kind version)" || true + echo "bootstrap cluster:" + kubectl version || true + echo "deployed cluster:" + kubectl --kubeconfig=${PWD}/kubeconfig version || true + echo "" + + # dump all the info from the CAPI related CRDs + kubectl get \ + clusters,openstackclusters,machines,openstackmachines,kubeadmconfigs,machinedeployments,openstackmachinetemplates,kubeadmconfigtemplates,machinesets \ + --all-namespaces -o yaml >> "${ARTIFACTS}/logs/capo.info" || true + + # dump images info + echo "images in docker" >> "${ARTIFACTS}/logs/images.info" + docker images >> "${ARTIFACTS}/logs/images.info" + echo "images from bootstrap using containerd CLI" >> "${ARTIFACTS}/logs/images.info" + docker exec clusterapi-control-plane ctr -n k8s.io images list >> "${ARTIFACTS}/logs/images.info" || true + echo "images in bootstrap cluster using kubectl CLI" >> "${ARTIFACTS}/logs/images.info" + (kubectl get pods --all-namespaces -o json \ + | jq --raw-output '.items[].spec.containers[].image' | sort) >> "${ARTIFACTS}/logs/images.info" || true + echo "images in deployed cluster using kubectl CLI" >> "${ARTIFACTS}/logs/images.info" + (kubectl --kubeconfig="${PWD}"/kubeconfig get pods --all-namespaces -o json \ + | jq --raw-output '.items[].spec.containers[].image' | sort) >> "${ARTIFACTS}/logs/images.info" || true + + # dump cluster info for kind + kubectl cluster-info dump > "${ARTIFACTS}/logs/kind-cluster.info" || true + + # export all logs from kind + kind "export" logs --name="clusterapi" "${ARTIFACTS}/logs" || true + + # dump OpenStack info + echo "" > "${ARTIFACTS}/logs/openstack-cluster.info" + echo "=== OpenStack compute instances list ===" >> "${ARTIFACTS}/logs/openstack-cluster.info" || true + openstack server list >> "${ARTIFACTS}/logs/openstack-cluster.info" || true + echo "=== OpenStack compute instances show ===" >> "${ARTIFACTS}/logs/openstack-cluster.info" || true + openstack server list -f value -c Name | xargs -I% openstack server show % >> "${ARTIFACTS}/logs/openstack-cluster.info" || true + echo "=== cluster-info dump ===" >> "${ARTIFACTS}/logs/openstack-cluster.info" || true + kubectl --kubeconfig=${PWD}/kubeconfig cluster-info dump >> "${ARTIFACTS}/logs/openstack-cluster.info" || true + + set -x + jump_node_name=$(openstack server list -f value -c Name | grep ${CLUSTER_NAME}-control-plane | head -n 1) + jump_node=$(openstack server show ${jump_node_name} -f value -c addresses | awk '{print $2}') + for node in $(openstack server list -f value -c Name) + do + echo "collecting logs from ${node} using jump host " + dir="${ARTIFACTS}/logs/${node}" + mkdir -p ${dir} + + openstack console log show "${node}" > "${dir}/console.log" || true + + ssh-to-node "${node}" "${jump_node}" "sudo chmod -R a+r /var/log" || true + + PROXY_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=30 -x -W %h:22 -i ${OPENSTACK_SSH_PRIVATE_KEY_PATH} capo@${jump_node}" + node=$(openstack port show ${node} -f json -c fixed_ips | jq '.fixed_ips[0].ip_address' -r) + scp -r -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=30 -o ProxyCommand="${PROXY_COMMAND}" -i ${OPENSTACK_SSH_PRIVATE_KEY_PATH} \ + "capo@${node}:/var/log/cloud-init.log" "capo@${node}:/var/log/cloud-init-output.log" \ + "capo@${node}:/var/log/pods" "capo@${node}:/var/log/containers" \ + "${dir}" || true + + ssh-to-node "${node}" "${jump_node}" "sudo journalctl --output=short-precise -k" > "${dir}/kern.log" || true + ssh-to-node "${node}" "${jump_node}" "sudo journalctl --output=short-precise" > "${dir}/systemd.log" || true + ssh-to-node "${node}" "${jump_node}" "sudo crictl version && sudo crictl info" > "${dir}/containerd.info" || true + ssh-to-node "${node}" "${jump_node}" "sudo journalctl --no-pager -u cloud-final" > "${dir}/cloud-final.log" || true + ssh-to-node "${node}" "${jump_node}" "sudo journalctl --no-pager -u kubelet.service" > "${dir}/kubelet.log" || true + ssh-to-node "${node}" "${jump_node}" "sudo journalctl --no-pager -u containerd.service" > "${dir}/containerd.log" || true + ssh-to-node "${node}" "${jump_node}" "sudo top -b -n 1" > "${dir}/top.txt" || true + ssh-to-node "${node}" "${jump_node}" "sudo crictl ps" > "${dir}/crictl-ps.log" || true + ssh-to-node "${node}" "${jump_node}" "sudo crictl pods" > "${dir}/crictl-pods.log" || true + done + set +x +} + +# SSH to a node by name ($1) via jump server ($2) and run a command ($3). +function ssh-to-node() { + local node="$1" + local jump="$2" + local cmd="$3" + + ssh_params="-o LogLevel=quiet -o ConnectTimeout=30 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + scp $ssh_params -i "${OPENSTACK_SSH_PRIVATE_KEY_PATH}" "${OPENSTACK_SSH_PRIVATE_KEY_PATH}" "capo@${jump}:${OPENSTACK_SSH_PRIVATE_KEY_PATH}" + ssh $ssh_params -i "${OPENSTACK_SSH_PRIVATE_KEY_PATH}" \ + -o "ProxyCommand ssh $ssh_params -W %h:%p -i ${OPENSTACK_SSH_PRIVATE_KEY_PATH} capo@${jump}" \ + capo@"${node}" "${cmd}" +} + +# cleanup all resources we use +cleanup() { + # KIND_IS_UP is true once we: kind create + if [[ "${KIND_IS_UP:-}" = true ]]; then + timeout 600 kubectl \ + delete cluster "${CLUSTER_NAME}" || true + timeout 600 kubectl \ + wait --for=delete -n ${CLUSTER_NAME} cluster/"${CLUSTER_NAME}" || true + make kind-reset || true + fi + # clean up e2e.test symlink + (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && rm -f _output/bin/e2e.test) || true +} + +# our exit handler (trap) +exit-handler() { + dump-logs + cleanup +} + +upload_image() { + echo "Upload image" + + # Remove old image if we don't want to reuse it + if [[ "${REUSE_OLD_IMAGES:-true}" == "false" ]]; then + image_id=$(openstack image list --name=${OPENSTACK_IMAGE_NAME} -f value -c ID) + if [[ ! -z "$image_id" ]]; then + echo "Deleting old image ${OPENSTACK_IMAGE_NAME} with id: ${image_id}" + openstack image delete ${image_id} + fi + fi + + image=$(openstack image list --name=${OPENSTACK_IMAGE_NAME} -f value -c Name) + if [[ ! -z "$image" ]]; then + echo "Image ${OPENSTACK_IMAGE_NAME} already exists" + return + fi + + tmp_source_image=/tmp/ubuntu-1910.ova.qcow2 + wget -q https://github.com/sbueringer/image-builder/releases/download/v1.17.3-04/ubuntu-1910-kube-v1.17.3.qcow2 -O ${tmp_source_image} + + echo "Uploading image ${tmp_source_image} as ${OPENSTACK_IMAGE_NAME}" + openstack image create --disk-format qcow2 \ + --private \ + --container-format bare \ + --file "${tmp_source_image}" ${OPENSTACK_IMAGE_NAME} +} + +install_prereqs() { + # Install Docker + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo apt-key fingerprint 0EBFCD88 + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io jq + # docker socket already works because OpenLab runs via root + + # Install go + sudo snap install go --classic + + # Install yq + GO111MODULE=on go get github.com/mikefarah/yq/v2 + + source "${REPO_ROOT}/hack/ensure-go.sh" + source "${REPO_ROOT}/hack/ensure-kubectl.sh" + source "${REPO_ROOT}/hack/ensure-kind.sh" + source "${REPO_ROOT}/hack/ensure-kustomize.sh" + + # Kubernetes does not build with modern bazel + #sudo apt install curl + #curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - + #echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list + #sudo apt update && sudo apt install bazel + wget -q https://github.com/bazelbuild/bazel/releases/download/0.28.1/bazel-0.28.1-installer-linux-x86_64.sh + chmod +x bazel-0.28.1-installer-linux-x86_64.sh + ./bazel-0.28.1-installer-linux-x86_64.sh --user + + # Bazel is installed in /root/bin + export PATH="/root/bin:${PATH}" +} + +# build kubernetes / node image, e2e binaries +build() { + # possibly enable bazel build caching before building kubernetes + if [[ "${BAZEL_REMOTE_CACHE_ENABLED:-false}" == "true" ]]; then + create_bazel_cache_rcs.sh || true + fi + + if [[ ! -d "$(go env GOPATH)/src/k8s.io/kubernetes" ]]; then + mkdir -p $(go env GOPATH)/src/k8s.io + cd $(go env GOPATH)/src/k8s.io + + git clone https://github.com/kubernetes/kubernetes.git + cd kubernetes + git checkout release-1.17 + fi + + pushd "$(go env GOPATH)/src/k8s.io/kubernetes" + + # make sure we have e2e requirements + bazel build //cmd/kubectl //vendor/github.com/onsi/ginkgo/ginkgo + + # ensure the e2e script will find our binaries ... + mkdir -p "${PWD}/_output/bin/" + rm -f "${PWD}/_output/bin/e2e.test" + if bazel --version | grep "0.28.1" + then + bazel build //test/e2e:e2e.test + cp "${PWD}/bazel-bin/test/e2e/e2e.test" "${PWD}/_output/bin/e2e.test" + else + go test -c ./test/e2e/ + cp "./e2e.test" "${PWD}/_output/bin/e2e.test" + fi + PATH="$(dirname "$(find "${PWD}/bazel-bin/" -name kubectl -type f)"):${PATH}" + export PATH + + # attempt to release some memory after building + sync || true + sudo sh -c "echo 1 > /proc/sys/vm/drop_caches" || true + + popd +} + +# generate manifests needed for creating the GCP cluster to run the tests +generate_manifests() { + cd ${REPO_ROOT} + if ! command -v kustomize >/dev/null 2>&1; then + (cd ./hack/tools/ && GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v3) + fi + + echo "Build Docker Images & Generating Manifests" + + # Enable the bits to inject a script that can pull newer versions of kubernetes + if [[ -n ${CI_VERSION:-} || -n ${USE_CI_ARTIFACTS:-} ]]; then + if ! grep -i -wq "patchesStrategicMerge" "templates/kustomization.yaml"; then + echo "patchesStrategicMerge:" >> "templates/kustomization.yaml" + echo "- kustomizeversions.yaml" >> "templates/kustomization.yaml" + fi + fi + + make modules docker-build +} + +# up a cluster with kind +create_cluster() { + # actually create the cluster + KIND_IS_UP=true + + if [[ ! -f ${OPENSTACK_SSH_AUTHORIZED_KEY_PATH} ]] + then + ssh-keygen -t rsa -f ${OPENSTACK_SSH_PRIVATE_KEY_PATH} -N "" + chmod 0400 ${OPENSTACK_SSH_AUTHORIZED_KEY_PATH} + fi + + # exports the b64 env vars used below + source ${REPO_ROOT}/templates/env.rc ${OPENSTACK_CLOUD_YAML_FILE} ${CLUSTER_NAME} + + OPENSTACK_CLOUD_CACERT_B64=${OPENSTACK_CLOUD_CACERT_B64} \ + OPENSTACK_CLOUD_PROVIDER_CONF_B64=${OPENSTACK_CLOUD_PROVIDER_CONF_B64} \ + OPENSTACK_CLOUD_YAML_B64=${OPENSTACK_CLOUD_YAML_B64} \ + OPENSTACK_EXTERNAL_NETWORK_ID=$(openstack network show "public" -f value -c id) \ + OPENSTACK_IMAGE_NAME=${OPENSTACK_IMAGE_NAME} \ + OPENSTACK_SSH_AUTHORIZED_KEY="$(cat ${OPENSTACK_SSH_AUTHORIZED_KEY_PATH})" \ + KUBERNETES_VERSION=${KUBERNETES_VERSION} \ + make create-cluster + + # Wait till all machines are running (bail out at 30 mins) + attempt=0 + while true; do + kubectl get machines -n "${CLUSTER_NAME}" + read running total <<< $(kubectl get machines -n "${CLUSTER_NAME}" \ + -o json | jq -r '.items[].status.phase' | awk 'BEGIN{count=0} /(r|R)unning/{count++} END{print count " " NR}') ; + if [[ ${total} == ${running} ]]; then + return 0 + fi + read failed total <<< $(kubectl get machines -n "${CLUSTER_NAME}" \ + -o json | jq -r '.items[].status.phase' | awk 'BEGIN{count=0} /(f|F)ailed/{count++} END{print count " " NR}') ; + if [[ ! ${failed} -eq 0 ]]; then + echo "$failed machines (out of $total) in cluster failed ... bailing out" + exit 1 + fi + timestamp=$(date +"[%H:%M:%S]") + if [[ ${attempt} -gt 180 ]]; then + echo "cluster did not start in 30 mins ... bailing out!" + exit 1 + fi + echo "$timestamp Total machines : $total / Running : $running .. waiting for 10 seconds" + sleep 10 + attempt=$((attempt+1)) + done +} + +# run e2es with kubetest +run_tests() { + # export the KUBECONFIG + KUBECONFIG="${PWD}/kubeconfig" + export KUBECONFIG + + # ginkgo regexes + SKIP="${SKIP:-}" + FOCUS="${FOCUS:-"\\[Conformance\\]"}" + # if we set PARALLEL=true, skip serial tests set --ginkgo-parallel + if [[ "${PARALLEL:-false}" == "true" ]]; then + export GINKGO_PARALLEL=y + echo "Running tests in parallel" + if [[ -z "${SKIP}" ]]; then + SKIP="\\[Serial\\]" + else + SKIP="\\[Serial\\]|${SKIP}" + fi + fi + + # get the number of worker nodes + # TODO(bentheelder): this is kinda gross + NUM_NODES="$(kubectl get nodes --kubeconfig="$KUBECONFIG" \ + -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.taints}{"\n"}{end}' \ + | grep -cv "node-role.kubernetes.io/master" )" + + # wait for all the nodes to be ready + kubectl wait --for=condition=Ready node --kubeconfig="$KUBECONFIG" --all || true + + # setting this env prevents ginkgo e2e from trying to run provider setup + export KUBERNETES_CONFORMANCE_TEST="y" + # run the tests + (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && ./hack/ginkgo-e2e.sh \ + '--provider=skeleton' "--num-nodes=${NUM_NODES}" \ + "--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \ + "--report-dir=${ARTIFACTS}" '--disable-log-dump=true') + + unset KUBECONFIG + unset KUBERNETES_CONFORMANCE_TEST +} + +# setup kind, build kubernetes, create a cluster, run the e2es +main() { + for arg in "$@" + do + if [[ "$arg" == "--verbose" ]]; then + set -o xtrace + fi + if [[ "$arg" == "--clean" ]]; then + cleanup + return 0 + fi + if [[ "$arg" == "--install-prereqs" ]]; then + INSTALL_PREREQS="1" + fi + if [[ "$arg" == "--skip-cleanup" ]]; then + SKIP_CLEANUP="1" + fi + if [[ "$arg" == "--use-ci-artifacts" ]]; then + USE_CI_ARTIFACTS="1" + fi + if [[ "$arg" == "--skip-upload-image" ]]; then + SKIP_UPLOAD_IMAGE="1" + fi + if [[ "$arg" == "--skip-run-tests" ]]; then + SKIP_RUN_TESTS="1" + fi + if [[ "$arg" == "--run-tests-parallel" ]]; then + export PARALLEL="true" + fi + done + + # create temp dir and setup cleanup + SKIP_CLEANUP=${SKIP_CLEANUP:-""} + if [[ -z "${SKIP_CLEANUP}" ]]; then + trap exit-handler EXIT + fi + # ensure artifacts exists when not in CI + export ARTIFACTS + mkdir -p "${ARTIFACTS}/logs" + + source "${REPO_ROOT}/hack/ensure-go.sh" + source "${REPO_ROOT}/hack/ensure-kind.sh" + + export GOPATH=${GOPATH:-/home/ubuntu/go} + export PATH=$PATH:${GOPATH}/bin:/snap/bin:${HOME}/bin + + INSTALL_PREREQS=${INSTALL_PREREQS:-""} + if [[ "${INSTALL_PREREQS}" == "yes" || "${INSTALL_PREREQS}" == "1" ]]; then + echo "Install prereqs..." + install_prereqs + fi + if [[ -z "${SKIP_UPLOAD_IMAGE:-}" ]]; then + echo "Uploading image..." + upload_image + fi + + build + generate_manifests + create_cluster + + if [[ -z "${SKIP_RUN_TESTS:-}" ]]; then + echo "Running tests..." + run_tests + fi +} + +main "$@" diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 7e17e23d44..71f93b4f43 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -4,19 +4,15 @@ cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.2 h1:6AWuh3uWrsZJcNoCHrCF/+g4aKPCU39kaMO6/qrnK/4= -github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0 h1:CxTzQrySOxDnKpLjFJeZAS5Qrv/qFPkgLjx5bOAi//I= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= @@ -24,6 +20,7 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -51,10 +48,12 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs= github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM= github.com/bombsimon/wsl/v2 v2.0.0 h1:+Vjcn+/T5lSrO8Bjzhk4v14Un/2UyCA1E3V5j9nwTkQ= github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -84,24 +83,20 @@ github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfm github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= -github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 h1:pEtiCjIXx3RvGjlUJuCNxNOw0MNblyR9Wi+vJGBFh+8= -github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= -github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= @@ -114,10 +109,10 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= -github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-critic/go-critic v0.4.1 h1:4DTQfT1wWwLg/hzxwD9bkdhDQrdJtxe6DUTadPlrIeE= github.com/go-critic/go-critic v0.4.1/go.mod h1:7/14rZGnZbY6E38VEGk2kVhoq6itzc1E68facVDK23g= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= @@ -143,7 +138,6 @@ github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= @@ -159,9 +153,11 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo= github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw= github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= @@ -213,9 +209,8 @@ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= -github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= @@ -239,17 +234,13 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps= github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk= -github.com/golangci/golangci-lint v1.23.3 h1:wkACDEoy+b0CVqnSK8BbWrVkN2tsVLUA1+SIkGSm4o0= -github.com/golangci/golangci-lint v1.23.3/go.mod h1:LNexeEyqT5hQH7v47e67JekL0V51lXFUjbPkopxNSK4= +github.com/golangci/golangci-lint v1.23.8 h1:NlD+Ld2TKH8qVmADy4iEvPxVmXaqPIeQu3d1cGQP4jc= github.com/golangci/golangci-lint v1.23.8/go.mod h1:g/38bxfhp4rI7zeWSxcdIeHTQGS58TCak8FYcyCmavQ= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= @@ -271,50 +262,42 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= -github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gophercloud/gophercloud v0.6.0 h1:Xb2lcqZtml1XjgYZxbeayEemq7ASbeTp09m36gQFpEU= -github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= -github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -364,6 +347,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= @@ -379,7 +363,6 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -393,6 +376,7 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= @@ -400,6 +384,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -410,7 +395,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -420,6 +404,7 @@ github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1 github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -428,6 +413,7 @@ github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -449,7 +435,9 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete/v2 v2.0.1-alpha.12 h1:0wvkuDfHb5vSZlNBYgpEH4XQHpF46MjLPHav8XC77Nc= github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRqBXoBUecJu3ILQPgbJiMQ= +github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w= github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -468,17 +456,15 @@ github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1: github.com/raviqqe/liche v0.0.0-20200229003944-f57a5d1c5be4/go.mod h1:MPBuzBAJcp9B/3xrqfgR+ieBgpMzDqTeieaRP3ESJhk= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk= -github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE= github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do= github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83 h1:AtnWoOvTioyDXFvu96MWEeE8qj4COSQnJogzLy/u41A= github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= @@ -534,7 +520,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -545,7 +530,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -560,6 +544,7 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -582,7 +567,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -618,7 +602,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= @@ -652,7 +635,6 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24 h1:R8bzl0244nw47n1xKs1MUMAaTNgjavKcN/aX2Ss3+Fo= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -666,9 +648,8 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -696,8 +677,6 @@ golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -779,6 +758,7 @@ k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= k8s.io/cli-runtime v0.0.0-20191214191754-e6dc6d5c8724/go.mod h1:wzlq80lvjgHW9if6MlE4OIGC86MDKsy5jtl9nxz/IYY= +k8s.io/cli-runtime v0.17.0 h1:XEuStbJBHCQlEKFyTQmceDKEWOSYHZkcYWKp3SsQ9Hk= k8s.io/cli-runtime v0.17.0/go.mod h1:1E5iQpMODZq2lMWLUJELwRu2MLWIzwvMgDBpn3Y81Qo= k8s.io/client-go v0.0.0-20191214190045-a32a6f7a3052/go.mod h1:tAaoc/sYuIL0+njJefSAmE28CIcxyaFV4kbIujBlY2s= k8s.io/client-go v0.0.0-20191219150334-0b8da7416048/go.mod h1:ZEe8ZASDUAuqVGJ+UN0ka0PfaR+b6a6E1PGsSNZRui8= @@ -805,9 +785,9 @@ k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLy k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd h1:nZX5+wEqTu/EBIYjrZlFOA63z4+Zcy96lDkCZPU9a9c= k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd/go.mod h1:9ehGcuUGjXVZh0qbYSB0vvofQw2JQe6c6cO0k4wu/Oo= k8s.io/metrics v0.0.0-20191214191643-6b1944c9f765/go.mod h1:5V7rewilItwK0cz4nomU0b3XCcees2Ka5EBYWS1HBeM= -k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= @@ -821,33 +801,28 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphD mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= -sigs.k8s.io/cluster-api/hack/tools v0.0.0-20200305133038-7711f9528dc2 h1:uY/92OlJvq6of92F5gg09v51mc0Bj2E16j0xy/pFxv8= -sigs.k8s.io/cluster-api/hack/tools v0.0.0-20200305133038-7711f9528dc2/go.mod h1:WF3aMZ1CbnmwMuLqGVkwskKKmT35v6AdDdszViYrD7k= +sigs.k8s.io/cluster-api/hack/tools v0.0.0-20200311152736-bfcc6f37fe12 h1:oRsyG+v4hme6VxjqeRpFPJ6y7glydtwNsI2/xH0cIUk= sigs.k8s.io/cluster-api/hack/tools v0.0.0-20200311152736-bfcc6f37fe12/go.mod h1:8aibHfbqxNthBN4KKnk8UV9xTHKB3TE5nzxeNkH9xrU= -sigs.k8s.io/controller-tools v0.2.6-0.20200226180227-d6efdcdd90e2 h1:emZ+AFmD8mTciBRbEj2Qfk/RDI8uApvizRrFpmHf7IQ= -sigs.k8s.io/controller-tools v0.2.6-0.20200226180227-d6efdcdd90e2/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/controller-tools v0.2.6 h1:4Fo36alFw5+Fffz7HhNd2EFBFvckqVkb4ePnIj0uLUc= sigs.k8s.io/controller-tools v0.2.6/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= -sigs.k8s.io/kubebuilder/docs/book/utils v0.0.0-20190903174343-de03361a00cb/go.mod h1:zSW4yXo5uN/rRueruOtX8TWM8wBPJippG9OAOygomc8= sigs.k8s.io/kubebuilder/docs/book/utils v0.0.0-20200226075303-ed8438ec10a4/go.mod h1:nyAxPBUS04gN3IRuEQ0elG4mVeto/d/qQRsW2PsyAy4= +sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/kustomize/api v0.2.0 h1:e++6JpysnnlUbHmFrv6jvfF5rFlgQ103bS1DO7r5bWA= -sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4= +sigs.k8s.io/kustomize/api v0.3.2 h1:64gvYVAvqe2fNfcTevtXh/GmLwVwHIcJ2Z5HBMfjncs= sigs.k8s.io/kustomize/api v0.3.2/go.mod h1:A+ATnlHqzictQfQC1q3KB/T6MSr0UWQsrrLxMWkge2E= +sigs.k8s.io/kustomize/cmd/config v0.0.5 h1:mFJowsk9IGvwm5dUpVB+ZM63on2JjgaCy+YcVsFaVxU= sigs.k8s.io/kustomize/cmd/config v0.0.5/go.mod h1:L47nDnZDfGFQG3gnPJLG2UABn0nVb9v+ndceyMH0jjU= +sigs.k8s.io/kustomize/cmd/kubectl v0.0.3 h1:cXn6GqRnOQtp4EC1+NiJKdUHE/aQ+5HhtAB28R4sVXA= sigs.k8s.io/kustomize/cmd/kubectl v0.0.3/go.mod h1:JnS9HnTjUUMOE44WNboy/wi89J/K/XbAoU7O/iPXqqE= -sigs.k8s.io/kustomize/kustomize/v3 v3.4.0 h1:K42+RIA5jYI/6mgP3yWt50s/2g5UNR3gXTm+cTSQ1GE= -sigs.k8s.io/kustomize/kustomize/v3 v3.4.0/go.mod h1:i7bGnbXAlZKomwwc0uOLETK3bIXnoWXrsxhHTF6UJNU= +sigs.k8s.io/kustomize/kustomize/v3 v3.5.4 h1:hoxkY1BhjghNjy3BEu/kCx4BI/LutAXs2hJcsuM9mFA= sigs.k8s.io/kustomize/kustomize/v3 v3.5.4/go.mod h1:tr4IIKWojBx6vFr6TUDoMMREcwrth1sV8BQ8VhlXxnI= sigs.k8s.io/kustomize/kyaml v0.0.5/go.mod h1:waxTrzQRK9i6/5fR5HNo8xa4YwvWn8t85vMnOGFEZik= +sigs.k8s.io/kustomize/kyaml v0.0.6 h1:KhQr7JwpCseFTSWCwqp4CJ4mY6Kx+i34tF4e0eNkcXw= sigs.k8s.io/kustomize/kyaml v0.0.6/go.mod h1:tDOfJjL6slQVBLHJ76XfXAFgAOEdfm04AW2HehYOp8k= -sigs.k8s.io/kustomize/pseudo/k8s v0.1.0 h1:otg4dLFc03c3gzl+2CV8GPGcd1kk8wjXwD+UhhcCn5I= -sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/testing_frameworks v0.1.1 h1:cP2l8fkA3O9vekpy5Ks8mmA0NW/F7yBdXf8brkWhVrs= -sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= +sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM= sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh index 13765c871c..fc3d3ab7eb 100755 --- a/hack/verify-boilerplate.sh +++ b/hack/verify-boilerplate.sh @@ -17,7 +17,7 @@ set -o errexit set -o nounset set -o pipefail -#set -o verbose +set -o verbose KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. diff --git a/pkg/cloud/services/compute/availabilityzone.go b/pkg/cloud/services/compute/availabilityzone.go new file mode 100644 index 0000000000..16f8c29813 --- /dev/null +++ b/pkg/cloud/services/compute/availabilityzone.go @@ -0,0 +1,36 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compute + +import ( + "fmt" + + "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones" +) + +func (s *Service) GetAvailabilityZones() ([]availabilityzones.AvailabilityZone, error) { + allPages, err := availabilityzones.List(s.computeClient).AllPages() + if err != nil { + return nil, fmt.Errorf("error getting availability zone list: %v", err) + } + availabilityZoneList, err := availabilityzones.ExtractAvailabilityZones(allPages) + if err != nil { + return nil, fmt.Errorf("error extracting availability zone list: %v", err) + } + + return availabilityZoneList, nil +} diff --git a/pkg/cloud/services/compute/instance.go b/pkg/cloud/services/compute/instance.go index 0e4d1c730e..3a06bb105f 100644 --- a/pkg/cloud/services/compute/instance.go +++ b/pkg/cloud/services/compute/instance.go @@ -20,14 +20,7 @@ import ( "fmt" "time" - "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" - - "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking" - clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/controllers/noderefutil" - - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" - + "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/common/extensions" "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces" "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/bootfromvolume" @@ -37,11 +30,16 @@ import ( "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" netext "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/trunks" "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/controllers/noderefutil" "sigs.k8s.io/cluster-api/util" ) @@ -102,37 +100,10 @@ func (s *Service) InstanceCreate(clusterName string, machine *clusterv1.Machine, // Get all network UUIDs var nets []ServerNetwork if len(openStackMachine.Spec.Networks) > 0 { - for _, net := range openStackMachine.Spec.Networks { - opts := networks.ListOpts(net.Filter) - opts.ID = net.UUID - ids, err := networking.GetNetworkIDsByFilter(s.networkClient, &opts) - if err != nil { - return nil, err - } - for _, netID := range ids { - if net.Subnets == nil { - nets = append(nets, ServerNetwork{ - networkID: netID, - }) - continue - } - - for _, subnet := range net.Subnets { - subnetOpts := subnets.ListOpts(subnet.Filter) - subnetOpts.ID = subnet.UUID - subnetOpts.NetworkID = netID - subnetsByFilter, err := networking.GetSubnetsByFilter(s.networkClient, &subnetOpts) - if err != nil { - return nil, err - } - for _, subnetByFilter := range subnetsByFilter { - nets = append(nets, ServerNetwork{ - networkID: subnetByFilter.NetworkID, - subnetID: subnetByFilter.ID, - }) - } - } - } + var err error + nets, err = getServerNetworks(s.networkClient, openStackMachine.Spec.Networks) + if err != nil { + return nil, err } } else { if openStackCluster.Status.Network == nil { @@ -240,11 +211,14 @@ func (s *Service) InstanceCreate(clusterName string, machine *clusterv1.Machine, return nil, fmt.Errorf("create new server err: %v", err) } + if machine.Spec.FailureDomain == nil { + return nil, fmt.Errorf("create new server err: failure domain not set") + } var serverCreateOpts servers.CreateOptsBuilder = servers.CreateOpts{ Name: openStackMachine.Name, ImageRef: imageID, FlavorName: openStackMachine.Spec.Flavor, - AvailabilityZone: openStackMachine.Spec.AvailabilityZone, + AvailabilityZone: *machine.Spec.FailureDomain, Networks: portsList, UserData: []byte(userData), SecurityGroups: securityGroups, @@ -329,6 +303,43 @@ func getSecurityGroups(is *Service, securityGroupParams []infrav1.SecurityGroupP return sgIDs, nil } +func getServerNetworks(networkClient *gophercloud.ServiceClient, networkParams []infrav1.NetworkParam) ([]ServerNetwork, error) { + var nets []ServerNetwork + for _, net := range networkParams { + opts := networks.ListOpts(net.Filter) + opts.ID = net.UUID + ids, err := networking.GetNetworkIDsByFilter(networkClient, &opts) + if err != nil { + return nil, err + } + for _, netID := range ids { + if net.Subnets == nil { + nets = append(nets, ServerNetwork{ + networkID: netID, + }) + continue + } + + for _, subnet := range net.Subnets { + subnetOpts := subnets.ListOpts(subnet.Filter) + subnetOpts.ID = subnet.UUID + subnetOpts.NetworkID = netID + subnetsByFilter, err := networking.GetSubnetsByFilter(networkClient, &subnetOpts) + if err != nil { + return nil, err + } + for _, subnetByFilter := range subnetsByFilter { + nets = append(nets, ServerNetwork{ + networkID: subnetByFilter.NetworkID, + subnetID: subnetByFilter.ID, + }) + } + } + } + } + return nets, nil +} + func isDuplicate(list []string, name string) bool { if len(list) == 0 { return false diff --git a/pkg/cloud/services/loadbalancer/loadbalancer.go b/pkg/cloud/services/loadbalancer/loadbalancer.go index db368d2d9f..0116887eff 100644 --- a/pkg/cloud/services/loadbalancer/loadbalancer.go +++ b/pkg/cloud/services/loadbalancer/loadbalancer.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package loadbalancer import ( @@ -52,7 +68,7 @@ func (s *Service) ReconcileLoadBalancer(clusterName string, openStackCluster *in if err != nil { return fmt.Errorf("error creating loadbalancer: %s", err) } - err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lb.ID, "ACTIVE") + err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lb.ID) if err != nil { return err } @@ -111,7 +127,7 @@ func (s *Service) ReconcileLoadBalancer(clusterName string, openStackCluster *in if err != nil { return fmt.Errorf("error creating listener: %s", err) } - err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lb.ID, "ACTIVE") + err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lb.ID) if err != nil { return err } @@ -138,7 +154,7 @@ func (s *Service) ReconcileLoadBalancer(clusterName string, openStackCluster *in if err != nil { return fmt.Errorf("error creating pool: %s", err) } - err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lb.ID, "ACTIVE") + err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lb.ID) if err != nil { return err } @@ -163,7 +179,7 @@ func (s *Service) ReconcileLoadBalancer(clusterName string, openStackCluster *in if err != nil { return fmt.Errorf("error creating monitor: %s", err) } - err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lb.ID, "ACTIVE") + err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lb.ID) if err != nil { return err } @@ -229,7 +245,7 @@ func (s *Service) ReconcileLoadBalancerMember(clusterName string, machine *clust s.logger.Info("Deleting lb member (because the IP of the machine changed)", "name", name) // lb member changed so let's delete it so we can create it again with the correct IP - err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lbID, "ACTIVE") + err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lbID) if err != nil { return err } @@ -237,7 +253,7 @@ func (s *Service) ReconcileLoadBalancerMember(clusterName string, machine *clust if err != nil { return fmt.Errorf("error deleting lbmember: %s", err) } - err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lbID, "ACTIVE") + err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lbID) if err != nil { return err } @@ -253,13 +269,13 @@ func (s *Service) ReconcileLoadBalancerMember(clusterName string, machine *clust SubnetID: subnetID, } - if err := waitForLoadBalancer(s.logger, s.loadbalancerClient, lbID, "ACTIVE"); err != nil { + if err := waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lbID); err != nil { return err } if _, err := pools.CreateMember(s.loadbalancerClient, pool.ID, lbMemberOpts).Extract(); err != nil { return fmt.Errorf("error create lbmember: %s", err) } - if err := waitForLoadBalancer(s.logger, s.loadbalancerClient, lbID, "ACTIVE"); err != nil { + if err := waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lbID); err != nil { return err } } @@ -287,10 +303,8 @@ func (s *Service) DeleteLoadBalancer(clusterName string, openStackCluster *infra if err != nil { return fmt.Errorf("error deleting loadbalancer: %s", err) } - } else { - if err := s.deleteLoadBalancerNeutronV2(lb.ID); err != nil { - return fmt.Errorf("error deleting loadbalancer: %s", err) - } + } else if err := s.deleteLoadBalancerNeutronV2(lb.ID); err != nil { + return fmt.Errorf("error deleting loadbalancer: %s", err) } // floating ip @@ -336,7 +350,7 @@ func (s *Service) deleteLoadBalancerNeutronV2(id string) error { if err != nil { return fmt.Errorf("error deleting lbaas monitor %s: %v", pool.MonitorID, err) } - if err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lb.ID, "ACTIVE"); err != nil { + if err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lb.ID); err != nil { return fmt.Errorf("loadbalancer %s did not get back to %s state in time", lb.ID, "Active") } } @@ -357,7 +371,7 @@ func (s *Service) deleteLoadBalancerNeutronV2(id string) error { if err != nil { return fmt.Errorf("error deleting lbaas member %s on pool %s: %v", member.ID, pool.ID, err) } - if err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lb.ID, "ACTIVE"); err != nil { + if err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lb.ID); err != nil { return fmt.Errorf("loadbalancer %s did not get back to %s state in time", lb.ID, "ACTIVE") } } @@ -368,7 +382,7 @@ func (s *Service) deleteLoadBalancerNeutronV2(id string) error { if err != nil { return fmt.Errorf("error deleting lbaas pool %s: %v", pool.ID, err) } - if err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lb.ID, "ACTIVE"); err != nil { + if err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lb.ID); err != nil { return fmt.Errorf("loadbalancer %s did not get back to %s state in time", lb.ID, "ACTIVE") } } @@ -380,7 +394,7 @@ func (s *Service) deleteLoadBalancerNeutronV2(id string) error { if err != nil { return fmt.Errorf("error deleting lbaas listener %s: %v", listener.ID, err) } - if err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lb.ID, "ACTIVE"); err != nil { + if err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lb.ID); err != nil { return fmt.Errorf("loadbalancer %s did not get back to %s state in time", lb.ID, "ACTIVE") } } @@ -428,7 +442,7 @@ func (s *Service) DeleteLoadBalancerMember(clusterName string, machine *clusterv if lbMember != nil { // lb member changed so let's delete it so we can create it again with the correct IP - err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lbID, "ACTIVE") + err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lbID) if err != nil { return err } @@ -436,7 +450,7 @@ func (s *Service) DeleteLoadBalancerMember(clusterName string, machine *clusterv if err != nil { return fmt.Errorf("error deleting lbmember: %s", err) } - err = waitForLoadBalancer(s.logger, s.loadbalancerClient, lbID, "ACTIVE") + err = waitForLoadBalancerActive(s.logger, s.loadbalancerClient, lbID) if err != nil { return err } @@ -543,14 +557,14 @@ var backoff = wait.Backoff{ } // Possible LoadBalancer states are documented here: https://developer.openstack.org/api-ref/network/v2/?expanded=show-load-balancer-status-tree-detail#load-balancer-statuses -func waitForLoadBalancer(logger logr.Logger, client *gophercloud.ServiceClient, id, target string) error { - logger.Info("Waiting for loadbalancer", "id", id, "targetStatus", target) +func waitForLoadBalancerActive(logger logr.Logger, client *gophercloud.ServiceClient, id string) error { + logger.Info("Waiting for loadbalancer", "id", id, "targetStatus", "ACTIVE") return wait.ExponentialBackoff(backoff, func() (bool, error) { lb, err := loadbalancers.Get(client, id).Extract() if err != nil { return false, err } - return lb.ProvisioningStatus == target, nil + return lb.ProvisioningStatus == "ACTIVE", nil }) } diff --git a/pkg/cloud/services/networking/floatingip.go b/pkg/cloud/services/networking/floatingip.go index 6cca0e1b16..823860c161 100644 --- a/pkg/cloud/services/networking/floatingip.go +++ b/pkg/cloud/services/networking/floatingip.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package networking import ( diff --git a/pkg/cloud/services/networking/network.go b/pkg/cloud/services/networking/network.go index 4f77329b58..d145f7f1e3 100644 --- a/pkg/cloud/services/networking/network.go +++ b/pkg/cloud/services/networking/network.go @@ -115,10 +115,13 @@ func (s *Service) ReconcileSubnet(clusterName string, openStackCluster *infrav1. } var observedSubnet infrav1.Subnet + if len(subnetList) > 1 { // Not panicing here, because every other cluster might work. return fmt.Errorf("found more than 1 network with the expected name (%d) and CIDR (%s), which should not be able to exist in OpenStack", len(subnetList), openStackCluster.Spec.NodeCIDR) - } else if len(subnetList) == 0 { + } + + if len(subnetList) == 0 { opts := subnets.CreateOpts{ NetworkID: openStackCluster.Status.Network.ID, Name: subnetName, diff --git a/pkg/cloud/services/networking/securitygroups.go b/pkg/cloud/services/networking/securitygroups.go index 945828a2d3..bbf43e9766 100644 --- a/pkg/cloud/services/networking/securitygroups.go +++ b/pkg/cloud/services/networking/securitygroups.go @@ -73,16 +73,17 @@ func (s *Service) ReconcileSecurityGroups(clusterName string, openStackCluster * } if observedSecGroups[k].ID != "" { - if matchGroups(&desiredSecGroup, observedSecGroups[k]) { + if matchGroups(desiredSecGroup, *observedSecGroups[k]) { s.logger.V(6).Info("Group matched, have nothing to do.", "name", desiredSecGroup.Name) continue } s.logger.V(6).Info("Group didn't match, reconciling...", "name", desiredSecGroup.Name) - observedSecGroups[k], err = s.reconcileGroup(&desiredSecGroup, observedSecGroups[k]) + observedSecGroup, err := s.reconcileGroup(desiredSecGroup, *observedSecGroups[k]) if err != nil { return err } + observedSecGroups[k] = &observedSecGroup continue } @@ -199,7 +200,7 @@ func generateGlobalGroup(clusterName string) infrav1.SecurityGroup { } // matchGroups will check if security groups match. -func matchGroups(desired, observed *infrav1.SecurityGroup) bool { +func matchGroups(desired, observed infrav1.SecurityGroup) bool { // If they have differing amount of rules they obviously don't match. if len(desired.Rules) != len(observed.Rules) { return false @@ -228,13 +229,13 @@ func matchGroups(desired, observed *infrav1.SecurityGroup) bool { // reconcileGroup reconciles an already existing observed group by essentially emptying out all the rules and // recreating them. -func (s *Service) reconcileGroup(desired, observed *infrav1.SecurityGroup) (*infrav1.SecurityGroup, error) { +func (s *Service) reconcileGroup(desired, observed infrav1.SecurityGroup) (infrav1.SecurityGroup, error) { s.logger.V(6).Info("Deleting all rules for group", "name", observed.Name) for _, rule := range observed.Rules { s.logger.V(6).Info("Deleting rule", "ruleID", rule.ID, "groupName", observed.Name) err := rules.Delete(s.client, rule.ID).ExtractErr() if err != nil { - return &infrav1.SecurityGroup{}, err + return infrav1.SecurityGroup{}, err } } recreatedRules := make([]infrav1.SecurityGroupRule, 0, len(desired.Rules)) @@ -247,7 +248,7 @@ func (s *Service) reconcileGroup(desired, observed *infrav1.SecurityGroup) (*inf } newRule, err := s.createRule(r) if err != nil { - return &infrav1.SecurityGroup{}, err + return infrav1.SecurityGroup{}, err } recreatedRules = append(recreatedRules, newRule) } diff --git a/pkg/cloud/services/provider/provider.go b/pkg/cloud/services/provider/provider.go index 6f7e015dbc..378a5ffbbd 100644 --- a/pkg/cloud/services/provider/provider.go +++ b/pkg/cloud/services/provider/provider.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package provider import ( diff --git a/templates/cluster-template-without-lb.yaml b/templates/cluster-template-without-lb.yaml new file mode 100644 index 0000000000..1557e3a521 --- /dev/null +++ b/templates/cluster-template-without-lb.yaml @@ -0,0 +1,199 @@ +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] # CIDR block used by Calico. + serviceDomain: "cluster.local" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: OpenStackCluster + name: ${CLUSTER_NAME} + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + name: ${CLUSTER_NAME}-control-plane +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: OpenStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + cloudName: ${OPENSTACK_CLOUD} + cloudsSecret: + name: ${CLUSTER_NAME}-cloud-config + namespace: ${CLUSTER_NAME} + controlPlaneEndpoint: + host: ${OPENSTACK_CONTROLPLANE_IP} + port: 6443 + nodeCidr: 10.6.0.0/24 + dnsNameservers: + - ${OPENSTACK_DNS_NAMESERVERS} + externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID} + disablePortSecurity: true + disableServerTags: true + useOctavia: false +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: 1 + infrastructureTemplate: + kind: OpenStackMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-provider: openstack + cloud-config: /etc/kubernetes/cloud.conf + clusterConfiguration: + controlPlaneEndpoint: "${OPENSTACK_CONTROLPLANE_IP}:6443" + imageRepository: k8s.gcr.io + apiServer: + extraArgs: + cloud-provider: openstack + cloud-config: /etc/kubernetes/cloud.conf + extraVolumes: + - name: cloud + hostPath: /etc/kubernetes/cloud.conf + mountPath: /etc/kubernetes/cloud.conf + readOnly: true + controllerManager: + extraArgs: + cloud-provider: openstack + cloud-config: /etc/kubernetes/cloud.conf + extraVolumes: + - name: cloud + hostPath: /etc/kubernetes/cloud.conf + mountPath: /etc/kubernetes/cloud.conf + readOnly: true + - name: cacerts + hostPath: /etc/certs/cacert + mountPath: /etc/certs/cacert + readOnly: true + joinConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + files: + - path: /etc/kubernetes/cloud.conf + owner: root + permissions: "0600" + content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + - path: /etc/certs/cacert + owner: root + permissions: "0600" + content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 + ntp: + servers: [] + users: + - name: capo + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${OPENSTACK_SSH_AUTHORIZED_KEY}" + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + floatingIP: ${OPENSTACK_CONTROLPLANE_IP} + flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR} + image: ${OPENSTACK_IMAGE_NAME} + cloudName: ${OPENSTACK_CLOUD} + cloudsSecret: + name: ${CLUSTER_NAME}-cloud-config + namespace: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + failureDomain: ${OPENSTACK_FAILURE_DOMAIN} + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: OpenStackMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + cloudsSecret: + name: ${CLUSTER_NAME}-cloud-config + namespace: ${CLUSTER_NAME} + flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR} + image: ${OPENSTACK_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: + - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + owner: root + path: /etc/kubernetes/cloud.conf + permissions: "0600" + - content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 + owner: root + path: /etc/certs/cacert + permissions: "0600" + joinConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + ntp: + servers: [] + users: + - name: capo + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${OPENSTACK_SSH_AUTHORIZED_KEY}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-cloud-config +data: + clouds.yaml: ${OPENSTACK_CLOUD_YAML_B64} + cacert: ${OPENSTACK_CLOUD_CACERT_B64} diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml new file mode 100644 index 0000000000..95fa304c75 --- /dev/null +++ b/templates/cluster-template.yaml @@ -0,0 +1,200 @@ +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] # CIDR block used by Calico. + serviceDomain: "cluster.local" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: OpenStackCluster + name: ${CLUSTER_NAME} + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + name: ${CLUSTER_NAME}-control-plane +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: OpenStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + cloudName: ${OPENSTACK_CLOUD} + cloudsSecret: + name: ${CLUSTER_NAME}-cloud-config + namespace: ${CLUSTER_NAME} + managedAPIServerLoadBalancer: true + apiServerLoadBalancerFloatingIP: ${OPENSTACK_CONTROLPLANE_IP} + apiServerLoadBalancerPort: 6443 + apiServerLoadBalancerAdditionalPorts: + - 22 + nodeCidr: 10.6.0.0/24 + dnsNameservers: + - ${OPENSTACK_DNS_NAMESERVERS} + externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID} + disablePortSecurity: true + disableServerTags: true + useOctavia: true +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + infrastructureTemplate: + kind: OpenStackMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-provider: openstack + cloud-config: /etc/kubernetes/cloud.conf + clusterConfiguration: + controlPlaneEndpoint: "${OPENSTACK_CONTROLPLANE_IP}:6443" + imageRepository: k8s.gcr.io + apiServer: + extraArgs: + cloud-provider: openstack + cloud-config: /etc/kubernetes/cloud.conf + extraVolumes: + - name: cloud + hostPath: /etc/kubernetes/cloud.conf + mountPath: /etc/kubernetes/cloud.conf + readOnly: true + controllerManager: + extraArgs: + cloud-provider: openstack + cloud-config: /etc/kubernetes/cloud.conf + extraVolumes: + - name: cloud + hostPath: /etc/kubernetes/cloud.conf + mountPath: /etc/kubernetes/cloud.conf + readOnly: true + - name: cacerts + hostPath: /etc/certs/cacert + mountPath: /etc/certs/cacert + readOnly: true + joinConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + files: + - path: /etc/kubernetes/cloud.conf + owner: root + permissions: "0600" + content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + - path: /etc/certs/cacert + owner: root + permissions: "0600" + content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 + ntp: + servers: [] + users: + - name: capo + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${OPENSTACK_SSH_AUTHORIZED_KEY}" + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR} + image: ${OPENSTACK_IMAGE_NAME} + cloudName: ${OPENSTACK_CLOUD} + cloudsSecret: + name: ${CLUSTER_NAME}-cloud-config + namespace: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + failureDomain: ${OPENSTACK_FAILURE_DOMAIN} + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: OpenStackMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + cloudsSecret: + name: ${CLUSTER_NAME}-cloud-config + namespace: ${CLUSTER_NAME} + flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR} + image: ${OPENSTACK_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: + - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + owner: root + path: /etc/kubernetes/cloud.conf + permissions: "0600" + - content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 + owner: root + path: /etc/certs/cacert + permissions: "0600" + joinConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + ntp: + servers: [] + users: + - name: capo + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${OPENSTACK_SSH_AUTHORIZED_KEY}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-cloud-config +data: + clouds.yaml: ${OPENSTACK_CLOUD_YAML_B64} + cacert: ${OPENSTACK_CLOUD_CACERT_B64} diff --git a/templates/env.rc b/templates/env.rc new file mode 100755 index 0000000000..36c3c7f98f --- /dev/null +++ b/templates/env.rc @@ -0,0 +1,122 @@ +#!/bin/bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT=$(basename "$0") +while test $# -gt 0; do + case "$1" in + -h|--help) + echo "$SCRIPT - sources env vars for clusterctl init from an OpenStack clouds.yaml file" + echo " " + echo "source $SCRIPT [options] " + echo " " + echo "options:" + echo "-h, --help show brief help" + exit 0 + ;; + *) + break + ;; + esac +done + +# Check if clouds.yaml file provided +if [[ -n "${1-}" ]] && [[ $1 != -* ]] && [[ $1 != --* ]];then + CLOUDS_PATH="$1" +else + echo "Error: No clouds.yaml provided" + echo "You must provide a valid clouds.yaml script to generate a cloud.conf" + echo "" + exit 1 +fi + +# Check if os cloud is provided +if [[ -n "${2-}" ]] && [[ $2 != -* ]] && [[ $2 != --* ]]; then + export CLOUD=$2 +else + echo "Error: No cloud specified" + echo "You must specify which cloud you want to use." + echo "" + exit 1 +fi + +yq_type=$(file "$(which yq)") +if [[ ${yq_type} == *"Python script"* ]]; then + echo "Wrong version of 'yq' installed, please install the one from https://github.com/mikefarah/yq" + echo "" + exit 1 +fi + +CLOUDS_PATH=${CLOUDS_PATH:-""} +OPENSTACK_CLOUD_YAML_CONTENT=$(cat "$CLOUDS_PATH") + +# Just blindly parse the cloud.yaml here, overwriting old vars. +AUTH_URL=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.auth_url) +USERNAME=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.username) +PASSWORD=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.password) +REGION=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.region_name) +PROJECT_ID=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.project_id) +PROJECT_NAME=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.project_name) +DOMAIN_NAME=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.user_domain_name) +if [[ "$DOMAIN_NAME" = "null" ]]; then + DOMAIN_NAME=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.domain_name) +fi +DOMAIN_ID=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.user_domain_id) +if [[ "$DOMAIN_ID" = "null" ]]; then + DOMAIN_ID=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.auth.domain_id) +fi +CACERT_ORIGINAL=$(echo "$OPENSTACK_CLOUD_YAML_CONTENT" | yq r - clouds.${CLOUD}.cacert) + +# Build OPENSTACK_CLOUD_YAML_B64 +OPENSTACK_CLOUD_YAML_SELECTED_CLOUD_B64=$(echo "${OPENSTACK_CLOUD_YAML_CONTENT}" | yq r - clouds.${CLOUD} | yq p - clouds.${CLOUD} | base64 --wrap=0) +export OPENSTACK_CLOUD_YAML_B64="${OPENSTACK_CLOUD_YAML_SELECTED_CLOUD_B64}" + +# Build OPENSTACK_CLOUD_PROVIDER_CONF_B64 +# Basic cloud.conf, no LB configuration as that data is not known yet. +cloud_provider_conf_tmp=$(mktemp /tmp/cloud.confXXX) +OPENSTACK_CLOUD_PROVIDER_CONF="" +cat >> ${cloud_provider_conf_tmp} << EOF +[Global] +auth-url=$AUTH_URL +username="$USERNAME" +password="$PASSWORD" +EOF + +if [[ "$PROJECT_ID" != "null" ]]; then + echo "tenant-id=\"${PROJECT_ID}\"" >> ${cloud_provider_conf_tmp} +fi +if [[ "$PROJECT_NAME" != "null" ]]; then + echo "tenant-name=\"${PROJECT_NAME}\"" >> ${cloud_provider_conf_tmp} +fi +if [[ "$DOMAIN_NAME" != "null" ]]; then + echo "domain-name=\"${DOMAIN_NAME}\"" >> ${cloud_provider_conf_tmp} +fi +if [[ "$DOMAIN_ID" != "null" ]]; then + echo "domain-id=\"${DOMAIN_ID}\"" >> ${cloud_provider_conf_tmp} +fi + +if [[ "$CACERT_ORIGINAL" != "null" ]]; then + echo "ca-file=\"${CACERT_ORIGINAL}\"" >> ${cloud_provider_conf_tmp} +fi +if [[ "$REGION" != "null" ]]; then + echo "region=\"${REGION}\"" >> ${cloud_provider_conf_tmp} +fi +export OPENSTACK_CLOUD_PROVIDER_CONF_B64="$(cat ${cloud_provider_conf_tmp} | base64 --wrap=0)" + +# Build OPENSTACK_CLOUD_CACERT_B64 +OPENSTACK_CLOUD_CACERT="" +if [[ "$CACERT_ORIGINAL" != "null" ]]; then + OPENSTACK_CLOUD_CACERT=$(cat "$CACERT_ORIGINAL") +fi +export OPENSTACK_CLOUD_CACERT_B64="$(echo ${OPENSTACK_CLOUD_CACERT} | base64 --wrap=0)" diff --git a/templates/kustomization.yaml b/templates/kustomization.yaml new file mode 100644 index 0000000000..5e058e123c --- /dev/null +++ b/templates/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - cluster-template.yaml diff --git a/templates/kustomizeversions.yaml b/templates/kustomizeversions.yaml new file mode 100644 index 0000000000..25c2d885f9 --- /dev/null +++ b/templates/kustomizeversions.yaml @@ -0,0 +1,167 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: "ci/latest" + preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh + files: + - path: /tmp/kubeadm-bootstrap.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + GSUTIL=gsutil + if ! command -v ${GSUTIL} > /dev/null; then + curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1 + GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil + fi + ${GSUTIL} version + + # This test installs debian packages that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + + LINE_SEPARATOR="*************************************************" + echo "$LINE_SEPARATOR" + + CI_VERSION=${CI_VERSION:-""} + if [[ "${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $CI_DIR + CI_URL="gs://kubernetes-release-dev/ci/$CI_VERSION-bazel/bin/linux/amd64" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + PACKAGE_EXT="deb" + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + + echo "* testing CI version $CI_VERSION" + + for CI_PACKAGE in "${PACKAGES_TO_TEST[@]}"; do + echo "* downloading package: $CI_URL/$CI_PACKAGE.$PACKAGE_EXT" + ${GSUTIL} cp "$CI_URL/$CI_PACKAGE.$PACKAGE_EXT" "$CI_DIR/$CI_PACKAGE.$PACKAGE_EXT" + ${SUDO} dpkg -i "$CI_DIR/$CI_PACKAGE.$PACKAGE_EXT" || echo "* ignoring expected 'dpkg -i' result" + done + + for CI_CONTAINER in "${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $CI_URL/$CI_CONTAINER.$CONTAINER_EXT" + ${GSUTIL} cp "$CI_URL/$CI_CONTAINER.$CONTAINER_EXT" "$CI_DIR/$CI_CONTAINER.$CONTAINER_EXT" + ${SUDO} ctr -n k8s.io images import "$CI_DIR/$CI_CONTAINER.$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + ${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$CI_CONTAINER-amd64:"${CI_VERSION//+/_}" k8s.gcr.io/$CI_CONTAINER:"${CI_VERSION//+/_}" + ${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$CI_CONTAINER-amd64:"${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$CI_CONTAINER:"${CI_VERSION//+/_}" + done + fi + + echo "* checking binary versions" + + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + + echo "$LINE_SEPARATOR" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: OpenStackMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: ${CLUSTER_NAME} +spec: + template: + spec: + image: ${OPENSTACK_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${CLUSTER_NAME} +spec: + template: + spec: + preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh + files: + - path: /tmp/kubeadm-bootstrap.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + GSUTIL=gsutil + if ! command -v ${GSUTIL} > /dev/null; then + curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1 + GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil + fi + ${GSUTIL} version + + # This test installs debian packages that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + + LINE_SEPARATOR="*************************************************" + echo "$LINE_SEPARATOR" + + CI_VERSION=${CI_VERSION:-""} + if [[ "${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $CI_DIR + CI_URL="gs://kubernetes-release-dev/ci/$CI_VERSION-bazel/bin/linux/amd64" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + PACKAGE_EXT="deb" + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + + echo "* testing CI version $CI_VERSION" + + for CI_PACKAGE in "${PACKAGES_TO_TEST[@]}"; do + echo "* downloading package: $CI_URL/$CI_PACKAGE.$PACKAGE_EXT" + ${GSUTIL} cp "$CI_URL/$CI_PACKAGE.$PACKAGE_EXT" "$CI_DIR/$CI_PACKAGE.$PACKAGE_EXT" + ${SUDO} dpkg -i "$CI_DIR/$CI_PACKAGE.$PACKAGE_EXT" || echo "* ignoring expected 'dpkg -i' result" + done + + for CI_CONTAINER in "${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $CI_URL/$CI_CONTAINER.$CONTAINER_EXT" + ${GSUTIL} cp "$CI_URL/$CI_CONTAINER.$CONTAINER_EXT" "$CI_DIR/$CI_CONTAINER.$CONTAINER_EXT" + ${SUDO} ctr -n k8s.io images import "$CI_DIR/$CI_CONTAINER.$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + ${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$CI_CONTAINER-amd64:"${CI_VERSION//+/_}" k8s.gcr.io/$CI_CONTAINER:"${CI_VERSION//+/_}" + ${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$CI_CONTAINER-amd64:"${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$CI_CONTAINER:"${CI_VERSION//+/_}" + done + fi + + echo "* checking binary versions" + + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + + echo "$LINE_SEPARATOR" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${CLUSTER_NAME} +spec: + template: + spec: + image: ${OPENSTACK_IMAGE_NAME} From fdeb0fc382d60346f6b7f4605260fc65c58ba935 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Wed, 18 Mar 2020 07:23:52 +0100 Subject: [PATCH 2/3] review fixes --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cd9ca56750..d5d2f94d4e 100644 --- a/Makefile +++ b/Makefile @@ -308,7 +308,7 @@ OPENSTACK_CLUSTER_TEMPLATE ?= "./templates/cluster-template-without-lb.yaml" KUBERNETES_VERSION ?= "v1.17.3" CONTROL_PLANE_MACHINE_COUNT ?= "1" WORKER_MACHINE_COUNT ?= "3" -LOAD_IMAGE=$(CONTROLLER_IMG)-$(ARCH):dev +LOAD_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) .PHONY: create-cluster create-cluster: $(CLUSTERCTL) $(ENVSUBST) ## Create a development Kubernetes cluster on OpenStack in a KIND management cluster. From a233311a5dff8ac8256ee4931e157473c8252c96 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Wed, 18 Mar 2020 07:30:07 +0100 Subject: [PATCH 3/3] review fixes --- README.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index 189052b93d..0ffe2b6cf1 100644 --- a/README.md +++ b/README.md @@ -37,13 +37,7 @@ previous cluster managers such as [kops][kops] and This provider's versions are compatible with the following versions of Cluster API: -| abc | def | adfa |adf |asdf | -|---|---|---|---|---| -| sdf | sdfsdf |sfddsf |sdfsd |sdf | -| | | | | | -| | | | | | - -| | Cluster API v1alpha1 (v0.1) | Cluster API v1alpha2 (v0.2) | Cluster API v1alpha2 (v0.3) | +| | Cluster API v1alpha1 (v0.1) | Cluster API v1alpha2 (v0.2) | Cluster API v1alpha3 (v0.3) | |-------------------------------------------------|---|---|---| | OpenStack Provider v1alpha1 (release-0.1 branch)| ✓ | | | | OpenStack Provider v1alpha2 (v0.2) | | ✓ | |