diff --git a/Makefile b/Makefile index a58e2b4f40..af8f0cdd98 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ E2E_DATA_DIR ?= $(REPO_ROOT)/test/e2e/data E2E_CONF_PATH ?= $(E2E_DATA_DIR)/e2e_conf.yaml KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml) KUBETEST_FAST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance-fast.yaml) +GO_INSTALL := ./scripts/go_install.sh # Binaries. CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen @@ -53,6 +54,17 @@ KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize MOCKGEN := $(TOOLS_BIN_DIR)/mockgen RELEASE_NOTES := $(TOOLS_BIN_DIR)/release-notes +# Setup-envtest +SETUP_ENVTEST_VER := v0.0.0-20211110210527-619e6b92dab9 +SETUP_ENVTEST_BIN := setup-envtest +SETUP_ENVTEST := $(abspath $(TOOLS_BIN_DIR)/$(SETUP_ENVTEST_BIN)-$(SETUP_ENVTEST_VER)) +SETUP_ENVTEST_PKG := sigs.k8s.io/controller-runtime/tools/setup-envtest + +# Kubebuilder +export KUBEBUILDER_ENVTEST_KUBERNETES_VERSION ?= 1.23.3 +export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT ?= 60s +export KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT ?= 60s + PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) export PATH export DOCKER_CLI_EXPERIMENTAL=enabled @@ -116,9 +128,15 @@ E2E_ARGS ?= $(ARTIFACTS): mkdir -p $@ +ifeq ($(shell go env GOOS),darwin) # Use the darwin/amd64 binary until an arm64 version is available + KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path --arch amd64 $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION)) +else + KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION)) +endif + .PHONY: test -test: ## Run tests - go test -v ./... +test: $(SETUP_ENVTEST) ## Run tests + KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -v ./... $(TEST_ARGS) # Can be run manually, e.g. via: # export OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml" @@ -167,6 +185,12 @@ managers: manager-openstack-infrastructure: ## Build manager binary. CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "${LDFLAGS} -extldflags '-static'" -o $(BIN_DIR)/manager . +$(SETUP_ENVTEST): # Build setup-envtest from tools folder. + GOBIN=$(abspath $(TOOLS_BIN_DIR)) $(GO_INSTALL) $(SETUP_ENVTEST_PKG) $(SETUP_ENVTEST_BIN) $(SETUP_ENVTEST_VER) + +.PHONY: $(SETUP_ENVTEST_BIN) + $(SETUP_ENVTEST_BIN): $(SETUP_ENVTEST) ## Build a local copy of setup-envtest. + ## -------------------------------------- ## Linting ## -------------------------------------- diff --git a/api/v1alpha3/conversion.go b/api/v1alpha3/conversion.go index 36450d694d..ff383b6e83 100644 --- a/api/v1alpha3/conversion.go +++ b/api/v1alpha3/conversion.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha3 import ( + unsafe "unsafe" + corev1 "k8s.io/api/core/v1" conversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -117,6 +119,10 @@ func Convert_v1alpha3_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(in *O Name: in.CloudsSecret.Name, } } + out.APIServerLoadBalancer = infrav1.APIServerLoadBalancer{ + Enabled: in.ManagedAPIServerLoadBalancer, + AdditionalPorts: *(*[]int)(unsafe.Pointer(&in.APIServerLoadBalancerAdditionalPorts)), + } return autoConvert_v1alpha3_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(in, out, s) } @@ -139,6 +145,10 @@ func Convert_v1beta1_OpenStackClusterSpec_To_v1alpha3_OpenStackClusterSpec(in *i Name: in.Bastion.Instance.IdentityRef.Name, } } + + out.ManagedAPIServerLoadBalancer = in.APIServerLoadBalancer.Enabled + out.APIServerLoadBalancerAdditionalPorts = *(*[]int)(unsafe.Pointer(&in.APIServerLoadBalancer.AdditionalPorts)) + return autoConvert_v1beta1_OpenStackClusterSpec_To_v1alpha3_OpenStackClusterSpec(in, out, s) } diff --git a/api/v1alpha3/conversion_test.go b/api/v1alpha3/conversion_test.go index 6c334dfe4f..31bb961d4f 100644 --- a/api/v1alpha3/conversion_test.go +++ b/api/v1alpha3/conversion_test.go @@ -25,10 +25,93 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" utilconversion "sigs.k8s.io/cluster-api/util/conversion" + ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" ) +func TestConvertTo(t *testing.T) { + g := gomega.NewWithT(t) + scheme := runtime.NewScheme() + g.Expect(AddToScheme(scheme)).To(gomega.Succeed()) + g.Expect(infrav1.AddToScheme(scheme)).To(gomega.Succeed()) + + tests := []struct { + name string + spoke ctrlconversion.Convertible + hub ctrlconversion.Hub + want ctrlconversion.Hub + }{ + { + name: "APIServer LoadBalancer Configuration", + spoke: &OpenStackCluster{ + Spec: OpenStackClusterSpec{ + ManagedAPIServerLoadBalancer: true, + APIServerLoadBalancerAdditionalPorts: []int{80, 443}, + }, + }, + hub: &infrav1.OpenStackCluster{}, + want: &infrav1.OpenStackCluster{ + Spec: infrav1.OpenStackClusterSpec{ + APIServerLoadBalancer: infrav1.APIServerLoadBalancer{ + Enabled: true, + AdditionalPorts: []int{80, 443}, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.spoke.ConvertTo(tt.hub) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(tt.hub).To(gomega.Equal(tt.want)) + }) + } +} + +func TestConvertFrom(t *testing.T) { + g := gomega.NewWithT(t) + scheme := runtime.NewScheme() + g.Expect(AddToScheme(scheme)).To(gomega.Succeed()) + g.Expect(infrav1.AddToScheme(scheme)).To(gomega.Succeed()) + + tests := []struct { + name string + spoke ctrlconversion.Convertible + hub ctrlconversion.Hub + want ctrlconversion.Convertible + }{ + { + name: "APIServer LoadBalancer Configuration", + spoke: &OpenStackCluster{}, + hub: &infrav1.OpenStackCluster{ + Spec: infrav1.OpenStackClusterSpec{ + APIServerLoadBalancer: infrav1.APIServerLoadBalancer{ + Enabled: true, + AdditionalPorts: []int{80, 443}, + }, + }, + }, + want: &OpenStackCluster{ + Spec: OpenStackClusterSpec{ + ManagedAPIServerLoadBalancer: true, + APIServerLoadBalancerAdditionalPorts: []int{80, 443}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.spoke.ConvertFrom(tt.hub) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(tt.spoke).To(gomega.Equal(tt.want)) + }) + } +} + func TestFuzzyConversion(t *testing.T) { g := gomega.NewWithT(t) scheme := runtime.NewScheme() diff --git a/api/v1alpha3/openstackcluster_types.go b/api/v1alpha3/openstackcluster_types.go index e71ea86fea..0200075701 100644 --- a/api/v1alpha3/openstackcluster_types.go +++ b/api/v1alpha3/openstackcluster_types.go @@ -107,6 +107,10 @@ type OpenStackClusterSpec struct { ControlPlaneAvailabilityZones []string `json:"controlPlaneAvailabilityZones,omitempty"` // Bastion is the OpenStack instance to login the nodes + // + // As a rolling update is not ideal during a bastion host session, we + // prevent changes to a running bastion configuration. Set `enabled: false` to + // make changes. //+optional Bastion *Bastion `json:"bastion,omitempty"` } diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index 45c9bee8c4..6c39e5a51f 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -676,10 +676,10 @@ func autoConvert_v1alpha3_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(i } out.ExternalNetworkID = in.ExternalNetworkID // WARNING: in.UseOctavia requires manual conversion: does not exist in peer-type - out.ManagedAPIServerLoadBalancer = in.ManagedAPIServerLoadBalancer + // WARNING: in.ManagedAPIServerLoadBalancer requires manual conversion: does not exist in peer-type out.APIServerFloatingIP = in.APIServerFloatingIP out.APIServerPort = in.APIServerPort - out.APIServerLoadBalancerAdditionalPorts = *(*[]int)(unsafe.Pointer(&in.APIServerLoadBalancerAdditionalPorts)) + // WARNING: in.APIServerLoadBalancerAdditionalPorts requires manual conversion: does not exist in peer-type out.ManagedSecurityGroups = in.ManagedSecurityGroups out.DisablePortSecurity = in.DisablePortSecurity out.Tags = *(*[]string)(unsafe.Pointer(&in.Tags)) @@ -721,12 +721,11 @@ func autoConvert_v1beta1_OpenStackClusterSpec_To_v1alpha3_OpenStackClusterSpec(i out.ExternalRouterIPs = nil } out.ExternalNetworkID = in.ExternalNetworkID - out.ManagedAPIServerLoadBalancer = in.ManagedAPIServerLoadBalancer + // WARNING: in.APIServerLoadBalancer requires manual conversion: does not exist in peer-type // WARNING: in.DisableAPIServerFloatingIP requires manual conversion: does not exist in peer-type out.APIServerFloatingIP = in.APIServerFloatingIP // WARNING: in.APIServerFixedIP requires manual conversion: does not exist in peer-type out.APIServerPort = in.APIServerPort - out.APIServerLoadBalancerAdditionalPorts = *(*[]int)(unsafe.Pointer(&in.APIServerLoadBalancerAdditionalPorts)) out.ManagedSecurityGroups = in.ManagedSecurityGroups // WARNING: in.AllowAllInClusterTraffic requires manual conversion: does not exist in peer-type out.DisablePortSecurity = in.DisablePortSecurity diff --git a/api/v1alpha4/conversion_test.go b/api/v1alpha4/conversion_test.go index 6364261b16..2bffec9dc3 100644 --- a/api/v1alpha4/conversion_test.go +++ b/api/v1alpha4/conversion_test.go @@ -70,6 +70,24 @@ func TestConvertTo(t *testing.T) { }, }, }, + { + name: "APIServer LoadBalancer Configuration", + spoke: &OpenStackCluster{ + Spec: OpenStackClusterSpec{ + ManagedAPIServerLoadBalancer: true, + APIServerLoadBalancerAdditionalPorts: []int{80, 443}, + }, + }, + hub: &infrav1.OpenStackCluster{}, + want: &infrav1.OpenStackCluster{ + Spec: infrav1.OpenStackClusterSpec{ + APIServerLoadBalancer: infrav1.APIServerLoadBalancer{ + Enabled: true, + AdditionalPorts: []int{80, 443}, + }, + }, + }, + }, } for _, tt := range tests { @@ -121,6 +139,24 @@ func TestConvertFrom(t *testing.T) { }, }, }, + { + name: "APIServer LoadBalancer Configuration", + spoke: &OpenStackCluster{}, + hub: &infrav1.OpenStackCluster{ + Spec: infrav1.OpenStackClusterSpec{ + APIServerLoadBalancer: infrav1.APIServerLoadBalancer{ + Enabled: true, + AdditionalPorts: []int{80, 443}, + }, + }, + }, + want: &OpenStackCluster{ + Spec: OpenStackClusterSpec{ + ManagedAPIServerLoadBalancer: true, + APIServerLoadBalancerAdditionalPorts: []int{80, 443}, + }, + }, + }, } for _, tt := range tests { diff --git a/api/v1alpha4/openstackcluster_conversion.go b/api/v1alpha4/openstackcluster_conversion.go index 6d4ec3da2d..5d97000cb3 100644 --- a/api/v1alpha4/openstackcluster_conversion.go +++ b/api/v1alpha4/openstackcluster_conversion.go @@ -17,9 +17,13 @@ limitations under the License. package v1alpha4 import ( + unsafe "unsafe" + "k8s.io/apimachinery/pkg/conversion" clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" ) // Convert_v1alpha4_APIEndpoint_To_v1beta1_APIEndpoint is an autogenerated conversion function. @@ -31,3 +35,17 @@ func Convert_v1alpha4_APIEndpoint_To_v1beta1_APIEndpoint(in *clusterv1alpha4.API func Convert_v1beta1_APIEndpoint_To_v1alpha4_APIEndpoint(in *clusterv1.APIEndpoint, out *clusterv1alpha4.APIEndpoint, s conversion.Scope) error { return clusterv1alpha4.Convert_v1beta1_APIEndpoint_To_v1alpha4_APIEndpoint(in, out, s) } + +func Convert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(in *OpenStackClusterSpec, out *infrav1.OpenStackClusterSpec, s conversion.Scope) error { + out.APIServerLoadBalancer.Enabled = in.ManagedAPIServerLoadBalancer + out.APIServerLoadBalancer.AdditionalPorts = *(*[]int)(unsafe.Pointer(&in.APIServerLoadBalancerAdditionalPorts)) + + return autoConvert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(in, out, s) +} + +func Convert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(in *infrav1.OpenStackClusterSpec, out *OpenStackClusterSpec, s conversion.Scope) error { + out.ManagedAPIServerLoadBalancer = in.APIServerLoadBalancer.Enabled + out.APIServerLoadBalancerAdditionalPorts = *(*[]int)(unsafe.Pointer(&in.APIServerLoadBalancer.AdditionalPorts)) + + return autoConvert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(in, out, s) +} diff --git a/api/v1alpha4/openstackcluster_types.go b/api/v1alpha4/openstackcluster_types.go index e06241196f..3c9d48c9e4 100644 --- a/api/v1alpha4/openstackcluster_types.go +++ b/api/v1alpha4/openstackcluster_types.go @@ -130,6 +130,10 @@ type OpenStackClusterSpec struct { ControlPlaneAvailabilityZones []string `json:"controlPlaneAvailabilityZones,omitempty"` // Bastion is the OpenStack instance to login the nodes + // + // As a rolling update is not ideal during a bastion host session, we + // prevent changes to a running bastion configuration. Set `enabled: false` to + // make changes. //+optional Bastion *Bastion `json:"bastion,omitempty"` diff --git a/api/v1alpha4/zz_generated.conversion.go b/api/v1alpha4/zz_generated.conversion.go index 83cfc3c9c3..ab72440d56 100644 --- a/api/v1alpha4/zz_generated.conversion.go +++ b/api/v1alpha4/zz_generated.conversion.go @@ -120,16 +120,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*OpenStackClusterSpec)(nil), (*v1beta1.OpenStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(a.(*OpenStackClusterSpec), b.(*v1beta1.OpenStackClusterSpec), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.OpenStackClusterSpec)(nil), (*OpenStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(a.(*v1beta1.OpenStackClusterSpec), b.(*OpenStackClusterSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*OpenStackClusterStatus)(nil), (*v1beta1.OpenStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_OpenStackClusterStatus_To_v1beta1_OpenStackClusterStatus(a.(*OpenStackClusterStatus), b.(*v1beta1.OpenStackClusterStatus), scope) }); err != nil { @@ -360,6 +350,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*OpenStackClusterSpec)(nil), (*v1beta1.OpenStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(a.(*OpenStackClusterSpec), b.(*v1beta1.OpenStackClusterSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*OpenStackMachineSpec)(nil), (*v1beta1.OpenStackMachineSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_OpenStackMachineSpec_To_v1beta1_OpenStackMachineSpec(a.(*OpenStackMachineSpec), b.(*v1beta1.OpenStackMachineSpec), scope) }); err != nil { @@ -400,6 +395,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.OpenStackClusterSpec)(nil), (*OpenStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(a.(*v1beta1.OpenStackClusterSpec), b.(*OpenStackClusterSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.OpenStackMachineSpec)(nil), (*OpenStackMachineSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_OpenStackMachineSpec_To_v1alpha4_OpenStackMachineSpec(a.(*v1beta1.OpenStackMachineSpec), b.(*OpenStackMachineSpec), scope) }); err != nil { @@ -810,12 +810,12 @@ func autoConvert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(i out.ExternalRouterIPs = nil } out.ExternalNetworkID = in.ExternalNetworkID - out.ManagedAPIServerLoadBalancer = in.ManagedAPIServerLoadBalancer + // WARNING: in.ManagedAPIServerLoadBalancer requires manual conversion: does not exist in peer-type out.DisableAPIServerFloatingIP = in.DisableAPIServerFloatingIP out.APIServerFloatingIP = in.APIServerFloatingIP out.APIServerFixedIP = in.APIServerFixedIP out.APIServerPort = in.APIServerPort - out.APIServerLoadBalancerAdditionalPorts = *(*[]int)(unsafe.Pointer(&in.APIServerLoadBalancerAdditionalPorts)) + // WARNING: in.APIServerLoadBalancerAdditionalPorts requires manual conversion: does not exist in peer-type out.ManagedSecurityGroups = in.ManagedSecurityGroups out.AllowAllInClusterTraffic = in.AllowAllInClusterTraffic out.DisablePortSecurity = in.DisablePortSecurity @@ -835,11 +835,6 @@ func autoConvert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(i return nil } -// Convert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec is an autogenerated conversion function. -func Convert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(in *OpenStackClusterSpec, out *v1beta1.OpenStackClusterSpec, s conversion.Scope) error { - return autoConvert_v1alpha4_OpenStackClusterSpec_To_v1beta1_OpenStackClusterSpec(in, out, s) -} - func autoConvert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(in *v1beta1.OpenStackClusterSpec, out *OpenStackClusterSpec, s conversion.Scope) error { out.CloudName = in.CloudName out.NodeCIDR = in.NodeCIDR @@ -862,12 +857,11 @@ func autoConvert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(i out.ExternalRouterIPs = nil } out.ExternalNetworkID = in.ExternalNetworkID - out.ManagedAPIServerLoadBalancer = in.ManagedAPIServerLoadBalancer + // WARNING: in.APIServerLoadBalancer requires manual conversion: does not exist in peer-type out.DisableAPIServerFloatingIP = in.DisableAPIServerFloatingIP out.APIServerFloatingIP = in.APIServerFloatingIP out.APIServerFixedIP = in.APIServerFixedIP out.APIServerPort = in.APIServerPort - out.APIServerLoadBalancerAdditionalPorts = *(*[]int)(unsafe.Pointer(&in.APIServerLoadBalancerAdditionalPorts)) out.ManagedSecurityGroups = in.ManagedSecurityGroups out.AllowAllInClusterTraffic = in.AllowAllInClusterTraffic out.DisablePortSecurity = in.DisablePortSecurity @@ -887,11 +881,6 @@ func autoConvert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(i return nil } -// Convert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec is an autogenerated conversion function. -func Convert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(in *v1beta1.OpenStackClusterSpec, out *OpenStackClusterSpec, s conversion.Scope) error { - return autoConvert_v1beta1_OpenStackClusterSpec_To_v1alpha4_OpenStackClusterSpec(in, out, s) -} - func autoConvert_v1alpha4_OpenStackClusterStatus_To_v1beta1_OpenStackClusterStatus(in *OpenStackClusterStatus, out *v1beta1.OpenStackClusterStatus, s conversion.Scope) error { out.Ready = in.Ready if in.Network != nil { diff --git a/api/v1beta1/openstackcluster_types.go b/api/v1beta1/openstackcluster_types.go index 2dd0ea526a..75708ece4c 100644 --- a/api/v1beta1/openstackcluster_types.go +++ b/api/v1beta1/openstackcluster_types.go @@ -57,10 +57,10 @@ type OpenStackClusterSpec struct { // +optional ExternalNetworkID string `json:"externalNetworkId,omitempty"` - // ManagedAPIServerLoadBalancer defines whether a LoadBalancer for the - // APIServer should be created. + // APIServerLoadBalancer configures the optional LoadBalancer for the APIServer. + // It must be activated by setting `enabled: true`. // +optional - ManagedAPIServerLoadBalancer bool `json:"managedAPIServerLoadBalancer"` + APIServerLoadBalancer APIServerLoadBalancer `json:"apiServerLoadBalancer,omitempty"` // DisableAPIServerFloatingIP determines whether or not to attempt to attach a floating // IP to the API server. This allows for the creation of clusters when attaching a floating @@ -97,9 +97,6 @@ type OpenStackClusterSpec struct { // will be created APIServerPort int `json:"apiServerPort,omitempty"` - // APIServerLoadBalancerAdditionalPorts adds additional ports to the APIServerLoadBalancer - APIServerLoadBalancerAdditionalPorts []int `json:"apiServerLoadBalancerAdditionalPorts,omitempty"` - // ManagedSecurityGroups determines whether OpenStack security groups for the cluster // will be managed by the OpenStack provider or whether pre-existing security groups will // be specified as part of the configuration. @@ -130,6 +127,10 @@ type OpenStackClusterSpec struct { ControlPlaneAvailabilityZones []string `json:"controlPlaneAvailabilityZones,omitempty"` // Bastion is the OpenStack instance to login the nodes + // + // As a rolling update is not ideal during a bastion host session, we + // prevent changes to a running bastion configuration. Set `enabled: false` to + // make changes. //+optional Bastion *Bastion `json:"bastion,omitempty"` diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 22c981dcf4..bf5d68aa24 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -296,3 +296,10 @@ type Bastion struct { //+optional AvailabilityZone string `json:"availabilityZone,omitempty"` } + +type APIServerLoadBalancer struct { + // Enabled defines whether a LoadBalancer should be created. + Enabled bool `json:"enabled,omitempty"` + // AdditionalPorts adds additional tcp ports to the Loadbalacner + AdditionalPorts []int `json:"additionalPorts,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d2c9de389f..98a8d24628 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -28,6 +28,26 @@ import ( "sigs.k8s.io/cluster-api/errors" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIServerLoadBalancer) DeepCopyInto(out *APIServerLoadBalancer) { + *out = *in + if in.AdditionalPorts != nil { + in, out := &in.AdditionalPorts, &out.AdditionalPorts + *out = make([]int, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerLoadBalancer. +func (in *APIServerLoadBalancer) DeepCopy() *APIServerLoadBalancer { + if in == nil { + return nil + } + out := new(APIServerLoadBalancer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AddressPair) DeepCopyInto(out *AddressPair) { *out = *in @@ -317,11 +337,7 @@ func (in *OpenStackClusterSpec) DeepCopyInto(out *OpenStackClusterSpec) { *out = make([]ExternalRouterIPParam, len(*in)) copy(*out, *in) } - if in.APIServerLoadBalancerAdditionalPorts != nil { - in, out := &in.APIServerLoadBalancerAdditionalPorts, &out.APIServerLoadBalancerAdditionalPorts - *out = make([]int, len(*in)) - copy(*out, *in) - } + in.APIServerLoadBalancer.DeepCopyInto(&out.APIServerLoadBalancer) if in.Tags != nil { in, out := &in.Tags, &out.Tags *out = make([]string, len(*in)) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml index f5b3bd5898..07d05741e0 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml @@ -82,7 +82,10 @@ spec: APIServer will be created type: integer bastion: - description: Bastion is the OpenStack instance to login the nodes + description: "Bastion is the OpenStack instance to login the nodes + \n As a rolling update is not ideal during a bastion host session, + we prevent changes to a running bastion configuration. Set `enabled: + false` to make changes." properties: availabilityZone: type: string @@ -1106,7 +1109,10 @@ spec: APIServer will be created type: integer bastion: - description: Bastion is the OpenStack instance to login the nodes + description: "Bastion is the OpenStack instance to login the nodes + \n As a rolling update is not ideal during a bastion host session, + we prevent changes to a running bastion configuration. Set `enabled: + false` to make changes." properties: availabilityZone: type: string @@ -2499,18 +2505,30 @@ spec: already exist. If not specified, a new floatingIP is allocated. This field is not used if DisableAPIServerFloatingIP is set to true. type: string - apiServerLoadBalancerAdditionalPorts: - description: APIServerLoadBalancerAdditionalPorts adds additional - ports to the APIServerLoadBalancer - items: - type: integer - type: array + apiServerLoadBalancer: + description: 'APIServerLoadBalancer configures the optional LoadBalancer + for the APIServer. It must be activated by setting `enabled: true`.' + properties: + additionalPorts: + description: AdditionalPorts adds additional tcp ports to the + Loadbalacner + items: + type: integer + type: array + enabled: + description: Enabled defines whether a LoadBalancer should be + created. + type: boolean + type: object apiServerPort: description: APIServerPort is the port on which the listener on the APIServer will be created type: integer bastion: - description: Bastion is the OpenStack instance to login the nodes + description: "Bastion is the OpenStack instance to login the nodes + \n As a rolling update is not ideal during a bastion host session, + we prevent changes to a running bastion configuration. Set `enabled: + false` to make changes." properties: availabilityZone: type: string @@ -2999,10 +3017,6 @@ spec: - kind - name type: object - managedAPIServerLoadBalancer: - description: ManagedAPIServerLoadBalancer defines whether a LoadBalancer - for the APIServer should be created. - type: boolean managedSecurityGroups: description: ManagedSecurityGroups determines whether OpenStack security groups for the cluster will be managed by the OpenStack provider diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml index e6b01f4b60..0203d5189f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml @@ -87,8 +87,10 @@ spec: on the APIServer will be created type: integer bastion: - description: Bastion is the OpenStack instance to login the - nodes + description: "Bastion is the OpenStack instance to login the + nodes \n As a rolling update is not ideal during a bastion + host session, we prevent changes to a running bastion configuration. + Set `enabled: false` to make changes." properties: availabilityZone: type: string @@ -764,19 +766,31 @@ spec: a new floatingIP is allocated. This field is not used if DisableAPIServerFloatingIP is set to true. type: string - apiServerLoadBalancerAdditionalPorts: - description: APIServerLoadBalancerAdditionalPorts adds additional - ports to the APIServerLoadBalancer - items: - type: integer - type: array + apiServerLoadBalancer: + description: 'APIServerLoadBalancer configures the optional + LoadBalancer for the APIServer. It must be activated by + setting `enabled: true`.' + properties: + additionalPorts: + description: AdditionalPorts adds additional tcp ports + to the Loadbalacner + items: + type: integer + type: array + enabled: + description: Enabled defines whether a LoadBalancer should + be created. + type: boolean + type: object apiServerPort: description: APIServerPort is the port on which the listener on the APIServer will be created type: integer bastion: - description: Bastion is the OpenStack instance to login the - nodes + description: "Bastion is the OpenStack instance to login the + nodes \n As a rolling update is not ideal during a bastion + host session, we prevent changes to a running bastion configuration. + Set `enabled: false` to make changes." properties: availabilityZone: type: string @@ -1283,10 +1297,6 @@ spec: - kind - name type: object - managedAPIServerLoadBalancer: - description: ManagedAPIServerLoadBalancer defines whether - a LoadBalancer for the APIServer should be created. - type: boolean managedSecurityGroups: description: ManagedSecurityGroups determines whether OpenStack security groups for the cluster will be managed by the OpenStack diff --git a/controllers/openstackcluster_controller.go b/controllers/openstackcluster_controller.go index ac23a9aa35..bf7e0deb12 100644 --- a/controllers/openstackcluster_controller.go +++ b/controllers/openstackcluster_controller.go @@ -141,7 +141,7 @@ func reconcileDelete(ctx context.Context, scope *scope.Scope, patchHelper *patch clusterName := fmt.Sprintf("%s-%s", cluster.Namespace, cluster.Name) - if openStackCluster.Spec.ManagedAPIServerLoadBalancer { + if openStackCluster.Spec.APIServerLoadBalancer.Enabled { loadBalancerService, err := loadbalancer.NewService(scope) if err != nil { return reconcile.Result{}, err @@ -222,8 +222,8 @@ func deleteBastion(scope *scope.Scope, cluster *clusterv1.Cluster, openStackClus } } - machineSpec := &openStackCluster.Spec.Bastion.Instance - if err = computeService.DeleteInstance(openStackCluster, machineSpec, instanceName, instanceStatus); err != nil { + instanceSpec := bastionToInstanceSpec(openStackCluster, cluster.Name) + if err = computeService.DeleteInstance(openStackCluster, instanceSpec, instanceStatus); err != nil { handleUpdateOSCError(openStackCluster, errors.Errorf("failed to delete bastion: %v", err)) return errors.Errorf("failed to delete bastion: %v", err) } @@ -320,7 +320,8 @@ func reconcileBastion(scope *scope.Scope, cluster *clusterv1.Cluster, openStackC return nil } - instanceStatus, err = computeService.CreateBastion(openStackCluster, cluster.Name) + instanceSpec := bastionToInstanceSpec(openStackCluster, cluster.Name) + instanceStatus, err = computeService.CreateInstance(openStackCluster, openStackCluster, instanceSpec, cluster.Name) if err != nil { return errors.Errorf("failed to reconcile bastion: %v", err) } @@ -356,6 +357,31 @@ func reconcileBastion(scope *scope.Scope, cluster *clusterv1.Cluster, openStackC return nil } +func bastionToInstanceSpec(openStackCluster *infrav1.OpenStackCluster, clusterName string) *compute.InstanceSpec { + name := fmt.Sprintf("%s-bastion", clusterName) + instanceSpec := &compute.InstanceSpec{ + Name: name, + Flavor: openStackCluster.Spec.Bastion.Instance.Flavor, + SSHKeyName: openStackCluster.Spec.Bastion.Instance.SSHKeyName, + Image: openStackCluster.Spec.Bastion.Instance.Image, + ImageUUID: openStackCluster.Spec.Bastion.Instance.ImageUUID, + FailureDomain: openStackCluster.Spec.Bastion.AvailabilityZone, + RootVolume: openStackCluster.Spec.Bastion.Instance.RootVolume, + } + + instanceSpec.SecurityGroups = openStackCluster.Spec.Bastion.Instance.SecurityGroups + if openStackCluster.Spec.ManagedSecurityGroups { + instanceSpec.SecurityGroups = append(instanceSpec.SecurityGroups, infrav1.SecurityGroupParam{ + UUID: openStackCluster.Status.BastionSecurityGroup.ID, + }) + } + + instanceSpec.Networks = openStackCluster.Spec.Bastion.Instance.Networks + instanceSpec.Ports = openStackCluster.Spec.Bastion.Instance.Ports + + return instanceSpec +} + func reconcileNetworkComponents(scope *scope.Scope, cluster *clusterv1.Cluster, openStackCluster *infrav1.OpenStackCluster) error { clusterName := fmt.Sprintf("%s-%s", cluster.Namespace, cluster.Name) @@ -448,7 +474,7 @@ func reconcileNetworkComponents(scope *scope.Scope, cluster *clusterv1.Cluster, apiServerPort = 6443 } - if openStackCluster.Spec.ManagedAPIServerLoadBalancer { + if openStackCluster.Spec.APIServerLoadBalancer.Enabled { loadBalancerService, err := loadbalancer.NewService(scope) if err != nil { return err @@ -465,7 +491,7 @@ func reconcileNetworkComponents(scope *scope.Scope, cluster *clusterv1.Cluster, var host string // If there is a load balancer use the floating IP for it if set, falling back to the internal IP switch { - case openStackCluster.Spec.ManagedAPIServerLoadBalancer: + case openStackCluster.Spec.APIServerLoadBalancer.Enabled: if openStackCluster.Status.Network.APIServerLoadBalancer.IP != "" { host = openStackCluster.Status.Network.APIServerLoadBalancer.IP } else { diff --git a/controllers/openstackmachine_controller.go b/controllers/openstackmachine_controller.go index 37e3d609ad..ca9c61c269 100644 --- a/controllers/openstackmachine_controller.go +++ b/controllers/openstackmachine_controller.go @@ -212,7 +212,7 @@ func (r *OpenStackMachineReconciler) reconcileDelete(ctx context.Context, scope return ctrl.Result{}, err } - if openStackCluster.Spec.ManagedAPIServerLoadBalancer { + if openStackCluster.Spec.APIServerLoadBalancer.Enabled { loadBalancerService, err := loadbalancer.NewService(scope) if err != nil { return ctrl.Result{}, err @@ -228,7 +228,7 @@ func (r *OpenStackMachineReconciler) reconcileDelete(ctx context.Context, scope if err != nil { return ctrl.Result{}, err } - if !openStackCluster.Spec.ManagedAPIServerLoadBalancer && util.IsControlPlaneMachine(machine) && openStackCluster.Spec.APIServerFloatingIP == "" { + if !openStackCluster.Spec.APIServerLoadBalancer.Enabled && util.IsControlPlaneMachine(machine) && openStackCluster.Spec.APIServerFloatingIP == "" { if instanceStatus != nil { instanceNS, err := instanceStatus.NetworkStatus() if err != nil { @@ -248,7 +248,14 @@ func (r *OpenStackMachineReconciler) reconcileDelete(ctx context.Context, scope } } - if err := computeService.DeleteInstance(openStackMachine, &openStackMachine.Spec, openStackMachine.Name, instanceStatus); err != nil { + instanceSpec, err := machineToInstanceSpec(openStackCluster, machine, openStackMachine, "") + if err != nil { + err = errors.Errorf("machine spec is invalid: %v", err) + handleUpdateMachineError(scope.Logger, openStackMachine, err) + return ctrl.Result{}, err + } + + if err := computeService.DeleteInstance(openStackMachine, instanceSpec, instanceStatus); err != nil { handleUpdateMachineError(scope.Logger, openStackMachine, errors.Errorf("error deleting OpenStack instance %s with ID %s: %v", instanceStatus.Name(), instanceStatus.ID(), err)) return ctrl.Result{}, nil } @@ -351,7 +358,7 @@ func (r *OpenStackMachineReconciler) reconcileNormal(ctx context.Context, scope return ctrl.Result{RequeueAfter: waitForInstanceBecomeActiveToReconcile}, nil } - if openStackCluster.Spec.ManagedAPIServerLoadBalancer { + if openStackCluster.Spec.APIServerLoadBalancer.Enabled { err = r.reconcileLoadBalancerMember(scope, openStackCluster, machine, openStackMachine, instanceNS, clusterName) if err != nil { handleUpdateMachineError(scope.Logger, openStackMachine, errors.Errorf("LoadBalancerMember cannot be reconciled: %v", err)) @@ -392,7 +399,14 @@ func (r *OpenStackMachineReconciler) getOrCreate(logger logr.Logger, cluster *cl if instanceStatus == nil { logger.Info("Machine not exist, Creating Machine", "Machine", openStackMachine.Name) - instanceStatus, err = computeService.CreateInstance(openStackCluster, machine, openStackMachine, cluster.Name, userData) + instanceSpec, err := machineToInstanceSpec(openStackCluster, machine, openStackMachine, userData) + if err != nil { + err = errors.Errorf("machine spec is invalid: %v", err) + handleUpdateMachineError(logger, openStackMachine, err) + return nil, err + } + + instanceStatus, err = computeService.CreateInstance(openStackMachine, openStackCluster, instanceSpec, cluster.Name) if err != nil { return nil, errors.Errorf("error creating Openstack instance: %v", err) } @@ -401,6 +415,75 @@ func (r *OpenStackMachineReconciler) getOrCreate(logger logr.Logger, cluster *cl return instanceStatus, nil } +func machineToInstanceSpec(openStackCluster *infrav1.OpenStackCluster, machine *clusterv1.Machine, openStackMachine *infrav1.OpenStackMachine, userData string) (*compute.InstanceSpec, error) { + if openStackMachine == nil { + return nil, fmt.Errorf("create Options need be specified to create instace") + } + + if machine.Spec.FailureDomain == nil { + return nil, fmt.Errorf("failure domain not set") + } + + instanceSpec := compute.InstanceSpec{ + Name: openStackMachine.Name, + Image: openStackMachine.Spec.Image, + ImageUUID: openStackMachine.Spec.ImageUUID, + Flavor: openStackMachine.Spec.Flavor, + SSHKeyName: openStackMachine.Spec.SSHKeyName, + UserData: userData, + Metadata: openStackMachine.Spec.ServerMetadata, + ConfigDrive: openStackMachine.Spec.ConfigDrive != nil && *openStackMachine.Spec.ConfigDrive, + FailureDomain: *machine.Spec.FailureDomain, + RootVolume: openStackMachine.Spec.RootVolume, + Subnet: openStackMachine.Spec.Subnet, + ServerGroupID: openStackMachine.Spec.ServerGroupID, + Trunk: openStackMachine.Spec.Trunk, + } + + machineTags := []string{} + + // Append machine specific tags + machineTags = append(machineTags, openStackMachine.Spec.Tags...) + + // Append cluster scope tags + machineTags = append(machineTags, openStackCluster.Spec.Tags...) + + // tags need to be unique or the "apply tags" call will fail. + deduplicate := func(tags []string) []string { + seen := make(map[string]struct{}, len(machineTags)) + unique := make([]string, 0, len(machineTags)) + for _, tag := range tags { + if _, ok := seen[tag]; !ok { + seen[tag] = struct{}{} + unique = append(unique, tag) + } + } + return unique + } + machineTags = deduplicate(machineTags) + + instanceSpec.Tags = machineTags + + instanceSpec.SecurityGroups = openStackMachine.Spec.SecurityGroups + if openStackCluster.Spec.ManagedSecurityGroups { + var managedSecurityGroup string + if util.IsControlPlaneMachine(machine) { + managedSecurityGroup = openStackCluster.Status.ControlPlaneSecurityGroup.ID + } else { + managedSecurityGroup = openStackCluster.Status.WorkerSecurityGroup.ID + } + + instanceSpec.SecurityGroups = append(instanceSpec.SecurityGroups, infrav1.SecurityGroupParam{ + UUID: managedSecurityGroup, + }) + } + + instanceSpec.Networks = openStackMachine.Spec.Networks + instanceSpec.Ports = openStackMachine.Spec.Ports + + return &instanceSpec, nil +} + func handleUpdateMachineError(logger logr.Logger, openstackMachine *infrav1.OpenStackMachine, message error) { err := capierrors.UpdateMachineError openstackMachine.Status.FailureReason = &err diff --git a/controllers/openstackmachine_controller_test.go b/controllers/openstackmachine_controller_test.go new file mode 100644 index 0000000000..d795378e8a --- /dev/null +++ b/controllers/openstackmachine_controller_test.go @@ -0,0 +1,226 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/compute" +) + +const ( + networkUUID = "d412171b-9fd7-41c1-95a6-c24e5953974d" + subnetUUID = "d2d8d98d-b234-477e-a547-868b7cb5d6a5" + extraSecurityGroupUUID = "514bb2d8-3390-4a3b-86a7-7864ba57b329" + controlPlaneSecurityGroupUUID = "c9817a91-4821-42db-8367-2301002ab659" + workerSecurityGroupUUID = "9c6c0d28-03c9-436c-815d-58440ac2c1c8" + serverGroupUUID = "7b940d62-68ef-4e42-a76a-1a62e290509c" + + openStackMachineName = "test-openstack-machine" + namespace = "test-namespace" + imageName = "test-image" + flavorName = "test-flavor" + sshKeyName = "test-ssh-key" + failureDomain = "test-failure-domain" +) + +func getDefaultOpenStackCluster() *infrav1.OpenStackCluster { + return &infrav1.OpenStackCluster{ + Spec: infrav1.OpenStackClusterSpec{}, + Status: infrav1.OpenStackClusterStatus{ + Network: &infrav1.Network{ + ID: networkUUID, + Subnet: &infrav1.Subnet{ + ID: subnetUUID, + }, + }, + ControlPlaneSecurityGroup: &infrav1.SecurityGroup{ID: controlPlaneSecurityGroupUUID}, + WorkerSecurityGroup: &infrav1.SecurityGroup{ID: workerSecurityGroupUUID}, + }, + } +} + +func getDefaultMachine() *clusterv1.Machine { + return &clusterv1.Machine{ + Spec: clusterv1.MachineSpec{ + FailureDomain: pointer.StringPtr(failureDomain), + }, + } +} + +func getDefaultOpenStackMachine() *infrav1.OpenStackMachine { + return &infrav1.OpenStackMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: openStackMachineName, + Namespace: namespace, + }, + Spec: infrav1.OpenStackMachineSpec{ + // ProviderID is set by the controller + // InstanceID is set by the controller + // FloatingIP is only used by the cluster controller for the Bastion + // TODO: Test Networks, Ports, Subnet, and Trunk separately + CloudName: "test-cloud", + Flavor: flavorName, + Image: imageName, + SSHKeyName: sshKeyName, + Tags: []string{"test-tag"}, + ServerMetadata: map[string]string{ + "test-metadata": "test-value", + }, + ConfigDrive: pointer.BoolPtr(true), + ServerGroupID: serverGroupUUID, + }, + } +} + +func getDefaultInstanceSpec() *compute.InstanceSpec { + return &compute.InstanceSpec{ + Name: openStackMachineName, + Image: imageName, + Flavor: flavorName, + SSHKeyName: sshKeyName, + UserData: "user-data", + Metadata: map[string]string{ + "test-metadata": "test-value", + }, + ConfigDrive: *pointer.BoolPtr(true), + FailureDomain: *pointer.StringPtr(failureDomain), + ServerGroupID: serverGroupUUID, + Tags: []string{"test-tag"}, + } +} + +func Test_machineToInstanceSpec(t *testing.T) { + RegisterTestingT(t) + + tests := []struct { + name string + openStackCluster func() *infrav1.OpenStackCluster + machine func() *clusterv1.Machine + openStackMachine func() *infrav1.OpenStackMachine + wantInstanceSpec func() *compute.InstanceSpec + wantErr bool + }{ + { + name: "Defaults", + openStackCluster: getDefaultOpenStackCluster, + machine: getDefaultMachine, + openStackMachine: getDefaultOpenStackMachine, + wantInstanceSpec: getDefaultInstanceSpec, + wantErr: false, + }, + { + name: "Control plane security group", + openStackCluster: func() *infrav1.OpenStackCluster { + c := getDefaultOpenStackCluster() + c.Spec.ManagedSecurityGroups = true + return c + }, + machine: func() *clusterv1.Machine { + m := getDefaultMachine() + m.Labels = map[string]string{ + clusterv1.MachineControlPlaneLabelName: "true", + } + return m + }, + openStackMachine: getDefaultOpenStackMachine, + wantInstanceSpec: func() *compute.InstanceSpec { + i := getDefaultInstanceSpec() + i.SecurityGroups = []infrav1.SecurityGroupParam{{UUID: controlPlaneSecurityGroupUUID}} + return i + }, + wantErr: false, + }, + { + name: "Worker security group", + openStackCluster: func() *infrav1.OpenStackCluster { + c := getDefaultOpenStackCluster() + c.Spec.ManagedSecurityGroups = true + return c + }, + machine: getDefaultMachine, + openStackMachine: getDefaultOpenStackMachine, + wantInstanceSpec: func() *compute.InstanceSpec { + i := getDefaultInstanceSpec() + i.SecurityGroups = []infrav1.SecurityGroupParam{{UUID: workerSecurityGroupUUID}} + return i + }, + wantErr: false, + }, + { + name: "Extra security group", + openStackCluster: func() *infrav1.OpenStackCluster { + c := getDefaultOpenStackCluster() + c.Spec.ManagedSecurityGroups = true + return c + }, + machine: getDefaultMachine, + openStackMachine: func() *infrav1.OpenStackMachine { + m := getDefaultOpenStackMachine() + m.Spec.SecurityGroups = []infrav1.SecurityGroupParam{{UUID: extraSecurityGroupUUID}} + return m + }, + wantInstanceSpec: func() *compute.InstanceSpec { + i := getDefaultInstanceSpec() + i.SecurityGroups = []infrav1.SecurityGroupParam{ + {UUID: extraSecurityGroupUUID}, + {UUID: workerSecurityGroupUUID}, + } + return i + }, + wantErr: false, + }, + { + name: "Tags", + openStackCluster: func() *infrav1.OpenStackCluster { + c := getDefaultOpenStackCluster() + c.Spec.Tags = []string{"cluster-tag", "duplicate-tag"} + return c + }, + machine: getDefaultMachine, + openStackMachine: func() *infrav1.OpenStackMachine { + m := getDefaultOpenStackMachine() + m.Spec.Tags = []string{"machine-tag", "duplicate-tag"} + return m + }, + wantInstanceSpec: func() *compute.InstanceSpec { + i := getDefaultInstanceSpec() + i.Tags = []string{"machine-tag", "duplicate-tag", "cluster-tag"} + return i + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := machineToInstanceSpec(tt.openStackCluster(), tt.machine(), tt.openStackMachine(), "user-data") + if tt.wantErr { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).NotTo(HaveOccurred()) + } + + Expect(got).To(Equal(tt.wantInstanceSpec())) + }) + } +} diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 812840a108..a81adebe43 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -17,13 +17,18 @@ limitations under the License. package controllers import ( + "context" "path/filepath" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" @@ -41,10 +46,6 @@ var ( ) func TestAPIs(t *testing.T) { - // TODO(sbueringer) controller don't work yet because kubebuilder is not installed correctly - // and therefore no etcd is available inside the path - t.Skip() - RegisterFailHandler(Fail) RunSpecsWithDefaultAndCustomReporters(t, @@ -83,3 +84,32 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).ToNot(HaveOccurred()) }) + +var _ = Describe("EnvTest sanity check", func() { + ctx := context.TODO() + It("should be able to create a namespace", func() { + testNamespace := "capo-test" + namespacedName := types.NamespacedName{ + Name: testNamespace, + } + namespaceInput := framework.CreateNamespaceInput{ + Creator: k8sClient, + Name: testNamespace, + } + + // Create the namespace + namespace := framework.CreateNamespace(ctx, namespaceInput) + // Check the result + namespaceResult := &corev1.Namespace{} + err := k8sClient.Get(ctx, namespacedName, namespaceResult) + Expect(err).To(BeNil()) + Expect(namespaceResult).To(Equal(namespace)) + + // Clean up + foregroundDeletePropagation := metav1.DeletePropagationForeground + err = k8sClient.Delete(ctx, namespace, &client.DeleteOptions{PropagationPolicy: &foregroundDeletePropagation}) + Expect(err).To(BeNil()) + // Note: Since the controller-manager is not part of envtest the namespace + // will actually stay in "Terminating" state and never be completely gone. + }) +}) diff --git a/pkg/cloud/services/compute/bastion.go b/pkg/cloud/services/compute/bastion.go deleted file mode 100644 index abce789f5e..0000000000 --- a/pkg/cloud/services/compute/bastion.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package compute - -import ( - "fmt" - - infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" -) - -func (s *Service) CreateBastion(openStackCluster *infrav1.OpenStackCluster, clusterName string) (*InstanceStatus, error) { - name := fmt.Sprintf("%s-bastion", clusterName) - instanceSpec := &InstanceSpec{ - Name: name, - Flavor: openStackCluster.Spec.Bastion.Instance.Flavor, - SSHKeyName: openStackCluster.Spec.Bastion.Instance.SSHKeyName, - Image: openStackCluster.Spec.Bastion.Instance.Image, - ImageUUID: openStackCluster.Spec.Bastion.Instance.ImageUUID, - FailureDomain: openStackCluster.Spec.Bastion.AvailabilityZone, - RootVolume: openStackCluster.Spec.Bastion.Instance.RootVolume, - } - - securityGroups, err := s.networkingService.GetSecurityGroups(openStackCluster.Spec.Bastion.Instance.SecurityGroups) - if err != nil { - return nil, err - } - if openStackCluster.Spec.ManagedSecurityGroups { - securityGroups = append(securityGroups, openStackCluster.Status.BastionSecurityGroup.ID) - } - instanceSpec.SecurityGroups = securityGroups - - var nets []infrav1.Network - if len(openStackCluster.Spec.Bastion.Instance.Networks) > 0 { - var err error - nets, err = s.getServerNetworks(openStackCluster.Spec.Bastion.Instance.Networks) - if err != nil { - return nil, err - } - } else { - nets = []infrav1.Network{{ - ID: openStackCluster.Status.Network.ID, - Subnet: &infrav1.Subnet{ - ID: openStackCluster.Status.Network.Subnet.ID, - }, - }} - } - instanceSpec.Networks = nets - - return s.createInstance(openStackCluster, clusterName, instanceSpec, retryIntervalInstanceStatus) -} diff --git a/pkg/cloud/services/compute/instance.go b/pkg/cloud/services/compute/instance.go index d170287888..8db1bbec42 100644 --- a/pkg/cloud/services/compute/instance.go +++ b/pkg/cloud/services/compute/instance.go @@ -30,7 +30,6 @@ import ( "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" "k8s.io/apimachinery/pkg/runtime" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" @@ -44,108 +43,24 @@ const ( timeoutInstanceDelete = 5 * time.Minute ) -func (s *Service) CreateInstance(openStackCluster *infrav1.OpenStackCluster, machine *clusterv1.Machine, openStackMachine *infrav1.OpenStackMachine, clusterName string, userData string) (instance *InstanceStatus, err error) { - return s.createInstanceImpl(openStackCluster, machine, openStackMachine, clusterName, userData, retryIntervalInstanceStatus) -} - -func (s *Service) createInstanceImpl(openStackCluster *infrav1.OpenStackCluster, machine *clusterv1.Machine, openStackMachine *infrav1.OpenStackMachine, clusterName string, userData string, retryInterval time.Duration) (instance *InstanceStatus, err error) { - if openStackMachine == nil { - return nil, fmt.Errorf("create Options need be specified to create instace") - } - - if machine.Spec.FailureDomain == nil { - return nil, fmt.Errorf("failure domain not set") - } - - instanceSpec := InstanceSpec{ - Name: openStackMachine.Name, - Image: openStackMachine.Spec.Image, - ImageUUID: openStackMachine.Spec.ImageUUID, - Flavor: openStackMachine.Spec.Flavor, - SSHKeyName: openStackMachine.Spec.SSHKeyName, - UserData: userData, - Metadata: openStackMachine.Spec.ServerMetadata, - ConfigDrive: openStackMachine.Spec.ConfigDrive != nil && *openStackMachine.Spec.ConfigDrive, - FailureDomain: *machine.Spec.FailureDomain, - RootVolume: openStackMachine.Spec.RootVolume, - Subnet: openStackMachine.Spec.Subnet, - ServerGroupID: openStackMachine.Spec.ServerGroupID, - } - - // verify that trunk is supported if set at instance level. - if openStackMachine.Spec.Trunk { - trunkSupported, err := s.isTrunkExtSupported() - if err != nil { - return nil, err - } - if !trunkSupported { - return nil, fmt.Errorf("there is no trunk support. please ensure that the trunk extension is enabled in your OpenStack deployment") - } - instanceSpec.Trunk = true - } - machineTags := []string{} - - // Append machine specific tags - machineTags = append(machineTags, openStackMachine.Spec.Tags...) - - // Append cluster scope tags - machineTags = append(machineTags, openStackCluster.Spec.Tags...) - - // tags need to be unique or the "apply tags" call will fail. - machineTags = deduplicate(machineTags) - - instanceSpec.Tags = machineTags - - // Get security groups - securityGroups, err := s.networkingService.GetSecurityGroups(openStackMachine.Spec.SecurityGroups) - if err != nil { - return nil, err - } - if openStackCluster.Spec.ManagedSecurityGroups { - if util.IsControlPlaneMachine(machine) { - securityGroups = append(securityGroups, openStackCluster.Status.ControlPlaneSecurityGroup.ID) - } else { - securityGroups = append(securityGroups, openStackCluster.Status.WorkerSecurityGroup.ID) - } - } - instanceSpec.SecurityGroups = securityGroups +// constructNetworks builds an array of networks from the network, subnet and ports items in the instance spec. +// If no networks or ports are in the spec, returns a single network item for a network connection to the default cluster network. +func (s *Service) constructNetworks(openStackCluster *infrav1.OpenStackCluster, instanceSpec *InstanceSpec) ([]infrav1.Network, error) { + trunkRequired := false - nets, err := s.constructNetworks(openStackCluster, openStackMachine) + nets, err := s.getServerNetworks(instanceSpec.Networks) if err != nil { return nil, err } - trunkConfigured := s.isTrunkConfigured(nets, instanceSpec.Trunk) - if trunkConfigured { - trunkSupported, err := s.isTrunkExtSupported() - if err != nil { - return nil, err - } - if !trunkSupported { - return nil, fmt.Errorf("there is no trunk support. please ensure that the trunk extension is enabled in your OpenStack deployment") - } - } - instanceSpec.Networks = nets - - return s.createInstance(openStackMachine, clusterName, &instanceSpec, retryInterval) -} - -// constructNetworks builds an array of networks from the network, subnet and ports items in the machine spec. -// If no networks or ports are in the spec, returns a single network item for a network connection to the default cluster network. -func (s *Service) constructNetworks(openStackCluster *infrav1.OpenStackCluster, openStackMachine *infrav1.OpenStackMachine) ([]infrav1.Network, error) { - var nets []infrav1.Network - if len(openStackMachine.Spec.Networks) > 0 { - var err error - nets, err = s.getServerNetworks(openStackMachine.Spec.Networks) - if err != nil { - return nil, err - } - } - for i, port := range openStackMachine.Spec.Ports { - pOpts := &openStackMachine.Spec.Ports[i] + for i := range instanceSpec.Ports { + port := &instanceSpec.Ports[i] // No Trunk field specified for the port, inherit openStackMachine.Spec.Trunk. - if pOpts.Trunk == nil { - pOpts.Trunk = &openStackMachine.Spec.Trunk + if port.Trunk == nil { + port.Trunk = &instanceSpec.Trunk + } + if *port.Trunk { + trunkRequired = true } if port.Network != nil { netID := port.Network.ID @@ -164,7 +79,7 @@ func (s *Service) constructNetworks(openStackCluster *infrav1.OpenStackCluster, nets = append(nets, infrav1.Network{ ID: netID, Subnet: &infrav1.Subnet{}, - PortOpts: pOpts, + PortOpts: port, }) } else { nets = append(nets, infrav1.Network{ @@ -172,10 +87,11 @@ func (s *Service) constructNetworks(openStackCluster *infrav1.OpenStackCluster, Subnet: &infrav1.Subnet{ ID: openStackCluster.Status.Network.Subnet.ID, }, - PortOpts: pOpts, + PortOpts: port, }) } } + // no networks or ports found in the spec, so create a port on the cluster network if len(nets) == 0 { nets = []infrav1.Network{{ @@ -184,14 +100,30 @@ func (s *Service) constructNetworks(openStackCluster *infrav1.OpenStackCluster, ID: openStackCluster.Status.Network.Subnet.ID, }, PortOpts: &infrav1.PortOpts{ - Trunk: &openStackMachine.Spec.Trunk, + Trunk: &instanceSpec.Trunk, }, }} + trunkRequired = instanceSpec.Trunk + } + + if trunkRequired { + trunkSupported, err := s.isTrunkExtSupported() + if err != nil { + return nil, err + } + if !trunkSupported { + return nil, fmt.Errorf("there is no trunk support. please ensure that the trunk extension is enabled in your OpenStack deployment") + } } + return nets, nil } -func (s *Service) createInstance(eventObject runtime.Object, clusterName string, instanceSpec *InstanceSpec, retryInterval time.Duration) (*InstanceStatus, error) { +func (s *Service) CreateInstance(eventObject runtime.Object, openStackCluster *infrav1.OpenStackCluster, instanceSpec *InstanceSpec, clusterName string) (*InstanceStatus, error) { + return s.createInstanceImpl(eventObject, openStackCluster, instanceSpec, clusterName, retryIntervalInstanceStatus) +} + +func (s *Service) createInstanceImpl(eventObject runtime.Object, openStackCluster *infrav1.OpenStackCluster, instanceSpec *InstanceSpec, clusterName string, retryInterval time.Duration) (*InstanceStatus, error) { var server *ServerExt accessIPv4 := "" portList := []servers.Network{} @@ -217,11 +149,21 @@ func (s *Service) createInstance(eventObject runtime.Object, clusterName string, } if err := s.deletePorts(eventObject, portList); err != nil { - s.scope.Logger.V(4).Error(err, "failed to clean up ports after failure", "cluster", clusterName, "machine", instanceSpec.Name) + s.scope.Logger.V(4).Error(err, "Failed to clean up ports after failure") } }() - for i, network := range instanceSpec.Networks { + nets, err := s.constructNetworks(openStackCluster, instanceSpec) + if err != nil { + return nil, err + } + + securityGroups, err := s.networkingService.GetSecurityGroups(instanceSpec.SecurityGroups) + if err != nil { + return nil, fmt.Errorf("error getting security groups: %v", err) + } + + for i, network := range nets { if network.ID == "" { return nil, fmt.Errorf("no network was found or provided. Please check your machine configuration and try again") } @@ -230,7 +172,7 @@ func (s *Service) createInstance(eventObject runtime.Object, clusterName string, iTags = instanceSpec.Tags } portName := getPortName(instanceSpec.Name, network.PortOpts, i) - port, err := s.networkingService.GetOrCreatePort(eventObject, clusterName, portName, network, &instanceSpec.SecurityGroups, iTags) + port, err := s.networkingService.GetOrCreatePort(eventObject, clusterName, portName, network, &securityGroups, iTags) if err != nil { return nil, err } @@ -253,7 +195,7 @@ func (s *Service) createInstance(eventObject runtime.Object, clusterName string, AvailabilityZone: instanceSpec.FailureDomain, Networks: portList, UserData: []byte(instanceSpec.UserData), - SecurityGroups: instanceSpec.SecurityGroups, + SecurityGroups: securityGroups, Tags: instanceSpec.Tags, Metadata: instanceSpec.Metadata, ConfigDrive: &instanceSpec.ConfigDrive, @@ -565,7 +507,7 @@ func (s *Service) GetManagementPort(openStackCluster *infrav1.OpenStackCluster, return &allPorts[0], nil } -func (s *Service) DeleteInstance(eventObject runtime.Object, openStackMachineSpec *infrav1.OpenStackMachineSpec, instanceName string, instanceStatus *InstanceStatus) error { +func (s *Service) DeleteInstance(eventObject runtime.Object, instanceSpec *InstanceSpec, instanceStatus *InstanceStatus) error { if instanceStatus == nil { /* We create a boot-from-volume instance in 2 steps: @@ -585,9 +527,9 @@ func (s *Service) DeleteInstance(eventObject runtime.Object, openStackMachineSpe Note that we don't need to separately delete the root volume when deleting the instance because DeleteOnTermination will ensure it is deleted in that case. */ - rootVolume := openStackMachineSpec.RootVolume + rootVolume := instanceSpec.RootVolume if hasRootVolume(rootVolume) { - name := rootVolumeName(instanceName) + name := rootVolumeName(instanceSpec.Name) volume, err := s.getVolumeByName(name) if err != nil { return err @@ -753,23 +695,6 @@ func (s *Service) GetInstanceStatusByName(eventObject runtime.Object, name strin return nil, nil } -// deduplicate takes a slice of input strings and filters out any duplicate -// string occurrences, for example making ["a", "b", "a", "c"] become ["a", "b", -// "c"]. -func deduplicate(sequence []string) []string { - var unique []string - set := make(map[string]bool) - - for _, s := range sequence { - if _, ok := set[s]; !ok { - unique = append(unique, s) - set[s] = true - } - } - - return unique -} - func getTimeout(name string, timeout int) time.Duration { if v := os.Getenv(name); v != "" { timeout, err := strconv.Atoi(v) @@ -791,19 +716,3 @@ func (s *Service) isTrunkExtSupported() (trunknSupported bool, err error) { } return true, nil } - -// isTrunkConfigured verifies trunk configuration at instance and port levels, useful for avoiding multple api calls to verify trunk support. -func (s *Service) isTrunkConfigured(nets []infrav1.Network, instanceLevelTrunk bool) bool { - if instanceLevelTrunk { - return true - } - for _, net := range nets { - port := net.PortOpts - if port != nil { - if port.Trunk != nil && *port.Trunk { - return true - } - } - } - return false -} diff --git a/pkg/cloud/services/compute/instance_test.go b/pkg/cloud/services/compute/instance_test.go index cccb1418c6..6dc2db6e9f 100644 --- a/pkg/cloud/services/compute/instance_test.go +++ b/pkg/cloud/services/compute/instance_test.go @@ -33,7 +33,6 @@ import ( "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/trunks" "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" @@ -41,10 +40,8 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" gomegatypes "github.com/onsi/gomega/types" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/pointer" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking" @@ -525,7 +522,6 @@ const ( imageUUID = "652b5a05-27fa-41d4-ac82-3e63cf6f7ab7" flavorUUID = "6dc820db-f912-454e-a1e3-1081f3b8cc72" instanceUUID = "383a8ec1-b6ea-4493-99dd-fc790da04ba9" - extraSecurityGroupUUID = "514bb2d8-3390-4a3b-86a7-7864ba57b329" controlPlaneSecurityGroupUUID = "c9817a91-4821-42db-8367-2301002ab659" workerSecurityGroupUUID = "9c6c0d28-03c9-436c-815d-58440ac2c1c8" serverGroupUUID = "7b940d62-68ef-4e42-a76a-1a62e290509c" @@ -533,14 +529,12 @@ const ( openStackMachineName = "test-openstack-machine" portName = "test-openstack-machine-0" - namespace = "test-namespace" imageName = "test-image" flavorName = "test-flavor" sshKeyName = "test-ssh-key" + failureDomain = "test-failure-domain" ) -var failureDomain = "test-failure-domain" - func getDefaultOpenStackCluster() *infrav1.OpenStackCluster { return &infrav1.OpenStackCluster{ Spec: infrav1.OpenStackClusterSpec{}, @@ -557,40 +551,25 @@ func getDefaultOpenStackCluster() *infrav1.OpenStackCluster { } } -func getDefaultMachine() *clusterv1.Machine { - return &clusterv1.Machine{ - Spec: clusterv1.MachineSpec{ - FailureDomain: &failureDomain, - }, - } -} - -func getDefaultOpenStackMachine() *infrav1.OpenStackMachine { - return &infrav1.OpenStackMachine{ - ObjectMeta: metav1.ObjectMeta{ - Name: openStackMachineName, - Namespace: namespace, - }, - Spec: infrav1.OpenStackMachineSpec{ - // ProviderID is set by the controller - // InstanceID is set by the controller - // FloatingIP is only used by the cluster controller for the Bastion - // TODO: Test Networks, Ports, Subnet, and Trunk separately - CloudName: "test-cloud", - Flavor: flavorName, - Image: imageName, - SSHKeyName: sshKeyName, - Tags: []string{"test-tag"}, - ServerMetadata: map[string]string{ - "test-metadata": "test-value", - }, - ConfigDrive: pointer.BoolPtr(true), - ServerGroupID: serverGroupUUID, +func getDefaultInstanceSpec() *InstanceSpec { + return &InstanceSpec{ + Name: openStackMachineName, + Image: imageName, + Flavor: flavorName, + SSHKeyName: sshKeyName, + UserData: "user-data", + Metadata: map[string]string{ + "test-metadata": "test-value", }, + ConfigDrive: *pointer.BoolPtr(true), + FailureDomain: *pointer.StringPtr(failureDomain), + ServerGroupID: serverGroupUUID, + Tags: []string{"test-tag"}, + SecurityGroups: []infrav1.SecurityGroupParam{{UUID: workerSecurityGroupUUID}}, } } -func TestService_CreateInstance(t *testing.T) { +func TestService_ReconcileInstance(t *testing.T) { RegisterTestingT(t) getDefaultServerMap := func() map[string]interface{} { @@ -603,6 +582,9 @@ func TestService_CreateInstance(t *testing.T) { "imageRef": imageUUID, "flavorRef": flavorUUID, "availability_zone": failureDomain, + "security_groups": []map[string]interface{}{ + {"name": workerSecurityGroupUUID}, + }, "networks": []map[string]interface{}{ {"port": portUUID}, }, @@ -741,18 +723,14 @@ func TestService_CreateInstance(t *testing.T) { // ******************* tests := []struct { - name string - getMachine func() *clusterv1.Machine - getOpenStackCluster func() *infrav1.OpenStackCluster - getOpenStackMachine func() *infrav1.OpenStackMachine - expect func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) - wantErr bool + name string + getInstanceSpec func() *InstanceSpec + expect func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) + wantErr bool }{ { - name: "Defaults", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: getDefaultOpenStackMachine, + name: "Defaults", + getInstanceSpec: getDefaultInstanceSpec, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { expectUseExistingDefaultPort(networkRecorder) expectDefaultImageAndFlavor(computeRecorder) @@ -763,10 +741,8 @@ func TestService_CreateInstance(t *testing.T) { wantErr: false, }, { - name: "Delete ports on server create error", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: getDefaultOpenStackMachine, + name: "Delete ports on server create error", + getInstanceSpec: getDefaultInstanceSpec, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { expectUseExistingDefaultPort(networkRecorder) expectDefaultImageAndFlavor(computeRecorder) @@ -779,16 +755,14 @@ func TestService_CreateInstance(t *testing.T) { wantErr: true, }, { - name: "Delete previously created ports on port creation error", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: func() *infrav1.OpenStackMachine { - m := getDefaultOpenStackMachine() - m.Spec.Ports = []infrav1.PortOpts{ + name: "Delete previously created ports on port creation error", + getInstanceSpec: func() *InstanceSpec { + s := getDefaultInstanceSpec() + s.Ports = []infrav1.PortOpts{ {Description: "Test port 0"}, {Description: "Test port 1"}, } - return m + return s }, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { computeRecorder.ListImages(images.ListOpts{Name: imageName}).Return([]images.Image{{ID: imageUUID}}, nil) @@ -808,10 +782,8 @@ func TestService_CreateInstance(t *testing.T) { wantErr: true, }, { - name: "Poll until server is created", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: getDefaultOpenStackMachine, + name: "Poll until server is created", + getInstanceSpec: getDefaultInstanceSpec, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { expectUseExistingDefaultPort(networkRecorder) expectDefaultImageAndFlavor(computeRecorder) @@ -822,10 +794,8 @@ func TestService_CreateInstance(t *testing.T) { wantErr: false, }, { - name: "Server errors during creation", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: getDefaultOpenStackMachine, + name: "Server errors during creation", + getInstanceSpec: getDefaultInstanceSpec, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { expectUseExistingDefaultPort(networkRecorder) expectDefaultImageAndFlavor(computeRecorder) @@ -838,15 +808,13 @@ func TestService_CreateInstance(t *testing.T) { wantErr: true, }, { - name: "Boot from volume success", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: func() *infrav1.OpenStackMachine { - osMachine := getDefaultOpenStackMachine() - osMachine.Spec.RootVolume = &infrav1.RootVolume{ + name: "Boot from volume success", + getInstanceSpec: func() *InstanceSpec { + s := getDefaultInstanceSpec() + s.RootVolume = &infrav1.RootVolume{ Size: 50, } - return osMachine + return s }, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { expectUseExistingDefaultPort(networkRecorder) @@ -883,17 +851,15 @@ func TestService_CreateInstance(t *testing.T) { wantErr: false, }, { - name: "Boot from volume with explicit AZ and volume type", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: func() *infrav1.OpenStackMachine { - osMachine := getDefaultOpenStackMachine() - osMachine.Spec.RootVolume = &infrav1.RootVolume{ + name: "Boot from volume with explicit AZ and volume type", + getInstanceSpec: func() *InstanceSpec { + s := getDefaultInstanceSpec() + s.RootVolume = &infrav1.RootVolume{ Size: 50, AvailabilityZone: "test-alternate-az", VolumeType: "test-volume-type", } - return osMachine + return s }, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { expectUseExistingDefaultPort(networkRecorder) @@ -931,15 +897,13 @@ func TestService_CreateInstance(t *testing.T) { wantErr: false, }, { - name: "Boot from volume failure cleans up ports", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: func() *infrav1.OpenStackMachine { - osMachine := getDefaultOpenStackMachine() - osMachine.Spec.RootVolume = &infrav1.RootVolume{ + name: "Boot from volume failure cleans up ports", + getInstanceSpec: func() *InstanceSpec { + s := getDefaultInstanceSpec() + s.RootVolume = &infrav1.RootVolume{ Size: 50, } - return osMachine + return s }, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { expectUseExistingDefaultPort(networkRecorder) @@ -962,100 +926,14 @@ func TestService_CreateInstance(t *testing.T) { wantErr: true, }, { - name: "Set control plane security group", - getMachine: func() *clusterv1.Machine { - machine := getDefaultMachine() - machine.Labels = map[string]string{ - clusterv1.MachineControlPlaneLabelName: "true", - } - return machine - }, - getOpenStackCluster: func() *infrav1.OpenStackCluster { - osCluster := getDefaultOpenStackCluster() - osCluster.Spec.ManagedSecurityGroups = true - return osCluster - }, - getOpenStackMachine: getDefaultOpenStackMachine, - expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { - expectUseExistingDefaultPort(networkRecorder) - expectDefaultImageAndFlavor(computeRecorder) - - createMap := getDefaultServerMap() - serverMap := createMap["server"].(map[string]interface{}) - serverMap["security_groups"] = []map[string]interface{}{ - {"name": controlPlaneSecurityGroupUUID}, - } - expectCreateServer(computeRecorder, createMap, false) - expectServerPollSuccess(computeRecorder) - }, - wantErr: false, - }, - { - name: "Set worker security group", - getMachine: getDefaultMachine, - getOpenStackCluster: func() *infrav1.OpenStackCluster { - osCluster := getDefaultOpenStackCluster() - osCluster.Spec.ManagedSecurityGroups = true - return osCluster - }, - getOpenStackMachine: getDefaultOpenStackMachine, - expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { - expectUseExistingDefaultPort(networkRecorder) - expectDefaultImageAndFlavor(computeRecorder) - - createMap := getDefaultServerMap() - serverMap := createMap["server"].(map[string]interface{}) - serverMap["security_groups"] = []map[string]interface{}{ - {"name": workerSecurityGroupUUID}, - } - expectCreateServer(computeRecorder, createMap, false) - expectServerPollSuccess(computeRecorder) - }, - wantErr: false, - }, - { - name: "Set extra security group", - getMachine: getDefaultMachine, - getOpenStackCluster: func() *infrav1.OpenStackCluster { - osCluster := getDefaultOpenStackCluster() - osCluster.Spec.ManagedSecurityGroups = true - return osCluster - }, - getOpenStackMachine: func() *infrav1.OpenStackMachine { - osMachine := getDefaultOpenStackMachine() - osMachine.Spec.SecurityGroups = []infrav1.SecurityGroupParam{{UUID: extraSecurityGroupUUID}} - return osMachine - }, - expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { - expectUseExistingDefaultPort(networkRecorder) - expectDefaultImageAndFlavor(computeRecorder) - - // TODO: Shortcut this API call if security groups are passed by UUID - networkRecorder.ListSecGroup(groups.ListOpts{ID: extraSecurityGroupUUID}). - Return([]groups.SecGroup{{ID: extraSecurityGroupUUID}}, nil) - - createMap := getDefaultServerMap() - serverMap := createMap["server"].(map[string]interface{}) - serverMap["security_groups"] = []map[string]interface{}{ - {"name": extraSecurityGroupUUID}, - {"name": workerSecurityGroupUUID}, - } - expectCreateServer(computeRecorder, createMap, false) - expectServerPollSuccess(computeRecorder) - }, - wantErr: false, - }, - { - name: "Delete trunks on port creation error", - getMachine: getDefaultMachine, - getOpenStackCluster: getDefaultOpenStackCluster, - getOpenStackMachine: func() *infrav1.OpenStackMachine { - m := getDefaultOpenStackMachine() - m.Spec.Ports = []infrav1.PortOpts{ + name: "Delete trunks on port creation error", + getInstanceSpec: func() *InstanceSpec { + s := getDefaultInstanceSpec() + s.Ports = []infrav1.PortOpts{ {Description: "Test port 0", Trunk: pointer.BoolPtr(true)}, {Description: "Test port 1"}, } - return m + return s }, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { computeRecorder.ListImages(images.ListOpts{Name: imageName}).Return([]images.Image{{ID: imageUUID}}, nil) @@ -1129,7 +1007,7 @@ func TestService_CreateInstance(t *testing.T) { ), } // Call CreateInstance with a reduced retry interval to speed up the test - _, err := s.createInstanceImpl(tt.getOpenStackCluster(), tt.getMachine(), tt.getOpenStackMachine(), "cluster-name", "user-data", time.Second) + _, err := s.createInstanceImpl(&infrav1.OpenStackMachine{}, getDefaultOpenStackCluster(), tt.getInstanceSpec(), "cluster-name", time.Nanosecond) if (err != nil) != tt.wantErr { t.Errorf("Service.CreateInstance() error = %v, wantErr %v", err, tt.wantErr) return @@ -1141,15 +1019,6 @@ func TestService_CreateInstance(t *testing.T) { func TestService_DeleteInstance(t *testing.T) { RegisterTestingT(t) - const instanceUUID = "7b8a2800-c615-4f52-9b75-d2ba60a2af66" - const portUUID = "94f3e9cb-89d5-4313-ad6d-44035722342b" - - const instanceName = "test-instance" - - getEventObject := func() runtime.Object { - return &infrav1.OpenStackMachine{} - } - getDefaultInstanceStatus := func() *InstanceStatus { return &InstanceStatus{ server: &ServerExt{ @@ -1160,27 +1029,23 @@ func TestService_DeleteInstance(t *testing.T) { } } - getDefaultOpenStackMachineSpec := func() *infrav1.OpenStackMachineSpec { - return &getDefaultOpenStackMachine().Spec - } - // ******************* // START OF TEST CASES // ******************* tests := []struct { - name string - eventObject runtime.Object - getOpenStackMachineSpec func() *infrav1.OpenStackMachineSpec - getInstanceStatus func() *InstanceStatus - expect func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) - wantErr bool + name string + eventObject runtime.Object + instanceSpec func() *InstanceSpec + instanceStatus func() *InstanceStatus + expect func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) + wantErr bool }{ { - name: "Defaults", - eventObject: getEventObject(), - getOpenStackMachineSpec: getDefaultOpenStackMachineSpec, - getInstanceStatus: getDefaultInstanceStatus, + name: "Defaults", + eventObject: &infrav1.OpenStackMachine{}, + instanceSpec: getDefaultInstanceSpec, + instanceStatus: getDefaultInstanceStatus, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { computeRecorder.ListAttachedInterfaces(instanceUUID).Return([]attachinterfaces.Interface{ { @@ -1204,18 +1069,18 @@ func TestService_DeleteInstance(t *testing.T) { }, { name: "Dangling volume", - eventObject: getEventObject(), - getOpenStackMachineSpec: func() *infrav1.OpenStackMachineSpec { - spec := getDefaultOpenStackMachineSpec() + eventObject: &infrav1.OpenStackMachine{}, + instanceSpec: func() *InstanceSpec { + spec := getDefaultInstanceSpec() spec.RootVolume = &infrav1.RootVolume{ Size: 50, } return spec }, - getInstanceStatus: func() *InstanceStatus { return nil }, + instanceStatus: func() *InstanceStatus { return nil }, expect: func(computeRecorder *MockClientMockRecorder, networkRecorder *mock_networking.MockNetworkClientMockRecorder) { // Fetch volume by name - volumeName := fmt.Sprintf("%s-root", instanceName) + volumeName := fmt.Sprintf("%s-root", openStackMachineName) computeRecorder.ListVolumes(volumes.ListOpts{ AllTenants: false, Name: volumeName, @@ -1252,7 +1117,7 @@ func TestService_DeleteInstance(t *testing.T) { "", mockNetworkClient, logr.Discard(), ), } - if err := s.DeleteInstance(tt.eventObject, tt.getOpenStackMachineSpec(), instanceName, tt.getInstanceStatus()); (err != nil) != tt.wantErr { + if err := s.DeleteInstance(tt.eventObject, tt.instanceSpec(), tt.instanceStatus()); (err != nil) != tt.wantErr { t.Errorf("Service.DeleteInstance() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/pkg/cloud/services/compute/instance_types.go b/pkg/cloud/services/compute/instance_types.go index 495fca1f1c..7eefeb8384 100644 --- a/pkg/cloud/services/compute/instance_types.go +++ b/pkg/cloud/services/compute/instance_types.go @@ -46,8 +46,9 @@ type InstanceSpec struct { ServerGroupID string Trunk bool Tags []string - SecurityGroups []string - Networks []infrav1.Network + SecurityGroups []infrav1.SecurityGroupParam + Networks []infrav1.NetworkParam + Ports []infrav1.PortOpts } // InstanceIdentifier describes an instance which has not necessarily been fetched. diff --git a/pkg/cloud/services/loadbalancer/loadbalancer.go b/pkg/cloud/services/loadbalancer/loadbalancer.go index 42b53469f2..719ead1495 100644 --- a/pkg/cloud/services/loadbalancer/loadbalancer.go +++ b/pkg/cloud/services/loadbalancer/loadbalancer.go @@ -82,7 +82,7 @@ func (s *Service) ReconcileLoadBalancer(openStackCluster *infrav1.OpenStackClust } portList := []int{apiServerPort} - portList = append(portList, openStackCluster.Spec.APIServerLoadBalancerAdditionalPorts...) + portList = append(portList, openStackCluster.Spec.APIServerLoadBalancer.AdditionalPorts...) for _, port := range portList { lbPortObjectsName := fmt.Sprintf("%s-%d", loadBalancerName, port) listener, err := s.getOrCreateListener(openStackCluster, lbPortObjectsName, lb.ID, port) @@ -268,7 +268,7 @@ func (s *Service) ReconcileLoadBalancerMember(openStackCluster *infrav1.OpenStac lbID := openStackCluster.Status.Network.APIServerLoadBalancer.ID portList := []int{int(openStackCluster.Spec.ControlPlaneEndpoint.Port)} - portList = append(portList, openStackCluster.Spec.APIServerLoadBalancerAdditionalPorts...) + portList = append(portList, openStackCluster.Spec.APIServerLoadBalancer.AdditionalPorts...) for _, port := range portList { lbPortObjectsName := fmt.Sprintf("%s-%d", loadBalancerName, port) name := lbPortObjectsName + "-" + openStackMachine.Name @@ -392,7 +392,7 @@ func (s *Service) DeleteLoadBalancerMember(openStackCluster *infrav1.OpenStackCl lbID := lb.ID portList := []int{int(openStackCluster.Spec.ControlPlaneEndpoint.Port)} - portList = append(portList, openStackCluster.Spec.APIServerLoadBalancerAdditionalPorts...) + portList = append(portList, openStackCluster.Spec.APIServerLoadBalancer.AdditionalPorts...) for _, port := range portList { lbPortObjectsName := fmt.Sprintf("%s-%d", loadBalancerName, port) name := lbPortObjectsName + "-" + openStackMachine.Name diff --git a/pkg/cloud/services/networking/securitygroups.go b/pkg/cloud/services/networking/securitygroups.go index c339a629a4..7ed2b6968f 100644 --- a/pkg/cloud/services/networking/securitygroups.go +++ b/pkg/cloud/services/networking/securitygroups.go @@ -393,6 +393,15 @@ func (s *Service) generateDesiredSecGroups(openStackCluster *infrav1.OpenStackCl func (s *Service) GetSecurityGroups(securityGroupParams []infrav1.SecurityGroupParam) ([]string, error) { var sgIDs []string for _, sg := range securityGroupParams { + // Don't validate an explicit UUID if we were given one + if sg.UUID != "" { + if isDuplicate(sgIDs, sg.UUID) { + continue + } + sgIDs = append(sgIDs, sg.UUID) + continue + } + listOpts := groups.ListOpts(sg.Filter) if listOpts.ProjectID == "" { listOpts.ProjectID = s.projectID diff --git a/scripts/go_install.sh b/scripts/go_install.sh new file mode 100755 index 0000000000..415e06d801 --- /dev/null +++ b/scripts/go_install.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +if [ -z "${1}" ]; then + echo "must provide module as first parameter" + exit 1 +fi + +if [ -z "${2}" ]; then + echo "must provide binary name as second parameter" + exit 1 +fi + +if [ -z "${3}" ]; then + echo "must provide version as third parameter" + exit 1 +fi + +if [ -z "${GOBIN}" ]; then + echo "GOBIN is not set. Must set GOBIN to install the bin in a specified directory." + exit 1 +fi + +rm "${GOBIN}/${2}"* || true + +# install the golang module specified as the first argument +go install -tags tools "${1}@${3}" +mv "${GOBIN}/${2}" "${GOBIN}/${2}-${3}" +ln -sf "${GOBIN}/${2}-${3}" "${GOBIN}/${2}" diff --git a/templates/cluster-template-external-cloud-provider.yaml b/templates/cluster-template-external-cloud-provider.yaml index b7476163f3..be88a33234 100644 --- a/templates/cluster-template-external-cloud-provider.yaml +++ b/templates/cluster-template-external-cloud-provider.yaml @@ -26,7 +26,8 @@ spec: identityRef: name: ${CLUSTER_NAME}-cloud-config kind: Secret - managedAPIServerLoadBalancer: true + apiServerLoadBalancer: + enabled: true managedSecurityGroups: true nodeCidr: 10.6.0.0/24 dnsNameservers: diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index d384524841..86fc41342d 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -26,7 +26,8 @@ spec: identityRef: name: ${CLUSTER_NAME}-cloud-config kind: Secret - managedAPIServerLoadBalancer: true + apiServerLoadBalancer: + enabled: true managedSecurityGroups: true nodeCidr: 10.6.0.0/24 dnsNameservers: diff --git a/test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml b/test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml index c7cc2067ea..d87eb4bc2d 100644 --- a/test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml +++ b/test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml @@ -31,7 +31,8 @@ spec: kind: Secret controlPlaneAvailabilityZones: - ${OPENSTACK_FAILURE_DOMAIN} - managedAPIServerLoadBalancer: true + apiServerLoadBalancer: + enabled: true managedSecurityGroups: true allowAllInClusterTraffic: true nodeCidr: 10.6.0.0/24 diff --git a/test/e2e/data/infrastructure-openstack/cluster-template-multi-az.yaml b/test/e2e/data/infrastructure-openstack/cluster-template-multi-az.yaml index 51ecf3d3d9..2a9278a9ca 100644 --- a/test/e2e/data/infrastructure-openstack/cluster-template-multi-az.yaml +++ b/test/e2e/data/infrastructure-openstack/cluster-template-multi-az.yaml @@ -31,7 +31,8 @@ spec: controlPlaneAvailabilityZones: - ${OPENSTACK_FAILURE_DOMAIN} - ${OPENSTACK_FAILURE_DOMAIN_ALT} - managedAPIServerLoadBalancer: true + apiServerLoadBalancer: + enabled: true managedSecurityGroups: true nodeCidr: 10.6.0.0/24 dnsNameservers: diff --git a/test/e2e/data/infrastructure-openstack/cluster-template-multi-network.yaml b/test/e2e/data/infrastructure-openstack/cluster-template-multi-network.yaml index 03bf1600e6..b6be426468 100644 --- a/test/e2e/data/infrastructure-openstack/cluster-template-multi-network.yaml +++ b/test/e2e/data/infrastructure-openstack/cluster-template-multi-network.yaml @@ -30,7 +30,8 @@ spec: kind: Secret controlPlaneAvailabilityZones: - ${OPENSTACK_FAILURE_DOMAIN} - managedAPIServerLoadBalancer: true + apiServerLoadBalancer: + enabled: true managedSecurityGroups: true nodeCidr: 10.6.0.0/24 dnsNameservers: diff --git a/test/e2e/data/infrastructure-openstack/cluster-template.yaml b/test/e2e/data/infrastructure-openstack/cluster-template.yaml index a99aa45a7d..5ad34089bd 100644 --- a/test/e2e/data/infrastructure-openstack/cluster-template.yaml +++ b/test/e2e/data/infrastructure-openstack/cluster-template.yaml @@ -32,7 +32,8 @@ spec: kind: Secret controlPlaneAvailabilityZones: - ${OPENSTACK_FAILURE_DOMAIN} - managedAPIServerLoadBalancer: true + apiServerLoadBalancer: + enabled: true managedSecurityGroups: true nodeCidr: 10.6.0.0/24 dnsNameservers: