From c999a5ef3982142870ad2f0cec838fa49a6caaaa Mon Sep 17 00:00:00 2001 From: didil <1284255+didil@users.noreply.github.com> Date: Fri, 24 Feb 2023 11:22:09 +0000 Subject: [PATCH 1/6] init new kind --- PROJECT | 8 ++ api/v1alpha1/loadbalancerconfig_types.go | 64 +++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 89 +++++++++++++++++++ config/crd/kustomization.yaml | 3 + .../cainjection_in_loadbalancerconfigs.yaml | 7 ++ .../webhook_in_loadbalancerconfigs.yaml | 16 ++++ .../rbac/loadbalancerconfig_editor_role.yaml | 31 +++++++ .../rbac/loadbalancerconfig_viewer_role.yaml | 27 ++++++ .../lb_v1alpha1_loadbalancerconfig.yaml | 12 +++ go.mod | 4 +- 10 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 api/v1alpha1/loadbalancerconfig_types.go create mode 100644 config/crd/patches/cainjection_in_loadbalancerconfigs.yaml create mode 100644 config/crd/patches/webhook_in_loadbalancerconfigs.yaml create mode 100644 config/rbac/loadbalancerconfig_editor_role.yaml create mode 100644 config/rbac/loadbalancerconfig_viewer_role.yaml create mode 100644 config/samples/lb_v1alpha1_loadbalancerconfig.yaml diff --git a/PROJECT b/PROJECT index 23b8dff..6cd728a 100644 --- a/PROJECT +++ b/PROJECT @@ -17,4 +17,12 @@ resources: kind: LoadBalancer path: github.com/didil/paperlb/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + domain: paperlb.com + group: lb + kind: LoadBalancerConfig + path: github.com/didil/paperlb/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/v1alpha1/loadbalancerconfig_types.go b/api/v1alpha1/loadbalancerconfig_types.go new file mode 100644 index 0000000..c70d4bb --- /dev/null +++ b/api/v1alpha1/loadbalancerconfig_types.go @@ -0,0 +1,64 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// LoadBalancerConfigSpec defines the desired state of LoadBalancerConfig +type LoadBalancerConfigSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Foo is an example field of LoadBalancerConfig. Edit loadbalancerconfig_types.go to remove/update + Foo string `json:"foo,omitempty"` +} + +// LoadBalancerConfigStatus defines the observed state of LoadBalancerConfig +type LoadBalancerConfigStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// LoadBalancerConfig is the Schema for the loadbalancerconfigs API +type LoadBalancerConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LoadBalancerConfigSpec `json:"spec,omitempty"` + Status LoadBalancerConfigStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// LoadBalancerConfigList contains a list of LoadBalancerConfig +type LoadBalancerConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LoadBalancerConfig `json:"items"` +} + +func init() { + SchemeBuilder.Register(&LoadBalancerConfig{}, &LoadBalancerConfigList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index dca9b5e..83630a7 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -67,6 +67,95 @@ func (in *LoadBalancer) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfig) DeepCopyInto(out *LoadBalancerConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfig. +func (in *LoadBalancerConfig) DeepCopy() *LoadBalancerConfig { + if in == nil { + return nil + } + out := new(LoadBalancerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LoadBalancerConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigList) DeepCopyInto(out *LoadBalancerConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LoadBalancerConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigList. +func (in *LoadBalancerConfigList) DeepCopy() *LoadBalancerConfigList { + if in == nil { + return nil + } + out := new(LoadBalancerConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LoadBalancerConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigSpec) DeepCopyInto(out *LoadBalancerConfigSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigSpec. +func (in *LoadBalancerConfigSpec) DeepCopy() *LoadBalancerConfigSpec { + if in == nil { + return nil + } + out := new(LoadBalancerConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigStatus) DeepCopyInto(out *LoadBalancerConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigStatus. +func (in *LoadBalancerConfigStatus) DeepCopy() *LoadBalancerConfigStatus { + if in == nil { + return nil + } + out := new(LoadBalancerConfigStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerList) DeepCopyInto(out *LoadBalancerList) { *out = *in diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 7ecd1a5..4dfb23c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,17 +3,20 @@ # It should be run by config/default resources: - bases/lb.paperlb.com_loadbalancers.yaml +- bases/lb.paperlb.com_loadbalancerconfigs.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_loadbalancers.yaml +#- patches/webhook_in_loadbalancerconfigs.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_loadbalancers.yaml +#- patches/cainjection_in_loadbalancerconfigs.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_loadbalancerconfigs.yaml b/config/crd/patches/cainjection_in_loadbalancerconfigs.yaml new file mode 100644 index 0000000..90aca37 --- /dev/null +++ b/config/crd/patches/cainjection_in_loadbalancerconfigs.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: loadbalancerconfigs.lb.paperlb.com diff --git a/config/crd/patches/webhook_in_loadbalancerconfigs.yaml b/config/crd/patches/webhook_in_loadbalancerconfigs.yaml new file mode 100644 index 0000000..4e2af14 --- /dev/null +++ b/config/crd/patches/webhook_in_loadbalancerconfigs.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: loadbalancerconfigs.lb.paperlb.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/loadbalancerconfig_editor_role.yaml b/config/rbac/loadbalancerconfig_editor_role.yaml new file mode 100644 index 0000000..628115e --- /dev/null +++ b/config/rbac/loadbalancerconfig_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit loadbalancerconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: loadbalancerconfig-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: paperlb + app.kubernetes.io/part-of: paperlb + app.kubernetes.io/managed-by: kustomize + name: loadbalancerconfig-editor-role +rules: +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs/status + verbs: + - get diff --git a/config/rbac/loadbalancerconfig_viewer_role.yaml b/config/rbac/loadbalancerconfig_viewer_role.yaml new file mode 100644 index 0000000..8fc7816 --- /dev/null +++ b/config/rbac/loadbalancerconfig_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view loadbalancerconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: loadbalancerconfig-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: paperlb + app.kubernetes.io/part-of: paperlb + app.kubernetes.io/managed-by: kustomize + name: loadbalancerconfig-viewer-role +rules: +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs + verbs: + - get + - list + - watch +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs/status + verbs: + - get diff --git a/config/samples/lb_v1alpha1_loadbalancerconfig.yaml b/config/samples/lb_v1alpha1_loadbalancerconfig.yaml new file mode 100644 index 0000000..f3079d0 --- /dev/null +++ b/config/samples/lb_v1alpha1_loadbalancerconfig.yaml @@ -0,0 +1,12 @@ +apiVersion: lb.paperlb.com/v1alpha1 +kind: LoadBalancerConfig +metadata: + labels: + app.kubernetes.io/name: loadbalancerconfig + app.kubernetes.io/instance: loadbalancerconfig-sample + app.kubernetes.io/part-of: paperlb + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: paperlb + name: loadbalancerconfig-sample +spec: + # TODO(user): Add fields here diff --git a/go.mod b/go.mod index 68106b8..a019fe9 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/didil/paperlb go 1.19 require ( + github.com/go-logr/logr v1.2.3 + github.com/golang/mock v1.6.0 github.com/onsi/ginkgo/v2 v2.6.0 github.com/onsi/gomega v1.24.1 github.com/pkg/errors v0.9.1 @@ -20,14 +22,12 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect From 52839ef055bc8eea04eba0fe65137ae4d5617758 Mon Sep 17 00:00:00 2001 From: didil <1284255+didil@users.noreply.github.com> Date: Fri, 24 Feb 2023 12:51:46 +0000 Subject: [PATCH 2/6] wip adding load balancer config crd --- api/v1alpha1/loadbalancer_types.go | 3 + api/v1alpha1/loadbalancerconfig_types.go | 30 ++++++-- api/v1alpha1/zz_generated.deepcopy.go | 16 +++++ .../lb.paperlb.com_loadbalancerconfigs.yaml | 68 +++++++++++++++++++ .../bases/lb.paperlb.com_loadbalancers.yaml | 3 + config/rbac/role.yaml | 34 ++++++++++ controllers/service_controller.go | 63 +++++++++++++++-- demo/load_balancer_config.yaml | 13 ++++ 8 files changed, 219 insertions(+), 11 deletions(-) create mode 100644 config/crd/bases/lb.paperlb.com_loadbalancerconfigs.yaml create mode 100644 demo/load_balancer_config.yaml diff --git a/api/v1alpha1/loadbalancer_types.go b/api/v1alpha1/loadbalancer_types.go index e7d0931..0e649cb 100644 --- a/api/v1alpha1/loadbalancer_types.go +++ b/api/v1alpha1/loadbalancer_types.go @@ -25,6 +25,9 @@ import ( // LoadBalancerSpec defines the desired state of LoadBalancer type LoadBalancerSpec struct { + // ConfigName is the loadbalancer config name + // +kubebuilder:validation:Required + ConfigName string `json:"configName,omitempty"` // HTTPUpdater is the http updater // +kubebuilder:validation:Required HTTPUpdater HTTPUpdater `json:"httpUpdater,omitempty"` diff --git a/api/v1alpha1/loadbalancerconfig_types.go b/api/v1alpha1/loadbalancerconfig_types.go index c70d4bb..1442d52 100644 --- a/api/v1alpha1/loadbalancerconfig_types.go +++ b/api/v1alpha1/loadbalancerconfig_types.go @@ -20,22 +20,38 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. // LoadBalancerConfigSpec defines the desired state of LoadBalancerConfig type LoadBalancerConfigSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file + // Default defines if this config is the default config + // +kubebuilder:validation:Required + Default bool `json:"default,omitempty"` + // HTTPUpdaterURL is the http updater url + // +kubebuilder:validation:Required + HTTPUpdaterURL string `json:"httpUpdaterURL,omitempty"` + // Host is the load balancer host + // +kubebuilder:validation:Required + Host string `json:"host,omitempty"` + // PortRange is the load balancer port range + // +kubebuilder:validation:Required + PortRange PortRange `json:"portRange,omitempty"` +} - // Foo is an example field of LoadBalancerConfig. Edit loadbalancerconfig_types.go to remove/update - Foo string `json:"foo,omitempty"` +// PortRange defines the load balancer port range +type PortRange struct { + // Low is the lower limit of the port range + // +kubebuilder:validation:Required + // +kubebuilder:validation:Minimum=1 + Low int `json:"low,omitempty"` + // High is the higher limit of the port range + // +kubebuilder:validation:Required + // +kubebuilder:validation:Maximum=65535 + High int `json:"high,omitempty"` } // LoadBalancerConfigStatus defines the observed state of LoadBalancerConfig type LoadBalancerConfigStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 83630a7..0e3b6f4 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -129,6 +129,7 @@ func (in *LoadBalancerConfigList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerConfigSpec) DeepCopyInto(out *LoadBalancerConfigSpec) { *out = *in + out.PortRange = in.PortRange } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigSpec. @@ -224,6 +225,21 @@ func (in *LoadBalancerStatus) DeepCopy() *LoadBalancerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortRange) DeepCopyInto(out *PortRange) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRange. +func (in *PortRange) DeepCopy() *PortRange { + if in == nil { + return nil + } + out := new(PortRange) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Target) DeepCopyInto(out *Target) { *out = *in diff --git a/config/crd/bases/lb.paperlb.com_loadbalancerconfigs.yaml b/config/crd/bases/lb.paperlb.com_loadbalancerconfigs.yaml new file mode 100644 index 0000000..1ce33ea --- /dev/null +++ b/config/crd/bases/lb.paperlb.com_loadbalancerconfigs.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: loadbalancerconfigs.lb.paperlb.com +spec: + group: lb.paperlb.com + names: + kind: LoadBalancerConfig + listKind: LoadBalancerConfigList + plural: loadbalancerconfigs + singular: loadbalancerconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: LoadBalancerConfig is the Schema for the loadbalancerconfigs + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: LoadBalancerConfigSpec defines the desired state of LoadBalancerConfig + properties: + default: + description: Default defines if this config is the default config + type: boolean + host: + description: Host is the load balancer host + type: string + httpUpdaterURL: + description: HTTPUpdaterURL is the http updater url + type: string + portRange: + description: PortRange is the load balancer port range + properties: + high: + description: High is the higher limit of the port range + maximum: 65535 + type: integer + low: + description: Low is the lower limit of the port range + minimum: 1 + type: integer + type: object + type: object + status: + description: LoadBalancerConfigStatus defines the observed state of LoadBalancerConfig + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/lb.paperlb.com_loadbalancers.yaml b/config/crd/bases/lb.paperlb.com_loadbalancers.yaml index dbd24f0..1d8887f 100644 --- a/config/crd/bases/lb.paperlb.com_loadbalancers.yaml +++ b/config/crd/bases/lb.paperlb.com_loadbalancers.yaml @@ -51,6 +51,9 @@ spec: spec: description: LoadBalancerSpec defines the desired state of LoadBalancer properties: + configName: + description: ConfigName is the loadbalancer config name + type: string host: description: Host is the lb host type: string diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f43935e..aafc4ec 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,26 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs/status + verbs: + - get + - patch + - update - apiGroups: - lb.paperlb.com resources: @@ -31,6 +51,20 @@ rules: - get - patch - update +- apiGroups: + - v1 + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - v1 + resources: + - nodes/status + verbs: + - get - apiGroups: - v1 resources: diff --git a/controllers/service_controller.go b/controllers/service_controller.go index f9bc59a..183613d 100644 --- a/controllers/service_controller.go +++ b/controllers/service_controller.go @@ -37,7 +37,6 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" ) // ServiceReconciler reconciles a Service object @@ -49,8 +48,12 @@ type ServiceReconciler struct { //+kubebuilder:rbac:groups=v1,resources=services,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=v1,resources=services/status,verbs=get;update;patch //+kubebuilder:rbac:groups=v1,resources=services/finalizers,verbs=update +//+kubebuilder:rbac:groups=v1,resources=nodes,verbs=get;list;watch +//+kubebuilder:rbac:groups=v1,resources=nodes/status,verbs=get //+kubebuilder:rbac:groups=lb.paperlb.com,resources=loadbalancers,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=lb.paperlb.com,resources=loadbalancers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=lb.paperlb.com,resources=loadbalancerconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=lb.paperlb.com,resources=loadbalancerconfigs/status,verbs=get;update;patch func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) @@ -223,10 +226,10 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } -func (r *ServiceReconciler) findExternalIP(node *v1.Node) string { +func (r *ServiceReconciler) findExternalIP(node *corev1.Node) string { addrs := node.Status.Addresses for _, addr := range addrs { - if addr.Type == v1.NodeExternalIP { + if addr.Type == corev1.NodeExternalIP { return addr.Address } } @@ -244,7 +247,7 @@ func (r *ServiceReconciler) getTargets(logger logr.Logger, ctx context.Context, } // get nodes - nodes := &v1.NodeList{} + nodes := &corev1.NodeList{} err := r.List(ctx, nodes) if err != nil { logger.Error(err, "Failed to get nodes") @@ -341,9 +344,16 @@ func (r *ServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { return errors.Wrapf(err, "failed to create service type index") } + if err := r.createLoadBalancerConfigNameIndex(mgr); err != nil { + return errors.Wrapf(err, "failed to create load balancer config name index") + } + return ctrl.NewControllerManagedBy(mgr). For(&corev1.Service{}). + // watches node changes to be able to update targets Watches(&source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(r.mapNodeToServices)). + // watches config changes to be able to update load balancer params + Watches(&source.Kind{Type: &lbv1alpha1.LoadBalancerConfig{}}, handler.EnqueueRequestsFromMapFunc(r.mapLoadBalancerConfigToServices)). Owns(&lbv1alpha1.LoadBalancer{}). Complete(r) } @@ -361,6 +371,19 @@ func (r *ServiceReconciler) createServiceTypeIndex(mgr ctrl.Manager) error { }) } +const loadBalancerConfigNameIndexField = ".spec.ConfigName" + +func (r *ServiceReconciler) createLoadBalancerConfigNameIndex(mgr ctrl.Manager) error { + return mgr.GetFieldIndexer().IndexField( + context.Background(), + &lbv1alpha1.LoadBalancer{}, + loadBalancerConfigNameIndexField, + func(object client.Object) []string { + lb := object.(*lbv1alpha1.LoadBalancer) + return []string{string(lb.Spec.ConfigName)} + }) +} + func (r *ServiceReconciler) mapNodeToServices(object client.Object) []reconcile.Request { node := object.(*corev1.Node) @@ -384,5 +407,37 @@ func (r *ServiceReconciler) mapNodeToServices(object client.Object) []reconcile. } return requests +} + +func (r *ServiceReconciler) mapLoadBalancerConfigToServices(object client.Object) []reconcile.Request { + lbConfig := object.(*lbv1alpha1.LoadBalancerConfig) + + ctx := context.Background() + logger := log.FromContext(ctx) + + lbList := &lbv1alpha1.LoadBalancerList{} + + err := r.List(context.Background(), lbList, client.MatchingFields{loadBalancerConfigNameIndexField: string(lbConfig.Name)}) + if err != nil { + logger.Error(err, "could not list load balancers", "lbConfigName", lbConfig.Name) + return nil + } + + requests := make([]reconcile.Request, 0, len(lbList.Items)) + + for _, lb := range lbList.Items { + ownerReferences := lb.GetOwnerReferences() + for _, ownerReference := range ownerReferences { + if ownerReference.Kind == "Service" { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: lb.Namespace, + Name: ownerReference.Name, + }, + }) + } + } + } + return requests } diff --git a/demo/load_balancer_config.yaml b/demo/load_balancer_config.yaml new file mode 100644 index 0000000..6145101 --- /dev/null +++ b/demo/load_balancer_config.yaml @@ -0,0 +1,13 @@ +apiVersion: lb.paperlb.com/v1alpha1 +kind: LoadBalancerConfig +meta: + name: default-lb-config + namespace: paperlb-system +spec: + default: true + httpUpdaterUrl: "http://192.168.64.1:3000/api/v1/lb" + host: "192.168.64.1" + portRange: + low: "8000" + high: "8100" + \ No newline at end of file From a44c101ac7900fa0e8d5ebf5ce6304d09f39b96d Mon Sep 17 00:00:00 2001 From: didil <1284255+didil@users.noreply.github.com> Date: Fri, 24 Feb 2023 16:01:19 +0000 Subject: [PATCH 3/6] implementation with config crd --- controllers/service_controller.go | 165 ++++++++++++++++++++----- controllers/service_controller_test.go | 43 +++++-- 2 files changed, 165 insertions(+), 43 deletions(-) diff --git a/controllers/service_controller.go b/controllers/service_controller.go index 183613d..08b7248 100644 --- a/controllers/service_controller.go +++ b/controllers/service_controller.go @@ -18,7 +18,8 @@ package controllers import ( "context" - "strconv" + "fmt" + "sort" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -85,58 +86,74 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } - httpUpdaterURL := svc.Annotations[loadBalancerHttpUpdaterURLKey] + var loadBalancerConfig *lbv1alpha1.LoadBalancerConfig + if configName := svc.Annotations[loadBalancerConfigNameKey]; configName != "" { + // find config with the name specified + loadBalancerConfig, err = r.getLoadBalancerConfigByName(ctx, configName) + if err != nil { + logger.Error(err, "Failed to get config by name", "loadBalancerConfigName", configName, "error", err) + return ctrl.Result{}, err + } + if loadBalancerConfig == nil { + logger.Error(err, "Specified load balancer config not found", "loadBalancerConfigName", configName) + return ctrl.Result{}, nil + } + } else { + // find default config + loadBalancerConfig, err = r.getDefaultLoadBalancerConfig(ctx) + if err != nil { + logger.Error(err, "Failed to get default config", "error", err) + return ctrl.Result{}, err + } + if loadBalancerConfig == nil { + logger.Error(err, "Default load balancer config not found") + return ctrl.Result{}, nil + } + } + + httpUpdaterURL := loadBalancerConfig.Spec.HTTPUpdaterURL if httpUpdaterURL == "" { // no http updater url set - logger.Info("No http updater url set for PaperLB load balancer") + logger.Info("No http updater url set in load balancer config", "loadBalancerConfigName", loadBalancerConfig.Name) return ctrl.Result{}, nil } - loadBalancerHost := svc.Annotations[loadBalancerHostKey] + loadBalancerHost := loadBalancerConfig.Spec.Host if loadBalancerHost == "" { // no host set - logger.Info("No Host Set for PaperLB load balancer") + logger.Info("No host set in load balancer config", "loadBalancerConfigName", loadBalancerConfig.Name) return ctrl.Result{}, nil } - loadBalancerPort := svc.Annotations[loadBalancerPortKey] - if loadBalancerPort == "" { - // no port set - logger.Info("No Port Set for PaperLB load balancer") + if len(svc.Spec.Ports) == 0 { + // no ports set + logger.Info("no ports set on service") return ctrl.Result{}, nil } - loadBalancerPortInt, err := strconv.ParseUint(loadBalancerPort, 10, 16) - if err != nil { - // port invalid - logger.Info("Invalid Port Set for PaperLB load balancer", "loadBalancerPort", loadBalancerPort) - return ctrl.Result{}, nil + loadBalancerProtocol := svc.Spec.Ports[0].Protocol + if loadBalancerProtocol == "" { + // defaults to TCP + loadBalancerProtocol = corev1.ProtocolTCP } - - loadBalancerProtocol := svc.Annotations[loadBalancerProtocolKey] // TCP, UDP or blank (defaults to TCP) are allowed - if loadBalancerProtocol != string(corev1.ProtocolTCP) && loadBalancerProtocol != string(corev1.ProtocolUDP) && loadBalancerProtocol != "" { + if loadBalancerProtocol != corev1.ProtocolTCP && loadBalancerProtocol != corev1.ProtocolUDP { // protocol invalid logger.Info("Invalid Protocol Set for PaperLB load balancer", "loadBalancerProtocol", loadBalancerProtocol) return ctrl.Result{}, nil } - if len(svc.Spec.Ports) == 0 { - // no ports set - logger.Info("no ports set on service") - return ctrl.Result{}, nil - } - targets, err := r.getTargets(logger, ctx, svc) if err != nil { return ctrl.Result{}, err } if targets == nil { // no targets, skip + return ctrl.Result{}, err } // Define new load balancer - lb, err := r.loadBalancerForService(svc, httpUpdaterURL, loadBalancerHost, int(loadBalancerPortInt), loadBalancerProtocol, targets) + lb, err := r.loadBalancerForService(svc, httpUpdaterURL, loadBalancerHost, string(loadBalancerProtocol), targets) if err != nil { logger.Error(err, "Failed to build new load balancer", "LoadBalancer.Name", svc.Name) return ctrl.Result{}, err @@ -147,6 +164,18 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct err = r.Get(ctx, types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}, existingLb) if err != nil { if apierrors.IsNotFound(err) { + loadBalancerPort, err := r.findAvailableLoadBalancerPort(ctx, loadBalancerConfig) + if err != nil { + logger.Error(err, "Failed to find available load balancer port", "loadBalancerConfigName", loadBalancerConfig.Name, "error", err) + return ctrl.Result{}, err + } + if loadBalancerPort == 0 { + logger.Info("No available load balancer port found", "loadBalancerConfigName", loadBalancerConfig.Name) + return ctrl.Result{}, nil + } + + lb.Spec.Port = loadBalancerPort + logger.Info("Creating a Load Balancer", "LoadBalancer.Name", lb.Name) err = r.Create(ctx, lb) if err != nil { @@ -162,7 +191,7 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, err } - if !equality.Semantic.DeepEqual(lb.Spec, existingLb.Spec) { + if r.lbNeedsUpdate(&logger, lb, existingLb, loadBalancerConfig) { logger.Info("Updating Load Balancer", "LoadBalancer.Name", existingLb.Name) existingLb.Spec = lb.Spec @@ -188,9 +217,9 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if existingLb.Status.Phase == lbv1alpha1.LoadBalancerPhaseReady { portStatus := corev1.PortStatus{} - portStatus.Port = int32(loadBalancerPortInt) + portStatus.Port = int32(existingLb.Spec.Port) - loadBalancerProtocol := svc.Annotations[loadBalancerProtocolKey] + loadBalancerProtocol := existingLb.Spec.Protocol switch corev1.Protocol(loadBalancerProtocol) { case corev1.ProtocolTCP: portStatus.Protocol = corev1.ProtocolTCP @@ -226,6 +255,80 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } +var paperLBSystemNamespaceName = "paperlb-system" + +func (r *ServiceReconciler) getLoadBalancerConfigByName(ctx context.Context, name string) (*lbv1alpha1.LoadBalancerConfig, error) { + config := &lbv1alpha1.LoadBalancerConfig{} + + err := r.Get(ctx, types.NamespacedName{Namespace: paperLBSystemNamespaceName, Name: name}, config) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, errors.Wrapf(err, "failed to fetch load balancer config") + } + + return config, nil +} + +func (r *ServiceReconciler) lbNeedsUpdate(logger *logr.Logger, lb, existingLb *lbv1alpha1.LoadBalancer, config *lbv1alpha1.LoadBalancerConfig) bool { + if existingLb.Spec.Port >= config.Spec.PortRange.Low && existingLb.Spec.Port <= config.Spec.PortRange.High { + // existing port is still in valid range + // keep the same port + lb.Spec.Port = existingLb.Spec.Port + } else { + logger.Info("Load balancer config update requires port change, this case is not supported at the moment, update skipped.") + return false + } + + return !equality.Semantic.DeepEqual(lb.Spec, existingLb.Spec) +} + +func (r *ServiceReconciler) getDefaultLoadBalancerConfig(ctx context.Context) (*lbv1alpha1.LoadBalancerConfig, error) { + configsList := &lbv1alpha1.LoadBalancerConfigList{} + err := r.List(ctx, configsList) + if err != nil { + return nil, errors.Wrapf(err, "failed to list configs") + } + if len(configsList.Items) == 0 { + return nil, nil + } + // sort configs by creation date to be able to take only the first one if multiple have default set to true + sort.Slice(configsList.Items, func(i, j int) bool { + return configsList.Items[i].CreationTimestamp.Before(&configsList.Items[j].CreationTimestamp) + }) + + for _, config := range configsList.Items { + if config.Spec.Default { + return &config, nil + } + } + + return nil, nil +} + +func (r *ServiceReconciler) findAvailableLoadBalancerPort(ctx context.Context, config *lbv1alpha1.LoadBalancerConfig) (int, error) { + lbList := &lbv1alpha1.LoadBalancerList{} + + err := r.List(context.Background(), lbList, client.MatchingFields{loadBalancerConfigNameIndexField: string(config.Name)}) + if err != nil { + return 0, fmt.Errorf("could not list load balancers for config name %s", config.Name) + } + used := map[int]bool{} + for _, lb := range lbList.Items { + used[lb.Spec.Port] = true + } + + for i := config.Spec.PortRange.Low; i <= config.Spec.PortRange.High; i++ { + if !used[i] { + return i, nil + } + } + + // no available port found + return 0, nil +} + func (r *ServiceReconciler) findExternalIP(node *corev1.Node) string { addrs := node.Status.Addresses for _, addr := range addrs { @@ -288,7 +391,7 @@ func (r *ServiceReconciler) isNodeReady(node *corev1.Node) bool { return false } -func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, httpUpdaterURL string, loadBalancerHost string, loadBalancerPortInt int, loadBalancerProtocol string, targets []lbv1alpha1.Target) (*lbv1alpha1.LoadBalancer, error) { +func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, httpUpdaterURL string, loadBalancerHost string, loadBalancerProtocol string, targets []lbv1alpha1.Target) (*lbv1alpha1.LoadBalancer, error) { lb := &lbv1alpha1.LoadBalancer{ ObjectMeta: metav1.ObjectMeta{ Name: svc.Name, @@ -299,7 +402,6 @@ func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, httpUpda URL: httpUpdaterURL, }, Host: loadBalancerHost, - Port: loadBalancerPortInt, Protocol: loadBalancerProtocol, Targets: targets, }, @@ -313,10 +415,7 @@ func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, httpUpda return lb, nil } -const loadBalancerHttpUpdaterURLKey = "lb.paperlb.com/http-updater-url" -const loadBalancerHostKey = "lb.paperlb.com/load-balancer-host" -const loadBalancerPortKey = "lb.paperlb.com/load-balancer-port" -const loadBalancerProtocolKey = "lb.paperlb.com/load-balancer-protocol" +const loadBalancerConfigNameKey = "lb.paperlb.com/config-name" const paperLBloadBalancerClass = "lb.paperlb.com/paperlb-class" diff --git a/controllers/service_controller_test.go b/controllers/service_controller_test.go index 5f13140..bd8ef03 100644 --- a/controllers/service_controller_test.go +++ b/controllers/service_controller_test.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - "strconv" "time" "github.com/golang/mock/gomock" @@ -31,9 +30,11 @@ var _ = Describe("Service controller", func() { var gomockController *gomock.Controller var httpLbUpdaterClient *mocks.MockHTTPLBUpdaterClient var service *corev1.Service + var loadBalancerConfig *lbv1alpha1.LoadBalancerConfig var loadBalancer *lbv1alpha1.LoadBalancer var node1 *corev1.Node var node2 *corev1.Node + var paperLBSystemNamespace *corev1.Namespace BeforeEach(func() { gomockController = gomock.NewController(GinkgoT()) @@ -51,13 +52,22 @@ var _ = Describe("Service controller", func() { loadBalancerName := "test-service" updaterURL := "http://example.com/api/v1/lb" lbHost := "192.168.55.99" - lbPort := 8888 + lbPortLow := 9000 + lbPortHigh := 9050 + lbProtocol := "TCP" port := 8000 targetPort := 8100 nodePort := 30100 + paperLBSystemNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: paperLBSystemNamespaceName, + }, + } + Expect(k8sClient.Create(ctx, paperLBSystemNamespace)).Should(Succeed()) + nodeHost1 := "1.2.3.4" node1 = &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -80,16 +90,27 @@ var _ = Describe("Service controller", func() { } Expect(k8sClient.Create(ctx, node1)).Should(Succeed()) + loadBalancerConfig = &lbv1alpha1.LoadBalancerConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-lb-config", + Namespace: paperLBSystemNamespaceName, + }, + Spec: lbv1alpha1.LoadBalancerConfigSpec{ + Default: true, + HTTPUpdaterURL: updaterURL, + Host: lbHost, + PortRange: lbv1alpha1.PortRange{ + Low: lbPortLow, + High: lbPortHigh, + }, + }, + } + Expect(k8sClient.Create(ctx, loadBalancerConfig)).Should(Succeed()) + service = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, Namespace: namespaceName, - Annotations: map[string]string{ - "lb.paperlb.com/http-updater-url": updaterURL, - "lb.paperlb.com/load-balancer-host": lbHost, - "lb.paperlb.com/load-balancer-port": strconv.Itoa(lbPort), - "lb.paperlb.com/load-balancer-protocol": lbProtocol, - }, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -124,7 +145,7 @@ var _ = Describe("Service controller", func() { Expect(loadBalancer.Spec.HTTPUpdater.URL).To(Equal(updaterURL)) Expect(loadBalancer.Spec.Host).To(Equal(lbHost)) - Expect(loadBalancer.Spec.Port).To(Equal(lbPort)) + Expect(loadBalancer.Spec.Port).To(Equal(lbPortLow)) Expect(loadBalancer.Spec.Protocol).To(Equal(lbProtocol)) Expect(loadBalancer.Spec.Targets).To(HaveLen(1)) Expect(loadBalancer.Spec.Targets[0]).To(Equal(lbv1alpha1.Target{ @@ -174,7 +195,7 @@ var _ = Describe("Service controller", func() { Expect(loadBalancer.Spec.HTTPUpdater.URL).To(Equal(updaterURL)) Expect(loadBalancer.Spec.Host).To(Equal(lbHost)) - Expect(loadBalancer.Spec.Port).To(Equal(lbPort)) + Expect(loadBalancer.Spec.Port).To(Equal(lbPortLow)) Expect(loadBalancer.Spec.Protocol).To(Equal(lbProtocol)) Expect(loadBalancer.Spec.Targets).To(ContainElements( lbv1alpha1.Target{ @@ -194,6 +215,8 @@ var _ = Describe("Service controller", func() { Expect(k8sClient.Delete(ctx, node1)).Should(Succeed()) Expect(k8sClient.Delete(ctx, node2)).Should(Succeed()) Expect(k8sClient.Delete(ctx, loadBalancer)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, loadBalancerConfig)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, paperLBSystemNamespace)).Should(Succeed()) gomockController.Finish() }) }) From 9029c22b5c13dca9d6a977925212fd19217d71f4 Mon Sep 17 00:00:00 2001 From: didil <1284255+didil@users.noreply.github.com> Date: Fri, 24 Feb 2023 16:04:23 +0000 Subject: [PATCH 4/6] remove annotations from demo --- demo/service.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/demo/service.yaml b/demo/service.yaml index d4abb37..3c63155 100644 --- a/demo/service.yaml +++ b/demo/service.yaml @@ -4,11 +4,6 @@ metadata: labels: app: k8s-pod-info-api name: k8s-pod-info-api-service - annotations: - lb.paperlb.com/http-updater-url: "http://192.168.64.1:3000/api/v1/lb" - lb.paperlb.com/load-balancer-host: "192.168.64.1" - lb.paperlb.com/load-balancer-port: "8100" - lb.paperlb.com/load-balancer-protocol: "TCP" spec: ports: - port: 5000 From 379b77b9c76adc9af04cb0ca12f10307d6dcbe70 Mon Sep 17 00:00:00 2001 From: didil <1284255+didil@users.noreply.github.com> Date: Fri, 24 Feb 2023 17:05:59 +0000 Subject: [PATCH 5/6] fixes --- README.md | 60 +++++++++++++++++---- controllers/loadbalancer_controller_test.go | 2 + controllers/service_controller.go | 10 ++-- controllers/service_controller_test.go | 2 + demo/load_balancer_config.yaml | 12 +++-- 5 files changed, 66 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 32efec6..8c341e2 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ You’ll need a kubernetes cluster to run against. You can use a local cluster f **Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). ### Usage +You can find the full example in the demo/ directory Service example: ````yaml apiVersion: v1 @@ -57,11 +58,9 @@ metadata: labels: app: k8s-pod-info-api name: k8s-pod-info-api-service - annotations: - lb.paperlb.com/http-updater-url: "http://192.168.64.1:3000/api/v1/lb" - lb.paperlb.com/load-balancer-host: "192.168.64.1" - lb.paperlb.com/load-balancer-port: "8100" - lb.paperlb.com/load-balancer-protocol: "TCP" + #optional annotation to use a config different than the default config + #annotations: + # lb.paperlb.com/config-name: "my-special-config" spec: ports: - port: 5000 @@ -70,14 +69,53 @@ spec: selector: app: k8s-pod-info-api type: LoadBalancer + ```` + +LoadBalancerConfig example: +````yaml +apiVersion: lb.paperlb.com/v1alpha1 +kind: LoadBalancerConfig +metadata: + name: default-lb-config + namespace: paperlb-system +spec: + default: true + httpUpdaterURL: "http://192.168.64.1:3000/api/v1/lb" + host: "192.168.64.1" + portRange: + low: 8100 + high: 8200 +```` + +LoadBalancerConfig fields: +- `.spec.default`: "true" if this should be the default config, false otherwise +- `.spec.httpUpdaterURL`: URL where the http lb updater instance can be called. The API is explained here: https://github.com/didil/nginx-lb-updater#api +- `.spec.host`: Load Balancer Host +- `.spec.portRange`: The controller will select a load balancer port from this range +- `.spec.portRange.low`: Lowest of the available ports on the load balancer +- `.spec.portRange.high`: Highest of the available ports on the load balancer + +When you apply these manifests, a load balancer resource should be created. To get the load balancer connection info you can run: +````bash +$ k get loadbalancer k8s-pod-info-api-service +NAME HOST PORT PROTOCOL TARGETCOUNT STATUS +k8s-pod-info-api-service 192.168.64.1 8100 TCP 2 READY ```` -Annotations: -- `lb.paperlb.com/http-updater-url`: URL where the http lb updater instance can be called. The API is explained here: https://github.com/didil/nginx-lb-updater#api -- `lb.paperlb.com/load-balancer-host`: Load Balancer Host -- `lb.paperlb.com/load-balancer-port`: Load Balancer Port -- `lb.paperlb.com/load-balancer-protocol`: Load Balancer Protocol (`TCP` or `UDP`) - +Testing with Curl +````bash +$ curl -s 192.168.64.1:8100/api/v1/info|jq + "pod": { + "name": "k8s-pod-info-api-84dc7c9bdd-mz74t", + "ip": "10.42.0.27", + "namespace": "default", + "serviceAccountName": "default" + }, + "node": { + "name": "k3s-local-server" + } +} +```` ### Run tests diff --git a/controllers/loadbalancer_controller_test.go b/controllers/loadbalancer_controller_test.go index a65d882..fbe7e13 100644 --- a/controllers/loadbalancer_controller_test.go +++ b/controllers/loadbalancer_controller_test.go @@ -42,6 +42,7 @@ var _ = Describe("LoadBalancer controller", func() { ctx := context.Background() + loadBalancerConfigName := "my-load-balancer-config" loadBalancerName := "my-test-service" updaterURL := "http://example.com/api/v1/lb" lbHost := "192.168.55.99" @@ -71,6 +72,7 @@ var _ = Describe("LoadBalancer controller", func() { Namespace: namespaceName, }, Spec: lbv1alpha1.LoadBalancerSpec{ + ConfigName: loadBalancerConfigName, HTTPUpdater: lbv1alpha1.HTTPUpdater{ URL: updaterURL, }, diff --git a/controllers/service_controller.go b/controllers/service_controller.go index 08b7248..72ad2c9 100644 --- a/controllers/service_controller.go +++ b/controllers/service_controller.go @@ -106,7 +106,7 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, err } if loadBalancerConfig == nil { - logger.Error(err, "Default load balancer config not found") + logger.Info("Default load balancer config not found") return ctrl.Result{}, nil } } @@ -153,7 +153,7 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } // Define new load balancer - lb, err := r.loadBalancerForService(svc, httpUpdaterURL, loadBalancerHost, string(loadBalancerProtocol), targets) + lb, err := r.loadBalancerForService(svc, loadBalancerConfig.Name, httpUpdaterURL, loadBalancerHost, string(loadBalancerProtocol), targets) if err != nil { logger.Error(err, "Failed to build new load balancer", "LoadBalancer.Name", svc.Name) return ctrl.Result{}, err @@ -277,6 +277,7 @@ func (r *ServiceReconciler) lbNeedsUpdate(logger *logr.Logger, lb, existingLb *l // keep the same port lb.Spec.Port = existingLb.Spec.Port } else { + // load balancers need to be deleted manually in case of incompatible port changes on the config logger.Info("Load balancer config update requires port change, this case is not supported at the moment, update skipped.") return false } @@ -286,7 +287,7 @@ func (r *ServiceReconciler) lbNeedsUpdate(logger *logr.Logger, lb, existingLb *l func (r *ServiceReconciler) getDefaultLoadBalancerConfig(ctx context.Context) (*lbv1alpha1.LoadBalancerConfig, error) { configsList := &lbv1alpha1.LoadBalancerConfigList{} - err := r.List(ctx, configsList) + err := r.List(ctx, configsList, client.InNamespace(paperLBSystemNamespaceName)) if err != nil { return nil, errors.Wrapf(err, "failed to list configs") } @@ -391,13 +392,14 @@ func (r *ServiceReconciler) isNodeReady(node *corev1.Node) bool { return false } -func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, httpUpdaterURL string, loadBalancerHost string, loadBalancerProtocol string, targets []lbv1alpha1.Target) (*lbv1alpha1.LoadBalancer, error) { +func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, configName string, httpUpdaterURL string, loadBalancerHost string, loadBalancerProtocol string, targets []lbv1alpha1.Target) (*lbv1alpha1.LoadBalancer, error) { lb := &lbv1alpha1.LoadBalancer{ ObjectMeta: metav1.ObjectMeta{ Name: svc.Name, Namespace: svc.Namespace, }, Spec: lbv1alpha1.LoadBalancerSpec{ + ConfigName: configName, HTTPUpdater: lbv1alpha1.HTTPUpdater{ URL: httpUpdaterURL, }, diff --git a/controllers/service_controller_test.go b/controllers/service_controller_test.go index bd8ef03..0618d49 100644 --- a/controllers/service_controller_test.go +++ b/controllers/service_controller_test.go @@ -143,6 +143,7 @@ var _ = Describe("Service controller", func() { Expect(loadBalancer.OwnerReferences).To(HaveLen(1)) Expect(loadBalancer.OwnerReferences[0].UID).To(Equal(service.UID)) + Expect(loadBalancer.Spec.ConfigName).To(Equal(loadBalancerConfig.Name)) Expect(loadBalancer.Spec.HTTPUpdater.URL).To(Equal(updaterURL)) Expect(loadBalancer.Spec.Host).To(Equal(lbHost)) Expect(loadBalancer.Spec.Port).To(Equal(lbPortLow)) @@ -193,6 +194,7 @@ var _ = Describe("Service controller", func() { Expect(loadBalancer.OwnerReferences).To(HaveLen(1)) Expect(loadBalancer.OwnerReferences[0].UID).To(Equal(service.UID)) + Expect(loadBalancer.Spec.ConfigName).To(Equal(loadBalancerConfig.Name)) Expect(loadBalancer.Spec.HTTPUpdater.URL).To(Equal(updaterURL)) Expect(loadBalancer.Spec.Host).To(Equal(lbHost)) Expect(loadBalancer.Spec.Port).To(Equal(lbPortLow)) diff --git a/demo/load_balancer_config.yaml b/demo/load_balancer_config.yaml index 6145101..ffd0d5f 100644 --- a/demo/load_balancer_config.yaml +++ b/demo/load_balancer_config.yaml @@ -1,13 +1,15 @@ apiVersion: lb.paperlb.com/v1alpha1 kind: LoadBalancerConfig -meta: +metadata: name: default-lb-config namespace: paperlb-system + #optional annotation to use a config different than the default config + #annotations: + # lb.paperlb.com/config-name: "my-special-config" spec: default: true - httpUpdaterUrl: "http://192.168.64.1:3000/api/v1/lb" + httpUpdaterURL: "http://192.168.64.1:3000/api/v1/lb" host: "192.168.64.1" portRange: - low: "8000" - high: "8100" - \ No newline at end of file + low: 8100 + high: 8200 \ No newline at end of file From fe0e8c148b2a362eeff8a489b6236bd97455ac29 Mon Sep 17 00:00:00 2001 From: didil <1284255+didil@users.noreply.github.com> Date: Fri, 24 Feb 2023 17:08:01 +0000 Subject: [PATCH 6/6] fix readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8c341e2..5ab9982 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,9 @@ PaperLB is implemented as a kubernetes "Operator": The idea is: -- You create a Kubernetes LoadBalancer type service and add some PaperLB annotations -- The controller notices the service and annotations and creates a "LoadBalancer" object -- The controller notices the "LoadBalancer" object and updates your network load balancer using the config data from the annotations + the service/nodes info +- You create a Kubernetes LoadBalancer type service and a LoadBalancerConfig configuration object +- The controller notices the service and LoadBalancerConfig and creates a "LoadBalancer" object +- The controller notices the "LoadBalancer" object and updates your network load balancer using the config data + the service/nodes info ## Features - Works with TCP or UDP L4 load balancers