diff --git a/PROJECT b/PROJECT index 23b8dff..6cd728a 100644 --- a/PROJECT +++ b/PROJECT @@ -17,4 +17,12 @@ resources: kind: LoadBalancer path: github.com/didil/paperlb/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + domain: paperlb.com + group: lb + kind: LoadBalancerConfig + path: github.com/didil/paperlb/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/README.md b/README.md index 32efec6..5ab9982 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,9 @@ PaperLB is implemented as a kubernetes "Operator": The idea is: -- You create a Kubernetes LoadBalancer type service and add some PaperLB annotations -- The controller notices the service and annotations and creates a "LoadBalancer" object -- The controller notices the "LoadBalancer" object and updates your network load balancer using the config data from the annotations + the service/nodes info +- You create a Kubernetes LoadBalancer type service and a LoadBalancerConfig configuration object +- The controller notices the service and LoadBalancerConfig and creates a "LoadBalancer" object +- The controller notices the "LoadBalancer" object and updates your network load balancer using the config data + the service/nodes info ## Features - Works with TCP or UDP L4 load balancers @@ -49,6 +49,7 @@ You’ll need a kubernetes cluster to run against. You can use a local cluster f **Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). ### Usage +You can find the full example in the demo/ directory Service example: ````yaml apiVersion: v1 @@ -57,11 +58,9 @@ metadata: labels: app: k8s-pod-info-api name: k8s-pod-info-api-service - annotations: - lb.paperlb.com/http-updater-url: "http://192.168.64.1:3000/api/v1/lb" - lb.paperlb.com/load-balancer-host: "192.168.64.1" - lb.paperlb.com/load-balancer-port: "8100" - lb.paperlb.com/load-balancer-protocol: "TCP" + #optional annotation to use a config different than the default config + #annotations: + # lb.paperlb.com/config-name: "my-special-config" spec: ports: - port: 5000 @@ -70,14 +69,53 @@ spec: selector: app: k8s-pod-info-api type: LoadBalancer + ```` + +LoadBalancerConfig example: +````yaml +apiVersion: lb.paperlb.com/v1alpha1 +kind: LoadBalancerConfig +metadata: + name: default-lb-config + namespace: paperlb-system +spec: + default: true + httpUpdaterURL: "http://192.168.64.1:3000/api/v1/lb" + host: "192.168.64.1" + portRange: + low: 8100 + high: 8200 +```` + +LoadBalancerConfig fields: +- `.spec.default`: "true" if this should be the default config, false otherwise +- `.spec.httpUpdaterURL`: URL where the http lb updater instance can be called. The API is explained here: https://github.com/didil/nginx-lb-updater#api +- `.spec.host`: Load Balancer Host +- `.spec.portRange`: The controller will select a load balancer port from this range +- `.spec.portRange.low`: Lowest of the available ports on the load balancer +- `.spec.portRange.high`: Highest of the available ports on the load balancer + +When you apply these manifests, a load balancer resource should be created. To get the load balancer connection info you can run: +````bash +$ k get loadbalancer k8s-pod-info-api-service +NAME HOST PORT PROTOCOL TARGETCOUNT STATUS +k8s-pod-info-api-service 192.168.64.1 8100 TCP 2 READY ```` -Annotations: -- `lb.paperlb.com/http-updater-url`: URL where the http lb updater instance can be called. The API is explained here: https://github.com/didil/nginx-lb-updater#api -- `lb.paperlb.com/load-balancer-host`: Load Balancer Host -- `lb.paperlb.com/load-balancer-port`: Load Balancer Port -- `lb.paperlb.com/load-balancer-protocol`: Load Balancer Protocol (`TCP` or `UDP`) - +Testing with Curl +````bash +$ curl -s 192.168.64.1:8100/api/v1/info|jq + "pod": { + "name": "k8s-pod-info-api-84dc7c9bdd-mz74t", + "ip": "10.42.0.27", + "namespace": "default", + "serviceAccountName": "default" + }, + "node": { + "name": "k3s-local-server" + } +} +```` ### Run tests diff --git a/api/v1alpha1/loadbalancer_types.go b/api/v1alpha1/loadbalancer_types.go index e7d0931..0e649cb 100644 --- a/api/v1alpha1/loadbalancer_types.go +++ b/api/v1alpha1/loadbalancer_types.go @@ -25,6 +25,9 @@ import ( // LoadBalancerSpec defines the desired state of LoadBalancer type LoadBalancerSpec struct { + // ConfigName is the loadbalancer config name + // +kubebuilder:validation:Required + ConfigName string `json:"configName,omitempty"` // HTTPUpdater is the http updater // +kubebuilder:validation:Required HTTPUpdater HTTPUpdater `json:"httpUpdater,omitempty"` diff --git a/api/v1alpha1/loadbalancerconfig_types.go b/api/v1alpha1/loadbalancerconfig_types.go new file mode 100644 index 0000000..1442d52 --- /dev/null +++ b/api/v1alpha1/loadbalancerconfig_types.go @@ -0,0 +1,80 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// LoadBalancerConfigSpec defines the desired state of LoadBalancerConfig +type LoadBalancerConfigSpec struct { + // Default defines if this config is the default config + // +kubebuilder:validation:Required + Default bool `json:"default,omitempty"` + // HTTPUpdaterURL is the http updater url + // +kubebuilder:validation:Required + HTTPUpdaterURL string `json:"httpUpdaterURL,omitempty"` + // Host is the load balancer host + // +kubebuilder:validation:Required + Host string `json:"host,omitempty"` + // PortRange is the load balancer port range + // +kubebuilder:validation:Required + PortRange PortRange `json:"portRange,omitempty"` +} + +// PortRange defines the load balancer port range +type PortRange struct { + // Low is the lower limit of the port range + // +kubebuilder:validation:Required + // +kubebuilder:validation:Minimum=1 + Low int `json:"low,omitempty"` + // High is the higher limit of the port range + // +kubebuilder:validation:Required + // +kubebuilder:validation:Maximum=65535 + High int `json:"high,omitempty"` +} + +// LoadBalancerConfigStatus defines the observed state of LoadBalancerConfig +type LoadBalancerConfigStatus struct { +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// LoadBalancerConfig is the Schema for the loadbalancerconfigs API +type LoadBalancerConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LoadBalancerConfigSpec `json:"spec,omitempty"` + Status LoadBalancerConfigStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// LoadBalancerConfigList contains a list of LoadBalancerConfig +type LoadBalancerConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LoadBalancerConfig `json:"items"` +} + +func init() { + SchemeBuilder.Register(&LoadBalancerConfig{}, &LoadBalancerConfigList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index dca9b5e..0e3b6f4 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -67,6 +67,96 @@ func (in *LoadBalancer) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfig) DeepCopyInto(out *LoadBalancerConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfig. +func (in *LoadBalancerConfig) DeepCopy() *LoadBalancerConfig { + if in == nil { + return nil + } + out := new(LoadBalancerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LoadBalancerConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigList) DeepCopyInto(out *LoadBalancerConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LoadBalancerConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigList. +func (in *LoadBalancerConfigList) DeepCopy() *LoadBalancerConfigList { + if in == nil { + return nil + } + out := new(LoadBalancerConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LoadBalancerConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigSpec) DeepCopyInto(out *LoadBalancerConfigSpec) { + *out = *in + out.PortRange = in.PortRange +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigSpec. +func (in *LoadBalancerConfigSpec) DeepCopy() *LoadBalancerConfigSpec { + if in == nil { + return nil + } + out := new(LoadBalancerConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigStatus) DeepCopyInto(out *LoadBalancerConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigStatus. +func (in *LoadBalancerConfigStatus) DeepCopy() *LoadBalancerConfigStatus { + if in == nil { + return nil + } + out := new(LoadBalancerConfigStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerList) DeepCopyInto(out *LoadBalancerList) { *out = *in @@ -135,6 +225,21 @@ func (in *LoadBalancerStatus) DeepCopy() *LoadBalancerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortRange) DeepCopyInto(out *PortRange) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRange. +func (in *PortRange) DeepCopy() *PortRange { + if in == nil { + return nil + } + out := new(PortRange) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Target) DeepCopyInto(out *Target) { *out = *in diff --git a/config/crd/bases/lb.paperlb.com_loadbalancerconfigs.yaml b/config/crd/bases/lb.paperlb.com_loadbalancerconfigs.yaml new file mode 100644 index 0000000..1ce33ea --- /dev/null +++ b/config/crd/bases/lb.paperlb.com_loadbalancerconfigs.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: loadbalancerconfigs.lb.paperlb.com +spec: + group: lb.paperlb.com + names: + kind: LoadBalancerConfig + listKind: LoadBalancerConfigList + plural: loadbalancerconfigs + singular: loadbalancerconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: LoadBalancerConfig is the Schema for the loadbalancerconfigs + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: LoadBalancerConfigSpec defines the desired state of LoadBalancerConfig + properties: + default: + description: Default defines if this config is the default config + type: boolean + host: + description: Host is the load balancer host + type: string + httpUpdaterURL: + description: HTTPUpdaterURL is the http updater url + type: string + portRange: + description: PortRange is the load balancer port range + properties: + high: + description: High is the higher limit of the port range + maximum: 65535 + type: integer + low: + description: Low is the lower limit of the port range + minimum: 1 + type: integer + type: object + type: object + status: + description: LoadBalancerConfigStatus defines the observed state of LoadBalancerConfig + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/lb.paperlb.com_loadbalancers.yaml b/config/crd/bases/lb.paperlb.com_loadbalancers.yaml index dbd24f0..1d8887f 100644 --- a/config/crd/bases/lb.paperlb.com_loadbalancers.yaml +++ b/config/crd/bases/lb.paperlb.com_loadbalancers.yaml @@ -51,6 +51,9 @@ spec: spec: description: LoadBalancerSpec defines the desired state of LoadBalancer properties: + configName: + description: ConfigName is the loadbalancer config name + type: string host: description: Host is the lb host type: string diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 7ecd1a5..4dfb23c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,17 +3,20 @@ # It should be run by config/default resources: - bases/lb.paperlb.com_loadbalancers.yaml +- bases/lb.paperlb.com_loadbalancerconfigs.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_loadbalancers.yaml +#- patches/webhook_in_loadbalancerconfigs.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_loadbalancers.yaml +#- patches/cainjection_in_loadbalancerconfigs.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_loadbalancerconfigs.yaml b/config/crd/patches/cainjection_in_loadbalancerconfigs.yaml new file mode 100644 index 0000000..90aca37 --- /dev/null +++ b/config/crd/patches/cainjection_in_loadbalancerconfigs.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: loadbalancerconfigs.lb.paperlb.com diff --git a/config/crd/patches/webhook_in_loadbalancerconfigs.yaml b/config/crd/patches/webhook_in_loadbalancerconfigs.yaml new file mode 100644 index 0000000..4e2af14 --- /dev/null +++ b/config/crd/patches/webhook_in_loadbalancerconfigs.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: loadbalancerconfigs.lb.paperlb.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/loadbalancerconfig_editor_role.yaml b/config/rbac/loadbalancerconfig_editor_role.yaml new file mode 100644 index 0000000..628115e --- /dev/null +++ b/config/rbac/loadbalancerconfig_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit loadbalancerconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: loadbalancerconfig-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: paperlb + app.kubernetes.io/part-of: paperlb + app.kubernetes.io/managed-by: kustomize + name: loadbalancerconfig-editor-role +rules: +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs/status + verbs: + - get diff --git a/config/rbac/loadbalancerconfig_viewer_role.yaml b/config/rbac/loadbalancerconfig_viewer_role.yaml new file mode 100644 index 0000000..8fc7816 --- /dev/null +++ b/config/rbac/loadbalancerconfig_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view loadbalancerconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: loadbalancerconfig-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: paperlb + app.kubernetes.io/part-of: paperlb + app.kubernetes.io/managed-by: kustomize + name: loadbalancerconfig-viewer-role +rules: +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs + verbs: + - get + - list + - watch +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f43935e..aafc4ec 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,26 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - lb.paperlb.com + resources: + - loadbalancerconfigs/status + verbs: + - get + - patch + - update - apiGroups: - lb.paperlb.com resources: @@ -31,6 +51,20 @@ rules: - get - patch - update +- apiGroups: + - v1 + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - v1 + resources: + - nodes/status + verbs: + - get - apiGroups: - v1 resources: diff --git a/config/samples/lb_v1alpha1_loadbalancerconfig.yaml b/config/samples/lb_v1alpha1_loadbalancerconfig.yaml new file mode 100644 index 0000000..f3079d0 --- /dev/null +++ b/config/samples/lb_v1alpha1_loadbalancerconfig.yaml @@ -0,0 +1,12 @@ +apiVersion: lb.paperlb.com/v1alpha1 +kind: LoadBalancerConfig +metadata: + labels: + app.kubernetes.io/name: loadbalancerconfig + app.kubernetes.io/instance: loadbalancerconfig-sample + app.kubernetes.io/part-of: paperlb + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: paperlb + name: loadbalancerconfig-sample +spec: + # TODO(user): Add fields here diff --git a/controllers/loadbalancer_controller_test.go b/controllers/loadbalancer_controller_test.go index a65d882..fbe7e13 100644 --- a/controllers/loadbalancer_controller_test.go +++ b/controllers/loadbalancer_controller_test.go @@ -42,6 +42,7 @@ var _ = Describe("LoadBalancer controller", func() { ctx := context.Background() + loadBalancerConfigName := "my-load-balancer-config" loadBalancerName := "my-test-service" updaterURL := "http://example.com/api/v1/lb" lbHost := "192.168.55.99" @@ -71,6 +72,7 @@ var _ = Describe("LoadBalancer controller", func() { Namespace: namespaceName, }, Spec: lbv1alpha1.LoadBalancerSpec{ + ConfigName: loadBalancerConfigName, HTTPUpdater: lbv1alpha1.HTTPUpdater{ URL: updaterURL, }, diff --git a/controllers/service_controller.go b/controllers/service_controller.go index f9bc59a..72ad2c9 100644 --- a/controllers/service_controller.go +++ b/controllers/service_controller.go @@ -18,7 +18,8 @@ package controllers import ( "context" - "strconv" + "fmt" + "sort" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -37,7 +38,6 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" ) // ServiceReconciler reconciles a Service object @@ -49,8 +49,12 @@ type ServiceReconciler struct { //+kubebuilder:rbac:groups=v1,resources=services,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=v1,resources=services/status,verbs=get;update;patch //+kubebuilder:rbac:groups=v1,resources=services/finalizers,verbs=update +//+kubebuilder:rbac:groups=v1,resources=nodes,verbs=get;list;watch +//+kubebuilder:rbac:groups=v1,resources=nodes/status,verbs=get //+kubebuilder:rbac:groups=lb.paperlb.com,resources=loadbalancers,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=lb.paperlb.com,resources=loadbalancers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=lb.paperlb.com,resources=loadbalancerconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=lb.paperlb.com,resources=loadbalancerconfigs/status,verbs=get;update;patch func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) @@ -82,58 +86,74 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } - httpUpdaterURL := svc.Annotations[loadBalancerHttpUpdaterURLKey] + var loadBalancerConfig *lbv1alpha1.LoadBalancerConfig + if configName := svc.Annotations[loadBalancerConfigNameKey]; configName != "" { + // find config with the name specified + loadBalancerConfig, err = r.getLoadBalancerConfigByName(ctx, configName) + if err != nil { + logger.Error(err, "Failed to get config by name", "loadBalancerConfigName", configName, "error", err) + return ctrl.Result{}, err + } + if loadBalancerConfig == nil { + logger.Error(err, "Specified load balancer config not found", "loadBalancerConfigName", configName) + return ctrl.Result{}, nil + } + } else { + // find default config + loadBalancerConfig, err = r.getDefaultLoadBalancerConfig(ctx) + if err != nil { + logger.Error(err, "Failed to get default config", "error", err) + return ctrl.Result{}, err + } + if loadBalancerConfig == nil { + logger.Info("Default load balancer config not found") + return ctrl.Result{}, nil + } + } + + httpUpdaterURL := loadBalancerConfig.Spec.HTTPUpdaterURL if httpUpdaterURL == "" { // no http updater url set - logger.Info("No http updater url set for PaperLB load balancer") + logger.Info("No http updater url set in load balancer config", "loadBalancerConfigName", loadBalancerConfig.Name) return ctrl.Result{}, nil } - loadBalancerHost := svc.Annotations[loadBalancerHostKey] + loadBalancerHost := loadBalancerConfig.Spec.Host if loadBalancerHost == "" { // no host set - logger.Info("No Host Set for PaperLB load balancer") + logger.Info("No host set in load balancer config", "loadBalancerConfigName", loadBalancerConfig.Name) return ctrl.Result{}, nil } - loadBalancerPort := svc.Annotations[loadBalancerPortKey] - if loadBalancerPort == "" { - // no port set - logger.Info("No Port Set for PaperLB load balancer") + if len(svc.Spec.Ports) == 0 { + // no ports set + logger.Info("no ports set on service") return ctrl.Result{}, nil } - loadBalancerPortInt, err := strconv.ParseUint(loadBalancerPort, 10, 16) - if err != nil { - // port invalid - logger.Info("Invalid Port Set for PaperLB load balancer", "loadBalancerPort", loadBalancerPort) - return ctrl.Result{}, nil + loadBalancerProtocol := svc.Spec.Ports[0].Protocol + if loadBalancerProtocol == "" { + // defaults to TCP + loadBalancerProtocol = corev1.ProtocolTCP } - - loadBalancerProtocol := svc.Annotations[loadBalancerProtocolKey] // TCP, UDP or blank (defaults to TCP) are allowed - if loadBalancerProtocol != string(corev1.ProtocolTCP) && loadBalancerProtocol != string(corev1.ProtocolUDP) && loadBalancerProtocol != "" { + if loadBalancerProtocol != corev1.ProtocolTCP && loadBalancerProtocol != corev1.ProtocolUDP { // protocol invalid logger.Info("Invalid Protocol Set for PaperLB load balancer", "loadBalancerProtocol", loadBalancerProtocol) return ctrl.Result{}, nil } - if len(svc.Spec.Ports) == 0 { - // no ports set - logger.Info("no ports set on service") - return ctrl.Result{}, nil - } - targets, err := r.getTargets(logger, ctx, svc) if err != nil { return ctrl.Result{}, err } if targets == nil { // no targets, skip + return ctrl.Result{}, err } // Define new load balancer - lb, err := r.loadBalancerForService(svc, httpUpdaterURL, loadBalancerHost, int(loadBalancerPortInt), loadBalancerProtocol, targets) + lb, err := r.loadBalancerForService(svc, loadBalancerConfig.Name, httpUpdaterURL, loadBalancerHost, string(loadBalancerProtocol), targets) if err != nil { logger.Error(err, "Failed to build new load balancer", "LoadBalancer.Name", svc.Name) return ctrl.Result{}, err @@ -144,6 +164,18 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct err = r.Get(ctx, types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}, existingLb) if err != nil { if apierrors.IsNotFound(err) { + loadBalancerPort, err := r.findAvailableLoadBalancerPort(ctx, loadBalancerConfig) + if err != nil { + logger.Error(err, "Failed to find available load balancer port", "loadBalancerConfigName", loadBalancerConfig.Name, "error", err) + return ctrl.Result{}, err + } + if loadBalancerPort == 0 { + logger.Info("No available load balancer port found", "loadBalancerConfigName", loadBalancerConfig.Name) + return ctrl.Result{}, nil + } + + lb.Spec.Port = loadBalancerPort + logger.Info("Creating a Load Balancer", "LoadBalancer.Name", lb.Name) err = r.Create(ctx, lb) if err != nil { @@ -159,7 +191,7 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, err } - if !equality.Semantic.DeepEqual(lb.Spec, existingLb.Spec) { + if r.lbNeedsUpdate(&logger, lb, existingLb, loadBalancerConfig) { logger.Info("Updating Load Balancer", "LoadBalancer.Name", existingLb.Name) existingLb.Spec = lb.Spec @@ -185,9 +217,9 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if existingLb.Status.Phase == lbv1alpha1.LoadBalancerPhaseReady { portStatus := corev1.PortStatus{} - portStatus.Port = int32(loadBalancerPortInt) + portStatus.Port = int32(existingLb.Spec.Port) - loadBalancerProtocol := svc.Annotations[loadBalancerProtocolKey] + loadBalancerProtocol := existingLb.Spec.Protocol switch corev1.Protocol(loadBalancerProtocol) { case corev1.ProtocolTCP: portStatus.Protocol = corev1.ProtocolTCP @@ -223,10 +255,85 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } -func (r *ServiceReconciler) findExternalIP(node *v1.Node) string { +var paperLBSystemNamespaceName = "paperlb-system" + +func (r *ServiceReconciler) getLoadBalancerConfigByName(ctx context.Context, name string) (*lbv1alpha1.LoadBalancerConfig, error) { + config := &lbv1alpha1.LoadBalancerConfig{} + + err := r.Get(ctx, types.NamespacedName{Namespace: paperLBSystemNamespaceName, Name: name}, config) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, errors.Wrapf(err, "failed to fetch load balancer config") + } + + return config, nil +} + +func (r *ServiceReconciler) lbNeedsUpdate(logger *logr.Logger, lb, existingLb *lbv1alpha1.LoadBalancer, config *lbv1alpha1.LoadBalancerConfig) bool { + if existingLb.Spec.Port >= config.Spec.PortRange.Low && existingLb.Spec.Port <= config.Spec.PortRange.High { + // existing port is still in valid range + // keep the same port + lb.Spec.Port = existingLb.Spec.Port + } else { + // load balancers need to be deleted manually in case of incompatible port changes on the config + logger.Info("Load balancer config update requires port change, this case is not supported at the moment, update skipped.") + return false + } + + return !equality.Semantic.DeepEqual(lb.Spec, existingLb.Spec) +} + +func (r *ServiceReconciler) getDefaultLoadBalancerConfig(ctx context.Context) (*lbv1alpha1.LoadBalancerConfig, error) { + configsList := &lbv1alpha1.LoadBalancerConfigList{} + err := r.List(ctx, configsList, client.InNamespace(paperLBSystemNamespaceName)) + if err != nil { + return nil, errors.Wrapf(err, "failed to list configs") + } + if len(configsList.Items) == 0 { + return nil, nil + } + // sort configs by creation date to be able to take only the first one if multiple have default set to true + sort.Slice(configsList.Items, func(i, j int) bool { + return configsList.Items[i].CreationTimestamp.Before(&configsList.Items[j].CreationTimestamp) + }) + + for _, config := range configsList.Items { + if config.Spec.Default { + return &config, nil + } + } + + return nil, nil +} + +func (r *ServiceReconciler) findAvailableLoadBalancerPort(ctx context.Context, config *lbv1alpha1.LoadBalancerConfig) (int, error) { + lbList := &lbv1alpha1.LoadBalancerList{} + + err := r.List(context.Background(), lbList, client.MatchingFields{loadBalancerConfigNameIndexField: string(config.Name)}) + if err != nil { + return 0, fmt.Errorf("could not list load balancers for config name %s", config.Name) + } + used := map[int]bool{} + for _, lb := range lbList.Items { + used[lb.Spec.Port] = true + } + + for i := config.Spec.PortRange.Low; i <= config.Spec.PortRange.High; i++ { + if !used[i] { + return i, nil + } + } + + // no available port found + return 0, nil +} + +func (r *ServiceReconciler) findExternalIP(node *corev1.Node) string { addrs := node.Status.Addresses for _, addr := range addrs { - if addr.Type == v1.NodeExternalIP { + if addr.Type == corev1.NodeExternalIP { return addr.Address } } @@ -244,7 +351,7 @@ func (r *ServiceReconciler) getTargets(logger logr.Logger, ctx context.Context, } // get nodes - nodes := &v1.NodeList{} + nodes := &corev1.NodeList{} err := r.List(ctx, nodes) if err != nil { logger.Error(err, "Failed to get nodes") @@ -285,18 +392,18 @@ func (r *ServiceReconciler) isNodeReady(node *corev1.Node) bool { return false } -func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, httpUpdaterURL string, loadBalancerHost string, loadBalancerPortInt int, loadBalancerProtocol string, targets []lbv1alpha1.Target) (*lbv1alpha1.LoadBalancer, error) { +func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, configName string, httpUpdaterURL string, loadBalancerHost string, loadBalancerProtocol string, targets []lbv1alpha1.Target) (*lbv1alpha1.LoadBalancer, error) { lb := &lbv1alpha1.LoadBalancer{ ObjectMeta: metav1.ObjectMeta{ Name: svc.Name, Namespace: svc.Namespace, }, Spec: lbv1alpha1.LoadBalancerSpec{ + ConfigName: configName, HTTPUpdater: lbv1alpha1.HTTPUpdater{ URL: httpUpdaterURL, }, Host: loadBalancerHost, - Port: loadBalancerPortInt, Protocol: loadBalancerProtocol, Targets: targets, }, @@ -310,10 +417,7 @@ func (r *ServiceReconciler) loadBalancerForService(svc *corev1.Service, httpUpda return lb, nil } -const loadBalancerHttpUpdaterURLKey = "lb.paperlb.com/http-updater-url" -const loadBalancerHostKey = "lb.paperlb.com/load-balancer-host" -const loadBalancerPortKey = "lb.paperlb.com/load-balancer-port" -const loadBalancerProtocolKey = "lb.paperlb.com/load-balancer-protocol" +const loadBalancerConfigNameKey = "lb.paperlb.com/config-name" const paperLBloadBalancerClass = "lb.paperlb.com/paperlb-class" @@ -341,9 +445,16 @@ func (r *ServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { return errors.Wrapf(err, "failed to create service type index") } + if err := r.createLoadBalancerConfigNameIndex(mgr); err != nil { + return errors.Wrapf(err, "failed to create load balancer config name index") + } + return ctrl.NewControllerManagedBy(mgr). For(&corev1.Service{}). + // watches node changes to be able to update targets Watches(&source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(r.mapNodeToServices)). + // watches config changes to be able to update load balancer params + Watches(&source.Kind{Type: &lbv1alpha1.LoadBalancerConfig{}}, handler.EnqueueRequestsFromMapFunc(r.mapLoadBalancerConfigToServices)). Owns(&lbv1alpha1.LoadBalancer{}). Complete(r) } @@ -361,6 +472,19 @@ func (r *ServiceReconciler) createServiceTypeIndex(mgr ctrl.Manager) error { }) } +const loadBalancerConfigNameIndexField = ".spec.ConfigName" + +func (r *ServiceReconciler) createLoadBalancerConfigNameIndex(mgr ctrl.Manager) error { + return mgr.GetFieldIndexer().IndexField( + context.Background(), + &lbv1alpha1.LoadBalancer{}, + loadBalancerConfigNameIndexField, + func(object client.Object) []string { + lb := object.(*lbv1alpha1.LoadBalancer) + return []string{string(lb.Spec.ConfigName)} + }) +} + func (r *ServiceReconciler) mapNodeToServices(object client.Object) []reconcile.Request { node := object.(*corev1.Node) @@ -384,5 +508,37 @@ func (r *ServiceReconciler) mapNodeToServices(object client.Object) []reconcile. } return requests +} + +func (r *ServiceReconciler) mapLoadBalancerConfigToServices(object client.Object) []reconcile.Request { + lbConfig := object.(*lbv1alpha1.LoadBalancerConfig) + + ctx := context.Background() + logger := log.FromContext(ctx) + lbList := &lbv1alpha1.LoadBalancerList{} + + err := r.List(context.Background(), lbList, client.MatchingFields{loadBalancerConfigNameIndexField: string(lbConfig.Name)}) + if err != nil { + logger.Error(err, "could not list load balancers", "lbConfigName", lbConfig.Name) + return nil + } + + requests := make([]reconcile.Request, 0, len(lbList.Items)) + + for _, lb := range lbList.Items { + ownerReferences := lb.GetOwnerReferences() + for _, ownerReference := range ownerReferences { + if ownerReference.Kind == "Service" { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: lb.Namespace, + Name: ownerReference.Name, + }, + }) + } + } + } + + return requests } diff --git a/controllers/service_controller_test.go b/controllers/service_controller_test.go index 5f13140..0618d49 100644 --- a/controllers/service_controller_test.go +++ b/controllers/service_controller_test.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - "strconv" "time" "github.com/golang/mock/gomock" @@ -31,9 +30,11 @@ var _ = Describe("Service controller", func() { var gomockController *gomock.Controller var httpLbUpdaterClient *mocks.MockHTTPLBUpdaterClient var service *corev1.Service + var loadBalancerConfig *lbv1alpha1.LoadBalancerConfig var loadBalancer *lbv1alpha1.LoadBalancer var node1 *corev1.Node var node2 *corev1.Node + var paperLBSystemNamespace *corev1.Namespace BeforeEach(func() { gomockController = gomock.NewController(GinkgoT()) @@ -51,13 +52,22 @@ var _ = Describe("Service controller", func() { loadBalancerName := "test-service" updaterURL := "http://example.com/api/v1/lb" lbHost := "192.168.55.99" - lbPort := 8888 + lbPortLow := 9000 + lbPortHigh := 9050 + lbProtocol := "TCP" port := 8000 targetPort := 8100 nodePort := 30100 + paperLBSystemNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: paperLBSystemNamespaceName, + }, + } + Expect(k8sClient.Create(ctx, paperLBSystemNamespace)).Should(Succeed()) + nodeHost1 := "1.2.3.4" node1 = &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -80,16 +90,27 @@ var _ = Describe("Service controller", func() { } Expect(k8sClient.Create(ctx, node1)).Should(Succeed()) + loadBalancerConfig = &lbv1alpha1.LoadBalancerConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-lb-config", + Namespace: paperLBSystemNamespaceName, + }, + Spec: lbv1alpha1.LoadBalancerConfigSpec{ + Default: true, + HTTPUpdaterURL: updaterURL, + Host: lbHost, + PortRange: lbv1alpha1.PortRange{ + Low: lbPortLow, + High: lbPortHigh, + }, + }, + } + Expect(k8sClient.Create(ctx, loadBalancerConfig)).Should(Succeed()) + service = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, Namespace: namespaceName, - Annotations: map[string]string{ - "lb.paperlb.com/http-updater-url": updaterURL, - "lb.paperlb.com/load-balancer-host": lbHost, - "lb.paperlb.com/load-balancer-port": strconv.Itoa(lbPort), - "lb.paperlb.com/load-balancer-protocol": lbProtocol, - }, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -122,9 +143,10 @@ var _ = Describe("Service controller", func() { Expect(loadBalancer.OwnerReferences).To(HaveLen(1)) Expect(loadBalancer.OwnerReferences[0].UID).To(Equal(service.UID)) + Expect(loadBalancer.Spec.ConfigName).To(Equal(loadBalancerConfig.Name)) Expect(loadBalancer.Spec.HTTPUpdater.URL).To(Equal(updaterURL)) Expect(loadBalancer.Spec.Host).To(Equal(lbHost)) - Expect(loadBalancer.Spec.Port).To(Equal(lbPort)) + Expect(loadBalancer.Spec.Port).To(Equal(lbPortLow)) Expect(loadBalancer.Spec.Protocol).To(Equal(lbProtocol)) Expect(loadBalancer.Spec.Targets).To(HaveLen(1)) Expect(loadBalancer.Spec.Targets[0]).To(Equal(lbv1alpha1.Target{ @@ -172,9 +194,10 @@ var _ = Describe("Service controller", func() { Expect(loadBalancer.OwnerReferences).To(HaveLen(1)) Expect(loadBalancer.OwnerReferences[0].UID).To(Equal(service.UID)) + Expect(loadBalancer.Spec.ConfigName).To(Equal(loadBalancerConfig.Name)) Expect(loadBalancer.Spec.HTTPUpdater.URL).To(Equal(updaterURL)) Expect(loadBalancer.Spec.Host).To(Equal(lbHost)) - Expect(loadBalancer.Spec.Port).To(Equal(lbPort)) + Expect(loadBalancer.Spec.Port).To(Equal(lbPortLow)) Expect(loadBalancer.Spec.Protocol).To(Equal(lbProtocol)) Expect(loadBalancer.Spec.Targets).To(ContainElements( lbv1alpha1.Target{ @@ -194,6 +217,8 @@ var _ = Describe("Service controller", func() { Expect(k8sClient.Delete(ctx, node1)).Should(Succeed()) Expect(k8sClient.Delete(ctx, node2)).Should(Succeed()) Expect(k8sClient.Delete(ctx, loadBalancer)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, loadBalancerConfig)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, paperLBSystemNamespace)).Should(Succeed()) gomockController.Finish() }) }) diff --git a/demo/load_balancer_config.yaml b/demo/load_balancer_config.yaml new file mode 100644 index 0000000..ffd0d5f --- /dev/null +++ b/demo/load_balancer_config.yaml @@ -0,0 +1,15 @@ +apiVersion: lb.paperlb.com/v1alpha1 +kind: LoadBalancerConfig +metadata: + name: default-lb-config + namespace: paperlb-system + #optional annotation to use a config different than the default config + #annotations: + # lb.paperlb.com/config-name: "my-special-config" +spec: + default: true + httpUpdaterURL: "http://192.168.64.1:3000/api/v1/lb" + host: "192.168.64.1" + portRange: + low: 8100 + high: 8200 \ No newline at end of file diff --git a/demo/service.yaml b/demo/service.yaml index d4abb37..3c63155 100644 --- a/demo/service.yaml +++ b/demo/service.yaml @@ -4,11 +4,6 @@ metadata: labels: app: k8s-pod-info-api name: k8s-pod-info-api-service - annotations: - lb.paperlb.com/http-updater-url: "http://192.168.64.1:3000/api/v1/lb" - lb.paperlb.com/load-balancer-host: "192.168.64.1" - lb.paperlb.com/load-balancer-port: "8100" - lb.paperlb.com/load-balancer-protocol: "TCP" spec: ports: - port: 5000 diff --git a/go.mod b/go.mod index 68106b8..a019fe9 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/didil/paperlb go 1.19 require ( + github.com/go-logr/logr v1.2.3 + github.com/golang/mock v1.6.0 github.com/onsi/ginkgo/v2 v2.6.0 github.com/onsi/gomega v1.24.1 github.com/pkg/errors v0.9.1 @@ -20,14 +22,12 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect