feat: add kong proxy controller codes (#170)

* add kong type

* add kong proxy

* add release event handler

* add kong status

* use ingress class to set node label instead of lb.Name

* fix kong ingress class of proxy.status

* optimize codes

* add kong crds

* fix missing orphan release error

* vendor update to add kong type

* delete ingresses when loadbalancer was deleted

* use patch method to update release

* format codes

Gopkg.toml

@@ -30,7 +30,7 @@
 
 [[override]]
   name = "github.com/caicloud/clientset"
-  branch = "release-1.12"
+  branch = "release-1.12-oem-starbucks"
 
 [[override]]
   name = "github.com/json-iterator/go"

pkg/controller/loadbalancer.go

@@ -39,6 +39,10 @@ import (
 	log "k8s.io/klog"
 )
 
+const (
+	ingressClassKey = "loadbalance.caicloud.io/ingress.class"
+)
+
 // LoadBalancerController is responsible for synchronizing LoadBalancer objects stored
 // in the system with actual running proxies and providers.
 type LoadBalancerController struct {
@@ -275,3 +279,16 @@ func (lbc *LoadBalancerController) deleteLoadBalancer(obj interface{}) {
 
 	lbc.queue.Enqueue(lb)
 }
+
+func getIngressClassFromLoadbalancer(lb *lbapi.LoadBalancer) string {
+	// get ingress class from annotation of loadbalancer
+	annotations := lb.GetAnnotations()
+	if annotations != nil {
+		ingressClass := annotations[ingressClassKey]
+		if ingressClass != "" {
+			return ingressClass
+		}
+	}
+
+	return lb.Name
+}

pkg/controller/nodes.go

@@ -62,8 +62,9 @@ func (nc *nodeController) syncNodes(lb *lbapi.LoadBalancer) error {
 }
 
 func (nc *nodeController) getNodesForLoadBalancer(lb *lbapi.LoadBalancer) ([]*apiv1.Node, error) {
+	ingressClass := getIngressClassFromLoadbalancer(lb)
 	// list old nodes
-	labelkey := fmt.Sprintf(lbapi.UniqueLabelKeyFormat, lb.Namespace, lb.Name)
+	labelkey := fmt.Sprintf(lbapi.UniqueLabelKeyFormat, lb.Namespace, ingressClass)
 	selector := labels.Set{labelkey: "true"}.AsSelector()
 	return nc.nodeLister.List(selector)
 }
@@ -77,8 +78,10 @@ func (nc *nodeController) getVerifiedNodes(lb *lbapi.LoadBalancer, oldNodes []*a
 		Labels:         map[string]string{},
 	}
 
+	ingressClass := getIngressClassFromLoadbalancer(lb)
 	ran.Labels = map[string]string{
-		fmt.Sprintf(lbapi.UniqueLabelKeyFormat, lb.Namespace, lb.Name): "true",
+		fmt.Sprintf(lbapi.UniqueLabelKeyFormat, lb.Namespace, lb.Name):      "true",
+		fmt.Sprintf(lbapi.UniqueLabelKeyFormat, lb.Namespace, ingressClass): "true",
 	}
 
 	if len(lb.Spec.Nodes.Names) == 0 {

pkg/proxy/kong/crds.go

@@ -0,0 +1,279 @@
+package kong
+
+import (
+	// "github.com/caicloud/clientset/kubernetes"
+	// apiextv1beta1 "github.com/caicloud/clientset/pkg/apis/apiextensions/v1beta1"
+
+	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/clientcmd"
+	log "k8s.io/klog"
+)
+
+const (
+	kongCrdGroup = "configuration.konghq.com"
+)
+
+var kongCrds = []apiextv1beta1.CustomResourceDefinition{
+	kongclusterplugins,
+	kongconsumers,
+	kongcredentials,
+	kongingresses,
+	kongplugins,
+	tcpingresses,
+}
+
+func installKongCrds() error {
+	// create client
+	// caicloud clientset can't create crd with AdditionalPrinterColumns and Validation
+	kubeconfig, err := clientcmd.BuildConfigFromFlags("", "")
+	if err != nil {
+		log.Errorf("build kubeconfig error for kong crd, error %v", err)
+		return err
+	}
+	client, err := apiextensionsv1beta1.NewForConfig(kubeconfig)
+	if err != nil {
+		log.Errorf("create apiextensionsv1beta1 clientset error %v", err)
+		return err
+	}
+	// all crds
+	for _, crd := range kongCrds {
+		//
+		_, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				// create the crd
+				log.Infof("Create kong crd %v", crd.Name)
+				if _, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Create(&crd); err != nil {
+					log.Errorf("Create crd %v error %v", crd.Name, err)
+					return err
+				}
+			} else {
+				log.Errorf("Get crd %v error %v", crd.Name, err)
+				return err
+			}
+		}
+		log.Infof("Get crd %v installed", crd.Name)
+	}
+	return nil
+}
+
+// kongclusterplugins
+var kongclusterplugins = apiextv1beta1.CustomResourceDefinition{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "kongclusterplugins.configuration.konghq.com",
+	},
+	Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+		AdditionalPrinterColumns: []apiextv1beta1.CustomResourceColumnDefinition{
+			{
+				JSONPath:    ".plugin",
+				Description: "Name of the plugin",
+				Name:        "Plugin-Type",
+				Type:        "string",
+			},
+			{
+				JSONPath:    ".metadata.creationTimestamp",
+				Description: "Age",
+				Name:        "Age",
+				Type:        "date",
+			},
+			{
+				JSONPath:    ".disabled",
+				Description: "Indicates if the plugin is disabled",
+				Name:        "Disabled",
+				Priority:    1,
+				Type:        "boolean",
+			},
+			{
+				JSONPath:    ".config",
+				Description: "Configuration of the plugin",
+				Name:        "Config",
+				Priority:    1,
+				Type:        "string",
+			},
+		},
+		Group: kongCrdGroup,
+		Names: apiextv1beta1.CustomResourceDefinitionNames{
+			Kind:   "KongClusterPlugin",
+			Plural: "kongclusterplugins",
+			ShortNames: []string{
+				"kcp",
+			},
+		},
+		Scope: apiextv1beta1.ClusterScoped,
+		// Validation
+		Version: "v1",
+	},
+}
+
+// kongconsumers
+var kongconsumers = apiextv1beta1.CustomResourceDefinition{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "kongconsumers.configuration.konghq.com",
+	},
+	Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+		AdditionalPrinterColumns: []apiextv1beta1.CustomResourceColumnDefinition{
+			{
+				JSONPath:    ".username",
+				Description: "Username of a Kong Consumer",
+				Name:        "Username",
+				Type:        "string",
+			},
+			{
+				JSONPath:    ".metadata.creationTimestamp",
+				Description: "Age",
+				Name:        "Age",
+				Type:        "date",
+			},
+		},
+		Group: kongCrdGroup,
+		Names: apiextv1beta1.CustomResourceDefinitionNames{
+			Kind:   "KongConsumer",
+			Plural: "kongconsumers",
+			ShortNames: []string{
+				"kc",
+			},
+		},
+		Scope: apiextv1beta1.NamespaceScoped,
+		// Validation
+		Version: "v1",
+	},
+}
+
+// kongcredentials
+var kongcredentials = apiextv1beta1.CustomResourceDefinition{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "kongcredentials.configuration.konghq.com",
+	},
+	Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+		AdditionalPrinterColumns: []apiextv1beta1.CustomResourceColumnDefinition{
+			{
+				JSONPath:    ".type",
+				Description: "Type of credential",
+				Name:        "Credential-type",
+				Type:        "string",
+			},
+			{
+				JSONPath:    ".metadata.creationTimestamp",
+				Description: "Age",
+				Name:        "Age",
+				Type:        "date",
+			},
+			{
+				JSONPath:    ".consumerRef",
+				Description: "Owner of the credential",
+				Name:        "Consumer-Ref",
+				Type:        "string",
+			},
+		},
+		Group: kongCrdGroup,
+		Names: apiextv1beta1.CustomResourceDefinitionNames{
+			Kind:   "KongCredential",
+			Plural: "kongcredentials",
+		},
+		Scope: apiextv1beta1.NamespaceScoped,
+		// Validation
+		Version: "v1",
+	},
+}
+
+// kongingresses
+var kongingresses = apiextv1beta1.CustomResourceDefinition{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "kongingresses.configuration.konghq.com",
+	},
+	Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+		Group: kongCrdGroup,
+		Names: apiextv1beta1.CustomResourceDefinitionNames{
+			Kind:   "KongIngress",
+			Plural: "kongingresses",
+			ShortNames: []string{
+				"ki",
+			},
+		},
+		Scope: apiextv1beta1.NamespaceScoped,
+		// Validation
+		Version: "v1",
+	},
+}
+
+// kongplugins
+var kongplugins = apiextv1beta1.CustomResourceDefinition{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "kongplugins.configuration.konghq.com",
+	},
+	Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+		AdditionalPrinterColumns: []apiextv1beta1.CustomResourceColumnDefinition{
+			{
+				JSONPath:    ".plugin",
+				Description: "Name of the plugin",
+				Name:        "Plugin-Type",
+				Type:        "string",
+			},
+			{
+				JSONPath:    ".metadata.creationTimestamp",
+				Description: "Age",
+				Name:        "Age",
+				Type:        "date",
+			},
+			{
+				JSONPath:    ".disabled",
+				Description: "Indicates if the plugin is disabled",
+				Name:        "Disabled",
+				Priority:    1,
+				Type:        "boolean",
+			},
+			{
+				JSONPath:    ".config",
+				Description: "Configuration of the plugin",
+				Name:        "Config",
+				Priority:    1,
+				Type:        "string",
+			},
+		},
+		Group: kongCrdGroup,
+		Names: apiextv1beta1.CustomResourceDefinitionNames{
+			Kind:   "KongPlugin",
+			Plural: "kongplugins",
+			ShortNames: []string{
+				"kp",
+			},
+		},
+		Scope: apiextv1beta1.NamespaceScoped,
+		// Validation
+		Version: "v1",
+	},
+}
+
+// tcpingresses
+var tcpingresses = apiextv1beta1.CustomResourceDefinition{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "tcpingresses.configuration.konghq.com",
+	},
+	Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+		AdditionalPrinterColumns: []apiextv1beta1.CustomResourceColumnDefinition{
+			{
+				JSONPath:    ".status.loadBalancer.ingress[*].ip",
+				Description: "Address of the load balancer",
+				Name:        "Address",
+				Type:        "string",
+			},
+			{
+				JSONPath:    ".metadata.creationTimestamp",
+				Description: "Age",
+				Name:        "Age",
+				Type:        "date",
+			},
+		},
+		Group: kongCrdGroup,
+		Names: apiextv1beta1.CustomResourceDefinitionNames{
+			Kind:   "TCPIngress",
+			Plural: "tcpingresses",
+		},
+		Scope: apiextv1beta1.NamespaceScoped,
+		// Validation
+		Version: "v1beta1",
+	},
+}