From 67a40ef96438bc409df2de9a0c9b7eb6a628bbf2 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Tue, 31 Aug 2021 17:05:47 -0700 Subject: [PATCH 01/10] builtin/k8s: Add Ingress as a release This commit introduces a new resource to manage a Waypoint release with using Ingress. It creates an ingress resource that's associated with a service backend for a deployment. --- builtin/k8s/plugin.pb.go | 84 +++++++- builtin/k8s/plugin.proto | 3 + builtin/k8s/releaser.go | 437 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 509 insertions(+), 15 deletions(-) diff --git a/builtin/k8s/plugin.pb.go b/builtin/k8s/plugin.pb.go index 8dc2f752032..312c3d64576 100644 --- a/builtin/k8s/plugin.pb.go +++ b/builtin/k8s/plugin.pb.go @@ -329,6 +329,53 @@ func (x *Resource_Service) GetName() string { return "" } +type Resource_Ingress struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *Resource_Ingress) Reset() { + *x = Resource_Ingress{} + if protoimpl.UnsafeEnabled { + mi := &file_waypoint_builtin_k8s_plugin_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Resource_Ingress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Resource_Ingress) ProtoMessage() {} + +func (x *Resource_Ingress) ProtoReflect() protoreflect.Message { + mi := &file_waypoint_builtin_k8s_plugin_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Resource_Ingress.ProtoReflect.Descriptor instead. +func (*Resource_Ingress) Descriptor() ([]byte, []int) { + return file_waypoint_builtin_k8s_plugin_proto_rawDescGZIP(), []int{2, 2} +} + +func (x *Resource_Ingress) GetName() string { + if x != nil { + return x.Name + } + return "" +} + var File_waypoint_builtin_k8s_plugin_proto protoreflect.FileDescriptor var file_waypoint_builtin_k8s_plugin_proto_rawDesc = []byte{ @@ -351,15 +398,17 @@ var file_waypoint_builtin_k8s_plugin_proto_rawDesc = []byte{ 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, - 0x4b, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x1a, 0x20, 0x0a, 0x0a, 0x44, + 0x6a, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x1a, 0x20, 0x0a, 0x0a, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x1a, 0x1d, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x1a, 0x0a, 0x08, - 0x54, 0x61, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x42, 0x16, 0x5a, 0x14, 0x77, 0x61, 0x79, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x2f, 0x6b, 0x38, 0x73, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x1a, 0x1d, 0x0a, 0x07, + 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x1a, 0x0a, 0x08, 0x54, + 0x61, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x42, 0x16, 0x5a, 0x14, 0x77, 0x61, 0x79, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x2f, 0x6b, 0x38, 0x73, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -374,7 +423,7 @@ func file_waypoint_builtin_k8s_plugin_proto_rawDescGZIP() []byte { return file_waypoint_builtin_k8s_plugin_proto_rawDescData } -var file_waypoint_builtin_k8s_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_waypoint_builtin_k8s_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_waypoint_builtin_k8s_plugin_proto_goTypes = []interface{}{ (*Deployment)(nil), // 0: k8s.Deployment (*Release)(nil), // 1: k8s.Release @@ -382,11 +431,12 @@ var file_waypoint_builtin_k8s_plugin_proto_goTypes = []interface{}{ (*TaskInfo)(nil), // 3: k8s.TaskInfo (*Resource_Deployment)(nil), // 4: k8s.Resource.Deployment (*Resource_Service)(nil), // 5: k8s.Resource.Service - (*anypb.Any)(nil), // 6: google.protobuf.Any + (*Resource_Ingress)(nil), // 6: k8s.Resource.Ingress + (*anypb.Any)(nil), // 7: google.protobuf.Any } var file_waypoint_builtin_k8s_plugin_proto_depIdxs = []int32{ - 6, // 0: k8s.Deployment.resource_state:type_name -> google.protobuf.Any - 6, // 1: k8s.Release.resource_state:type_name -> google.protobuf.Any + 7, // 0: k8s.Deployment.resource_state:type_name -> google.protobuf.Any + 7, // 1: k8s.Release.resource_state:type_name -> google.protobuf.Any 2, // [2:2] is the sub-list for method output_type 2, // [2:2] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name @@ -472,6 +522,18 @@ func file_waypoint_builtin_k8s_plugin_proto_init() { return nil } } + file_waypoint_builtin_k8s_plugin_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Resource_Ingress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -479,7 +541,7 @@ func file_waypoint_builtin_k8s_plugin_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_waypoint_builtin_k8s_plugin_proto_rawDesc, NumEnums: 0, - NumMessages: 6, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/builtin/k8s/plugin.proto b/builtin/k8s/plugin.proto index d079a5d9785..a92d46c4cca 100644 --- a/builtin/k8s/plugin.proto +++ b/builtin/k8s/plugin.proto @@ -27,6 +27,9 @@ message Resource { message Service { string name = 1; } + message Ingress { + string name = 1; + } } // This represents the state of the TaskLaunch implementation. diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 66e2692a2a1..6983fcf834b 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -8,16 +8,18 @@ import ( "strings" "time" - "github.com/hashicorp/go-hclog" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/waypoint-plugin-sdk/component" "github.com/hashicorp/waypoint-plugin-sdk/docs" "github.com/hashicorp/waypoint-plugin-sdk/framework/resource" @@ -65,6 +67,13 @@ func (r *Releaser) resourceManager(log hclog.Logger, dcr *component.DeclaredReso resource.WithDestroy(r.resourceServiceDestroy), resource.WithStatus(r.resourceServiceStatus), )), + resource.WithResource(resource.NewResource( + resource.WithName("ingress"), + resource.WithState(&Resource_Ingress{}), + resource.WithCreate(r.resourceIngressCreate), + resource.WithDestroy(r.resourceIngressDestroy), + resource.WithStatus(r.resourceIngressStatus), + )), ) } @@ -123,7 +132,7 @@ func (r *Releaser) resourceServiceStatus( serviceResource.Name = serviceResp.Name serviceResource.CreatedTime = timestamppb.New(serviceResp.CreationTimestamp.Time) serviceResource.Health = sdk.StatusReport_READY // If we found the service, then it's ready. It doesn't have any other conditions. - serviceResource.HealthMessage = "exists" + serviceResource.HealthMessage = fmt.Sprintf("Service %q exists and is ready", serviceResource.Name) serviceResource.StateJson = string(serviceJson) } @@ -136,7 +145,6 @@ func (r *Releaser) resourceServiceCreate( ctx context.Context, log hclog.Logger, target *Deployment, - result *Release, state *Resource_Service, csinfo *clientsetInfo, @@ -181,7 +189,7 @@ func (r *Releaser) resourceServiceCreate( } if (r.config.Port != 0 || r.config.NodePort != 0) && r.config.Ports != nil { - return fmt.Errorf("cannot define both 'ports' and 'port' or 'node_port'." + + return status.Errorf(codes.FailedPrecondition, "Cannot define both 'ports' and 'port' or 'node_port'."+ " Use 'ports' for configuring multiple service ports") } else if r.config.Ports == nil && (r.config.Port != 0 || r.config.NodePort != 0) { r.config.Ports = make([]map[string]string, 1) @@ -339,10 +347,308 @@ func (r *Releaser) resourceServiceDestroy( return err } + step.Update("Service deleted") step.Done() return nil } +func (r *Releaser) resourceIngressCreate( + ctx context.Context, + log hclog.Logger, + target *Deployment, + result *Release, + state *Resource_Ingress, + serviceState *Resource_Service, + csinfo *clientsetInfo, + sg terminal.StepGroup, +) error { + // Preflight config checks + if r.config.IngressConfig != nil { + if r.config.LoadBalancer { + return status.Error(codes.FailedPrecondition, "A LoadBalancer type is not "+ + "compatible with an Ingress config. Please pick one or the other for your release") + } + if r.config.NodePort != 0 { + return status.Error(codes.FailedPrecondition, "A NodePort type is not "+ + "compatible with an Ingress config. Please pick one or the other for your release") + } + + if r.config.IngressConfig.ClassName != "http" { + return status.Error(codes.FailedPrecondition, "An ingress stanza must be "+ + "of type \"http\".") + } + } else { + // No ingress config was set, we're not configuring an ingress resource + return nil + } + + // Prepare our namespace and override if set. + ns := csinfo.Namespace + if r.config.Namespace != "" { + ns = r.config.Namespace + } + + step := sg.Add("Preparing ingress resource...") + defer func() { step.Abort() }() // Defer in func in case more steps are added to this func in the future + + clientSet := csinfo.Clientset + serviceclient := clientSet.CoreV1().Services(ns) + ingressClient := clientSet.NetworkingV1().Ingresses(ns) + + // Determine if we have a deployment that we manage already + serviceBackend, err := serviceclient.Get(ctx, serviceState.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + // We expect resourceServiceCreate to have created a Service prior to + // creating an ingress resource. Otherwise there is no service backend + // the ingress resource can refer to + return status.Errorf(codes.FailedPrecondition, "A service must exist before "+ + "an ingress resource can be created: %s", err) + } + if err != nil { + return err + } + + var ingressResource *networkingv1.Ingress + create := false + ingressResource, err = ingressClient.Get(ctx, serviceState.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + // We haven't created an ingress resource yet... + log.Debug("no ingress resource found, will create one") + + err = nil + create = true + + // basic ingress resource + ingressResource = &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Ingress", + }, + + ObjectMeta: metav1.ObjectMeta{ + Name: result.ServiceName, + }, + } + } + if err != nil { + return err + } + + // Set the ingress resource state name to match the service name + state.Name = result.ServiceName + + var ingressTls networkingv1.IngressTLS + if r.config.IngressConfig.TlsConfig != nil { + ingressTls = networkingv1.IngressTLS{ + Hosts: r.config.IngressConfig.TlsConfig.Hosts, + SecretName: r.config.IngressConfig.TlsConfig.SecretName, + } + } + + // Apply any annotations to the ingress resource + if r.config.IngressConfig.Annotations != nil { + ingressResource = &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: r.config.IngressConfig.Annotations, + }, + } + } + + // Define the ingress resources backend service it should route traffic to + ingressBackend := networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: serviceState.Name, + Port: networkingv1.ServiceBackendPort{ + Name: serviceBackend.Spec.Ports[0].Name, + Number: serviceBackend.Spec.Ports[0].Port, + }, + }, + } + + // NOTE, the k8s go api doesn't seem to like being set directly from the + // string config in our releaser, so we manually set it with this switch + var pathType networkingv1.PathType + switch r.config.IngressConfig.PathType { + case "Exact": + pathType = "Exact" + case "Prefix": + pathType = "Prefix" + case "ImplementationSpecific": + pathType = "ImplementationSpecific" + default: + if r.config.IngressConfig.PathType != "" { + return status.Errorf(codes.FailedPrecondition, + "Invalid PathType given for ingress resource: %q", + r.config.IngressConfig.PathType) + } + + // Default Path Type if nothing is configured in an ingress releaser + pathType = "Prefix" + } + + // Set the default path to '/' if not set and path type is Prefix + if pathType == "Prefix" && r.config.IngressConfig.Path == "" { + r.config.IngressConfig.Path = "/" + } + + httpIngressPath := networkingv1.HTTPIngressPath{ + Path: r.config.IngressConfig.Path, + PathType: &pathType, + Backend: ingressBackend, + } + + ingressRule := networkingv1.IngressRule{ + Host: r.config.IngressConfig.Host, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{httpIngressPath}, + }, + }, + } + + ingressResource.Spec = networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ingressRule}, + } + + if r.config.IngressConfig.TlsConfig != nil { + ingressResource.Spec.TLS = []networkingv1.IngressTLS{ingressTls} + } + + if r.config.IngressConfig.DefaultBackend { + ingressResource.Spec.DefaultBackend = &ingressBackend + } + + // create or update the ingress resource + if create { + step.Update("Creating ingress resource...") + _, err = ingressClient.Create(ctx, ingressResource, metav1.CreateOptions{}) + } else { + step.Update("Updating existing ingress resource...") + _, err = ingressClient.Update(ctx, ingressResource, metav1.UpdateOptions{}) + } + if err != nil { + return err + } + + step.Update("Waiting for ingress resource to become ready...") + + // Wait on load balancer + var ingress *networkingv1.Ingress + err = wait.PollImmediate(2*time.Second, 10*time.Minute, func() (bool, error) { + ingress, err = ingressClient.Get(ctx, state.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return len(ingress.Status.LoadBalancer.Ingress) > 0, nil + }) + if err != nil { + return err + } + + step.Update("Ingress resource is ready!") + step.Done() + + if len(ingress.Status.LoadBalancer.Ingress) > 0 { + protocol := "http://" + if r.config.IngressConfig.TlsConfig != nil { + protocol = "https://" + } + lbIngress := ingress.Status.LoadBalancer.Ingress[0] + result.Url = protocol + lbIngress.IP + if lbIngress.Hostname != "" { + result.Url = protocol + lbIngress.Hostname + } + + if serviceBackend.Spec.Ports[0].Port != 80 { + result.Url = fmt.Sprintf("%s:%d", result.Url, serviceBackend.Spec.Ports[0].Port) + } + + if r.config.IngressConfig.Path != "/" { + result.Url = fmt.Sprintf("%s%s", result.Url, r.config.IngressConfig.Path) + } + + } // else we show the cluster IP URL setup by the service resource + + return nil +} + +func (r *Releaser) resourceIngressDestroy( + ctx context.Context, + state *Resource_Ingress, + sg terminal.StepGroup, + csinfo *clientsetInfo, +) error { + if state.Name == "" { + // we didn't create an ingress resource, so we can't delete it either + return nil + } + + step := sg.Add("Initializing Kubernetes client...") + defer step.Abort() + + // Prepare our namespace and override if set. + ns := csinfo.Namespace + if r.config.Namespace != "" { + ns = r.config.Namespace + } + + clientSet := csinfo.Clientset + ingressClient := clientSet.NetworkingV1().Ingresses(ns) + + step.Update("Kubernetes client connected to %s with namespace %s", csinfo.Config.Host, ns) + step.Done() + + // get ingress first + _, err := ingressClient.Get(ctx, state.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + step = sg.Add("No Ingress resource found to delete") + step.Done() + return nil + } + if err != nil { + return err + } + + step = sg.Add("Deleting ingress resource...") + if err = ingressClient.Delete(ctx, state.Name, metav1.DeleteOptions{}); err != nil { + return err + } + + step.Update("Ingress resource deleted") + step.Done() + return nil +} + +func (r *Releaser) resourceIngressStatus( + ctx context.Context, + log hclog.Logger, + sg terminal.StepGroup, + state *Resource_Ingress, + clientset *clientsetInfo, + sr *resource.StatusResponse, +) error { + if state.Name == "" { + log.Debug("skipping ingress status, no resource state found") + // we didn't create an ingress resource, so no use building a report for it + return nil + } + + s := sg.Add("Checking status of Kubernetes ingress resource...") + defer s.Abort() + + namespace := r.config.Namespace + if namespace == "" { + namespace = clientset.Namespace + } + + // TODO: write me + + s.Update("Finished building report for Kubernetes ingress resource") + s.Done() + return nil +} + // getClientset is a value provider for our resource manager and provides // the connection information used by resources to interact with Kubernetes. func (r *Releaser) getClientset() (*clientsetInfo, error) { @@ -408,6 +714,9 @@ func (r *Releaser) Destroy( rm.Resource("service").SetState(&Resource_Service{ Name: release.ServiceName, }) + rm.Resource("ingress").SetState(&Resource_Service{ + Name: release.ServiceName, + }) } else { // Load our set state if err := rm.LoadState(release.ResourceState); err != nil { @@ -436,6 +745,9 @@ func (r *Releaser) Status( rm.Resource("service").SetState(&Resource_Service{ Name: release.ServiceName, }) + rm.Resource("ingress").SetState(&Resource_Service{ + Name: release.ServiceName, + }) } else { // Load our set state if err := rm.LoadState(release.ResourceState); err != nil { @@ -502,6 +814,9 @@ type ReleaserConfig struct { // Annotations to be applied to the kube service. Annotations map[string]string `hcl:"annotations,optional"` + // Ingress represents an config for setting up an ingress resource. + IngressConfig *IngressConfig `hcl:"ingress,block"` + // KubeconfigPath is the path to the kubeconfig file. If this is // blank then we default to the home directory. KubeconfigPath string `hcl:"kubeconfig,optional"` @@ -532,6 +847,58 @@ type ReleaserConfig struct { Namespace string `hcl:"namespace,optional"` } +// IngressConfig holds various options to configure an Ingress resource with +// during a release. It currently only spports 'http' based route rules. +type IngressConfig struct { + // Annotations to be applied to the ingress service. + Annotations map[string]string `hcl:"annotations,optional"` + + // Currently Waypoint only supports "HTTP" rule-backed ingress resources. + // We include this stanza label in the future for when Kubernetes has other + // kinds of rule types. + ClassName string `hcl:",label"` + + // If set, this will configure the given ingress resources backend service + // as the default service that accepts traffic if no route rules match from + // the inbound request. + // Defaults to false + DefaultBackend bool `hcl:"default,optional"` + + // If set, this option will configure the ingress controller to accept + // traffic from the defined hostname. IPs are not allowed, nor are `:` delimiters. + // Wildcards are allowed to a certain extent. For more details, check out the + // k8s go client package. + Host string `hcl:"host,optional"` + + // Defines the kind of rule the path will be. Possible values are: + // 'Exact', 'Prefix', and 'ImplementationSpecific'. + // We believe most users expect a 'Prefix' type, so we default to 'Prefix' if + // not specified. + PathType string `hcl:"path_type,optional"` + + // Path represents a rule to route requests to. I.e. if an inbound request + // matches a route like `/foo`, the ingress controller would see that this + // resource is configured for that rule, and would route traffic to this + // ingress resources service backend. + // maybe not, each path looks to represent a different service backend in kubectl + Path string `hcl:"path,optional"` + + // TlsConfig is an optional config that users can set to enable HTTPS traffic + TlsConfig *IngressTls `hcl:"tls,block"` +} + +// IngressTls holds options required to configure an ingress resource with TLS. +type IngressTls struct { + // Hosts is a list of hosts included in the TLS certificate + Hosts []string `hcl:"hosts,optional"` + + // SecretName references the name of the secret created inside Kubernetes + // associated with the TLS certificate information. If cert-manager is used, + // this name will refer to the secret cert-manager should create when generating + // certficiates for the secret. + SecretName string `hcl:"secret_name,optional"` +} + func (r *Releaser) Documentation() (*docs.Documentation, error) { doc, err := docs.New(docs.FromConfig(&ReleaserConfig{})) if err != nil { @@ -540,6 +907,11 @@ func (r *Releaser) Documentation() (*docs.Documentation, error) { doc.Description("Manipulates the Kubernetes Service activate Deployments") + doc.SetField( + "annotations", + "Annotations to be applied to the kube service", + ) + doc.SetField( "kubeconfig", "path to the kubeconfig file to use", @@ -598,6 +970,63 @@ func (r *Releaser) Documentation() (*docs.Documentation, error) { ), ) + doc.SetField( + "ingress", + "Configuration to set up an ingress resource to route traffic to the given "+ + "application from an ingress controller", + docs.SubFields(func(doc *docs.SubFieldDoc) { + doc.SetField( + "annotations", + "Annotations to be applied to the ingress resource", + ) + + doc.SetField( + "default", + "sets the ingress resource to be the default backend for any traffic "+ + "that doesn't match existing ingress rule paths", + docs.Default("false"), + ) + + doc.SetField( + "host", + "If set, will configure the ingress resource to have the ingress controller "+ + "route traffic for any inbound requests that match this host. IP addresses "+ + "are not allowed, nor are ':' delimiters. Wildcards are allowed to a "+ + "certain extent. For more details check out the Kubernetes documentation", + ) + + doc.SetField( + "path_type", + "defines the kind of rule the path will be for the ingress controller. "+ + "Valid path types are 'Exact', 'Prefix', and 'ImplementationSpecific'.", + docs.Default("Prefix"), + ) + + doc.SetField( + "path", + "The route rule that should be used to route requests to this ingress resource. "+ + "A path must begin with a '/'.", + docs.Default("/"), + ) + + doc.SetField( + "tls", + "A stanza of TLS configuration options for traffic to the ingress resource", + docs.SubFields(func(doc *docs.SubFieldDoc) { + doc.SetField( + "hosts", + "A list of hosts included in the TLS certificate", + ) + + doc.SetField( + "secret_name", + "The Kubernetes secret name that should be used to look up or store TLS configs", + ) + }), + ) + }), + ) + return doc, nil } From 47cc10cdc49cf740c1103d59461877a797c6a57e Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Fri, 3 Sep 2021 14:56:20 -0700 Subject: [PATCH 02/10] website autogen --- .../components/releasemanager-kubernetes.mdx | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/website/content/partials/components/releasemanager-kubernetes.mdx b/website/content/partials/components/releasemanager-kubernetes.mdx index 0f65ce7de38..e85f11cd4a9 100644 --- a/website/content/partials/components/releasemanager-kubernetes.mdx +++ b/website/content/partials/components/releasemanager-kubernetes.mdx @@ -6,7 +6,61 @@ Manipulates the Kubernetes Service activate Deployments. ### Required Parameters -This plugin has no required parameters. +These parameters are used in the [`use` stanza](/docs/waypoint-hcl/use) for this plugin. + +#### ingress (category) + +Configuration to set up an ingress resource to route traffic to the given application from an ingress controller. + +##### ingress.annotations + +Annotations to be applied to the ingress resource. + +- Type: **map of string to string** +- **Optional** + +##### ingress.default + +Sets the ingress resource to be the default backend for any traffic that doesn't match existing ingress rule paths. + +- Type: **bool** +- **Optional** +- Default: false + +##### ingress.host + +If set, will configure the ingress resource to have the ingress controller route traffic for any inbound requests that match this host. IP addresses are not allowed, nor are ':' delimiters. Wildcards are allowed to a certain extent. For more details check out the Kubernetes documentation. + +- Type: **string** +- **Optional** + +##### ingress.path + +The route rule that should be used to route requests to this ingress resource. A path must begin with a '/'. + +- Type: **string** +- **Optional** +- Default: / + +##### ingress.path_type + +Defines the kind of rule the path will be for the ingress controller. Valid path types are 'Exact', 'Prefix', and 'ImplementationSpecific'. + +- Type: **string** +- **Optional** +- Default: Prefix + +##### ingress.tls (category) + +A stanza of TLS configuration options for traffic to the ingress resource. + +###### ingress.tls.hosts + +A list of hosts included in the TLS certificate. + +###### ingress.tls.secret_name + +The Kubernetes secret name that should be used to look up or store TLS configs. ### Optional Parameters @@ -14,6 +68,8 @@ These parameters are used in the [`use` stanza](/docs/waypoint-hcl/use) for this #### annotations +Annotations to be applied to the kube service. + - Type: **map of string to string** - **Optional** From a6696ec53f54e80a1c5ca76c136a5d49f796a533 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 8 Sep 2021 09:16:25 -0700 Subject: [PATCH 03/10] Add status func for ingress resource --- builtin/k8s/releaser.go | 79 +++++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 6983fcf834b..0a0948c38c8 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -85,7 +85,7 @@ func (r *Releaser) resourceServiceStatus( clientset *clientsetInfo, sr *resource.StatusResponse, ) error { - s := sg.Add("Checking status of Kubernetes service resource...") + s := sg.Add("Checking status of Kubernetes service resource %q...", state.Name) defer s.Abort() namespace := r.config.Namespace @@ -105,6 +105,11 @@ func (r *Releaser) resourceServiceStatus( if !errors.IsNotFound(err) { return err } else { + s.Update("No service resource was found") + s.Status(terminal.StatusError) + s.Done() + s = sg.Add("") + serviceResource.Name = state.Name serviceResource.Health = sdk.StatusReport_MISSING serviceResource.HealthMessage = sdk.StatusReport_MISSING.String() @@ -131,7 +136,8 @@ func (r *Releaser) resourceServiceStatus( serviceResource.Id = fmt.Sprintf("%s", serviceResp.UID) serviceResource.Name = serviceResp.Name serviceResource.CreatedTime = timestamppb.New(serviceResp.CreationTimestamp.Time) - serviceResource.Health = sdk.StatusReport_READY // If we found the service, then it's ready. It doesn't have any other conditions. + // If we found the service, then it's ready. It doesn't have any other conditions. + serviceResource.Health = sdk.StatusReport_READY serviceResource.HealthMessage = fmt.Sprintf("Service %q exists and is ready", serviceResource.Name) serviceResource.StateJson = string(serviceJson) } @@ -628,13 +634,7 @@ func (r *Releaser) resourceIngressStatus( clientset *clientsetInfo, sr *resource.StatusResponse, ) error { - if state.Name == "" { - log.Debug("skipping ingress status, no resource state found") - // we didn't create an ingress resource, so no use building a report for it - return nil - } - - s := sg.Add("Checking status of Kubernetes ingress resource...") + s := sg.Add("Checking status of Kubernetes ingress resource %q...", state.Name) defer s.Abort() namespace := r.config.Namespace @@ -642,7 +642,66 @@ func (r *Releaser) resourceIngressStatus( namespace = clientset.Namespace } - // TODO: write me + ingressResource := sdk.StatusReport_Resource{ + CategoryDisplayHint: sdk.ResourceCategoryDisplayHint_ROUTER, + } + sr.Resources = append(sr.Resources, &ingressResource) + + ingressClient := clientset.Clientset.NetworkingV1().Ingresses(namespace) + ingressResp, err := ingressClient.Get(ctx, state.Name, metav1.GetOptions{}) + if ingressResp == nil { + return status.Errorf(codes.FailedPrecondition, + "kubernetes ingress resource response returned nothing for %q", state.Name) + } else if err != nil { + if !errors.IsNotFound(err) { + return err + } else { + s.Update("No ingress resource named %q was found", state.Name) + s.Status(terminal.StatusError) + s.Done() + s = sg.Add("") + + ingressResource.Name = state.Name + ingressResource.Health = sdk.StatusReport_MISSING + ingressResource.HealthMessage = sdk.StatusReport_MISSING.String() + } + } else { + // An ingress resource exists + s.Update("Building status report for ingress resource %q...", state.Name) + + lbIngress := ingressResp.Status.LoadBalancer.Ingress[0] + + var ipAddress string + if lbIngress.IP != "" { + ipAddress = lbIngress.IP + } + var hostname string + if lbIngress.Hostname != "" { + hostname = lbIngress.Hostname + } + // we only configure 1 rule, so grab the first service resource + serviceBackend := ingressResp.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].Backend.Service + + ingressJson, err := json.Marshal(map[string]interface{}{ + "ingress": ingressResp, + "ipAddress": ipAddress, + "hostname": hostname, + "serviceBackend": serviceBackend, + }) + if err != nil { + return status.Errorf(codes.Internal, + "failed to marshal ingress resource definition to json: %s", err) + } + + ingressResource.Id = fmt.Sprintf("%s", ingressResp.UID) + ingressResource.Name = ingressResp.Name + ingressResource.CreatedTime = timestamppb.New(ingressResp.CreationTimestamp.Time) + // If we found the ingress resource, then it's ready. It doesn't have any + // other conditions. + ingressResource.Health = sdk.StatusReport_READY + ingressResource.HealthMessage = fmt.Sprintf("Ingress resource %q exists and is ready", ingressResource.Name) + ingressResource.StateJson = string(ingressJson) + } s.Update("Finished building report for Kubernetes ingress resource") s.Done() From 7a5ea791e6b503bc63dd192471c9ac2df6858e13 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 8 Sep 2021 10:01:27 -0700 Subject: [PATCH 04/10] Only build ingress status if created --- builtin/k8s/releaser.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 0a0948c38c8..54fad003854 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -634,6 +634,11 @@ func (r *Releaser) resourceIngressStatus( clientset *clientsetInfo, sr *resource.StatusResponse, ) error { + if state.Name == "" { + log.Debug("no state found for ingress resource, cannot build status report") + return nil + } + s := sg.Add("Checking status of Kubernetes ingress resource %q...", state.Name) defer s.Abort() From 4108d057d6a63337bc28d9d06378c5dd12b64b19 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 8 Sep 2021 10:57:01 -0700 Subject: [PATCH 05/10] Update docs summary for ingress resource option --- builtin/k8s/releaser.go | 10 ++++++++-- .../partials/components/releasemanager-kubernetes.mdx | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 54fad003854..f6f3742b19d 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -371,11 +371,11 @@ func (r *Releaser) resourceIngressCreate( // Preflight config checks if r.config.IngressConfig != nil { if r.config.LoadBalancer { - return status.Error(codes.FailedPrecondition, "A LoadBalancer type is not "+ + return status.Error(codes.FailedPrecondition, "A LoadBalancer service type is not "+ "compatible with an Ingress config. Please pick one or the other for your release") } if r.config.NodePort != 0 { - return status.Error(codes.FailedPrecondition, "A NodePort type is not "+ + return status.Error(codes.FailedPrecondition, "A NodePort service type is not "+ "compatible with an Ingress config. Please pick one or the other for your release") } @@ -1038,6 +1038,12 @@ func (r *Releaser) Documentation() (*docs.Documentation, error) { "ingress", "Configuration to set up an ingress resource to route traffic to the given "+ "application from an ingress controller", + docs.Summary( + "An ingress resource can be created on release that will route traffic "+ + "to the Kubernetes service. Note that before this happens, the Kubernetes "+ + "cluster must already be configured with an Ingress controller. Otherwise "+ + "there won't be a way for inbound traffic to be routed to the ingress resource.", + ), docs.SubFields(func(doc *docs.SubFieldDoc) { doc.SetField( "annotations", diff --git a/website/content/partials/components/releasemanager-kubernetes.mdx b/website/content/partials/components/releasemanager-kubernetes.mdx index e85f11cd4a9..d37f32efc64 100644 --- a/website/content/partials/components/releasemanager-kubernetes.mdx +++ b/website/content/partials/components/releasemanager-kubernetes.mdx @@ -12,6 +12,8 @@ These parameters are used in the [`use` stanza](/docs/waypoint-hcl/use) for this Configuration to set up an ingress resource to route traffic to the given application from an ingress controller. +An ingress resource can be created on release that will route traffic to the Kubernetes service. Note that before this happens, the Kubernetes cluster must already be configured with an Ingress controller. Otherwise there won't be a way for inbound traffic to be routed to the ingress resource. + ##### ingress.annotations Annotations to be applied to the ingress resource. From ef48405107cab55ac03e9aa54e7af78f21703a61 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 8 Sep 2021 13:22:17 -0700 Subject: [PATCH 06/10] Add changelog --- .changelog/2261.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .changelog/2261.txt diff --git a/.changelog/2261.txt b/.changelog/2261.txt new file mode 100644 index 00000000000..62093cd6e8f --- /dev/null +++ b/.changelog/2261.txt @@ -0,0 +1,4 @@ +```release-note:improvement +plugin/k8s: Add new ability to release by creating an ingress resource to route +traffic to a service backend from an ingress controller. +``` From 0ec9fafc754f91abc5497b865c4e4b8827f63b78 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 8 Sep 2021 15:02:01 -0700 Subject: [PATCH 07/10] boolean zen Simplify the if-statement preflight config checks --- builtin/k8s/releaser.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index f6f3742b19d..2c7809eb169 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -369,25 +369,26 @@ func (r *Releaser) resourceIngressCreate( sg terminal.StepGroup, ) error { // Preflight config checks - if r.config.IngressConfig != nil { - if r.config.LoadBalancer { - return status.Error(codes.FailedPrecondition, "A LoadBalancer service type is not "+ - "compatible with an Ingress config. Please pick one or the other for your release") - } - if r.config.NodePort != 0 { - return status.Error(codes.FailedPrecondition, "A NodePort service type is not "+ - "compatible with an Ingress config. Please pick one or the other for your release") - } - - if r.config.IngressConfig.ClassName != "http" { - return status.Error(codes.FailedPrecondition, "An ingress stanza must be "+ - "of type \"http\".") - } - } else { - // No ingress config was set, we're not configuring an ingress resource + if r.config.IngressConfig == nil { + // No ingress config, we're not going to configure one return nil } + if r.config.LoadBalancer { + return status.Error(codes.FailedPrecondition, "A LoadBalancer service type is not "+ + "compatible with an Ingress config. Please pick one or the other for your release") + } + + if r.config.NodePort != 0 { + return status.Error(codes.FailedPrecondition, "A NodePort service type is not "+ + "compatible with an Ingress config. Please pick one or the other for your release") + } + + if r.config.IngressConfig.ClassName != "http" { + return status.Error(codes.FailedPrecondition, "An ingress stanza must be "+ + "of type \"http\".") + } + // Prepare our namespace and override if set. ns := csinfo.Namespace if r.config.Namespace != "" { From aab92139c20fe0ca82428273b53f3d158eb28f5d Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 8 Sep 2021 15:07:04 -0700 Subject: [PATCH 08/10] Delete old comment left over from todo --- builtin/k8s/releaser.go | 1 - 1 file changed, 1 deletion(-) diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 2c7809eb169..1729ba163af 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -945,7 +945,6 @@ type IngressConfig struct { // matches a route like `/foo`, the ingress controller would see that this // resource is configured for that rule, and would route traffic to this // ingress resources service backend. - // maybe not, each path looks to represent a different service backend in kubectl Path string `hcl:"path,optional"` // TlsConfig is an optional config that users can set to enable HTTPS traffic From 8c22b1451ca218e9f6dd50ae4a9bfd554e98cb9a Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Thu, 9 Sep 2021 15:19:24 -0700 Subject: [PATCH 09/10] Use struct vars for PathType --- builtin/k8s/releaser.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 1729ba163af..eb970533a2c 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -477,11 +477,11 @@ func (r *Releaser) resourceIngressCreate( var pathType networkingv1.PathType switch r.config.IngressConfig.PathType { case "Exact": - pathType = "Exact" + pathType = networkingv1.PathTypeExact case "Prefix": - pathType = "Prefix" + pathType = networkingv1.PathTypePrefix case "ImplementationSpecific": - pathType = "ImplementationSpecific" + pathType = networkingv1.PathTypeImplementationSpecific default: if r.config.IngressConfig.PathType != "" { return status.Errorf(codes.FailedPrecondition, @@ -490,11 +490,11 @@ func (r *Releaser) resourceIngressCreate( } // Default Path Type if nothing is configured in an ingress releaser - pathType = "Prefix" + pathType = networkingv1.PathTypePrefix } // Set the default path to '/' if not set and path type is Prefix - if pathType == "Prefix" && r.config.IngressConfig.Path == "" { + if pathType == networkingv1.PathTypePrefix && r.config.IngressConfig.Path == "" { r.config.IngressConfig.Path = "/" } From 30bff63730a5e24d37a0c164f4461fce2782dc9e Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Fri, 10 Sep 2021 10:08:59 -0700 Subject: [PATCH 10/10] Simplify setting PathType from Waypoint config --- builtin/k8s/releaser.go | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index eb970533a2c..f835b623b3e 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -271,7 +271,7 @@ func (r *Releaser) resourceServiceCreate( step.Update("Creating service...") service, err = serviceclient.Create(ctx, service, metav1.CreateOptions{}) } else { - step.Update("Updating service...") + step.Update("Updating existing service...") service, err = serviceclient.Update(ctx, service, metav1.UpdateOptions{}) } if err != nil { @@ -472,26 +472,10 @@ func (r *Releaser) resourceIngressCreate( }, } - // NOTE, the k8s go api doesn't seem to like being set directly from the - // string config in our releaser, so we manually set it with this switch - var pathType networkingv1.PathType - switch r.config.IngressConfig.PathType { - case "Exact": - pathType = networkingv1.PathTypeExact - case "Prefix": - pathType = networkingv1.PathTypePrefix - case "ImplementationSpecific": - pathType = networkingv1.PathTypeImplementationSpecific - default: - if r.config.IngressConfig.PathType != "" { - return status.Errorf(codes.FailedPrecondition, - "Invalid PathType given for ingress resource: %q", - r.config.IngressConfig.PathType) - } - - // Default Path Type if nothing is configured in an ingress releaser - pathType = networkingv1.PathTypePrefix + if r.config.IngressConfig.PathType == "" { + r.config.IngressConfig.PathType = "Prefix" } + pathType := networkingv1.PathType(r.config.IngressConfig.PathType) // Set the default path to '/' if not set and path type is Prefix if pathType == networkingv1.PathTypePrefix && r.config.IngressConfig.Path == "" { @@ -537,7 +521,8 @@ func (r *Releaser) resourceIngressCreate( return err } - step.Update("Waiting for ingress resource to become ready...") + step.Done() + step = sg.Add("Waiting for ingress resource to become ready...") // Wait on load balancer var ingress *networkingv1.Ingress