[Enhancement] Refactoring: normalize label flags (k3s node & runtime) (…

…#598, @ejose19)

cmd/cluster/clusterCreate.go

@@ -283,8 +283,11 @@ func NewCmdClusterCreate() *cobra.Command {
 	cmd.Flags().StringArrayP("port", "p", nil, "Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent[0] -p 8081@agent[1]`")
 	_ = ppViper.BindPFlag("cli.ports", cmd.Flags().Lookup("port"))
 
-	cmd.Flags().StringArrayP("label", "l", nil, "Add label to node container (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -l \"my.label@agent[0,1]\" -l \"other.label=somevalue@server[0]\"`")
-	_ = ppViper.BindPFlag("cli.labels", cmd.Flags().Lookup("label"))
+	cmd.Flags().StringArrayP("k3s-node-label", "", nil, "Add label to k3s node (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --k3s-node-label \"my.label@agent[0,1]\" --k3s-node-label \"other.label=somevalue@server[0]\"`")
+	_ = ppViper.BindPFlag("cli.k3s-node-labels", cmd.Flags().Lookup("k3s-node-label"))
+
+	cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent[0,1]\" --runtime-label \"other.label=somevalue@server[0]\"`")
+	_ = ppViper.BindPFlag("cli.runtime-labels", cmd.Flags().Lookup("runtime-label"))
 
 	/* k3s */
 	cmd.Flags().StringArray("k3s-arg", nil, "Additional args passed to k3s command (Format: `ARG@NODEFILTER[;@NODEFILTER]`)\n - Example: `k3d cluster create --k3s-arg \"--disable=traefik@server[0]\"")
@@ -481,33 +484,63 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 
 	log.Tracef("PortFilterMap: %+v", portFilterMap)
 
-	// --label
-	// labelFilterMap will add container label to applied node filters
-	labelFilterMap := make(map[string][]string, 1)
-	for _, labelFlag := range ppViper.GetStringSlice("cli.labels") {
+	// --k3s-node-label
+	// k3sNodeLabelFilterMap will add k3s node label to applied node filters
+	k3sNodeLabelFilterMap := make(map[string][]string, 1)
+	for _, labelFlag := range ppViper.GetStringSlice("cli.k3s-node-labels") {
+
+		// split node filter from the specified label
+		label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		// create new entry or append filter to existing entry
+		if _, exists := k3sNodeLabelFilterMap[label]; exists {
+			k3sNodeLabelFilterMap[label] = append(k3sNodeLabelFilterMap[label], nodeFilters...)
+		} else {
+			k3sNodeLabelFilterMap[label] = nodeFilters
+		}
+	}
+
+	for label, nodeFilters := range k3sNodeLabelFilterMap {
+		cfg.Options.K3sOptions.NodeLabels = append(cfg.Options.K3sOptions.NodeLabels, conf.LabelWithNodeFilters{
+			Label:       label,
+			NodeFilters: nodeFilters,
+		})
+	}
+
+	log.Tracef("K3sNodeLabelFilterMap: %+v", k3sNodeLabelFilterMap)
+
+	// --runtime-label
+	// runtimeLabelFilterMap will add container runtime label to applied node filters
+	runtimeLabelFilterMap := make(map[string][]string, 1)
+	for _, labelFlag := range ppViper.GetStringSlice("cli.runtime-labels") {
 
 		// split node filter from the specified label
 		label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
 		if err != nil {
 			log.Fatalln(err)
 		}
 
+		cliutil.ValidateRuntimeLabelKey(strings.Split(label, "=")[0])
+
 		// create new entry or append filter to existing entry
-		if _, exists := labelFilterMap[label]; exists {
-			labelFilterMap[label] = append(labelFilterMap[label], nodeFilters...)
+		if _, exists := runtimeLabelFilterMap[label]; exists {
+			runtimeLabelFilterMap[label] = append(runtimeLabelFilterMap[label], nodeFilters...)
 		} else {
-			labelFilterMap[label] = nodeFilters
+			runtimeLabelFilterMap[label] = nodeFilters
 		}
 	}
 
-	for label, nodeFilters := range labelFilterMap {
-		cfg.Labels = append(cfg.Labels, conf.LabelWithNodeFilters{
+	for label, nodeFilters := range runtimeLabelFilterMap {
+		cfg.Options.Runtime.Labels = append(cfg.Options.Runtime.Labels, conf.LabelWithNodeFilters{
 			Label:       label,
 			NodeFilters: nodeFilters,
 		})
 	}
 
-	log.Tracef("LabelFilterMap: %+v", labelFilterMap)
+	log.Tracef("RuntimeLabelFilterMap: %+v", runtimeLabelFilterMap)
 
 	// --env
 	// envFilterMap will add container env vars to applied node filters

cmd/node/nodeCreate.go

@@ -30,6 +30,7 @@ import (
 
 	dockerunits "github.com/docker/go-units"
 	"github.com/rancher/k3d/v4/cmd/util"
+	cliutil "github.com/rancher/k3d/v4/cmd/util"
 	k3dc "github.com/rancher/k3d/v4/pkg/client"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
@@ -74,6 +75,7 @@ func NewCmdNodeCreate() *cobra.Command {
 	cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.")
 	cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.")
 
+	cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"")
 	cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"")
 
 	// done
@@ -127,17 +129,38 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 		log.Errorf("Provided memory limit value is invalid")
 	}
 
+	// --runtime-label
+	runtimeLabelsFlag, err := cmd.Flags().GetStringSlice("runtime-label")
+	if err != nil {
+		log.Errorln("No runtime-label specified")
+		log.Fatalln(err)
+	}
+
+	runtimeLabels := make(map[string]string, len(runtimeLabelsFlag)+1)
+	for _, label := range runtimeLabelsFlag {
+		labelSplitted := strings.Split(label, "=")
+		if len(labelSplitted) != 2 {
+			log.Fatalf("unknown runtime-label format format: %s, use format \"foo=bar\"", label)
+		}
+		cliutil.ValidateRuntimeLabelKey(labelSplitted[0])
+		runtimeLabels[labelSplitted[0]] = labelSplitted[1]
+	}
+
+	// Internal k3d runtime labels take precedence over user-defined labels
+	runtimeLabels[k3d.LabelRole] = roleStr
+
+	// --k3s-node-label
 	k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label")
 	if err != nil {
-		log.Errorln("No node-label specified")
+		log.Errorln("No k3s-node-label specified")
 		log.Fatalln(err)
 	}
 
 	k3sNodeLabels := make(map[string]string, len(k3sNodeLabelsFlag))
 	for _, label := range k3sNodeLabelsFlag {
 		labelSplitted := strings.Split(label, "=")
 		if len(labelSplitted) != 2 {
-			log.Fatalf("unknown label format format: %s, use format \"foo=bar\"", label)
+			log.Fatalf("unknown k3s-node-label format format: %s, use format \"foo=bar\"", label)
 		}
 		k3sNodeLabels[labelSplitted[0]] = labelSplitted[1]
 	}
@@ -146,13 +169,11 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 	nodes := []*k3d.Node{}
 	for i := 0; i < replicas; i++ {
 		node := &k3d.Node{
-			Name:  fmt.Sprintf("%s-%s-%d", k3d.DefaultObjectNamePrefix, args[0], i),
-			Role:  role,
-			Image: image,
-			Labels: map[string]string{
-				k3d.LabelRole: roleStr,
-			},
+			Name:          fmt.Sprintf("%s-%s-%d", k3d.DefaultObjectNamePrefix, args[0], i),
+			Role:          role,
+			Image:         image,
 			K3sNodeLabels: k3sNodeLabels,
+			RuntimeLabels: runtimeLabels,
 			Restart:       true,
 			Memory:        memory,
 		}

cmd/node/nodeList.go

@@ -88,7 +88,7 @@ func NewCmdNodeList() *cobra.Command {
 					fmt.Fprintf(tabwriter, "%s\t%s\t%s\t%s\n",
 						strings.TrimPrefix(node.Name, "/"),
 						string(node.Role),
-						node.Labels[k3d.LabelClusterName],
+						node.RuntimeLabels[k3d.LabelClusterName],
 						node.State.Status)
 				}))
 		},

cmd/registry/registryList.go

@@ -88,8 +88,8 @@ func NewCmdRegistryList() *cobra.Command {
 			util.PrintNodes(existingNodes, registryListFlags.output,
 				headers, util.NodePrinterFunc(func(tabwriter *tabwriter.Writer, node *k3d.Node) {
 					cluster := "*"
-					if _, ok := node.Labels[k3d.LabelClusterName]; ok {
-						cluster = node.Labels[k3d.LabelClusterName]
+					if _, ok := node.RuntimeLabels[k3d.LabelClusterName]; ok {
+						cluster = node.RuntimeLabels[k3d.LabelClusterName]
 					}
 					fmt.Fprintf(tabwriter, "%s\t%s\t%s\t%s\n",
 						strings.TrimPrefix(node.Name, "/"),

cmd/util/runtimeLabels.go

@@ -0,0 +1,35 @@
+/*
+Copyright © 2020 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package util
+
+import (
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// validateRuntimeLabelKey validates a given label key is not reserved for internal k3d usage
+func ValidateRuntimeLabelKey(labelKey string) {
+	if strings.HasPrefix(labelKey, "k3s.") || strings.HasPrefix(labelKey, "k3d.") || labelKey == "app" {
+		log.Fatalf("runtime label \"%s\" is reserved for internal usage", labelKey)
+	}
+}

pkg/client/cluster.go

@@ -381,13 +381,13 @@ ClusterCreatOpts:
 
 	nodeSetup := func(node *k3d.Node, suffix int) error {
 		// cluster specific settings
-		if node.Labels == nil {
-			node.Labels = make(map[string]string) // TODO: maybe create an init function?
+		if node.RuntimeLabels == nil {
+			node.RuntimeLabels = make(map[string]string) // TODO: maybe create an init function?
 		}
 
 		// ensure global labels
 		for k, v := range clusterCreateOpts.GlobalLabels {
-			node.Labels[k] = v
+			node.RuntimeLabels[k] = v
 		}
 
 		// ensure global env
@@ -404,15 +404,15 @@ ClusterCreatOpts:
 				cluster.Network.IPAM.IPsUsed = append(cluster.Network.IPAM.IPsUsed, ip) // make sure that we're not reusing the same IP next time
 				node.IP.Static = true
 				node.IP.IP = ip
-				node.Labels[k3d.LabelNodeStaticIP] = ip.String()
+				node.RuntimeLabels[k3d.LabelNodeStaticIP] = ip.String()
 			}
 
 			node.ServerOpts.KubeAPI = cluster.KubeAPI
 
 			// the cluster has an init server node, but its not this one, so connect it to the init node
 			if cluster.InitNode != nil && !node.ServerOpts.IsInit {
 				node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL))
-				node.Labels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
+				node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
 			}
 
 		} else if node.Role == k3d.AgentRole {
@@ -446,10 +446,10 @@ ClusterCreatOpts:
 	if cluster.InitNode != nil {
 		log.Infoln("Creating initializing server node")
 		cluster.InitNode.Args = append(cluster.InitNode.Args, "--cluster-init")
-		if cluster.InitNode.Labels == nil {
-			cluster.InitNode.Labels = map[string]string{}
+		if cluster.InitNode.RuntimeLabels == nil {
+			cluster.InitNode.RuntimeLabels = map[string]string{}
 		}
-		cluster.InitNode.Labels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server
+		cluster.InitNode.RuntimeLabels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server
 
 		// in case the LoadBalancer was disabled, expose the API Port on the initializing server node
 		if clusterCreateOpts.DisableLoadBalancer {
@@ -547,10 +547,10 @@ ClusterCreatOpts:
 				fmt.Sprintf("PORTS=%s", strings.Join(ports, ",")),
 				fmt.Sprintf("WORKER_PROCESSES=%d", len(ports)),
 			},
-			Role:     k3d.LoadBalancerRole,
-			Labels:   clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
-			Networks: []string{cluster.Network.Name},
-			Restart:  true,
+			Role:          k3d.LoadBalancerRole,
+			RuntimeLabels: clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
+			Networks:      []string{cluster.Network.Name},
+			Restart:       true,
 		}
 		if len(udp_ports) > 0 {
 			lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ",")))
@@ -673,7 +673,7 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 // ClusterList returns a list of all existing clusters
 func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, error) {
 	log.Traceln("Listing Clusters...")
-	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels)
+	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels)
 	if err != nil {
 		log.Errorln("Failed to get clusters")
 		return nil, err
@@ -691,7 +691,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
 	log.Tracef("Found %d cluster-internal nodes", len(nodes))
 	if log.GetLevel() == log.TraceLevel {
 		for _, node := range nodes {
-			log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.Labels[k3d.LabelClusterName])
+			log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.RuntimeLabels[k3d.LabelClusterName])
 		}
 	}
 
@@ -700,7 +700,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
 	for _, node := range nodes {
 		clusterExists := false
 		for _, cluster := range clusters {
-			if node.Labels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist
+			if node.RuntimeLabels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist
 				cluster.Nodes = append(cluster.Nodes, node)
 				clusterExists = true
 				break
@@ -709,7 +709,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
 		// cluster is not in the list yet, so we add it with the current node as its first member
 		if !clusterExists {
 			clusters = append(clusters, &k3d.Cluster{
-				Name:  node.Labels[k3d.LabelClusterName],
+				Name:  node.RuntimeLabels[k3d.LabelClusterName],
 				Nodes: []*k3d.Node{node},
 			})
 		}
@@ -734,15 +734,15 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
 
 		// get the name of the cluster network
 		if cluster.Network.Name == "" {
-			if networkName, ok := node.Labels[k3d.LabelNetwork]; ok {
+			if networkName, ok := node.RuntimeLabels[k3d.LabelNetwork]; ok {
 				cluster.Network.Name = networkName
 			}
 		}
 
 		// check if the network is external
 		// since the struct value is a bool, initialized as false, we cannot check if it's unset
 		if !cluster.Network.External && !networkExternalSet {
-			if networkExternalString, ok := node.Labels[k3d.LabelNetworkExternal]; ok {
+			if networkExternalString, ok := node.RuntimeLabels[k3d.LabelNetworkExternal]; ok {
 				if networkExternal, err := strconv.ParseBool(networkExternalString); err == nil {
 					cluster.Network.External = networkExternal
 					networkExternalSet = true
@@ -752,14 +752,14 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
 
 		// get image volume // TODO: enable external image volumes the same way we do it with networks
 		if cluster.ImageVolume == "" {
-			if imageVolumeName, ok := node.Labels[k3d.LabelImageVolume]; ok {
+			if imageVolumeName, ok := node.RuntimeLabels[k3d.LabelImageVolume]; ok {
 				cluster.ImageVolume = imageVolumeName
 			}
 		}
 
 		// get k3s cluster's token
 		if cluster.Token == "" {
-			if token, ok := node.Labels[k3d.LabelClusterToken]; ok {
+			if token, ok := node.RuntimeLabels[k3d.LabelClusterToken]; ok {
 				cluster.Token = token
 			}
 		}

pkg/client/kubeconfig.go

@@ -131,11 +131,11 @@ func KubeconfigGet(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.C
 	APIHost := k3d.DefaultAPIHost
 
 	for _, server := range serverNodes {
-		if _, ok := server.Labels[k3d.LabelServerAPIPort]; ok {
+		if _, ok := server.RuntimeLabels[k3d.LabelServerAPIPort]; ok {
 			chosenServer = server
-			APIPort = server.Labels[k3d.LabelServerAPIPort]
-			if _, ok := server.Labels[k3d.LabelServerAPIHost]; ok {
-				APIHost = server.Labels[k3d.LabelServerAPIHost]
+			APIPort = server.RuntimeLabels[k3d.LabelServerAPIPort]
+			if _, ok := server.RuntimeLabels[k3d.LabelServerAPIHost]; ok {
+				APIHost = server.RuntimeLabels[k3d.LabelServerAPIHost]
 			}
 			break
 		}