CFE-878: Add userLabels in Infrastructure to driver args list

assets/controller.yaml

@@ -50,7 +50,6 @@ spec:
             - --endpoint=$(CSI_ENDPOINT)
             - --logtostderr
             - --v=${LOG_LEVEL}
-            - --extra-labels=kubernetes-io-cluster-${CLUSTER_ID}=owned
           env:
             - name: GOOGLE_APPLICATION_CREDENTIALS
               value: "/etc/cloud-sa/service_account.json"

go.mod

@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/google/go-cmp v0.5.9
-	github.com/openshift/api v0.0.0-20230613151523-ba04973d3ed1
+	github.com/openshift/api v0.0.0-20230804173756-26b8597c4de2
 	github.com/openshift/build-machinery-go v0.0.0-20230306181456-d321ffa04533
 	github.com/openshift/client-go v0.0.0-20230503144108-75015d2347cb
 	github.com/openshift/library-go v0.0.0-20230626162119-954ade536d6d

go.sum

@@ -415,8 +415,8 @@ github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
-github.com/openshift/api v0.0.0-20230613151523-ba04973d3ed1 h1:sgr89m3ejIIKhSbTtHq7HEZ80et4IAXDrJlk+u+rYX8=
-github.com/openshift/api v0.0.0-20230613151523-ba04973d3ed1/go.mod h1:4VWG+W22wrB4HfBL88P40DxLEpSOaiBVxUnfalfJo9k=
+github.com/openshift/api v0.0.0-20230804173756-26b8597c4de2 h1:K7rBUJvIEa9Ei7tyAv4wDwDLpOFKa6nP84JnqxrY73o=
+github.com/openshift/api v0.0.0-20230804173756-26b8597c4de2/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs=
 github.com/openshift/build-machinery-go v0.0.0-20230306181456-d321ffa04533 h1:mh3ZYs7kPIIe3UUY6tJcTExmtjnXXUu0MrBuK2W/Qvw=
 github.com/openshift/build-machinery-go v0.0.0-20230306181456-d321ffa04533/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
 github.com/openshift/client-go v0.0.0-20230503144108-75015d2347cb h1:Nij5OnaECrkmcRQMAE9LMbQXPo95aqFnf+12B7SyFVI=

pkg/operator/starter.go

@@ -3,8 +3,10 @@ package operator
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/dynamic"
@@ -15,13 +17,15 @@ import (
 	opv1 "github.com/openshift/api/operator/v1"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	configinformers "github.com/openshift/client-go/config/informers/externalversions"
+	configlisters "github.com/openshift/client-go/config/listers/config/v1"
 	opclient "github.com/openshift/client-go/operator/clientset/versioned"
 	opinformers "github.com/openshift/client-go/operator/informers/externalversions"
 	"github.com/openshift/library-go/pkg/controller/controllercmd"
 	"github.com/openshift/library-go/pkg/controller/factory"
 	"github.com/openshift/library-go/pkg/operator/csi/csicontrollerset"
 	"github.com/openshift/library-go/pkg/operator/csi/csidrivercontrollerservicecontroller"
 	"github.com/openshift/library-go/pkg/operator/csi/csidrivernodeservicecontroller"
+	dc "github.com/openshift/library-go/pkg/operator/deploymentcontroller"
 	goc "github.com/openshift/library-go/pkg/operator/genericoperatorclient"
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
 
@@ -41,6 +45,13 @@ const (
 
 	diskEncryptionKMSKey  = "disk-encryption-kms-key"
 	defaultKMSKeyLocation = "global"
+
+	// globalInfrastructureName is the default name of the Infrastructure object
+	globalInfrastructureName = "cluster"
+
+	// ocpDefaultLabelFmt is the format string for the default label
+	// added to the OpenShift created GCP resources.
+	ocpDefaultLabelFmt = "kubernetes-io-cluster-%s=owned"
 )
 
 func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error {
@@ -167,6 +178,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
 			secretInformer,
 		),
 		csidrivercontrollerservicecontroller.WithReplicasHook(nodeInformer.Lister()),
+		withCustomLabels(infraInformer.Lister()),
 	).WithCSIDriverNodeService(
 		"GCPPDDriverNodeServiceController",
 		assets.ReadFile,
@@ -215,3 +227,38 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
 
 	return fmt.Errorf("stopped")
 }
+
+// withCustomLabels adds labels from Infrastructure.Status.PlatformStatus.GCP.ResourceLabels to the
+// driver command line as --extra-labels=<key1>=<value1>,<key2>=<value2>,...
+func withCustomLabels(infraLister configlisters.InfrastructureLister) dc.DeploymentHookFunc {
+	return func(spec *opv1.OperatorSpec, deployment *appsv1.Deployment) error {
+		infra, err := infraLister.Get(globalInfrastructureName)
+		if err != nil {
+			return fmt.Errorf("custom labels: failed to fetch global Infrastructure object: %w", err)
+		}
+
+		var labels []string
+		if infra.Status.PlatformStatus != nil &&
+			infra.Status.PlatformStatus.GCP != nil &&
+			infra.Status.PlatformStatus.GCP.ResourceLabels != nil {
+			labels = make([]string, len(infra.Status.PlatformStatus.GCP.ResourceLabels))
+			for i, label := range infra.Status.PlatformStatus.GCP.ResourceLabels {
+				labels[i] = fmt.Sprintf("%s=%s", label.Key, label.Value)
+			}
+		}
+
+		labels = append(labels, fmt.Sprintf(ocpDefaultLabelFmt, infra.Status.InfrastructureName))
+		labelsStr := strings.Join(labels, ",")
+		labelsArg := fmt.Sprintf("--extra-labels=%s", labelsStr)
+		klog.V(1).Infof("withCustomLabels: adding extra-labels arg to driver with value %s", labelsStr)
+
+		for i := range deployment.Spec.Template.Spec.Containers {
+			container := &deployment.Spec.Template.Spec.Containers[i]
+			if container.Name != "csi-driver" {
+				continue
+			}
+			container.Args = append(container.Args, labelsArg)
+		}
+		return nil
+	}
+}

pkg/operator/starter_test.go

@@ -0,0 +1,144 @@
+package operator
+
+import (
+	"fmt"
+	"testing"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	v1 "github.com/openshift/api/config/v1"
+	fakeconfig "github.com/openshift/client-go/config/clientset/versioned/fake"
+	configinformers "github.com/openshift/client-go/config/informers/externalversions"
+)
+
+func TestWithCustomLabels(t *testing.T) {
+
+	infraObj := &v1.Infrastructure{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "cluster",
+		},
+		Status: v1.InfrastructureStatus{
+			InfrastructureName: "test-vbc3g",
+			PlatformStatus: &v1.PlatformStatus{
+				GCP: &v1.GCPPlatformStatus{
+					ProjectID: "test",
+					Region:    "test",
+				},
+			},
+		},
+	}
+
+	tmplDeployObj := &appsv1.Deployment{
+		Spec: appsv1.DeploymentSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "csi-driver",
+							Image: "example.io/example-csi-driver",
+							Args: []string{
+								"--endpoint=$(CSI_ENDPOINT)",
+								"--logtostderr",
+								"--v=2",
+							},
+							Env: []corev1.EnvVar{
+								{
+									Name:  "GOOGLE_APPLICATION_CREDENTIALS",
+									Value: "/etc/cloud-sa/service_account.json",
+								},
+								{
+									Name:  "CSI_ENDPOINT",
+									Value: "unix:///var/lib/csi/sockets/pluginproxy/csi.sock",
+								},
+							},
+						},
+						{
+							Name:  "test-driver",
+							Image: "example.io/example-test-driver",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name          string
+		labels        []v1.GCPResourceLabel
+		expArgList    string
+		createInfraCR bool
+		wantErr       bool
+	}{
+		{
+			name:          "labels not configured",
+			labels:        []v1.GCPResourceLabel{},
+			expArgList:    fmt.Sprintf("--extra-labels=%s", fmt.Sprintf(ocpDefaultLabelFmt, infraObj.Status.InfrastructureName)),
+			createInfraCR: true,
+			wantErr:       false,
+		},
+		{
+			name: "labels configured",
+			labels: []v1.GCPResourceLabel{
+				{
+					Key:   "key1",
+					Value: "value1",
+				},
+				{
+					Key:   "key2",
+					Value: "value2",
+				},
+				{
+					Key:   "key3",
+					Value: "value3",
+				},
+			},
+			expArgList: fmt.Sprintf("--extra-labels=key1=value1,key2=value2,"+
+				"key3=value3,%s", fmt.Sprintf(ocpDefaultLabelFmt, infraObj.Status.InfrastructureName)),
+			createInfraCR: true,
+			wantErr:       false,
+		},
+		{
+			name:          "Infrastructure CR does not exist",
+			labels:        []v1.GCPResourceLabel{},
+			expArgList:    "",
+			createInfraCR: false,
+			wantErr:       true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			objs := make([]runtime.Object, 0)
+			if test.createInfraCR {
+				infraObj.Status.PlatformStatus.GCP.ResourceLabels = test.labels
+				objs = append(objs, infraObj)
+			}
+			configClient := fakeconfig.NewSimpleClientset(objs...)
+			configInformerFactory := configinformers.NewSharedInformerFactory(configClient, 0)
+			if test.createInfraCR {
+				configInformerFactory.Config().V1().Infrastructures().Informer().GetIndexer().Add(infraObj)
+			}
+
+			deployment := tmplDeployObj.DeepCopy()
+			updDeployment := tmplDeployObj.DeepCopy()
+			if test.expArgList != "" {
+				updDeployment.Spec.Template.Spec.Containers[0].Args = append(
+					updDeployment.Spec.Template.Spec.Containers[0].Args,
+					test.expArgList,
+				)
+			}
+
+			err := withCustomLabels(configInformerFactory.Config().V1().Infrastructures().Lister())(nil, deployment)
+			if (err != nil) != test.wantErr {
+				t.Errorf("unexpected error: %v", err)
+			}
+			if !equality.Semantic.DeepEqual(deployment, updDeployment) {
+				t.Errorf("unexpected deployment want: %+v got: %+v", updDeployment, deployment)
+			}
+		})
+	}
+}