STOR-1602: Rebase to upstream v1.14.1 for OCP 4.17

.ci-operator.yaml

@@ -1,4 +1,4 @@
 build_root_image:
   name: release
   namespace: openshift
-  tag: rhel-9-release-golang-1.21-openshift-4.16
+  tag: rhel-9-release-golang-1.22-openshift-4.17

CHANGELOG.md

@@ -1 +1 @@
-See the [1.13 changelog in the master branch](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/blob/master/CHANGELOG/CHANGELOG-1.13.md).
+See the [1.14 changelog in the master branch](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/blob/master/CHANGELOG/CHANGELOG-1.14.md).

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=$BUILDPLATFORM golang:1.21.5 as builder
+FROM --platform=$BUILDPLATFORM golang:1.22.4 as builder
 
 ARG STAGINGVERSION
 ARG TARGETPLATFORM
@@ -22,13 +22,15 @@ ADD . .
 RUN GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=$STAGINGVERSION make gce-pd-driver
 
 # Start from Kubernetes Debian base.
-FROM gke.gcr.io/debian-base:bullseye-v1.4.3-gke.5 as debian
+
+FROM gke.gcr.io/debian-base:bookworm-v1.0.3-gke.0 as debian
+
 # Install necessary dependencies
 # google_nvme_id script depends on the following packages: nvme-cli, xxd, bash
 RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash
 
 # Since we're leveraging apt to pull in dependencies, we use `gcr.io/distroless/base` because it includes glibc.
-FROM gcr.io/distroless/base-debian11 as distroless-base
+FROM gcr.io/distroless/base-debian12 as distroless-base
 
 # The distroless amd64 image has a target triplet of x86_64
 FROM distroless-base AS distroless-amd64
@@ -72,8 +74,7 @@ COPY --from=debian /bin/ln /bin/ln
 COPY --from=debian /bin/udevadm /bin/udevadm
 
 # Copy shared libraries into distroless base.
-COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \
+COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libe2p.so.2 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libcom_err.so.2 \
@@ -82,7 +83,21 @@ COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libgcc_s.so.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/liblzma.so.5 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libreadline.so.8 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libz.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libz.so.1 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libc.so.6 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/liburcu.so.8 \ 
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libcap.so.2 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libcrypto.so.3 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libdbus-1.so.3 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libgcrypt.so.20 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libjson-c.so.5 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/liblz4.so.1 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libm.so.6 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libnvme-mi.so.1 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libnvme.so.1 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libsystemd.so.0 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libgpg-error.so.0 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/
 
 COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libbsd.so.0 \
@@ -93,9 +108,9 @@ COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libacl.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libattr.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libedit.so.2 \
-                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicudata.so.67 \
-                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicui18n.so.67 \
-                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicuuc.so.67 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicudata.so.72 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicui18n.so.72 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicuuc.so.72 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libkmod.so.2 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmd.so.0 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre2-8.so.0 \

Dockerfile.Windows

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE
-FROM --platform=$BUILDPLATFORM golang:1.21.5 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.22.4 AS builder
 
 ARG TARGETPLATFORM
 ARG STAGINGVERSION

Dockerfile.debug

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.21.5 as builder
+FROM golang:1.22.4 as builder
 WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 ADD . .
 

Dockerfile.openshift

@@ -1,10 +1,10 @@
-FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.21-openshift-4.16 AS builder
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.17 AS builder
 WORKDIR /go/src/github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver
 COPY . .
 # Skip Windows driver build
 RUN make gce-pd-driver
 
-FROM registry.ci.openshift.org/ocp/4.16:base-rhel9
+FROM registry.ci.openshift.org/ocp/4.17:base-rhel9
 
 # Get all deps
 # nvme-cli and /usr/bin/xxd are deps of google_nvme_id

README.md

@@ -12,7 +12,7 @@ lifecycle of Google Compute Engine Persistent Disks.
 ## Project Status
 
 Status: GA
-Latest stable image: `registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1`
+Latest stable image: `registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.13.2`
 
 ### Test Status
 
@@ -35,10 +35,10 @@ versions.
 
 | Kubernetes Version | PD CSI Driver Version |
 |--------------------|-----------------------|
-| HEAD               | v1.10.x               |
+| HEAD               | v1.13.x               |
+| 1.29               | v1.12.x               |
+| 1.28               | v1.12.x               |
 | 1.27               | v1.10.x               |
-| 1.26               | v1.9.x                |
-| 1.25               | v1.8.x                |
 
 The manifest bundle which captures all the driver components (driver pod which includes the containers csi-provisioner, csi-resizer, csi-snapshotter, gce-pd-driver, csi-driver-registrar;
 csi driver object, rbacs, pod security policies etc) for the lastest stable
@@ -59,7 +59,8 @@ See Github [Issues](https://github.com/kubernetes-sigs/gcp-compute-persistent-di
 | disk-encryption-kms-key     | Fully qualified resource identifier for the key to use to encrypt new disks. | Empty string. | Encrypt disk using Customer Managed Encryption Key (CMEK). See [GKE Docs](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek#create_a_cmek_protected_attached_disk) for details. |
 | labels                      | `key1=value1,key2=value2` |               | Labels allow you to assign custom [GCE Disk labels](https://cloud.google.com/compute/docs/labeling-resources). |
 | provisioned-iops-on-create  | string (int64 format). Values typically between 10,000 and 120,000 |               | Indicates how many IOPS to provision for the disk. See the [Extreme persistent disk documentation](https://cloud.google.com/compute/docs/disks/extreme-persistent-disk) for details, including valid ranges for IOPS. |
-| provisioned-throughput-on-create  | string (int64 format). Values typically between 1 and 7,124 mb per second |               | Indicates how much throughput to provision for the disk. See the [hyperdisk documentation](TBD) for details, including valid ranges for throughput. |
+| provisioned-throughput-on-create  | string (int64 format). Values typically between 1 and 7,124 mb per second |               | Indicates how much throughput to provision for the disk. See the [hyperdisk documentation]([TBD](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/hyperdisk#create)) for details, including valid ranges for throughput. |
+| resource-tags               | `<parent_id1>/<tag_key1>/<tag_value1>,<parent_id2>/<tag_key2>/<tag_value2>` |               | Resource tags allow you to attach user-defined tags to each Compute Disk, Image and Snapshot. See [Tags overview](https://cloud.google.com/resource-manager/docs/tags/tags-overview), [Creating and managing tags](https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing). |
 
 ### Topology
 

cmd/gce-pd-csi-driver/main.go

@@ -17,14 +17,18 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
+	"fmt"
 	"math/rand"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
 	"time"
 
 	"k8s.io/klog/v2"
+	"k8s.io/utils/strings/slices"
 
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils"
@@ -38,7 +42,6 @@ import (
 var (
 	cloudConfigFilePath  = flag.String("cloud-config", "", "Path to GCE cloud provider config")
 	endpoint             = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
-	computeEndpoint      = flag.String("compute-endpoint", "", "If set, used as the endpoint for the GCE API.")
 	runControllerService = flag.Bool("run-controller-service", true, "If set to false then the CSI driver does not activate its controller service (default: true)")
 	runNodeService       = flag.Bool("run-node-service", true, "If set to false then the CSI driver does not activate its node service (default: true)")
 	httpEndpoint         = flag.String("http-endpoint", "", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled.")
@@ -67,10 +70,21 @@ var (
 
 	maxConcurrentFormatAndMount = flag.Int("max-concurrent-format-and-mount", 1, "If set then format and mount operations are serialized on each node. This is stronger than max-concurrent-format as it includes fsck and other mount operations")
 	formatAndMountTimeout       = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount")
+	fallbackRequisiteZonesFlag  = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk")
+	enableStoragePoolsFlag      = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools")
 
-	fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk")
+	multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable")
+	multiZoneVolumeHandleEnableFlag    = flag.Bool("multi-zone-volume-handle-enable", false, "If set to true, the multi-zone volumeHandle feature will be enabled")
 
-	enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools")
+	computeEnvironment        gce.Environment = gce.EnvironmentProduction
+	computeEndpoint           *url.URL
+	allowedComputeEnvironment = []gce.Environment{gce.EnvironmentStaging, gce.EnvironmentProduction}
+
+	useInstanceAPIOnWaitForAttachDiskTypesFlag     = flag.String("use-instance-api-to-poll-attachment-disk-types", "", "Comma separated list of disk types that should use instances.get API when polling for disk attach during ControllerPublish")
+	useInstanceAPIForListVolumesPublishedNodesFlag = flag.Bool("use-instance-api-to-list-volumes-published-nodes", false, "Enables using the instances.list API to determine published_node_ids in ListVolumes. When false (default), the disks.list API is used")
+	instancesListFiltersFlag                       = flag.String("instances-list-filters", "", "Comma separated list of filters to use when calling the instances.list API. By default instances.list fetches all instances in a region")
+
+	extraTagsStr = flag.String("extra-tags", "", "Extra tags to attach to each Compute Disk, Image, Snapshot created. It is a comma separated list of parent id, key and value like '<parent_id1>/<tag_key1>/<tag_value1>,...,<parent_idN>/<tag_keyN>/<tag_valueN>'. parent_id is the Organization or the Project ID or Project name where the tag key and the tag value resources exist. A maximum of 50 tags bindings is allowed for a resource. See https://cloud.google.com/resource-manager/docs/tags/tags-overview, https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details")
 
 	version string
 )
@@ -85,13 +99,16 @@ func init() {
 	// Use V(4) for general debug information logging
 	// Use V(5) for GCE Cloud Provider Call informational logging
 	// Use V(6) for extra repeated/polling information
+	stringEnumFlag(&computeEnvironment, "compute-environment", allowedComputeEnvironment, "Operating compute environment")
+	urlFlag(&computeEndpoint, "compute-endpoint", "Compute endpoint")
 	klog.InitFlags(flag.CommandLine)
 	flag.Set("logtostderr", "true")
 }
 
 func main() {
 	flag.Parse()
 	rand.Seed(time.Now().UnixNano())
+	klog.Infof("Operating compute environment set to: %s and computeEndpoint is set to: %v", computeEnvironment, computeEndpoint)
 	handle()
 	os.Exit(0)
 }
@@ -141,6 +158,14 @@ func handle() {
 		klog.Fatalf("Bad extra volume labels: %v", err.Error())
 	}
 
+	if len(*extraTagsStr) > 0 && !*runControllerService {
+		klog.Fatalf("Extra tags provided but not running controller")
+	}
+	extraTags, err := common.ConvertTagsStringToMap(*extraTagsStr)
+	if err != nil {
+		klog.Fatalf("Bad extra tags: %v", err.Error())
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -150,19 +175,41 @@ func handle() {
 	// Initialize identity server
 	identityServer := driver.NewIdentityServer(gceDriver)
 
-	// Initilaize requisite zones
-	fallbackRequisiteZones := strings.Split(*fallbackRequisiteZonesFlag, ",")
+	// Initialize requisite zones
+	fallbackRequisiteZones := parseCSVFlag(*fallbackRequisiteZonesFlag)
+
+	// Initialize multi-zone disk types
+	multiZoneVolumeHandleDiskTypes := parseCSVFlag(*multiZoneVolumeHandleDiskTypesFlag)
+	multiZoneVolumeHandleConfig := driver.MultiZoneVolumeHandleConfig{
+		Enable:    *multiZoneVolumeHandleEnableFlag,
+		DiskTypes: multiZoneVolumeHandleDiskTypes,
+	}
+
+	// Initialize waitForAttach config
+	useInstanceAPIOnWaitForAttachDiskTypes := parseCSVFlag(*useInstanceAPIOnWaitForAttachDiskTypesFlag)
+	waitForAttachConfig := gce.WaitForAttachConfig{
+		UseInstancesAPIForDiskTypes: useInstanceAPIOnWaitForAttachDiskTypes,
+	}
+
+	// Initialize listVolumes config
+	instancesListFilters := parseCSVFlag(*instancesListFiltersFlag)
+	listInstancesConfig := gce.ListInstancesConfig{
+		Filters: instancesListFilters,
+	}
+	listVolumesConfig := driver.ListVolumesConfig{
+		UseInstancesAPIForPublishedNodes: *useInstanceAPIForListVolumesPublishedNodesFlag,
+	}
 
 	// Initialize requirements for the controller service
 	var controllerServer *driver.GCEControllerServer
 	if *runControllerService {
-		cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath, *computeEndpoint)
+		cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath, computeEndpoint, computeEnvironment, waitForAttachConfig, listInstancesConfig)
 		if err != nil {
 			klog.Fatalf("Failed to get cloud provider: %v", err.Error())
 		}
 		initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond
 		maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond
-		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag)
+		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig)
 	} else if *cloudConfigFilePath != "" {
 		klog.Warningf("controller service is disabled but cloud config given - it has no effect")
 	}
@@ -186,7 +233,7 @@ func handle() {
 		}
 	}
 
-	err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, identityServer, controllerServer, nodeServer)
+	err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, extraTags, identityServer, controllerServer, nodeServer)
 	if err != nil {
 		klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error())
 	}
@@ -205,3 +252,62 @@ func handle() {
 
 	gceDriver.Run(*endpoint, *grpcLogCharCap, *enableOtelTracing)
 }
+
+func notEmpty(v string) bool {
+	return v != ""
+}
+
+func parseCSVFlag(list string) []string {
+	return slices.Filter(nil, strings.Split(list, ","), notEmpty)
+}
+
+type enumConverter[T any] interface {
+	convert(v string) (T, error)
+	eq(a, b T) bool
+}
+
+type stringConverter[T ~string] struct{}
+
+func (s stringConverter[T]) convert(v string) (T, error) {
+	return T(v), nil
+}
+
+func (s stringConverter[T]) eq(a, b T) bool {
+	return a == b
+}
+
+func stringEnumFlag[T ~string](target *T, name string, allowed []T, usage string) {
+	enumFlag(target, name, stringConverter[T]{}, allowed, usage)
+}
+
+func enumFlag[T any](target *T, name string, converter enumConverter[T], allowed []T, usage string) {
+	flag.Func(name, usage, func(flagValue string) error {
+		tValue, err := converter.convert(flagValue)
+		if err != nil {
+			return err
+		}
+		for _, allowedValue := range allowed {
+			if converter.eq(allowedValue, tValue) {
+				*target = tValue
+				return nil
+			}
+		}
+		errMsg := fmt.Sprintf(`must be one of %v`, allowedComputeEnvironment)
+		return errors.New(errMsg)
+	})
+}
+
+func urlFlag(target **url.URL, name string, usage string) {
+	flag.Func(name, usage, func(flagValue string) error {
+		if flagValue == "" {
+			return nil
+		}
+		computeURL, err := url.ParseRequestURI(flagValue)
+		if err == nil {
+			*target = computeURL
+			return nil
+		}
+		klog.Errorf("Error parsing endpoint compute endpoint %v", err)
+		return err
+	})
+}

deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml

@@ -48,6 +48,6 @@ metadata:
 imageTag:
   name: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver
   newName: gcr.io/k8s-staging-cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver
-  newTag: "v1.12.0-rc1"
+  newTag: "v1.13.3-rc1"
 ---