openshift · bharath-b-rh · Jul 19, 2023 · Jul 19, 2023 · Jul 25, 2023 · Aug 4, 2023
diff --git a/Dockerfile b/Dockerfile
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Build driver go binary
-FROM --platform=$BUILDPLATFORM golang:1.20.12 as builder
+FROM --platform=$BUILDPLATFORM golang:1.22.2 as builder
 
 ARG STAGINGVERSION
 ARG TARGETPLATFORM
@@ -25,7 +25,7 @@ RUN GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') make driver BINDIR=/bin GCP_
 # Install nfs packages
 # Note that the newer debian bullseye image does not work with nfs-common; I
 # believe that libcap needs extra configuration.
-FROM gke.gcr.io/debian-base:bullseye-v1.4.3-gke.5 as deps
+FROM gke.gcr.io/debian-base:bookworm-v1.0.2-gke.0 as deps
 ENV DEBIAN_FRONTEND noninteractive
 
 # The netbase package is needed to get rpcbind to work correctly,
@@ -51,6 +51,8 @@ ENV DEBIAN_FRONTEND noninteractive
 # Now in `nfs_services_start.sh` we call rpcbind start, this tries to source the `/lib/lsb/init-functions` file. This needs to be installed from the lsb-base package. In the debian-base image the lsb package is deleted (https://github.com/kubernetes/release/blob/v0.15.0/images/build/debian-base/bullseye/Dockerfile.build#L90). Hence using `apt-get install --reinstall` fixes the problem.
 RUN apt-get update && apt-get dist-upgrade -y && apt-mark unhold libcap2 && apt-get install --reinstall -y --no-install-recommends \
     lsb-base \
+    # New depenency of lsb-base in bookworm
+    sysvinit-utils \
     mount \
     rpcbind \
     netbase \
@@ -62,14 +64,14 @@ RUN apt-get update && apt-get dist-upgrade -y && apt-mark unhold libcap2 && apt-
 RUN mkdir /run/sendsigs.omit.d
 
 # Hold required packages to avoid breaking the installation of packages
-RUN apt-mark hold apt gnupg adduser passwd libsemanage1 libcap2 mount nfs-common init
+RUN apt-mark hold apt gnupg adduser passwd libsemanage2 libcap2 mount nfs-common init
 
 # Clean up unnecessary packages
-# This list is copied from
-# https://github.com/kubernetes/kubernetes/blob/master/build/debian-base/Dockerfile.build
-# and modified to keep nfs dependencies
-RUN echo "Yes, do as I say!" | apt-get purge \
-    # bash \
+# We don't need to remove the packages that are already removed from the base image: 
+# https://github.com/kubernetes/release/blob/78ecea5a708046ee2d4e71be5dc73393b8d7d7cc/images/build/debian-base/bookworm/Dockerfile.build#L44-L54. 
+# The commented out packages are nfs dependencies, and should not be removed.
+RUN echo "Yes, do as I say!" | apt-get purge -y --allow-remove-essential \
+   # bash \
     e2fslibs \
     e2fsprogs \
     # init \
@@ -79,13 +81,14 @@ RUN echo "Yes, do as I say!" | apt-get purge \
     # libsmartcols1 \
     # libudev1 \
     # libblkid1 \
-    libncursesw5 \
+    # Not able to be removed even though I don't think this is needed, but removing it causes:
+    # "Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages."
+    # libncursesw6 \
     libss2 \
     ncurses-base \
-    ncurses-bin \
+    ncurses-bin
     # systemd \
     # systemd-sysv \
-    tzdata
 
 # Cleanup cached and unnecessary files.
 RUN apt-get autoremove -y && \

diff --git a/README.md b/README.md
@@ -14,64 +14,19 @@ dynamically created and mounted by workloads.
 ## Project Status
 Status: GA
 
-Latest image: `registry.k8s.io/cloud-provider-gcp/gcp-filestore-csi-driver:v1.5.3`
+Latest image: `registry.k8s.io/cloud-provider-gcp/gcp-filestore-csi-driver:v1.6.13`
 
 Also see [known issues](KNOWN_ISSUES.md) and [CHANGELOG](CHANGELOG.md).
 
-### CSI Compatibility
-This plugin is compatible with CSI version 1.3.0.
-
-### Kubernetes Compatibility
-The following table captures the compatibility matrix of the core filestore driver binary
-`registry.k8s.io/cloud-provider-gcp/gcp-filestore-csi-driver`
-
-| Filestore CSI Driver\Kubernetes Version | 1.16 | 1.17+ |
-| --------------------------------------- | ---- | ----- |
-| v0.2.0 (alpha)                          |  no  |  no   |
-| v0.3.1 (beta)                           |  yes |  yes  |
-| v0.4.0 (beta)                           |  yes |  yes  |
-| v0.5.0 (beta)                           |  yes |  yes  |
-| v0.6.0 (beta)                           |  yes |  yes  |
-| v0.6.1 (beta)                           |  yes |  yes  |
-| v1.0.0 (GA)                             |  yes |  yes  |
-| v1.1.1 (GA)                             |  yes |  yes  |
-| v1.1.2 (GA)                             |  yes |  yes  |
-| v1.1.3 (GA)                             |  yes |  yes  |
-| v1.1.4 (GA)                             |  yes |  yes  |
-| v1.2.0 (GA)                             |  yes |  yes  |
-| v1.2.1 (GA)                             |  yes |  yes  |
-| v1.2.2 (GA)                             |  yes |  yes  |
-| v1.2.3 (GA)                             |  yes |  yes  |
-| v1.2.4 (GA)                             |  yes |  yes  |
-| v1.2.5 (GA)                             |  yes |  yes  |
-| v1.2.7 (GA)                             |  yes |  yes  |
-| v1.3.0 (GA)                             |  yes |  yes  |
-| v1.3.1 (GA)                             |  yes |  yes  |
-| v1.3.2 (GA)                             |  yes |  yes  |
-| v1.3.4 (GA)                             |  yes |  yes  |
-| v1.3.5 (GA)                             |  yes |  yes  |
-| v1.3.9 (GA)                             |  yes |  yes  |
-| v1.3.10 (GA)                            |  yes |  yes  |
-| v1.3.11 (GA)                            |  yes |  yes  |
-| v1.3.12 (GA)                            |  yes |  yes  |
-| v1.3.13 (GA)                            |  yes |  yes  |
-| v1.3.14 (GA)                            |  yes |  yes  |
-| v1.4.1 (GA)                             |  yes |  yes  |
-| v1.4.2 (GA)                             |  yes |  yes  |
-| v1.4.3 (GA)                             |  yes |  yes  |
-| v1.4.4 (GA)                             |  yes |  yes  |
-| v1.4.5 (GA)                             |  yes |  yes  |
-| v1.5.0 (GA)                             |  yes |  yes  |
-| v1.5.2 (GA)                             |  yes |  yes  |
-| v1.5.3 (GA)                             |  yes |  yes  |
-| master                                  |  yes |  yes  |
-
 The manifest bundle which captures all the driver components (driver pod which includes the containers csi-external-provisioner, csi-external-resizer, csi-external-snapshotter, gcp-filestore-driver, csi-driver-registrar, csi driver object, rbacs, pod security policies etc) can be picked up from the master branch [overlays](deploy/kubernetes/overlays) directory. We structure the overlays directory per minor version of kubernetes because not all driver components can be used with all kubernetes versions. For example volume snapshots are supported 1.17+ kubernetes versions thus [stable-1-16](deploy/kubernetes/overlays/stable-1-16) driver manifests does not contain the snapshotter sidecar. Read more about overlays [here](docs/release/overlays.md).
 
 Example:
 `stable-1-19` overlays bundle can be used to deploy all the components of the driver on kubernetes 1.19.
 `stable-master` overlays bundle can be used to deploy all the components of the driver on kubernetes master.
 
+### CSI Compatibility
+This plugin is compatible with CSI version 1.3.0.
+
 ## Plugin Features
 
 ### Supported CreateVolume parameters
@@ -80,7 +35,7 @@ volume. Customizable parameters for volume creation include:
 
 | Parameter         | Values                  | Default                                | Description |
 | ---------------   | ----------------------- |-----------                             | ----------- |
-| tier              | "standard"<br>"premium"<br>"enterprise" | "standard"             | storage performance tier |
+| tier              | "standard"/"basic_hdd"<br>"premium"/"basic_ssd"<br>"enterprise"<br>"high_scale_ssd"/"zonal" | "standard"             | storage performance tier |
 | network           | string                  | "default"                              | VPC name.<br>When using "PRIVATE_SERVICE_ACCESS" connect-mode, network needs to be the full VPC name. |
 | reserved-ipv4-cidr| string		              | ""                                     | CIDR range to allocate Filestore IP Ranges from.<br>The CIDR must be large enough to accommodate multiple Filestore IP Ranges of /29 each, /26 if enterprise tier is used. |
 | reserved-ip-range | string		              | ""                                     | IP range to allocate Filestore IP Ranges from.<br>This flag is used instead of "reserved-ipv4-cidr" when "connect-mode" is set to "PRIVATE_SERVICE_ACCESS" and the value must be an [allocated IP address range](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-internal-ip-address).<br>The IP range must be large enough to accommodate multiple Filestore IP Ranges of /29 each, /26 if enterprise tier is used. |
@@ -113,6 +68,9 @@ Note that non-default networks require extra [firewall setup](https://cloud.goog
 * Volume Restore: The CSI driver supports out-of-place restore of new GCP Filestore instance from a given GCP Filestore Backup. See user-guide restore steps [here](docs/kubernetes/backup.md) and GCP Filestore Backup restore documentation [here](https://cloud.google.com/filestore/docs/backup-restore). This feature needs kubernetes 1.17+.
 * Pre-provisioned Filestore instance: Pre-provisioned filestore instances can be leveraged and consumed by workloads by mapping a given filestore instance to a PersistentVolume and PersistentVolumeClaim. See user-guide [here](docs/kubernetes/pre-provisioned-pv.md) and filestore documentation [here](https://cloud.google.com/filestore/docs/accessing-fileshares)
 * FsGroup: [CSIVolumeFSGroupPolicy](https://kubernetes-csi.github.io/docs/support-fsgroup.html) is a Kubernetes feature in Beta is 1.20, which allows CSI drivers to opt into FSGroup policies. The stable-master [overlay](deploy/kubernetes/overlays/stable-master) of Filestore CSI driver now supports this. See the user-guide [here](docs/kubernetes/fsgroup.md) on how to apply fsgroup to volumes backed by filestore instances. For a workaround to apply fsgroup on clusters 1.19 (with CSIVolumeFSGroupPolicy feature gate disabled), and clusters <= 1.18 see user-guide [here](docs/kubernetes/fsgroup-workaround.md)
+* Resource Tags: Filestore supports resource tags for instance and backup resources, which is a map of key value pairs. Filestore CSI driver enables user defined tags to be attached to instance and backup resources created by the driver.
+  User can provide resource tags by using `resource-tags` key in StorageClass.parameters or using the `--resource-tags` command line option, and the tags should be defined as comma separated values of the form `<parent_id>/<tagKey_shortname>/<tagValue_shortname>` where, parentID is the ID of Organization or Project resource where tag key and tag value resources exist, tagKey_shortname is the shortName of the tag key resource, tagValue_shortname is the shortName of the tag value resource and a maximum of 50 tags can be attached to per resource. See https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for more details.
+  Please see storage class [example](examples/kubernetes/sc-tags.yaml) to define resource tags to be attached to the Filestore instance resources.
 
 ## Future Features
 * Non-root access: By default, GCFS instances are only writable by the root user
@@ -151,6 +109,23 @@ $ PROJECT=<your-gcp-project> GCFS_SA_DIR=<your-directory-to-store-credentials-by
   testing, and the `dev` overlay is for driver development. `./deploy/kubernetes/cluster-setup.sh`
   will install the driver pods, as well as necessary RBAC and resources.
 
+* If deploying new changes in the master branch update the overlay file with a new custom tag to identify this image.
+```
+apiVersion: builtin
+kind: ImageTagTransformer
+metadata:
+  name: imagetag-gce-fs-driver
+imageTag:
+  name: k8s.gcr.io/cloud-provider-gcp/gcp-filestore-csi-driver
+  newName: gcr.io/<your-project>/gcp-filestore-csi-driver # Add newName
+  newTag: "<your-custom-tag>" # Change to your custom tag
+```
+Make and build the image if desploying new master branch.
+
+```
+GCP_FS_CSI_STAGING_VERSION=<your-custom-tag> GCP_FS_CSI_STAGING_IMAGE=gcr.io/<your-project>/gcp-filestore-csi-driver make build-image-and-push
+```
+Once the image is pushed it can be verified by visiting `https://pantheon.corp.google.com/gcr/images/<your-project>/global/gcp-filestore-csi-driver`
 ```
 $ PROJECT=<your-gcp-project> DEPLOY_VERSION=<your-overlay-choice> GCFS_SA_DIR=<your-directory-to-store-credentials-by-default-home-dir> ./deploy/kubernetes/cluster_setup.sh
 ```

diff --git a/cmd/main.go b/cmd/main.go
@@ -49,6 +49,7 @@ var (
 	isRegional                      = flag.Bool("is-regional", false, "cluster is regional cluster")
 	gkeClusterName                  = flag.String("gke-cluster-name", "", "Cluster Name of the current GKE cluster driver is running on, required for multishare")
 	extraVolumeLabelsStr            = flag.String("extra-labels", "", "Extra labels to attach to each volume created. It is a comma separated list of key value pairs like '<key1>=<value1>,<key2>=<value2>'. See https://cloud.google.com/compute/docs/labeling-resources for details")
+	resourceTagsStr                 = flag.String("resource-tags", "", "Resource tags to attach to each volume created. It is a comma separated list of tags of the form '<parentID_1>/<tagKey_1>/<tagValue_1>...<parentID_N>/<tagKey_N>/<tagValue_N>' where, parentID is the ID of Organization or Project resource where tag key and value resources exist, tagKey is the shortName of the tag key resource, tagValue is the shortName of the tag value resource. See https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for more details.")
 
 	// Feature lock release specific parameters, only take effect when feature-lock-release is set to true.
 	featureLockRelease    = flag.Bool("feature-lock-release", false, "if set to true, the node driver will support Filestore lock release.")
@@ -61,7 +62,8 @@ var (
 	coreInformerResyncPeriod   = flag.Duration("core-informer-resync-repriod", 15*time.Minute, "Core informer resync period.")
 
 	// Feature multishare backups enabled
-	featureMultishareBackups = flag.Bool("feature-multishare-backups", false, "if set to true, the multishare backups will be enabled. enable-multishare must be set to true as well")
+	featureMultishareBackups        = flag.Bool("feature-multishare-backups", false, "if set to true, the multishare backups will be enabled. enable-multishare must be set to true as well")
+	featureNFSExportOptionsOnCreate = flag.Bool("feature-nfs-export-options", false, "if set to true, the driver will accpet nfs-export-options-on-create parameter and configure IP Access rules")
 
 	// Feature stateful CSI driver specific parameters
 	featureStateful      = flag.Bool("feature-stateful-multishare", false, "if set to true, the controller will run stateful multishare controller, if set to true, enable-multishare must be set to true as well")
@@ -94,6 +96,7 @@ func main() {
 	var meta metadata.Service
 	var mm *metrics.MetricsManager
 	var extraVolumeLabels map[string]string
+	var tagMgr cloud.TagService
 	if *runController {
 		if *httpEndpoint != "" && metrics.IsGKEComponentVersionAvailable() {
 			mm = metrics.NewMetricsManager()
@@ -114,13 +117,23 @@ func main() {
 		}
 
 		provider, err = cloud.NewCloud(ctx, version, *cloudConfigFilePath, *primaryFilestoreServiceEndpoint, *testFilestoreServiceEndpoint)
+
+		tagMgr = cloud.NewTagManager(provider)
+		tags, err := tagMgr.ValidateResourceTags(ctx, "command line", *resourceTagsStr)
+		if err != nil {
+			klog.Fatalf("failed to parse resource tags provided in command line: %v", err)
+		}
+		tagMgr.SetResourceTags(tags)
 	} else {
 		if *nodeID == "" {
 			klog.Fatalf("nodeid cannot be empty for node service")
 		}
 		if len(*extraVolumeLabelsStr) > 0 {
 			klog.Fatalf("Extra volume labels provided but not running controller")
 		}
+		if len(*resourceTagsStr) > 0 {
+			klog.Fatalf("Resource tags provided but not running controller")
+		}
 
 		meta, err = metadataservice.NewMetadataService()
 		if err != nil {
@@ -183,6 +196,9 @@ func main() {
 		FeatureMultishareBackups: &driver.FeatureMultishareBackups{
 			Enabled: *featureMultishareBackups,
 		},
+		FeatureNFSExportOptionsOnCreate: &driver.FeatureNFSExportOptionsOnCreate{
+			Enabled: *featureNFSExportOptionsOnCreate,
+		},
 	}
 
 	mounter := mount.New("")
@@ -202,6 +218,7 @@ func main() {
 		ClusterName:       *gkeClusterName,
 		FeatureOptions:    featureOptions,
 		ExtraVolumeLabels: extraVolumeLabels,
+		TagManager:        tagMgr,
 	}
 
 	gcfsDriver, err := driver.NewGCFSDriver(config)

diff --git a/cmd/webhook/Dockerfile b/cmd/webhook/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=$BUILDPLATFORM golang:1.20.12 as builder
+FROM --platform=$BUILDPLATFORM golang:1.22.2 as builder
 
 ARG TARGETPLATFORM
 

diff --git a/deploy/kubernetes/images/prow-gke-release-staging-rc-master/image.yaml b/deploy/kubernetes/images/prow-gke-release-staging-rc-master/image.yaml
@@ -41,5 +41,5 @@ metadata:
 imageTag:
   name: registry.k8s.io/cloud-provider-gcp/gcp-filestore-csi-driver
   newName: gcr.io/k8s-staging-cloud-provider-gcp/gcp-filestore-csi-driver
-  newTag: "v1.5.6-rc1"
+  newTag: "v1.6.13-rc1"
 ---
diff --git a/deploy/kubernetes/images/stable-master/image.yaml b/deploy/kubernetes/images/stable-master/image.yaml
@@ -40,5 +40,5 @@ metadata:
   name: imagetag-gce-fs-driver
 imageTag:
   name: registry.k8s.io/cloud-provider-gcp/gcp-filestore-csi-driver
-  newTag: "v1.5.3"
+  newTag: "v1.6.13"
 ---
diff --git a/docs/kubernetes/backup.md b/docs/kubernetes/backup.md
@@ -1,4 +1,4 @@
-# Filestore Backups User Guide (Beta)
+# Filestore Backups User Guide
 
 >**Attention:** Filestore Backup relies on CSI VolumeSnapshot which is a Beta feature in k8s enabled by default in
 Kubernetes 1.17+. CSI VolumeSnapshot should not be confused with Filestore Backups. Filestore CSI driver leverages CSI VolumeSnapshot capability to support Filestore Backups by specifying a `type` parameter in the VolumeSnapshotClass object.
@@ -31,7 +31,6 @@ The [CSI Snapshot](https://github.com/container-storage-interface/spec/blob/mast
     name: csi-filestore
     provisioner: filestore.csi.storage.gke.io
     parameters:
-      tier: enterprise
       network: <network name> # Change this network as per the deployment
     volumeBindingMode: WaitForFirstConsumer
     allowVolumeExpansion: true

diff --git a/examples/kubernetes/backups-restore/sc.yaml b/examples/kubernetes/backups-restore/sc.yaml
@@ -3,8 +3,6 @@ kind: StorageClass
 metadata:
   name: csi-filestore
 provisioner: filestore.csi.storage.gke.io
-parameters:
-  tier: enterprise
 volumeBindingMode: WaitForFirstConsumer
 reclaimPolicy: Delete
 allowVolumeExpansion: true
diff --git a/examples/kubernetes/sc-tags.yaml b/examples/kubernetes/sc-tags.yaml
@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-filestore
+provisioner: filestore.csi.storage.gke.io
+parameters:
+  resource-tags: parent1/tagKey1/tagValue1,parent2/tagKey2/tagValue2,...,parentN/tagKeyN/tagValueN
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true