Skip to content

Commit 407330d

Browse files
jsafranebertinatto
jsafrane
authored and
bertinatto
committed
UPSTREAM: <carry>: Add volume group snapshot test driver
Upstream enables volume group snapshots by editing yaml files in a shell script [1]. We can't use this script in openshift-tests. Create a brand new, OCP specific test driver based on csi-driver-hostpath, only with the --feature-gate=VolumeGroupSnapshot on external-snapshotter command line. We will need to carry this patch until the feature graduates to GA. I've chosen to create brand new files in this carry patch, so it can't conflict with the existing ones. 1: https://github.com/kubernetes/kubernetes/blob/91d6fd3455c4a071408df20c7f48df221f2b6d30/test/e2e/testing-manifests/storage-csi/external-snapshotter/volume-group-snapshots/run_group_snapshot_e2e.sh
1 parent 6fc9a39 commit 407330d

File tree

2 files changed

+327
-0
lines changed

2 files changed

+327
-0
lines changed

‎test/e2e/storage/drivers/openshift_group_snapshot_driver.go‎

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
package drivers
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
"github.com/onsi/ginkgo/v2"
9+
10+
appsv1 "k8s.io/api/apps/v1"
11+
v1 "k8s.io/api/core/v1"
12+
storagev1 "k8s.io/api/storage/v1"
13+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
14+
"k8s.io/apimachinery/pkg/util/sets"
15+
"k8s.io/kubernetes/test/e2e/framework"
16+
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
17+
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
18+
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
19+
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
20+
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
21+
"k8s.io/kubernetes/test/e2e/storage/utils"
22+
)
23+
24+
// Special test driver for volume group snapshots.
25+
//
26+
// Upstream uses a script to install csi-driver-hostpath with group snapshots enabled in its CSI sidecars.
27+
// We can't use that in OCP, so let's create a new test driver based on [Driver: csi-hospath],
28+
// only with the group snapshots enabled.
29+
30+
// The rest of the file is a copy of Kubernete's HostPath test driver from test/e2e/storage/drivers/csi.go
31+
// Differences:
32+
// - the tests driver name is: [Driver: csi-hospath-groupsnapshot].
33+
// - enabled group snapshots in the external-snapshotter sidecar.
34+
// - still use "csi-hostpath" as PatchCSIOptions.OldDriverName, because it's a name of a directory than needs to be replaced in the driver yaml files.
35+
36+
type groupSnapshotHostpathCSIDriver struct {
37+
driverInfo storageframework.DriverInfo
38+
manifests []string
39+
volumeAttributes []map[string]string
40+
}
41+
42+
func initGroupSnapshotHostpathCSIDriver(name string, capabilities map[storageframework.Capability]bool, volumeAttributes []map[string]string, manifests ...string) storageframework.TestDriver {
43+
return &groupSnapshotHostpathCSIDriver{
44+
driverInfo: storageframework.DriverInfo{
45+
Name: name,
46+
MaxFileSize: storageframework.FileSizeMedium,
47+
SupportedFsType: sets.NewString(
48+
"", // Default fsType
49+
),
50+
SupportedSizeRange: e2evolume.SizeRange{
51+
Min: "1Mi",
52+
},
53+
Capabilities: capabilities,
54+
StressTestOptions: &storageframework.StressTestOptions{
55+
NumPods: 10,
56+
NumRestarts: 10,
57+
},
58+
VolumeSnapshotStressTestOptions: &storageframework.VolumeSnapshotStressTestOptions{
59+
NumPods: 10,
60+
NumSnapshots: 10,
61+
},
62+
PerformanceTestOptions: &storageframework.PerformanceTestOptions{
63+
ProvisioningOptions: &storageframework.PerformanceTestProvisioningOptions{
64+
VolumeSize: "1Mi",
65+
Count: 300,
66+
// Volume provisioning metrics are compared to a high baseline.
67+
// Failure to pass would suggest a performance regression.
68+
ExpectedMetrics: &storageframework.Metrics{
69+
AvgLatency: 2 * time.Minute,
70+
Throughput: 0.5,
71+
},
72+
},
73+
},
74+
TestTags: []interface{}{"[OCPFeatureGate:VolumeGroupSnapshot]"},
75+
},
76+
manifests: manifests,
77+
volumeAttributes: volumeAttributes,
78+
}
79+
}
80+
81+
var _ storageframework.TestDriver = &groupSnapshotHostpathCSIDriver{}
82+
var _ storageframework.DynamicPVTestDriver = &groupSnapshotHostpathCSIDriver{}
83+
var _ storageframework.SnapshottableTestDriver = &groupSnapshotHostpathCSIDriver{}
84+
var _ storageframework.EphemeralTestDriver = &groupSnapshotHostpathCSIDriver{}
85+
86+
// InitgroupSnapshotHostpathCSIDriver returns groupSnapshotHostpathCSIDriver that implements TestDriver interface
87+
func InitGroupSnapshotHostpathCSIDriver() storageframework.TestDriver {
88+
capabilities := map[storageframework.Capability]bool{
89+
storageframework.CapPersistence: true,
90+
storageframework.CapSnapshotDataSource: true,
91+
storageframework.CapMultiPODs: true,
92+
storageframework.CapBlock: true,
93+
storageframework.CapPVCDataSource: true,
94+
storageframework.CapControllerExpansion: true,
95+
storageframework.CapOfflineExpansion: true,
96+
storageframework.CapOnlineExpansion: true,
97+
storageframework.CapSingleNodeVolume: true,
98+
storageframework.CapReadWriteOncePod: true,
99+
storageframework.CapMultiplePVsSameID: true,
100+
storageframework.CapFSResizeFromSourceNotSupported: true,
101+
storageframework.CapVolumeGroupSnapshot: true,
102+
103+
// This is needed for the
104+
// testsuites/volumelimits.go `should support volume limits`
105+
// test. --maxvolumespernode=10 gets
106+
// added when patching the deployment.
107+
storageframework.CapVolumeLimits: true,
108+
}
109+
// OCP specific code: a different driver name (csi-hostpath-groupsnapshot)
110+
return initGroupSnapshotHostpathCSIDriver("csi-hostpath-groupsnapshot",
111+
capabilities,
112+
// Volume attributes don't matter, but we have to provide at least one map.
113+
[]map[string]string{
114+
{"foo": "bar"},
115+
},
116+
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
117+
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
118+
"test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml",
119+
"test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml",
120+
"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
121+
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml",
122+
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml",
123+
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
124+
)
125+
}
126+
127+
func (h *groupSnapshotHostpathCSIDriver) GetDriverInfo() *storageframework.DriverInfo {
128+
return &h.driverInfo
129+
}
130+
131+
func (h *groupSnapshotHostpathCSIDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
132+
if pattern.VolType == storageframework.CSIInlineVolume && len(h.volumeAttributes) == 0 {
133+
e2eskipper.Skipf("%s has no volume attributes defined, doesn't support ephemeral inline volumes", h.driverInfo.Name)
134+
}
135+
}
136+
137+
func (h *groupSnapshotHostpathCSIDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
138+
provisioner := config.GetUniqueDriverName()
139+
parameters := map[string]string{}
140+
ns := config.Framework.Namespace.Name
141+
142+
return storageframework.GetStorageClass(provisioner, parameters, nil, ns)
143+
}
144+
145+
func (h *groupSnapshotHostpathCSIDriver) GetVolume(config *storageframework.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) {
146+
return h.volumeAttributes[volumeNumber%len(h.volumeAttributes)], false /* not shared */, false /* read-write */
147+
}
148+
149+
func (h *groupSnapshotHostpathCSIDriver) GetCSIDriverName(config *storageframework.PerTestConfig) string {
150+
return config.GetUniqueDriverName()
151+
}
152+
153+
func (h *groupSnapshotHostpathCSIDriver) GetSnapshotClass(ctx context.Context, config *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured {
154+
snapshotter := config.GetUniqueDriverName()
155+
ns := config.Framework.Namespace.Name
156+
157+
return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns)
158+
}
159+
160+
func (h *groupSnapshotHostpathCSIDriver) GetVolumeAttributesClass(_ context.Context, config *storageframework.PerTestConfig) *storagev1.VolumeAttributesClass {
161+
return storageframework.CopyVolumeAttributesClass(&storagev1.VolumeAttributesClass{
162+
DriverName: config.GetUniqueDriverName(),
163+
Parameters: map[string]string{
164+
hostpathCSIDriverMutableParameterName: hostpathCSIDriverMutableParameterValue,
165+
},
166+
}, config.Framework.Namespace.Name, "e2e-vac-hostpath")
167+
}
168+
func (h *groupSnapshotHostpathCSIDriver) GetVolumeGroupSnapshotClass(ctx context.Context, config *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured {
169+
snapshotter := config.GetUniqueDriverName()
170+
ns := config.Framework.Namespace.Name
171+
172+
return utils.GenerateVolumeGroupSnapshotClassSpec(snapshotter, parameters, ns)
173+
}
174+
175+
func (h *groupSnapshotHostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
176+
// Create secondary namespace which will be used for creating driver
177+
driverNamespace := utils.CreateDriverNamespace(ctx, f)
178+
driverns := driverNamespace.Name
179+
180+
ginkgo.By(fmt.Sprintf("deploying %s driver", h.driverInfo.Name))
181+
cancelLogging := utils.StartPodLogs(ctx, f, driverNamespace)
182+
cs := f.ClientSet
183+
184+
// The hostpath CSI driver only works when everything runs on the same node.
185+
node, err := e2enode.GetRandomReadySchedulableNode(ctx, cs)
186+
framework.ExpectNoError(err)
187+
config := &storageframework.PerTestConfig{
188+
Driver: h,
189+
Prefix: "hostpath",
190+
Framework: f,
191+
ClientNodeSelection: e2epod.NodeSelection{Name: node.Name},
192+
DriverNamespace: driverNamespace,
193+
}
194+
195+
patches := []utils.PatchCSIOptions{}
196+
197+
patches = append(patches, utils.PatchCSIOptions{
198+
OldDriverName: "csi-hostpath", // OCP: hardcode csi-hostpath here, it specifies directories in yaml files that need to be replaced with the unique driver name.
199+
NewDriverName: config.GetUniqueDriverName(),
200+
DriverContainerName: "hostpath",
201+
DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName(),
202+
// This is needed for the
203+
// testsuites/volumelimits.go `should support volume limits`
204+
// test.
205+
"--maxvolumespernode=10",
206+
// Enable volume lifecycle checks, to report failure if
207+
// the volume is not unpublished / unstaged correctly.
208+
"--check-volume-lifecycle=true",
209+
},
210+
ProvisionerContainerName: "csi-provisioner",
211+
SnapshotterContainerName: "csi-snapshotter",
212+
NodeName: node.Name,
213+
})
214+
215+
// VAC E2E HostPath patch
216+
// Enables ModifyVolume support in the hostpath CSI driver, and adds an enabled parameter name
217+
patches = append(patches, utils.PatchCSIOptions{
218+
DriverContainerName: "hostpath",
219+
DriverContainerArguments: []string{"--enable-controller-modify-volume=true", "--accepted-mutable-parameter-names=e2eVacTest"},
220+
})
221+
222+
// VAC E2E FeatureGate patches
223+
// TODO: These can be removed after the VolumeAttributesClass feature is default enabled
224+
patches = append(patches, utils.PatchCSIOptions{
225+
DriverContainerName: "csi-provisioner",
226+
DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"},
227+
})
228+
patches = append(patches, utils.PatchCSIOptions{
229+
DriverContainerName: "csi-resizer",
230+
DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"},
231+
})
232+
233+
// OCP specific code: enable group snapshot
234+
patches = append(patches, utils.PatchCSIOptions{
235+
DriverContainerName: "csi-snapshotter",
236+
DriverContainerArguments: []string{"--feature-gates=CSIVolumeGroupSnapshot=true"},
237+
})
238+
239+
err = utils.CreateFromManifests(ctx, config.Framework, driverNamespace, func(item interface{}) error {
240+
for _, o := range patches {
241+
if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
242+
return err
243+
}
244+
}
245+
246+
// Remove csi-external-health-monitor-agent and
247+
// csi-external-health-monitor-controller
248+
// containers. The agent is obsolete.
249+
// The controller is not needed for any of the
250+
// tests and is causing too much overhead when
251+
// running in a large cluster (see
252+
// https://github.com/kubernetes/kubernetes/issues/102452#issuecomment-856991009).
253+
switch item := item.(type) {
254+
case *appsv1.StatefulSet:
255+
var containers []v1.Container
256+
for _, container := range item.Spec.Template.Spec.Containers {
257+
switch container.Name {
258+
case "csi-external-health-monitor-agent", "csi-external-health-monitor-controller":
259+
// Remove these containers.
260+
default:
261+
// Keep the others.
262+
containers = append(containers, container)
263+
}
264+
}
265+
item.Spec.Template.Spec.Containers = containers
266+
}
267+
return nil
268+
}, h.manifests...)
269+
270+
if err != nil {
271+
framework.Failf("deploying %s driver: %v", h.driverInfo.Name, err)
272+
}
273+
274+
cleanupFunc := generateDriverCleanupFunc(
275+
f,
276+
h.driverInfo.Name,
277+
driverns,
278+
cancelLogging)
279+
ginkgo.DeferCleanup(cleanupFunc)
280+
281+
return config
282+
}

‎test/e2e/storage/openshift_csi_volumes.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// This is a copy of csi_volumes.go with OpenShift specific test driver.
18+
// Used a copy of the file to avoid conflicts when editing the existing file.
19+
package storage
20+
21+
import (
22+
"k8s.io/kubernetes/test/e2e/framework"
23+
"k8s.io/kubernetes/test/e2e/storage/drivers"
24+
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
25+
"k8s.io/kubernetes/test/e2e/storage/testsuites"
26+
"k8s.io/kubernetes/test/e2e/storage/utils"
27+
)
28+
29+
// List of testDrivers to be executed in below loop
30+
var ocpCSITestDrivers = []func() storageframework.TestDriver{
31+
drivers.InitGroupSnapshotHostpathCSIDriver,
32+
}
33+
34+
// This executes testSuites for csi volumes.
35+
var _ = utils.SIGDescribe("OCP CSI Volumes", func() {
36+
for _, initDriver := range ocpCSITestDrivers {
37+
curDriver := initDriver()
38+
39+
args := storageframework.GetDriverNameWithFeatureTags(curDriver)
40+
args = append(args, func() {
41+
storageframework.DefineTestSuites(curDriver, testsuites.CSISuites)
42+
})
43+
framework.Context(args...)
44+
}
45+
})

0 commit comments

Comments
 (0)