diff --git a/api/v1alpha1/kubevirtcluster_types.go b/api/v1alpha1/kubevirtcluster_types.go index 4dffb9667..c07f0701b 100644 --- a/api/v1alpha1/kubevirtcluster_types.go +++ b/api/v1alpha1/kubevirtcluster_types.go @@ -28,13 +28,17 @@ const ( ClusterFinalizer = "kubevirtcluster.infrastructure.cluster.x-k8s.io" ) -const ( +const ( //labels KubevirtMachineNameLabel = "capk.cluster.x-k8s.io/kubevirt-machine-name" KubevirtMachineNamespaceLabel = "capk.cluster.x-k8s.io/kubevirt-machine-namespace" KubevirtMachineVMTerminalLabel = "capk.cluster.x-k8s.io/vm-is-terminal" ) +const ( // annotations + VmiDeletionGraceTime = "capk.cluster.x-k8s.io/vmi-deletion-grace-time" +) + // KubevirtClusterSpec defines the desired state of KubevirtCluster. type KubevirtClusterSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 27dfaab16..41062cfd6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtmachines.yaml index 9aac02628..6fc7cea7f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtmachines.yaml @@ -163,18 +163,18 @@ spec: description: PVC is the PVC specification properties: accessModes: - description: 'AccessModes contains the desired + description: 'accessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array dataSource: - description: 'This field can be used to specify - either: * An existing VolumeSnapshot object - (snapshot.storage.k8s.io/VolumeSnapshot) * - An existing PVC (PersistentVolumeClaim) If - the provisioner or an external controller + description: 'dataSource field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource @@ -202,17 +202,17 @@ spec: - name type: object dataSourceRef: - description: 'Specifies the object from which - to populate the volume with data, if a non-empty - volume is desired. This may be any local object - from a non-empty API group (non core object) - or a PersistentVolumeClaim object. When this - field is specified, volume binding will only - succeed if the type of the specified object - matches some installed volume populator or - dynamic provisioner. This field will replace - the functionality of the DataSource field - and as such if both fields are non-empty, + description: 'dataSourceRef specifies the object + from which to populate the volume with data, + if a non-empty volume is desired. This may + be any local object from a non-empty API group + (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume + binding will only succeed if the type of the + specified object matches some installed volume + populator or dynamic provisioner. This field + will replace the functionality of the DataSource + field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value @@ -225,7 +225,7 @@ spec: objects. * While DataSource ignores disallowed values (dropping them), DataSourceRef preserves all values, and generates an error if a disallowed - value is specified. (Alpha) Using this field + value is specified. (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled.' properties: @@ -249,7 +249,7 @@ spec: - name type: object resources: - description: 'Resources represents the minimum + description: 'resources represents the minimum resources the volume should have. If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements that are lower than @@ -284,8 +284,8 @@ spec: type: object type: object selector: - description: A label query over volumes to consider - for binding. + description: selector is a label query over + volumes to consider for binding. properties: matchExpressions: description: matchExpressions is a list @@ -336,8 +336,9 @@ spec: type: object type: object storageClassName: - description: 'Name of the StorageClass required - by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + description: 'storageClassName is the name of + the StorageClass required by the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeMode: description: volumeMode defines what type of @@ -346,7 +347,7 @@ spec: claim spec. type: string volumeName: - description: VolumeName is the binding reference + description: volumeName is the binding reference to the PersistentVolume backing this claim. type: string type: object @@ -707,9 +708,22 @@ spec: description: Name is the name of the VirtualMachineFlavor or VirtualMachineClusterFlavor type: string - profile: - description: Profile is the name of a custom profile in - the flavor. If left empty, the default profile is used. + required: + - name + type: object + preference: + description: PreferenceMatcher references a set of preference + that is used to fill fields in Template + properties: + kind: + description: 'Kind specifies which preference resource + is referenced. Allowed values are: "VirtualMachinePreference" + and "VirtualMachineClusterPreference". If not specified, + "VirtualMachineClusterPreference" is used by default.' + type: string + name: + description: Name is the name of the VirtualMachinePreference + or VirtualMachineClusterPreference type: string required: - name @@ -1200,9 +1214,6 @@ spec: or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is - only honored when PodAffinityNamespaceSelector - feature is enabled. properties: matchExpressions: description: matchExpressions @@ -1273,7 +1284,7 @@ spec: field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -1397,10 +1408,7 @@ spec: field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) - matches all namespaces. This field - is beta-level and is only honored - when PodAffinityNamespaceSelector - feature is enabled. + matches all namespaces. properties: matchExpressions: description: matchExpressions is @@ -1466,7 +1474,7 @@ spec: in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -1594,9 +1602,6 @@ spec: or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is - only honored when PodAffinityNamespaceSelector - feature is enabled. properties: matchExpressions: description: matchExpressions @@ -1667,7 +1672,7 @@ spec: field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -1791,10 +1796,7 @@ spec: field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) - matches all namespaces. This field - is beta-level and is only honored - when PodAffinityNamespaceSelector - feature is enabled. + matches all namespaces. properties: matchExpressions: description: matchExpressions is @@ -1860,7 +1862,7 @@ spec: in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -2472,6 +2474,8 @@ spec: boot order are not tried. type: integer bridge: + description: InterfaceBridge connects + to a given network via a linux bridge. type: object dhcpOptions: description: If specified the network @@ -2526,8 +2530,15 @@ spec: DE-AD-00-00-BE-AF.' type: string macvtap: + description: InterfaceMacvtap connects + to a given network by extending the + Kubernetes node's L2 networks via + a macvtap interface. type: object masquerade: + description: InterfaceMasquerade connects + to a given network using netfilter + rules to nat the traffic. type: object model: description: 'Interface model. One of: @@ -2583,8 +2594,14 @@ spec: type: object type: array slirp: + description: InterfaceSlirp connects + to a given network using QEMU user + networking mode. type: object sriov: + description: InterfaceSRIOV connects + to a given network by passing-through + an SR-IOV PCI device via vfio. type: object tag: description: If specified, the virtual @@ -2625,6 +2642,9 @@ spec: required: - name type: object + tpm: + description: Whether to emulate a TPM device. + type: object useVirtioTransitional: description: Fall back to legacy virtio 0.9 support if virtio bus is selected on devices. @@ -3799,14 +3819,15 @@ spec: to the vmi via qemu. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: - description: 'ClaimName is the name + description: 'claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string readOnly: - description: Will force the ReadOnly - setting in VolumeMounts. Default false. + description: readOnly Will force the + ReadOnly setting in VolumeMounts. + Default false. type: boolean required: - claimName @@ -3840,6 +3861,28 @@ spec: - path - type type: object + memoryDump: + description: MemoryDump is attached to the virt + launcher and is populated with a memory dump + of the vmi + properties: + claimName: + description: 'claimName is the name of a + PersistentVolumeClaim in the same namespace + as the pod using this volume. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + hotpluggable: + description: Hotpluggable indicates whether + the volume can be hotplugged and hotunplugged. + type: boolean + readOnly: + description: readOnly Will force the ReadOnly + setting in VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object name: description: 'Volume''s name. Must be a DNS_LABEL and unique within the vmi. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' @@ -3851,7 +3894,7 @@ spec: the vmi via qemu. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: - description: 'ClaimName is the name of a + description: 'claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' @@ -3861,8 +3904,8 @@ spec: the volume can be hotplugged and hotunplugged. type: boolean readOnly: - description: Will force the ReadOnly setting - in VolumeMounts. Default false. + description: readOnly Will force the ReadOnly + setting in VolumeMounts. Default false. type: boolean required: - claimName diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtmachinetemplates.yaml index f16262634..2e4fc36df 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtmachinetemplates.yaml @@ -181,16 +181,16 @@ spec: description: PVC is the PVC specification properties: accessModes: - description: 'AccessModes contains the + description: 'accessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array dataSource: - description: 'This field can be used - to specify either: * An existing VolumeSnapshot - object (snapshot.storage.k8s.io/VolumeSnapshot) + description: 'dataSource field can be + used to specify either: * An existing + VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified @@ -223,18 +223,18 @@ spec: - name type: object dataSourceRef: - description: 'Specifies the object from - which to populate the volume with - data, if a non-empty volume is desired. - This may be any local object from - a non-empty API group (non core object) - or a PersistentVolumeClaim object. - When this field is specified, volume - binding will only succeed if the type - of the specified object matches some - installed volume populator or dynamic - provisioner. This field will replace - the functionality of the DataSource + description: 'dataSourceRef specifies + the object from which to populate + the volume with data, if a non-empty + volume is desired. This may be any + local object from a non-empty API + group (non core object) or a PersistentVolumeClaim + object. When this field is specified, + volume binding will only succeed if + the type of the specified object matches + some installed volume populator or + dynamic provisioner. This field will + replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, @@ -251,7 +251,7 @@ spec: disallowed values (dropping them), DataSourceRef preserves all values, and generates an error if a disallowed - value is specified. (Alpha) Using + value is specified. (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled.' properties: @@ -277,7 +277,7 @@ spec: - name type: object resources: - description: 'Resources represents the + description: 'resources represents the minimum resources the volume should have. If RecoverVolumeExpansionFailure feature is enabled users are allowed @@ -316,8 +316,8 @@ spec: type: object type: object selector: - description: A label query over volumes - to consider for binding. + description: selector is a label query + over volumes to consider for binding. properties: matchExpressions: description: matchExpressions is @@ -376,9 +376,9 @@ spec: type: object type: object storageClassName: - description: 'Name of the StorageClass - required by the claim. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + description: 'storageClassName is the + name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeMode: description: volumeMode defines what @@ -387,7 +387,7 @@ spec: when not included in claim spec. type: string volumeName: - description: VolumeName is the binding + description: volumeName is the binding reference to the PersistentVolume backing this claim. type: string @@ -787,10 +787,23 @@ spec: description: Name is the name of the VirtualMachineFlavor or VirtualMachineClusterFlavor type: string - profile: - description: Profile is the name of a custom profile - in the flavor. If left empty, the default profile - is used. + required: + - name + type: object + preference: + description: PreferenceMatcher references a set of + preference that is used to fill fields in Template + properties: + kind: + description: 'Kind specifies which preference + resource is referenced. Allowed values are: + "VirtualMachinePreference" and "VirtualMachineClusterPreference". + If not specified, "VirtualMachineClusterPreference" + is used by default.' + type: string + name: + description: Name is the name of the VirtualMachinePreference + or VirtualMachineClusterPreference type: string required: - name @@ -1365,10 +1378,6 @@ spec: means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level - and is only honored when - PodAffinityNamespaceSelector - feature is enabled. properties: matchExpressions: description: matchExpressions @@ -1455,7 +1464,7 @@ spec: the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -1600,10 +1609,7 @@ spec: namespaces list means "this pod's namespace". An empty selector ({}) matches all - namespaces. This field is - beta-level and is only honored - when PodAffinityNamespaceSelector - feature is enabled. + namespaces. properties: matchExpressions: description: matchExpressions @@ -1681,7 +1687,7 @@ spec: ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -1834,10 +1840,6 @@ spec: means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level - and is only honored when - PodAffinityNamespaceSelector - feature is enabled. properties: matchExpressions: description: matchExpressions @@ -1924,7 +1926,7 @@ spec: the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -2069,10 +2071,7 @@ spec: namespaces list means "this pod's namespace". An empty selector ({}) matches all - namespaces. This field is - beta-level and is only honored - when PodAffinityNamespaceSelector - feature is enabled. + namespaces. properties: matchExpressions: description: matchExpressions @@ -2150,7 +2149,7 @@ spec: ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -2833,6 +2832,9 @@ spec: order are not tried. type: integer bridge: + description: InterfaceBridge + connects to a given network + via a linux bridge. type: object dhcpOptions: description: If specified the @@ -2891,8 +2893,17 @@ spec: or DE-AD-00-00-BE-AF.' type: string macvtap: + description: InterfaceMacvtap + connects to a given network + by extending the Kubernetes + node's L2 networks via a macvtap + interface. type: object masquerade: + description: InterfaceMasquerade + connects to a given network + using netfilter rules to nat + the traffic. type: object model: description: 'Interface model. @@ -2957,8 +2968,16 @@ spec: type: object type: array slirp: + description: InterfaceSlirp + connects to a given network + using QEMU user networking + mode. type: object sriov: + description: InterfaceSRIOV + connects to a given network + by passing-through an SR-IOV + PCI device via vfio. type: object tag: description: If specified, the @@ -3003,6 +3022,10 @@ spec: required: - name type: object + tpm: + description: Whether to emulate a + TPM device. + type: object useVirtioTransitional: description: Fall back to legacy virtio 0.9 support if virtio bus is selected @@ -4285,15 +4308,15 @@ spec: More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: - description: 'ClaimName is the + description: 'claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string readOnly: - description: Will force the - ReadOnly setting in VolumeMounts. + description: readOnly Will force + the ReadOnly setting in VolumeMounts. Default false. type: boolean required: @@ -4330,6 +4353,31 @@ spec: - path - type type: object + memoryDump: + description: MemoryDump is attached + to the virt launcher and is populated + with a memory dump of the vmi + properties: + claimName: + description: 'claimName is the name + of a PersistentVolumeClaim in + the same namespace as the pod + using this volume. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + hotpluggable: + description: Hotpluggable indicates + whether the volume can be hotplugged + and hotunplugged. + type: boolean + readOnly: + description: readOnly Will force + the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object name: description: 'Volume''s name. Must be a DNS_LABEL and unique within the @@ -4342,7 +4390,7 @@ spec: to the vmi via qemu. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: - description: 'ClaimName is the name + description: 'claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: @@ -4354,9 +4402,9 @@ spec: and hotunplugged. type: boolean readOnly: - description: Will force the ReadOnly - setting in VolumeMounts. Default - false. + description: readOnly Will force + the ReadOnly setting in VolumeMounts. + Default false. type: boolean required: - claimName diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d386ac807..f4bd1a581 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -84,8 +84,11 @@ rules: resources: - virtualmachineinstances verbs: + - delete - get - list + - patch + - update - watch - apiGroups: - kubevirt.io diff --git a/controllers/kubevirtmachine_controller_test.go b/controllers/kubevirtmachine_controller_test.go index 4d6bb4f05..d66da2271 100644 --- a/controllers/kubevirtmachine_controller_test.go +++ b/controllers/kubevirtmachine_controller_test.go @@ -247,8 +247,6 @@ var _ = Describe("reconcile a kubevirt machine", func() { kubevirtMachine = testing.NewKubevirtMachine(kubevirtMachineName, machineName) kubevirtCluster = testing.NewKubevirtCluster(clusterName, machineName) - workloadClusterMock = workloadclustermock.NewMockWorkloadCluster(mockCtrl) - infraClusterMock = infraclustermock.NewMockInfraCluster(mockCtrl) machineFactoryMock = machinemocks.NewMockMachineFactory(mockCtrl) machineMock = machinemocks.NewMockMachineInterface(mockCtrl) @@ -422,7 +420,7 @@ var _ = Describe("reconcile a kubevirt machine", func() { Expect(apierrors.IsNotFound(err)).To(BeTrue()) //Check finalizer is removed from machine - Expect(len(machineContext.Machine.ObjectMeta.Finalizers)).To(Equal(0)) + Expect(machineContext.Machine.ObjectMeta.Finalizers).To(BeEmpty()) }) It("should ensure deletion of KubevirtMachine when bootstrap secret was never created", func() { @@ -445,7 +443,7 @@ var _ = Describe("reconcile a kubevirt machine", func() { Expect(out).To(Equal(ctrl.Result{Requeue: false, RequeueAfter: 0})) //Check finalizer is removed from machine - Expect(len(machineContext.Machine.ObjectMeta.Finalizers)).To(Equal(0)) + Expect(machineContext.Machine.ObjectMeta.Finalizers).To(BeEmpty()) }) It("should update userdata correctly at KubevirtMachine reconcile", func() { @@ -870,7 +868,7 @@ var _ = Describe("reconcile a kubevirt machine", func() { infraClusterMock.EXPECT().GenerateInfraClusterClient(kubevirtMachine.Spec.InfraClusterSecretRef, kubevirtMachine.Namespace, machineContext.Context).Return(fakeClient, kubevirtMachine.Namespace, nil) _, err := kubevirtMachineReconciler.reconcileNormal(machineContext) - Expect(err).ShouldNot(BeNil()) + Expect(err).Should(HaveOccurred()) conditions := machineContext.KubevirtMachine.GetConditions() Expect(conditions[0].Type).To(Equal(infrav1.VMProvisionedCondition)) diff --git a/controllers/vmi_eviction_controller.go b/controllers/vmi_eviction_controller.go new file mode 100644 index 000000000..3f91eb0b5 --- /dev/null +++ b/controllers/vmi_eviction_controller.go @@ -0,0 +1,276 @@ +package controllers + +import ( + goContext "context" + "fmt" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "time" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubedrain "k8s.io/kubectl/pkg/drain" + kubevirtv1 "kubevirt.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/noderefutil" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + infrav1 "sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha1" + context "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster" +) + +const ( + vmiDeleteGraceTimeoutDurationSeconds = 600 // 10 minutes +) + +type VmiEvictionReconciler struct { + client.Client + workloadCluster workloadcluster.WorkloadCluster +} + +// NewVmiEvictionReconciler creates a new VmiEvictionReconciler +func NewVmiEvictionReconciler(cl client.Client) *VmiEvictionReconciler { + return &VmiEvictionReconciler{Client: cl, workloadCluster: workloadcluster.New(cl)} +} + +// SetupWithManager will add watches for this controller. +func (r *VmiEvictionReconciler) SetupWithManager(ctx goContext.Context, mgr ctrl.Manager) error { + selector, err := getLabelPredicate() + + if err != nil { + return fmt.Errorf("can't setup the VMI eviction controller; %w", err) + } + + _, err = ctrl.NewControllerManagedBy(mgr). + For(&kubevirtv1.VirtualMachineInstance{}). + WithEventFilter(selector). + Build(r) + + return err +} + +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;machines,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch +// +kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances;,verbs=get;list;watch;patch;update;delete + +// Reconcile handles VMI events. +func (r VmiEvictionReconciler) Reconcile(ctx goContext.Context, req ctrl.Request) (ctrl.Result, error) { + logger := ctrl.LoggerFrom(ctx) + + vmi := &kubevirtv1.VirtualMachineInstance{} + err := r.Get(ctx, req.NamespacedName, vmi) + if err != nil { + if apierrors.IsNotFound(err) { + logger.V(4).Info(fmt.Sprintf("Can't find virtualMachineInstance %s; it was already deleted.", req.NamespacedName)) + return ctrl.Result{}, nil + } + logger.Error(err, fmt.Sprintf("failed to read VMI %s", req.Name)) + return ctrl.Result{}, err + } + + if !shouldGracefulDeleteVMI(vmi, logger, req.NamespacedName) { + return ctrl.Result{}, nil + } + + exceeded, err := r.drainGracePeriodExceeded(ctx, vmi, logger) + if err != nil { + return ctrl.Result{}, err + } + + if !exceeded { + cluster, err := r.getCluster(ctx, vmi) + if err != nil { + logger.Error(err, "Can't get the cluster form the VirtualMachineInstance", "VirtualMachineInstance name", req.NamespacedName) + return ctrl.Result{}, err + } + + nodeDrained, retryDuration, err := r.drainNode(ctx, cluster, vmi.Status.EvacuationNodeName, logger) + if err != nil { + return ctrl.Result{RequeueAfter: retryDuration}, err + } + + if !nodeDrained { + return ctrl.Result{RequeueAfter: retryDuration}, nil + } + } + + // now, when the node is drained (or vmiDeleteGraceTimeoutDurationSeconds has passed), we can delete the VMI + propagationPolicy := metav1.DeletePropagationForeground + err = r.Delete(ctx, vmi, &client.DeleteOptions{PropagationPolicy: &propagationPolicy}) + if err != nil { + logger.Error(err, "failed to delete VirtualMachineInstance", "VirtualMachineInstance name", req.NamespacedName) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func shouldGracefulDeleteVMI(vmi *kubevirtv1.VirtualMachineInstance, logger logr.Logger, namespacedName types.NamespacedName) bool { + if vmi.DeletionTimestamp != nil { + logger.V(4).Info("The virtualMachineInstance is already in deletion process. Nothing to do here", "VirtualMachineInstance name", namespacedName) + return false + } + + if vmi.Spec.EvictionStrategy == nil || *vmi.Spec.EvictionStrategy != kubevirtv1.EvictionStrategyExternal { + logger.V(4).Info("Graceful deletion is not supported for virtualMachineInstance. Nothing to do here", "VirtualMachineInstance name", namespacedName) + return false + } + + // KubeVirt will set the EvacuationNodeName field in case of guest node eviction. If the field is not set, there is + // nothing to do. + if len(vmi.Status.EvacuationNodeName) == 0 { + logger.V(4).Info("The virtualMachineInstance is not marked for deletion. Nothing to do here", "VirtualMachineInstance name", namespacedName) + return false + } + + return true +} + +func (r VmiEvictionReconciler) getCluster(ctx goContext.Context, vmi *kubevirtv1.VirtualMachineInstance) (*clusterv1.Cluster, error) { + // get cluster from vmi + clusterNS, ok := vmi.Labels[infrav1.KubevirtMachineNamespaceLabel] + if !ok { + return nil, fmt.Errorf("can't find the cluster namespace from the VM; missing %s label", infrav1.KubevirtMachineNamespaceLabel) + } + + clusterName, ok := vmi.Labels[clusterv1.ClusterLabelName] + if !ok { + return nil, fmt.Errorf("can't find the cluster name from the VM; missing %s label", clusterv1.ClusterLabelName) + } + + cluster := &clusterv1.Cluster{} + err := r.Get(ctx, client.ObjectKey{Namespace: clusterNS, Name: clusterName}, cluster) + if err != nil { + return nil, fmt.Errorf("can't find the cluster %s/%s; %w", clusterNS, clusterName, err) + } + + return cluster, nil +} + +// This functions drains a node from a tenant cluster. +// The function returns 3 values: +// * drain done - boolean +// * retry time, or 0 if not needed +// * error - to be returned if we want to retry +func (r VmiEvictionReconciler) drainNode(goctx goContext.Context, cluster *clusterv1.Cluster, nodeName string, logger logr.Logger) (bool, time.Duration, error) { + ctx := &context.MachineContext{Context: goctx, KubevirtCluster: &infrav1.KubevirtCluster{ObjectMeta: metav1.ObjectMeta{Namespace: cluster.Namespace, Name: cluster.Name}}} + kubeClient, err := r.workloadCluster.GenerateWorkloadClusterK8sClient(ctx) + if err != nil { + logger.Error(err, "Error creating a remote client while deleting Machine, won't retry") + return false, 0, nil + } + + node, err := kubeClient.CoreV1().Nodes().Get(goctx, nodeName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // If an admin deletes the node directly, we'll end up here. + logger.Error(err, "Could not find node from noderef, it may have already been deleted") + return true, 0, nil + } + return false, 0, fmt.Errorf("unable to get node %q: %w", nodeName, err) + } + + drainer := &kubedrain.Helper{ + Client: kubeClient, + Ctx: ctx, + Force: true, + IgnoreAllDaemonSets: true, + DeleteEmptyDirData: true, + GracePeriodSeconds: -1, + // If a pod is not evicted in 20 seconds, retry the eviction next time the + // machine gets reconciled again (to allow other machines to be reconciled). + Timeout: 20 * time.Second, + OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) { + verbStr := "Deleted" + if usingEviction { + verbStr = "Evicted" + } + logger.Info(fmt.Sprintf("%s pod from Node", verbStr), + "pod", fmt.Sprintf("%s/%s", pod.Name, pod.Namespace)) + }, + Out: writer{logger.Info}, + ErrOut: writer{func(msg string, keysAndValues ...interface{}) { + logger.Error(nil, msg, keysAndValues...) + }}, + } + + if noderefutil.IsNodeUnreachable(node) { + // When the node is unreachable and some pods are not evicted for as long as this timeout, we ignore them. + drainer.SkipWaitForDeleteTimeoutSeconds = 60 * 5 // 5 minutes + } + + if err = kubedrain.RunCordonOrUncordon(drainer, node, true); err != nil { + // Machine will be re-reconciled after a cordon failure. + logger.Error(err, "Cordon failed") + return false, 0, errors.Errorf("unable to cordon node %s: %v", nodeName, err) + } + + if err = kubedrain.RunNodeDrain(drainer, node.Name); err != nil { + // Machine will be re-reconciled after a drain failure. + logger.Error(err, "Drain failed, retry in 20s", "node name", nodeName) + return false, 20 * time.Second, nil + } + + logger.Info("Drain successful", "node name", nodeName) + return true, 0, nil +} + +// wait vmiDeleteGraceTimeoutDurationSeconds to the node to be drained. If this time had passed, don't wait anymore. +func (r VmiEvictionReconciler) drainGracePeriodExceeded(ctx goContext.Context, vmi *kubevirtv1.VirtualMachineInstance, logger logr.Logger) (bool, error) { + if graceTime, found := vmi.Annotations[infrav1.VmiDeletionGraceTime]; found { + deletionGraceTime, err := time.Parse(time.RFC3339, graceTime) + if err != nil { // wrong format - rewrite + if err = r.setVmiDeletionGraceTime(ctx, vmi, logger); err != nil { + return false, err + } + } else { + return time.Now().UTC().After(deletionGraceTime), nil + } + } else { + if err := r.setVmiDeletionGraceTime(ctx, vmi, logger); err != nil { + return false, err + } + } + + return false, nil +} + +func (r VmiEvictionReconciler) setVmiDeletionGraceTime(ctx goContext.Context, vmi *kubevirtv1.VirtualMachineInstance, logger logr.Logger) error { + logger.V(2).Info(fmt.Sprintf("setting the %s annotation", infrav1.VmiDeletionGraceTime)) + graceTime := time.Now().Add(vmiDeleteGraceTimeoutDurationSeconds * time.Second).UTC().Format(time.RFC3339) + patch := fmt.Sprintf(`{"metadata":{"annotations":{"%s": "%s"}}}`, infrav1.VmiDeletionGraceTime, graceTime) + patchRequest := client.RawPatch(types.MergePatchType, []byte(patch)) + + if err := r.Patch(ctx, vmi, patchRequest); err != nil { + return fmt.Errorf("failed to add the %s annotation to the VMI; %w", infrav1.VmiDeletionGraceTime, err) + } + + return nil +} + +func getLabelPredicate() (predicate.Predicate, error) { + return predicate.LabelSelectorPredicate( + metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: infrav1.KubevirtMachineNameLabel, + Operator: metav1.LabelSelectorOpExists, + Values: nil, + }}, + }) +} + +// writer implements io.Writer interface as a pass-through for klog. +type writer struct { + logFunc func(msg string, keysAndValues ...interface{}) +} + +// Write passes string(p) into writer's logFunc and always returns len(p). +func (w writer) Write(p []byte) (n int, err error) { + w.logFunc(string(p)) + return len(p), nil +} diff --git a/controllers/vmi_eviction_controller_test.go b/controllers/vmi_eviction_controller_test.go new file mode 100644 index 000000000..5e78f82c7 --- /dev/null +++ b/controllers/vmi_eviction_controller_test.go @@ -0,0 +1,465 @@ +package controllers + +import ( + gocontext "context" + "errors" + "time" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + kubevirtv1 "kubevirt.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + + infrav1 "sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha1" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/testing" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster/mock" +) + +var _ = Describe("Test VMI Controller", func() { + + const ( + clusterName = "test" + clusterNamespace = clusterName + "-cluster" + clusterInstanceName = clusterName + "-1234" + nodeName = "worker-node-1" + ) + + Context("Test VmiEviction reconciler", func() { + var ( + mockCtrl *gomock.Controller + fakeClient client.Client + vmi *kubevirtv1.VirtualMachineInstance + cluster *clusterv1.Cluster + wlCluster *mock.MockWorkloadCluster + ) + + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + + vmi = &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-cluster", + Name: nodeName, + Labels: map[string]string{ + infrav1.KubevirtMachineNamespaceLabel: clusterNamespace, + clusterv1.ClusterLabelName: clusterInstanceName, + }, + Annotations: make(map[string]string), + }, + } + + cluster = &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterNamespace, + Name: clusterInstanceName, + }, + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &corev1.ObjectReference{ + Kind: "Secret", + Namespace: clusterNamespace, + Name: clusterInstanceName, + }, + }, + } + + wlCluster = mock.NewMockWorkloadCluster(mockCtrl) + }) + + It("Should ignore vmi if it already deleted", func() { + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).Build() + + // make sure we never get into darin process, but exit earlier + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + + Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) + }) + + It("Should ignore vmi if its deletion process already started", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + now := metav1.Now() + vmi.DeletionTimestamp = &now + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + // make sure we never get into darin process, but exit earlier + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + + Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) + }) + + It("Should ignore vmi with no eviction strategy", func() { + vmi.Spec.EvictionStrategy = nil + vmi.Status.EvacuationNodeName = nodeName + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + // make sure we never get into darin process, but exit earlier + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + + Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) + }) + + It("Should ignore vmi with no eviction strategy != external", func() { + es := kubevirtv1.EvictionStrategyLiveMigrate + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + // make sure we never get into darin process, but exit earlier + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + + Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) + }) + + It("Should ignore non-evicted VMIs", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + // make sure we never get into darin process, but exit earlier + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + + Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) + }) + + It("Should drain node", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + } + + Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) + cl := k8sfake.NewSimpleClientset(node) + + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Return(cl, nil).Times(1) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + + Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) + + // check that the node was drained + readNode, err := cl.CoreV1().Nodes().Get(gocontext.TODO(), nodeName, metav1.GetOptions{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(readNode.Spec.Unschedulable).To(BeTrue()) + + // check that the VMI was removed + readVMI := &kubevirtv1.VirtualMachineInstance{} + err = fakeClient.Get(gocontext.TODO(), client.ObjectKey{Namespace: clusterNamespace, Name: nodeName}, readVMI) + Expect(apierrors.IsNotFound(err)).Should(BeTrue()) + }) + + It("Should skip drain if the node already deleted", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + + vmi.Status.EvacuationNodeName = nodeName + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) + cl := k8sfake.NewSimpleClientset() + + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Return(cl, nil).Times(1) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + + Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) + }) + + Context("Error cases", func() { + It("Should return error if the 'capk.cluster.x-k8s.io/kubevirt-machine-namespace' label is missing", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + delete(vmi.Labels, infrav1.KubevirtMachineNamespaceLabel) + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + // make sure we never get into darin process, but exit earlier + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + _, err := r.Reconcile(gocontext.TODO(), req) + Expect(err).Should(HaveOccurred()) + }) + + It("Should return error if the 'cluster.x-k8s.io/cluster-name' label is missing", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + delete(vmi.Labels, clusterv1.ClusterLabelName) + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + // make sure we never get into darin process, but exit earlier + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + _, err := r.Reconcile(gocontext.TODO(), req) + Expect(err).Should(HaveOccurred()) + }) + + It("Should return error if the cluster is missing", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi).Build() + + // make sure we never get into darin process, but exit earlier + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + _, err := r.Reconcile(gocontext.TODO(), req) + Expect(err).Should(HaveOccurred()) + }) + + It("Should return not error if can't get the external cluster client, but do not remove the VMI", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + + Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + } + + Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) + cl := k8sfake.NewSimpleClientset(node) + + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Return(nil, errors.New("fake error")).Times(1) + + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} + + _, err := r.Reconcile(gocontext.TODO(), req) + Expect(err).ShouldNot(HaveOccurred()) + + // check that the node was not drained + readNode, err := cl.CoreV1().Nodes().Get(gocontext.TODO(), nodeName, metav1.GetOptions{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(readNode.Spec.Unschedulable).To(BeFalse()) + + // check that the VMI was not deleted + readVMI := &kubevirtv1.VirtualMachineInstance{} + err = fakeClient.Get(gocontext.TODO(), client.ObjectKey{Namespace: clusterNamespace, Name: nodeName}, readVMI) + Expect(err).ShouldNot(HaveOccurred()) + Expect(readVMI).ToNot(BeNil()) + }) + }) + + Context("test drainGracePeriodExceeded", func() { + It("should add the annotation", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + ctx := gocontext.Background() + Expect(r.drainGracePeriodExceeded(ctx, vmi, ctrl.LoggerFrom(ctx))).To(BeFalse()) + timeoutAnnotation, found := vmi.Annotations[infrav1.VmiDeletionGraceTime] + Expect(found).To(BeTrue()) + timeout, err := time.Parse(time.RFC3339, timeoutAnnotation) + Expect(err).ToNot(HaveOccurred()) + Expect(timeout).To(And( + BeTemporally(">", time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds-1)*time.Second)), + BeTemporally("<", time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds+1)*time.Second)))) + }) + + It("should return false if timeout was not exceeded", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + + timeout := time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds / 2) * time.Second).Format(time.RFC3339) + vmi.Annotations[infrav1.VmiDeletionGraceTime] = timeout + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + ctx := gocontext.Background() + Expect(r.drainGracePeriodExceeded(ctx, vmi, ctrl.LoggerFrom(ctx))).To(BeFalse()) + timeoutAnnotation, found := vmi.Annotations[infrav1.VmiDeletionGraceTime] + Expect(found).To(BeTrue()) + Expect(timeoutAnnotation).To(Equal(timeout)) + }) + + It("should return true if timeout was exceeded", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + + timeout := time.Now().UTC().Add(-(time.Millisecond)).Format(time.RFC3339) + vmi.Annotations[infrav1.VmiDeletionGraceTime] = timeout + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + ctx := gocontext.Background() + Expect(r.drainGracePeriodExceeded(ctx, vmi, ctrl.LoggerFrom(ctx))).To(BeTrue()) + timeoutAnnotation, found := vmi.Annotations[infrav1.VmiDeletionGraceTime] + Expect(found).To(BeTrue()) + Expect(timeoutAnnotation).To(Equal(timeout)) + }) + + It("should fix the annotation if it's with a wrong format", func() { + es := kubevirtv1.EvictionStrategyExternal + vmi.Spec.EvictionStrategy = &es + vmi.Status.EvacuationNodeName = nodeName + + origTimeout := time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds / 2) * time.Second).Format(time.RFC850) + vmi.Annotations[infrav1.VmiDeletionGraceTime] = origTimeout + + fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() + r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} + ctx := gocontext.Background() + Expect(r.drainGracePeriodExceeded(ctx, vmi, ctrl.LoggerFrom(ctx))).To(BeFalse()) + timeoutAnnotation, found := vmi.Annotations[infrav1.VmiDeletionGraceTime] + Expect(found).To(BeTrue()) + timeout, err := time.Parse(time.RFC3339, timeoutAnnotation) + Expect(err).ToNot(HaveOccurred()) + Expect(timeout).To(And( + BeTemporally(">", time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds-1)*time.Second)), + BeTemporally("<", time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds+1)*time.Second)))) + }) + }) + }) + + Context("check the label predicate", func() { + sel, err := getLabelPredicate() + It("should successfully create the predicate", func() { + Expect(err).ToNot(HaveOccurred()) + }) + + It("should select if the label exist", func() { + Expect(sel.Create(event.CreateEvent{ + Object: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, + }, + }, + })).To(BeTrue()) + }) + + It("should select if the label exist and empty", func() { + Expect(sel.Create(event.CreateEvent{ + Object: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{infrav1.KubevirtMachineNameLabel: ""}, + }, + }, + })).To(BeTrue()) + }) + + It("should select if the label does not exist", func() { + Expect(sel.Create(event.CreateEvent{ + Object: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: nil, + }, + }, + })).To(BeFalse()) + }) + + It("should select if the label exist", func() { + Expect(sel.Update(event.UpdateEvent{ + ObjectOld: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, + }, + }, + ObjectNew: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, + }, + }, + })).To(BeTrue()) + }) + + It("should select if the label now exist", func() { + Expect(sel.Update(event.UpdateEvent{ + ObjectOld: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"foo": "bar"}, + }, + }, + ObjectNew: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, + }, + }, + })).To(BeTrue()) + + }) + + It("should select if the label now not exist", func() { + Expect(sel.Update(event.UpdateEvent{ + ObjectOld: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, + }, + }, + ObjectNew: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"foo": "bar"}, + }, + }, + })).To(BeFalse()) + }) + }) + +}) + +func setupRemoteScheme() *runtime.Scheme { + s := runtime.NewScheme() + if err := corev1.AddToScheme(s); err != nil { + panic(err) + } + return s +} diff --git a/e2e/create-cluster_test.go b/e2e/create-cluster_test.go index 1c80a957b..ebdb21d7c 100644 --- a/e2e/create-cluster_test.go +++ b/e2e/create-cluster_test.go @@ -15,11 +15,14 @@ import ( . "github.com/onsi/gomega" "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" + policy "k8s.io/api/policy/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" + "k8s.io/utils/pointer" kubevirtv1 "kubevirt.io/api/core/v1" "kubevirt.io/client-go/kubecli" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -33,6 +36,8 @@ import ( var virtClient kubecli.KubevirtClient +const testFinalizer = "holdForTestFinalizer" + var _ = Describe("CreateCluster", func() { var tmpDir string @@ -124,22 +129,25 @@ var _ = Describe("CreateCluster", func() { } } return nil - }, 10*time.Minute, 5*time.Second).Should(Succeed(), "kubevirt machines should have bootstrap succeeded condition") + }).WithOffset(1). + WithTimeout(10*time.Minute). + WithPolling(5*time.Second). + Should(Succeed(), "kubevirt machines should have bootstrap succeeded condition") } markExternalKubeVirtClusterReady := func(clusterName string, namespace string) { By("Ensuring no other controller is managing the kvcluster's status") - Consistently(func() error { + Consistently(func(g Gomega) error { kvCluster := &infrav1.KubevirtCluster{} key := client.ObjectKey{Namespace: namespace, Name: clusterName} err := k8sclient.Get(context.Background(), key, kvCluster) - Expect(err).ToNot(HaveOccurred()) + g.Expect(err).ToNot(HaveOccurred()) - Expect(kvCluster.Finalizers).To(BeEmpty()) - Expect(kvCluster.Status.Ready).To(BeFalse()) - Expect(kvCluster.Status.FailureDomains).To(BeEmpty()) - Expect(kvCluster.Status.Conditions).To(BeEmpty()) + g.Expect(kvCluster.Finalizers).To(BeEmpty()) + g.Expect(kvCluster.Status.Ready).To(BeFalse()) + g.Expect(kvCluster.Status.FailureDomains).To(BeEmpty()) + g.Expect(kvCluster.Status.Conditions).To(BeEmpty()) return nil }, 30*time.Second, 5*time.Second).Should(Succeed()) @@ -240,7 +248,10 @@ var _ = Describe("CreateCluster", func() { } return nil - }, 5*time.Minute, 5*time.Second).Should(Succeed(), "waiting for expected readiness.") + }).WithOffset(1). + WithTimeout(5*time.Minute). + WithPolling(5*time.Second). + Should(Succeed(), "waiting for expected readiness.") } waitForTenantPods := func() { @@ -253,9 +264,9 @@ var _ = Describe("CreateCluster", func() { clientSet, err := tenantAccessor.generateClient() Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { + Eventually(func(g Gomega) error { podList, err := clientSet.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{}) - Expect(err).ToNot(HaveOccurred()) + g.Expect(err).ToNot(HaveOccurred()) offlinePodList := []string{} for _, pod := range podList.Items { @@ -269,7 +280,10 @@ var _ = Describe("CreateCluster", func() { return fmt.Errorf("Waiting on tenant pods [%v] to reach a Running phase", offlinePodList) } return nil - }, 8*time.Minute, 5*time.Second).Should(Succeed(), "waiting for pods to hit Running phase.") + }).WithOffset(1). + WithTimeout(8*time.Minute). + WithPolling(5*time.Second). + Should(Succeed(), "waiting for pods to hit Running phase.") } @@ -291,7 +305,10 @@ var _ = Describe("CreateCluster", func() { } return nil - }, 5*time.Minute, 5*time.Second).Should(Succeed(), "waiting for expected readiness.") + }).WithOffset(1). + WithTimeout(5*time.Minute). + WithPolling(5*time.Second). + Should(Succeed(), "waiting for expected readiness.") return clientSet } @@ -305,10 +322,10 @@ var _ = Describe("CreateCluster", func() { clientSet, err := tenantAccessor.generateClient() Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { + Eventually(func(g Gomega) error { nodeList, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - Expect(err).ToNot(HaveOccurred()) + g.Expect(err).ToNot(HaveOccurred()) for _, node := range nodeList.Items { @@ -330,7 +347,10 @@ var _ = Describe("CreateCluster", func() { } return nil - }, 8*time.Minute, 5*time.Second).Should(Succeed(), "ensure healthy nodes.") + }).WithOffset(1). + WithTimeout(8*time.Minute). + WithPolling(5*time.Second). + Should(Succeed(), "ensure healthy nodes.") return clientSet } @@ -360,7 +380,7 @@ var _ = Describe("CreateCluster", func() { waitForControlPlane := func() { By("Waiting on cluster's control plane to initialize") - Eventually(func() error { + Eventually(func(g Gomega) error { cluster := &clusterv1.Cluster{} key := client.ObjectKey{Namespace: namespace, Name: "kvcluster"} err := k8sclient.Get(context.Background(), key, cluster) @@ -369,11 +389,14 @@ var _ = Describe("CreateCluster", func() { } if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { - return fmt.Errorf("still waiting on controlPlaneInitialized condition to be true") + return fmt.Errorf("still waiting on controlPlaneReady condition to be true") } return nil - }, 20*time.Minute, 5*time.Second).Should(Succeed(), "cluster should have control plane initialized") + }).WithOffset(1). + WithTimeout(20*time.Minute). + WithPolling(5*time.Second). + Should(Succeed(), "cluster should have control plane initialized") By("Waiting on cluster's control plane to be ready") Eventually(func() error { @@ -385,11 +408,14 @@ var _ = Describe("CreateCluster", func() { } if !conditions.IsTrue(cluster, clusterv1.ControlPlaneReadyCondition) { - return fmt.Errorf("still waiting on controlPlaneReady condition to be true") + return fmt.Errorf("still waiting on controlPlaneInitialized condition to be true") } return nil - }, 15*time.Minute, 5*time.Second).Should(Succeed(), "cluster should have control plane initialized") + }).WithOffset(1). + WithTimeout(15*time.Minute). + WithPolling(5*time.Second). + Should(Succeed(), "cluster should have control plane initialized") } injectKubevirtClusterExternallyManagedAnnotation := func(yamlStr string) string { @@ -779,7 +805,7 @@ var _ = Describe("CreateCluster", func() { waitForMachineReadiness(2, 0) By("Waiting for getting access to the tenant cluster") - waitForTenantAccess(2) + clientSet := waitForTenantAccess(2) By("posting calico CNI manifests to the guest cluster and waiting for network") installCalicoCNI() @@ -789,6 +815,34 @@ var _ = Describe("CreateCluster", func() { By("waiting all tenant Pods to be Ready") waitForTenantPods() + + vmiName := chosenVMI.Name + + By("read the VMI again after it was recreated") + recreatedVMI := getRecreatedVMI(vmiName, namespace, chosenVMI.GetUID()) + + Expect(*recreatedVMI.Spec.EvictionStrategy).Should(Equal(kubevirtv1.EvictionStrategyExternal)) + + By("Set a testFinalizer to hold the VMI deletion so we could query it after eviction") + recreatedVMI = addFinalizerFromVMI(recreatedVMI.Name, namespace) + + By("Get VMI's pod") + pod := getVMIPod(recreatedVMI) + + By("Try to evict the VMI pod; should fail, but trigger the VMI draining") + evictNode(pod) + + By("wait for a VMI to be marked for deletion") + waitForVMIDraining(vmiName, namespace) + + By("remove the test finalizer") + removeFinalizerFromVMI(recreatedVMI) + + By("wait for a new VMI to be created") + getRecreatedVMI(vmiName, namespace, recreatedVMI.GetUID()) + + By("Read the worker node from the tenant cluster, and validate its IP") + validateNewNodeIP(clientSet, vmiName, namespace) }) // This test will create a tenant cluster from `templates/cluster-template-ext-infra.yaml` template. @@ -844,3 +898,146 @@ var _ = Describe("CreateCluster", func() { waitForControlPlane() }) }) + +func waitForVMIDraining(vmiName, namespace string) { + var vmi *kubevirtv1.VirtualMachineInstance + var err error + + By("wait for VMI is marked for deletion") + Eventually(func(g Gomega) bool { + vmi, err = virtClient.VirtualMachineInstance(namespace).Get(vmiName, &metav1.GetOptions{}) + g.Expect(err).ShouldNot(HaveOccurred()) + + vmiDebugPrintout(vmi) + + g.Expect(vmi.Status.EvacuationNodeName).ShouldNot(BeEmpty()) + g.Expect(vmi.DeletionTimestamp).ShouldNot(BeNil()) + + return true + }).WithOffset(1). + WithTimeout(time.Minute * 2). + WithPolling(time.Second). + Should(BeTrue()) +} + +func evictNode(pod *corev1.Pod) { + + err := virtClient.CoreV1().Pods(pod.Namespace).EvictV1beta1(context.Background(), &policy.Eviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.Name, + }, + DeleteOptions: &metav1.DeleteOptions{ + GracePeriodSeconds: pointer.Int64(60 * 10), // 10 minutes + }, + }) + + ExpectWithOffset(1, k8serrors.IsTooManyRequests(err)).To(BeTrue(), "should return TooManyRequests error; got %v instead", err) +} + +func getRecreatedVMI(vmiName string, namespace string, originalUID types.UID) *kubevirtv1.VirtualMachineInstance { + var ( + vmi *kubevirtv1.VirtualMachineInstance + err error + ) + Eventually(func(g Gomega) types.UID { + vmi, err = virtClient.VirtualMachineInstance(namespace).Get(vmiName, &metav1.GetOptions{}) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(vmi).ShouldNot(BeNil()) + + vmiDebugPrintout(vmi) + + g.Expect(vmi.Status.EvacuationNodeName).Should(BeEmpty()) + g.Expect(vmi.DeletionTimestamp).Should(BeNil()) + + return vmi.GetUID() + + }).WithOffset(1). + WithTimeout(time.Minute * 5). + WithPolling(time.Second * 5). + ShouldNot(Equal(originalUID)) // make sure that a new VMI was created + + return vmi +} + +func validateNewNodeIP(cl *kubernetes.Clientset, vmiName, namespace string) { + Eventually(func(g Gomega) bool { + // reading the node and the VMI again and again, because it takes time to the IPs to be synchronized + node, err := cl.CoreV1().Nodes().Get(context.Background(), vmiName, metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + + var nodeIp string + for _, address := range node.Status.Addresses { + if address.Type == "InternalIP" { + nodeIp = address.Address + } + } + + g.Expect(nodeIp).ShouldNot(BeEmpty(), "node's IP is not set") + + vmi, err := virtClient.VirtualMachineInstance(namespace).Get(vmiName, &metav1.GetOptions{}) + + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(vmi).ShouldNot(BeNil()) + + for _, ifs := range vmi.Status.Interfaces { + for _, ip := range ifs.IPs { + if ip == nodeIp { + return true + } + } + } + return false + + }).WithTimeout(5 * time.Minute). + WithOffset(1). + WithPolling(10 * time.Second). + Should(BeTrue()) +} + +func vmiDebugPrintout(vmi *kubevirtv1.VirtualMachineInstance) { + GinkgoWriter.Printf(`[Debug] VMI: {"UID": "%v", "DeletionTimestamp": "%v", "EvacuationNodeName": "%s"}`+"\n", + vmi.UID, vmi.DeletionTimestamp, vmi.Status.EvacuationNodeName) +} + +func addFinalizerFromVMI(vmiName, namespace string) *kubevirtv1.VirtualMachineInstance { + var ( + vmi *kubevirtv1.VirtualMachineInstance + err error + ) + Eventually(func() error { + vmi, err = virtClient.VirtualMachineInstance(namespace).Get(vmiName, &metav1.GetOptions{}) + if err != nil { + return err + } + + vmi.Finalizers = append(vmi.Finalizers, testFinalizer) + _, err = virtClient.VirtualMachineInstance(vmi.Namespace).Update(vmi) + return err + }).WithOffset(1). + WithTimeout(time.Minute). + WithPolling(2 * time.Second). + Should(Succeed()) + + return vmi +} + +func removeFinalizerFromVMI(vmi *kubevirtv1.VirtualMachineInstance) { + index := -1 + for i, finalizer := range vmi.Finalizers { + if finalizer == testFinalizer { + index = i + break + } + } + ExpectWithOffset(1, index).To(BeNumerically(">=", 0)) + + patch := []byte(fmt.Sprintf(`[{"op": "remove", "path": "/metadata/finalizers/%d"}]`, index)) + + Eventually(func() error { + _, err := virtClient.VirtualMachineInstance(vmi.Namespace).Patch(vmi.Name, types.JSONPatchType, patch, &metav1.PatchOptions{}) + return err + }).WithOffset(1). + WithTimeout(time.Minute). + WithPolling(2 * time.Second). + Should(Succeed()) +} diff --git a/go.mod b/go.mod index e66b322ab..f5db8e06b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module sigs.k8s.io/cluster-api-provider-kubevirt go 1.18 require ( - github.com/go-logr/logr v1.2.0 + github.com/go-logr/logr v1.2.3 github.com/golang/glog v1.0.0 github.com/golang/mock v1.6.0 github.com/onsi/ginkgo/v2 v2.1.4 @@ -12,12 +12,14 @@ require ( github.com/spf13/cobra v1.3.0 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 - k8s.io/api v0.23.1 - k8s.io/apimachinery v0.23.1 + k8s.io/api v0.23.5 + k8s.io/apimachinery v0.23.5 k8s.io/client-go v12.0.0+incompatible k8s.io/component-base v0.23.1 - k8s.io/klog/v2 v2.30.0 - kubevirt.io/api v0.51.0 + k8s.io/klog/v2 v2.40.1 + k8s.io/kubectl v0.22.2 + k8s.io/utils v0.0.0-20211116205334-6203023598ed + kubevirt.io/api v0.53.0 kubevirt.io/client-go v0.0.0-00010101000000-000000000000 sigs.k8s.io/cluster-api v0.3.11-0.20210525210043-6c7878e7b4a9 sigs.k8s.io/controller-runtime v0.11.0-beta.0.0.20211110210527-619e6b92dab9 @@ -27,41 +29,66 @@ require ( require ( cloud.google.com/go v0.99.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/prometheus-operator v0.38.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-errors/errors v1.0.1 // indirect github.com/go-kit/kit v0.9.0 // indirect github.com/go-logfmt/logfmt v0.5.0 // indirect github.com/go-logr/zapr v1.2.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/swag v0.21.1 // indirect github.com/gobuffalo/flect v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/btree v1.0.1 // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.1.2 // indirect github.com/googleapis/gnostic v0.5.5 // indirect - github.com/gorilla/websocket v1.4.2 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20191119172530-79f836b90111 // indirect - github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1 // indirect + github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mitchellh/go-wordwrap v1.0.0 // indirect + github.com/moby/spdystream v0.2.0 // indirect + github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/openshift/api v0.0.0-20210105115604-44119421ec6b // indirect github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 // indirect - github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca // indirect + github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pborman/uuid v1.2.0 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.28.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect + github.com/russross/blackfriday v1.5.2 // indirect + github.com/stretchr/testify v1.7.0 // indirect + github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect + go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect @@ -78,12 +105,14 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/apiextensions-apiserver v0.23.1 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect - kubevirt.io/containerized-data-importer-api v1.42.0 // indirect + k8s.io/cli-runtime v0.23.1 // indirect + k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect + kubevirt.io/containerized-data-importer-api v1.47.0 // indirect kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 // indirect sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/kustomize/api v0.10.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect ) replace ( @@ -94,5 +123,5 @@ replace ( replace ( k8s.io/client-go => k8s.io/client-go v0.23.1 - kubevirt.io/client-go => kubevirt.io/client-go v0.51.0 + kubevirt.io/client-go => kubevirt.io/client-go v0.53.0 ) diff --git a/go.sum b/go.sum index f6784f53a..8a942ad05 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/Azure/azure-sdk-for-go v36.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -174,8 +175,6 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= -github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coredns/caddy v1.1.0 h1:ezvsPrT/tA/7pYDBZxu0cT0VmWk75AfIaf6GSYCNMf0= github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= @@ -203,6 +202,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -231,11 +231,13 @@ github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.10.0+incompatible h1:l6Soi8WCOOVAeCo4W98iBFC6Og7/X8bpRt51oNLZ2C8= github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.15.0+incompatible h1:8KpYO/Xl/ZudZs5RNOEhWMBY4hmzlZhhRd9cu+jrZP4= +github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -257,6 +259,7 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= @@ -281,6 +284,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -297,8 +301,10 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= @@ -327,8 +333,9 @@ github.com/go-openapi/jsonreference v0.17.2/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3Hfo github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -358,8 +365,9 @@ github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/ github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -428,6 +436,7 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= @@ -468,10 +477,12 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -500,10 +511,12 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f h1:ShTPMJQes6tubcjzGMODIVG5hlrCeImaBnZzKF2N8SM= github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -572,7 +585,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -620,15 +632,14 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= -github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= -github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1 h1:t5bmB3Y8nCaLA4aFrIpX0zjHEF/HUkJp6f5rm7BsVzM= -github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1/go.mod h1:dV5oB3U62KBdlf9ADWkMmjGd3USauqQtwIm2OZb5mqI= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kubevirt/containerized-data-importer-api v1.41.1-0.20211201033752-05520fb9f18d h1:4HESRlaHoCI+w9AtL/TAcGPCqwPOpz+sHUlYHoM78XE= github.com/kubevirt/containerized-data-importer-api v1.41.1-0.20211201033752-05520fb9f18d/go.mod h1:Ty5GJ+6nKlpcBKjeebb/e6IrF8bNFgOus9hfuMjEt6A= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leanovate/gopter v0.2.4/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.0/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= @@ -644,8 +655,9 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -684,6 +696,7 @@ github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXx github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= @@ -693,8 +706,10 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -704,6 +719,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/mozillazg/go-cos v0.13.0/go.mod h1:Zp6DvvXn0RUOXGJ2chmWt2bLEqRAnJnS3DnAZsJsoaE= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= @@ -729,7 +745,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -737,6 +752,7 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -748,6 +764,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -757,8 +775,9 @@ github.com/openshift/api v0.0.0-20210105115604-44119421ec6b/go.mod h1:aqU5Cq+kqK github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 h1:+TEY29DK0XhqB7HFC9OfV8qf3wffSyi7MWv3AP28DGQ= github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47/go.mod h1:u7NRAjtYVAKokiI9LouzTv4mhds8P4S1TwdVAfbjKSk= -github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca h1:F1MEnOMwSrTA0YAkO0he9ip9w0JhYzI/iCB2mXmaSPg= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= +github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= +github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/openshift/prom-label-proxy v0.1.1-0.20191016113035-b8153a7f39f1/go.mod h1:p5MuxzsYP1JPsNGwtjtcgRHHlGziCJJfztff91nNixw= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -773,6 +792,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -841,6 +861,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -855,6 +876,7 @@ github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= @@ -906,6 +928,7 @@ github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -931,6 +954,7 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -939,6 +963,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.elastic.co/apm v1.5.0/go.mod h1:OdB9sPtM6Vt7oz3VXt7+KR96i9li74qrxBGHTQygFvk= go.elastic.co/apm/module/apmhttp v1.5.0/go.mod h1:1FbmNuyD3ddauwzgVwFB0fqY6KbZt3JkV187tGCYYhY= go.elastic.co/apm/module/apmot v1.5.0/go.mod h1:d2KYwhJParTpyw2WnTNy8geNlHKKFX+4oK3YLlsesWE= @@ -983,6 +1008,7 @@ go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1069,6 +1095,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1125,6 +1152,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1132,7 +1160,9 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1209,7 +1239,6 @@ golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1228,6 +1257,7 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1248,6 +1278,7 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1265,8 +1296,11 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1290,6 +1324,7 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= @@ -1373,6 +1408,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1441,7 +1477,6 @@ google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBr google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1585,8 +1620,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1602,14 +1639,16 @@ k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ys k8s.io/api v0.0.0-20190813020757-36bff7324fb7/go.mod h1:3Iy+myeAORNCLgjd/Xu9ebwN7Vh59Bw0vh9jhoX+V58= k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= k8s.io/api v0.0.0-20191115095533-47f6de673b26/go.mod h1:iA/8arsvelvo4IDqIhX4IbjTEKBGgvsf2OraTuRtLFU= -k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= k8s.io/api v0.17.3/go.mod h1:YZ0OTkuw7ipbe305fMpIdf3GLXZKRigjtZaV5gzC2J0= +k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= k8s.io/api v0.20.0/go.mod h1:HyLC5l5eoS/ygQYl1BXBgFzWNlkHiAuyNAbevIn+FKg= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/api v0.23.1 h1:ncu/qfBfUoClqwkTGbeRqqOqBCRoUAflMuOaOD7J0c8= k8s.io/api v0.23.1/go.mod h1:WfXnOnwSqNtG62Y1CdjoMxh7r7u9QXGCkA1u0na2jgo= +k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= +k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA= +k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= k8s.io/apiextensions-apiserver v0.17.3/go.mod h1:CJbCyMfkKftAd/X/V6OTHYhVn7zXnDdnkUjS1h0GTeY= k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= @@ -1622,15 +1661,16 @@ k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8/go.mod h1:sBJWIJZfxLhp7mR k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010/go.mod h1:Waf/xTS2FGRrgXCkO5FP3XxTOWh0qLf2QhL1qFZZ/R8= k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= k8s.io/apimachinery v0.0.0-20191115015347-3c7067801da2/go.mod h1:dXFS2zaQR8fyzuvRdJDHw2Aerij/yVGJSre0bZQSVJA= -k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.17.1-beta.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.3/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g= +k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.23.1 h1:sfBjlDFwj2onG0Ijx5C+SrAoeUscPrmghm7wHP+uXlo= k8s.io/apimachinery v0.23.1/go.mod h1:SADt2Kl8/sttJ62RRsi9MIV4o8f5S3coArm0Iu3fBno= +k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0= +k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= k8s.io/apiserver v0.17.3/go.mod h1:iJtsPpu1ZpEnHaNawpSV0nYTGBhhX2dUlnn7/QS7QiY= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= @@ -1639,6 +1679,8 @@ k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/apiserver v0.23.1 h1:vWGf8LcV9Pk/z5rdLmCiBDqE21ccbe930dzrtVMhw9g= k8s.io/apiserver v0.23.1/go.mod h1:Bqt0gWbeM2NefS8CjWswwd2VNAKN6lUKR85Ft4gippY= k8s.io/cli-runtime v0.22.2/go.mod h1:tkm2YeORFpbgQHEK/igqttvPTRIHFRz5kATlw53zlMI= +k8s.io/cli-runtime v0.23.1 h1:vHUZrq1Oejs0WaJnxs09mLHKScvIIl2hMSthhS8o8Yo= +k8s.io/cli-runtime v0.23.1/go.mod h1:r9r8H/qfXo9w+69vwUL7LokKlLRKW5D6A8vUKCx+YL0= k8s.io/client-go v0.23.1 h1:Ma4Fhf/p07Nmj9yAB1H7UwbFHEBrSPg8lviR24U2GiQ= k8s.io/client-go v0.23.1/go.mod h1:6QSI8fEuqD4zgFK0xbdwfB/PthBsIxCJMa3s17WlcO0= k8s.io/cluster-bootstrap v0.22.2 h1:jP6Nkp3CdSfr50cAn/7WGsNS52zrwMhvr0V+E3Vkh/w= @@ -1646,15 +1688,15 @@ k8s.io/cluster-bootstrap v0.22.2/go.mod h1:ZkmQKprEqvrUccMnbRHISsMscA1dsQ8SffM9n k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204= k8s.io/code-generator v0.0.0-20190717022600-77f3a1fe56bb/go.mod h1:cDx5jQmWH25Ff74daM7NVYty9JWw9dvIS9zT9eIubCY= k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= -k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.17.3/go.mod h1:l8BLVwASXQZTo2xamW5mQNFCe1XPiAesVq7Y1t7PiQQ= +k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/code-generator v0.23.1/go.mod h1:V7yn6VNTCWW8GqodYCESVo95fuiEg713S8B7WacWZDA= +k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= -k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= k8s.io/component-base v0.17.3/go.mod h1:GeQf4BrgelWm64PXkIXiPh/XS0hnO42d9gx9BtbZRp8= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= @@ -1667,9 +1709,11 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190907103519-ebc107f98eab/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -1680,18 +1724,21 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.40.1 h1:P4RRucWk/lFOlDdkAr3mc7iWFkgKrZY9qZMAgek06S4= +k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20181114233023-0317810137be/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf h1:M9XBsiMslw2lb2ZzglC0TOkBPK5NQi0/noUrdnoFwUg= +k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kubectl v0.22.2 h1:KMyYNZoBshaL3XKx04X07DtpoD4vMrdkfiN/G2Qx/PU= k8s.io/kubectl v0.22.2/go.mod h1:BApg2j0edxLArCOfO0ievI27EeTQqBDMNU9VQH734iQ= -k8s.io/kubernetes v1.14.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/metrics v0.22.2/go.mod h1:GUcsBtpsqQD1tKFS/2wCKu4ZBowwRncLOJH1rgWs3uw= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= @@ -1700,12 +1747,13 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -kubevirt.io/api v0.51.0 h1:G7nQDBkEzAxdSgpAPzE9ldXBIv8m0OtsgKWIUc+tvOw= -kubevirt.io/api v0.51.0/go.mod h1:RPYFWI69OVi7i6YtW5gHN3fjYsjlRfRilKVNcpxEMmM= -kubevirt.io/client-go v0.51.0 h1:Bn6CAsDIMRMew6hN4a62emFTauHPawWIa7DtJ9ZadLU= -kubevirt.io/client-go v0.51.0/go.mod h1:C34HnV4vfllPhCKOWrP3aRmlvyuCqNZUgtvNlaO2Pu0= +k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +kubevirt.io/api v0.53.0 h1:BXr595WisL1dB59eUt/QbvuLKR8GfRzaZ0p5piTIL6k= +kubevirt.io/api v0.53.0/go.mod h1:mK8ilpVLcZraqgo7hv2OSNQ5vdsA3G9Pxn8LY2/1+IY= +kubevirt.io/client-go v0.53.0 h1:4vAeoEHz7YS6PUyj0KxxyZhMyqFzVkQezmBHgwWp8y8= +kubevirt.io/client-go v0.53.0/go.mod h1:3FN4GQaV8iJEaRZgc63FVJ+ytk82s2pRumFUB9xs+MY= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 h1:I1b14fnhwrVvQLmgksMo9vgje42hmH4QN5kqyYDqbMA= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1/go.mod h1:ZJhLceiY2Gl5CXFGSp5eMGt/sksOiJP0289nAZFCQf0= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= @@ -1730,15 +1778,21 @@ sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNza sigs.k8s.io/kind v0.11.0 h1:tBxAEht9B3Dln8+kLxDg+A23ViRWcXquhV1Fe195fbE= sigs.k8s.io/kind v0.11.0/go.mod h1:fRpgVhtqAWrtLB9ED7zQahUimpUXuG/iHT88xYqEGIA= sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= +sigs.k8s.io/kustomize/api v0.10.1 h1:KgU7hfYoscuqag84kxtzKdEC3mKMb99DPI3a0eaV1d0= +sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8= sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs= sigs.k8s.io/kustomize/kustomize/v4 v4.2.0/go.mod h1:MOkR6fmhwG7hEDRXBYELTi5GSFcLwfqwzTRHW3kv5go= sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM= +sigs.k8s.io/kustomize/kyaml v0.13.0 h1:9c+ETyNfSrVhxvphs+K2dzT3dh5oVPPEqPOE/cUpScY= +sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index 2b5d8d433..e7a7c0b4d 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -22,4 +22,4 @@ fi rm -rf $TEST_WORKING_DIR mkdir -p $TEST_WORKING_DIR -$BIN_DIR/e2e.test --kubectl-path $KUBECTL_PATH --clusterctl-path $CLUSTERCTL_PATH --working-dir $TEST_WORKING_DIR +$BIN_DIR/e2e.test -ginkgo.v --kubectl-path $KUBECTL_PATH --clusterctl-path $CLUSTERCTL_PATH --working-dir $TEST_WORKING_DIR diff --git a/kubevirtci b/kubevirtci index 9987c465a..8e12381a6 100755 --- a/kubevirtci +++ b/kubevirtci @@ -93,9 +93,11 @@ function kubevirtci::up() { export KUBECONFIG=$(cluster-up/cluster-up/kubeconfig.sh) echo "installing kubevirt..." ${_kubectl} apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml - ${_kubectl} apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml - echo "installing capi..." + curl -L https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml \ + | sed -e "s|\( \+\)\(featureGates:\).*$|\1\2\n\1- LiveMigration|" \ + | ${_kubectl} apply -f - + echo "installing capi..." cat << EOF > ${_default_bin_path}/clusterctl_config.yaml --- cert-manager: diff --git a/main.go b/main.go index 389faa53b..c3de12ece 100644 --- a/main.go +++ b/main.go @@ -163,6 +163,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { setupLog.Error(err, "unable to create controller", "controller", "reconciler") os.Exit(1) } + if err := (&controllers.KubevirtClusterReconciler{ Client: mgr.GetClient(), InfraCluster: infracluster.New(mgr.GetClient()), @@ -171,6 +172,11 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { setupLog.Error(err, "unable to create controller", "controller", "KubevirtCluster") os.Exit(1) } + + if err := (controllers.NewVmiEvictionReconciler(mgr.GetClient())).SetupWithManager(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "VirtualMachineInstance") + os.Exit(1) + } } func setupWebhooks(mgr ctrl.Manager) { diff --git a/pkg/workloadcluster/mock/workloadcluster_generated.go b/pkg/workloadcluster/mock/workloadcluster_generated.go index 53b63e9dc..3a7bc1345 100644 --- a/pkg/workloadcluster/mock/workloadcluster_generated.go +++ b/pkg/workloadcluster/mock/workloadcluster_generated.go @@ -8,9 +8,9 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" - client "sigs.k8s.io/controller-runtime/pkg/client" - + kubernetes "k8s.io/client-go/kubernetes" context "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" + client "sigs.k8s.io/controller-runtime/pkg/client" ) // MockWorkloadCluster is a mock of WorkloadCluster interface. @@ -50,3 +50,18 @@ func (mr *MockWorkloadClusterMockRecorder) GenerateWorkloadClusterClient(ctx int mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateWorkloadClusterClient", reflect.TypeOf((*MockWorkloadCluster)(nil).GenerateWorkloadClusterClient), ctx) } + +// GenerateWorkloadClusterK8sClient mocks base method. +func (m *MockWorkloadCluster) GenerateWorkloadClusterK8sClient(ctx *context.MachineContext) (kubernetes.Interface, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateWorkloadClusterK8sClient", ctx) + ret0, _ := ret[0].(kubernetes.Interface) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateWorkloadClusterK8sClient indicates an expected call of GenerateWorkloadClusterK8sClient. +func (mr *MockWorkloadClusterMockRecorder) GenerateWorkloadClusterK8sClient(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateWorkloadClusterK8sClient", reflect.TypeOf((*MockWorkloadCluster)(nil).GenerateWorkloadClusterK8sClient), ctx) +} diff --git a/pkg/workloadcluster/workloadcluster.go b/pkg/workloadcluster/workloadcluster.go index c1ae0e38c..0fd9f21b8 100644 --- a/pkg/workloadcluster/workloadcluster.go +++ b/pkg/workloadcluster/workloadcluster.go @@ -3,6 +3,7 @@ package workloadcluster import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + k8sclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" @@ -12,6 +13,7 @@ import ( //go:generate mockgen -source=./workloadcluster.go -destination=./mock/workloadcluster_generated.go -package=mock type WorkloadCluster interface { GenerateWorkloadClusterClient(ctx *context.MachineContext) (client.Client, error) + GenerateWorkloadClusterK8sClient(ctx *context.MachineContext) (k8sclient.Interface, error) } func New(client client.Client) WorkloadCluster { @@ -48,6 +50,29 @@ func (w *workloadCluster) GenerateWorkloadClusterClient(ctx *context.MachineCont return workloadClusterClient, nil } +// GenerateWorkloadClusterK8sClient creates a kubernetes client for workload cluster. +func (w *workloadCluster) GenerateWorkloadClusterK8sClient(ctx *context.MachineContext) (k8sclient.Interface, error) { + // get workload cluster kubeconfig + kubeConfig, err := w.getKubeconfigForWorkloadCluster(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to get kubeconfig for workload cluster") + } + + // generate REST config + restConfig, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeConfig)) + if err != nil { + return nil, errors.Wrap(err, "failed to create REST config") + } + + // create the client + workloadClusterClient, err := k8sclient.NewForConfig(restConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to create workload cluster client") + } + + return workloadClusterClient, nil +} + // getKubeconfigForWorkloadCluster fetches kubeconfig for workload cluster from the corresponding secret. func (w *workloadCluster) getKubeconfigForWorkloadCluster(ctx *context.MachineContext) (string, error) { // workload cluster kubeconfig can be found in a secret with suffix "-kubeconfig" diff --git a/templates/cluster-template-ext-infra.yaml b/templates/cluster-template-ext-infra.yaml index e0d417c1a..c9d457256 100644 --- a/templates/cluster-template-ext-infra.yaml +++ b/templates/cluster-template-ext-infra.yaml @@ -58,6 +58,7 @@ spec: - disk: bus: virtio name: containervolume + evictionStrategy: External volumes: - containerDisk: image: "${NODE_VM_IMAGE_TEMPLATE}" @@ -114,6 +115,7 @@ spec: - disk: bus: virtio name: containervolume + evictionStrategy: External volumes: - containerDisk: image: "${NODE_VM_IMAGE_TEMPLATE}" diff --git a/templates/cluster-template-persistent-storage.yaml b/templates/cluster-template-persistent-storage.yaml index c83a1636b..fc0d6407e 100644 --- a/templates/cluster-template-persistent-storage.yaml +++ b/templates/cluster-template-persistent-storage.yaml @@ -68,6 +68,7 @@ spec: - disk: bus: virtio name: dv-volume + evictionStrategy: External volumes: - dataVolume: name: "${CLUSTER_NAME}-boot-volume" @@ -140,6 +141,7 @@ spec: - disk: bus: virtio name: dv-volume + evictionStrategy: External volumes: - dataVolume: name: "${CLUSTER_NAME}-boot-volume" diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a56775d57..fc29f8418 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -58,6 +58,7 @@ spec: - disk: bus: virtio name: containervolume + evictionStrategy: External volumes: - containerDisk: image: "${NODE_VM_IMAGE_TEMPLATE}" @@ -116,6 +117,7 @@ spec: - disk: bus: virtio name: containervolume + evictionStrategy: External volumes: - containerDisk: image: "${NODE_VM_IMAGE_TEMPLATE}"