diff --git a/assets/performanceprofile/configs/99-runtimes.conf b/assets/performanceprofile/configs/99-runtimes.conf index 3a9e5e1073..4874370d9b 100644 --- a/assets/performanceprofile/configs/99-runtimes.conf +++ b/assets/performanceprofile/configs/99-runtimes.conf @@ -4,18 +4,11 @@ infra_ctr_cpuset = "{{.ReservedCpus}}" {{if .SharedCpus}}shared_cpuset = "{{.SharedCpus}}"{{end}} {{end}} -# We should copy paste the default runtime because this snippet will override the whole runtimes section -[crio.runtime.runtimes.runc] -runtime_path = "" -runtime_type = "oci" -runtime_root = "/run/runc" # The CRI-O will check the allowed_annotations under the runtime handler and apply high-performance hooks when one of # high-performance annotations presents under it. # We should provide the runtime_path because we need to inform that we want to re-use runc binary and we # do not have high-performance binary under the $PATH that will point to it. [crio.runtime.runtimes.high-performance] -runtime_path = "{{.RuntimePath}}" -runtime_type = "oci" -runtime_root = "{{.RuntimeRoot}}" +inherit_default_runtime = true allowed_annotations = ["cpu-load-balancing.crio.io", "cpu-quota.crio.io", "irq-load-balancing.crio.io", "cpu-c-states.crio.io", "cpu-freq-governor.crio.io"{{ if .CrioSharedCPUsAnnotation }}{{ printf ", %q" .CrioSharedCPUsAnnotation}}{{end}}] diff --git a/hack/render-sync.sh b/hack/render-sync.sh index a190a3e324..708ceda934 100755 --- a/hack/render-sync.sh +++ b/hack/render-sync.sh @@ -41,6 +41,5 @@ function rendersync() { rendersync manual-cluster/performance base/performance default rendersync bootstrap-cluster/performance pinned-cluster/default bootstrap/no-mcp rendersync bootstrap-cluster/performance pinned-cluster/default bootstrap-cluster/extra-mcp bootstrap/extra-mcp -rendersync bootstrap-cluster/performance pinned-cluster/default container-runtime-crun bootstrap/extra-ctrcfg rendersync --owner-ref none -- base/performance manual-cluster/performance no-ref rendersync --owner-ref none -- base/performance manual-cluster/cpuFrequency default/cpuFrequency \ No newline at end of file diff --git a/manifests/40-rbac.yaml b/manifests/40-rbac.yaml index 2a0826bdce..7268544c13 100644 --- a/manifests/40-rbac.yaml +++ b/manifests/40-rbac.yaml @@ -81,7 +81,7 @@ rules: verbs: ["create","get","delete","list","update","watch"] # Needed by the core operator functionality. - apiGroups: ["machineconfiguration.openshift.io"] - resources: ["machineconfigpools", "containerruntimeconfigs"] + resources: ["machineconfigpools"] verbs: ["get","list","watch"] # Needed by the leases mechanism. - apiGroups: ["coordination.k8s.io"] diff --git a/pkg/operator/hypershift.go b/pkg/operator/hypershift.go index 6144382a28..e196a9b519 100644 --- a/pkg/operator/hypershift.go +++ b/pkg/operator/hypershift.go @@ -33,10 +33,9 @@ const ( // TODO remove once HyperShift has switched to using new key. tunedConfigMapConfigKeyDeprecated = "tuned" - operatorGeneratedMachineConfig = "hypershift.openshift.io/nto-generated-machine-config" - mcConfigMapDataKey = "config" - generatedConfigMapPrefix = "nto-mc-" - HypershiftControllerGeneratedContainerRuntimeConfig = "hypershift.openshift.io/containerruntimeconfig-config" + operatorGeneratedMachineConfig = "hypershift.openshift.io/nto-generated-machine-config" + mcConfigMapDataKey = "config" + generatedConfigMapPrefix = "nto-mc-" ) // syncHostedClusterTuneds synchronizes Tuned objects embedded in ConfigMaps diff --git a/pkg/performanceprofile/cmd/render/render.go b/pkg/performanceprofile/cmd/render/render.go index b0ce8817c9..2115117765 100644 --- a/pkg/performanceprofile/cmd/render/render.go +++ b/pkg/performanceprofile/cmd/render/render.go @@ -100,7 +100,6 @@ func render(ownerRefMode, inputDir, outputDir string) error { mcPools []*mcfgv1.MachineConfigPool mcConfigs []*mcfgv1.MachineConfig infra *apicfgv1.Infrastructure - ctrcfgs []*mcfgv1.ContainerRuntimeConfig ) // Iterate through the file paths and read in desired files for _, path := range filePaths { @@ -137,8 +136,6 @@ func render(ownerRefMode, inputDir, outputDir string) error { if obj.Name == clusterConfigResourceName { infra = obj } - case *mcfgv1.ContainerRuntimeConfig: - ctrcfgs = append(ctrcfgs, obj) default: klog.Infof("skipping %q [%d] manifest because of unhandled %T", file.Name(), idx+1, obji) } @@ -184,17 +181,11 @@ func render(ownerRefMode, inputDir, outputDir string) error { continue } - defaultRuntime, err := getContainerRuntimeName(pp, mcp, ctrcfgs) - if err != nil { - return fmt.Errorf("render: could not determine high-performance runtime class container-runtime for profile %q; %w", pp.Name, err) - } - components, err := manifestset.GetNewComponents(pp, &performanceprofilecomponents.Options{ ProfileMCP: mcp, MachineConfig: performanceprofilecomponents.MachineConfigOptions{ - PinningMode: partitioningMode, - DefaultRuntime: defaultRuntime}, + PinningMode: partitioningMode}, }) if err != nil { return err @@ -388,28 +379,3 @@ func selectMachineConfigPool(pools []*mcfgv1.MachineConfigPool, selectors map[st return mcp, nil } - -func getContainerRuntimeName(profile *performancev2.PerformanceProfile, mcp *mcfgv1.MachineConfigPool, ctrcfgs []*mcfgv1.ContainerRuntimeConfig) (mcfgv1.ContainerRuntimeDefaultRuntime, error) { - mcpLabels := labels.Set(mcp.Labels) - var matchingCtrConfigs []*mcfgv1.ContainerRuntimeConfig - for _, ctrcfg := range ctrcfgs { - ctrcfgSelector, err := v1.LabelSelectorAsSelector(ctrcfg.Spec.MachineConfigPoolSelector) - if err != nil { - return "", err - } - if ctrcfgSelector.Matches(mcpLabels) { - matchingCtrConfigs = append(matchingCtrConfigs, ctrcfg) - } - } - - if len(matchingCtrConfigs) == 0 { - klog.Infof("no ContainerRuntimeConfig found that matches MCP labels %s that associated with performance profile %q; using default container runtime", mcpLabels.String(), profile.Name) - return mcfgv1.ContainerRuntimeDefaultRuntimeRunc, nil - } - - if len(matchingCtrConfigs) > 1 { - return "", fmt.Errorf("more than one ContainerRuntimeConfig found that matches MCP labels %s that associated with performance profile %q", mcpLabels.String(), profile.Name) - } - - return matchingCtrConfigs[0].Spec.ContainerRuntimeConfig.DefaultRuntime, nil -} diff --git a/pkg/performanceprofile/controller/performanceprofile/components/components.go b/pkg/performanceprofile/controller/performanceprofile/components/components.go index bd41f26112..c4243b209e 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/components.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/components.go @@ -42,7 +42,6 @@ type Options struct { type MachineConfigOptions struct { PinningMode *apiconfigv1.CPUPartitioningMode - DefaultRuntime mcov1.ContainerRuntimeDefaultRuntime MixedCPUsEnabled bool } diff --git a/pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go b/pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go index 880b04b8ff..75df6b6ee2 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go @@ -6,22 +6,18 @@ import ( corev1 "k8s.io/api/core/v1" k8serros "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - mcov1 "github.com/openshift/api/machineconfiguration/v1" performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/machineconfig" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/manifestset" profileutil "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/resources" - "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/status" ) var _ components.Handler = &handler{} @@ -49,13 +45,6 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record. // set missing options opts.MachineConfig.MixedCPUsEnabled = opts.MixedCPUsFeatureGateEnabled && profileutil.IsMixedCPUsEnabled(profile) - ctrRuntime, err := h.getContainerRuntimeName(ctx, profile, opts.ProfileMCP) - if err != nil { - return fmt.Errorf("could not determine high-performance runtime class container-runtime for profile %q; %w", profile.Name, err) - } - klog.V(2).Infof("using %q as high-performance runtime class container-runtime for profile %q", ctrRuntime, profile.Name) - - opts.MachineConfig.DefaultRuntime = ctrRuntime components, err := manifestset.GetNewComponents(profile, opts) if err != nil { return err @@ -170,45 +159,3 @@ func (h *handler) Exists(ctx context.Context, profileName string) bool { } return false } - -func (h *handler) getContainerRuntimeName(ctx context.Context, profile *performancev2.PerformanceProfile, mcp *mcov1.MachineConfigPool) (mcov1.ContainerRuntimeDefaultRuntime, error) { - ctrcfgList := &mcov1.ContainerRuntimeConfigList{} - if err := h.Client.List(ctx, ctrcfgList); err != nil { - return "", err - } - - if len(ctrcfgList.Items) == 0 { - return mcov1.ContainerRuntimeDefaultRuntimeRunc, nil - } - - var ctrcfgs []*mcov1.ContainerRuntimeConfig - mcpSetLabels := labels.Set(mcp.Labels) - for i := 0; i < len(ctrcfgList.Items); i++ { - ctrcfg := &ctrcfgList.Items[i] - ctrcfgSelector, err := metav1.LabelSelectorAsSelector(ctrcfg.Spec.MachineConfigPoolSelector) - if err != nil { - return "", err - } - if ctrcfgSelector.Matches(mcpSetLabels) { - ctrcfgs = append(ctrcfgs, ctrcfg) - } - } - - if len(ctrcfgs) == 0 { - klog.V(1).Infof("no ContainerRuntimeConfig found that matches MCP labels %s that associated with performance profile %q; using default container runtime", mcpSetLabels.String(), profile.Name) - return mcov1.ContainerRuntimeDefaultRuntimeRunc, nil - } - - if len(ctrcfgs) > 1 { - return "", fmt.Errorf("more than one ContainerRuntimeConfig found that matches MCP labels %s that associated with performance profile %q", mcpSetLabels.String(), profile.Name) - } - - condition := status.GetLatestContainerRuntimeConfigCondition(ctrcfgs[0].Status.Conditions) - if condition == nil { - return "", fmt.Errorf("ContainerRuntimeConfig: %q no conditions reported (yet)", ctrcfgs[0].Name) - } - if condition.Type != mcov1.ContainerRuntimeConfigSuccess || condition.Status != corev1.ConditionTrue { - return "", fmt.Errorf("ContainerRuntimeConfig: %q failed to be applied: message=%q", ctrcfgs[0].Name, condition.Message) - } - return ctrcfgs[0].Spec.ContainerRuntimeConfig.DefaultRuntime, nil -} diff --git a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go index 163a8edbcb..3f37aca48c 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go @@ -113,8 +113,6 @@ const ( templateOvsSliceDefinitionFile = "ovs.slice" templateOvsSliceUsageFile = "01-use-ovs-slice.conf" templateWorkload = "Workload" - templateRuntimePath = "RuntimePath" - templateRuntimeRoot = "RuntimeRoot" templateCrioSharedCPUsAnnotation = "CrioSharedCPUsAnnotation" ) @@ -573,10 +571,7 @@ func addContent(ignitionConfig *igntypes.Config, content []byte, dst string, mod } func renderCrioConfigSnippet(profile *performancev2.PerformanceProfile, src string, opts *components.MachineConfigOptions) ([]byte, error) { - templateArgs := map[string]string{ - templateRuntimePath: "/bin/runc", - templateRuntimeRoot: "/run/runc", - } + templateArgs := map[string]string{} if profile.Spec.CPU.Reserved != nil { templateArgs[templateReservedCpus] = string(*profile.Spec.CPU.Reserved) @@ -587,11 +582,6 @@ func renderCrioConfigSnippet(profile *performancev2.PerformanceProfile, src stri templateArgs[templateCrioSharedCPUsAnnotation] = "cpu-shared.crio.io" } - if opts.DefaultRuntime == machineconfigv1.ContainerRuntimeDefaultRuntimeCrun { - templateArgs[templateRuntimePath] = "/usr/bin/crun" - templateArgs[templateRuntimeRoot] = "/run/crun" - } - profileTemplate, err := template.ParseFS(assets.Configs, src) if err != nil { return nil, err diff --git a/pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go b/pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go index 0af7a5e582..1137d9e8e2 100644 --- a/pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go +++ b/pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go @@ -15,9 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - mcov1 "github.com/openshift/api/machineconfiguration/v1" performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" - ntoconfig "github.com/openshift/cluster-node-tuning-operator/pkg/config" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/manifestset" profileutil "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile" @@ -96,13 +94,6 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record. // set missing options options.MachineConfig.MixedCPUsEnabled = options.MixedCPUsFeatureGateEnabled && profileutil.IsMixedCPUsEnabled(profile) - ctrRuntime, err := h.getContainerRuntimeName(ctx, profile) - if err != nil { - return fmt.Errorf("could not determine high-performance runtime class container-runtime for profile %q; %w", profile.Name, err) - } - klog.Infof("using %q as high-performance runtime class container-runtime for profile %q", ctrRuntime, profile.Name) - options.MachineConfig.DefaultRuntime = ctrRuntime - mfs, err := manifestset.GetNewComponents(profile, options) if err != nil { return err @@ -190,50 +181,6 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record. return nil } -func (h *handler) getContainerRuntimeName(ctx context.Context, profile *performancev2.PerformanceProfile) (mcov1.ContainerRuntimeDefaultRuntime, error) { - cmList := &corev1.ConfigMapList{} - if err := h.controlPlaneClient.List(ctx, cmList, &client.ListOptions{ - Namespace: ntoconfig.OperatorNamespace(), - }); err != nil { - return "", err - } - var ctrcfgs []*mcov1.ContainerRuntimeConfig - for _, cm := range cmList.Items { - data, ok := cm.Data[hypershiftconsts.ConfigKey] - // container runtime config should be store in the Config key - if !ok { - continue - } - // ConfigMaps with the PerformanceProfileNameLabel label are generated by - // the controller itself - if _, ok = cm.Labels[hypershiftconsts.PerformanceProfileNameLabel]; ok { - continue - } - ctrcfg, err := validateAndExtractContainerRuntimeConfigFrom(h.scheme, []byte(data)) - if err != nil { - return "", fmt.Errorf("failed to get ContainerRuntime name %w", err) - } - if ctrcfg != nil { - ctrcfgs = append(ctrcfgs, ctrcfg) - } - } - if len(ctrcfgs) == 0 { - klog.V(1).Infof("no ContainerRuntimeConfig found that associated with performance profile %q; using default container runtime %q", profile.Name, mcov1.ContainerRuntimeDefaultRuntimeRunc) - return mcov1.ContainerRuntimeDefaultRuntimeRunc, nil - } - if len(ctrcfgs) > 1 { - return "", fmt.Errorf("more than one ContainerRuntimeConfig found that associated with performance profile %q", profile.Name) - } - // Ideally, the controller is supposed to check the ContainerRuntimeConfig status and return the value only if - // ContainerRuntimeConfig applied successfully. - // On hypershift, the controller does not have access to the status, so it would have to relay on hypershift operator - // to apply the ContainerRuntimeConfig configuration correctly. - // In case something goes wrong with the ContainerRuntimeConfig application, - // the hypershift operator won't be applying NTO controller's configuration either - // and will reflect that on the NodePool's status. - return ctrcfgs[0].Spec.ContainerRuntimeConfig.DefaultRuntime, nil -} - func EncapsulateObjInConfigMap(scheme *runtime.Scheme, instance *corev1.ConfigMap, object client.Object, profileName, dataKey string, objectLabels map[string]string) (*corev1.ConfigMap, error) { encodedObj, err := hypershift.EncodeManifest(object, scheme) if err != nil { @@ -328,12 +275,3 @@ func createOrUpdateConfigMap(ctx context.Context, cli client.Client, cm *corev1. } return nil } - -func validateAndExtractContainerRuntimeConfigFrom(scheme *runtime.Scheme, manifest []byte) (*mcov1.ContainerRuntimeConfig, error) { - ctrcfg := &mcov1.ContainerRuntimeConfig{} - ok, err := hypershift.DecodeManifest(manifest, scheme, ctrcfg) - if !ok { - return nil, err - } - return ctrcfg, err -} diff --git a/pkg/performanceprofile/controller/performanceprofile/hypershift/consts/consts.go b/pkg/performanceprofile/controller/performanceprofile/hypershift/consts/consts.go index 3fda198273..d82571c360 100644 --- a/pkg/performanceprofile/controller/performanceprofile/hypershift/consts/consts.go +++ b/pkg/performanceprofile/controller/performanceprofile/hypershift/consts/consts.go @@ -33,7 +33,7 @@ const ( TuningKey = "tuning" // ConfigKey is the key under ConfigMap.Data on which encoded - // machine-config, kubelet-config and container-runtime-config objects are stored. + // machine-config, kubelet-config objects are stored. ConfigKey = "config" // PerformanceProfileStatusKey is the key under ConfigMap.Data on which an encoded diff --git a/pkg/performanceprofile/controller/performanceprofile/hypershift/hypershift.go b/pkg/performanceprofile/controller/performanceprofile/hypershift/hypershift.go index 2ca0d89388..7809a49434 100644 --- a/pkg/performanceprofile/controller/performanceprofile/hypershift/hypershift.go +++ b/pkg/performanceprofile/controller/performanceprofile/hypershift/hypershift.go @@ -103,8 +103,7 @@ func GetObjectConfigMapDataKey(obj runtime.Object) string { case *performancev2.PerformanceProfile, *performancev2.PerformanceProfileList, *tunedv1.Tuned, *tunedv1.TunedList: return hypershiftconsts.TuningKey case *machineconfigv1.KubeletConfig, *machineconfigv1.KubeletConfigList, - *machineconfigv1.MachineConfig, *machineconfigv1.MachineConfigList, - *machineconfigv1.ContainerRuntimeConfig, *machineconfigv1.ContainerRuntimeConfigList: + *machineconfigv1.MachineConfig, *machineconfigv1.MachineConfigList: return hypershiftconsts.ConfigKey default: return "" diff --git a/pkg/performanceprofile/controller/performanceprofile/status/status.go b/pkg/performanceprofile/controller/performanceprofile/status/status.go index f2b77fb7f8..493c4c1cf5 100644 --- a/pkg/performanceprofile/controller/performanceprofile/status/status.go +++ b/pkg/performanceprofile/controller/performanceprofile/status/status.go @@ -246,16 +246,6 @@ func getLatestKubeletConfigCondition(conditions []mcov1.KubeletConfigCondition) return latestCondition } -func GetLatestContainerRuntimeConfigCondition(conditions []mcov1.ContainerRuntimeConfigCondition) *mcov1.ContainerRuntimeConfigCondition { - var latestCondition *mcov1.ContainerRuntimeConfigCondition - for i := 0; i < len(conditions); i++ { - if latestCondition == nil || latestCondition.LastTransitionTime.Before(&conditions[i].LastTransitionTime) { - latestCondition = &conditions[i] - } - } - return latestCondition -} - func removeUnMatchedTunedProfiles(nodes []corev1.Node, profiles []tunedv1.Profile) []tunedv1.Profile { filteredProfiles := make([]tunedv1.Profile, 0) for _, profile := range profiles { diff --git a/pkg/performanceprofile/controller/performanceprofile_controller.go b/pkg/performanceprofile/controller/performanceprofile_controller.go index 0f49658e8c..cb1fc5c235 100644 --- a/pkg/performanceprofile/controller/performanceprofile_controller.go +++ b/pkg/performanceprofile/controller/performanceprofile_controller.go @@ -31,7 +31,6 @@ import ( performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" ntoconfig "github.com/openshift/cluster-node-tuning-operator/pkg/config" - "github.com/openshift/cluster-node-tuning-operator/pkg/operator" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" profileutil "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile" hypershiftconsts "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift/consts" @@ -131,28 +130,6 @@ func (r *PerformanceProfileReconciler) SetupWithManager(mgr ctrl.Manager) error }, } - ctrcfgPredicates := predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - ctrcfg := e.Object.(*mcov1.ContainerRuntimeConfig) - return ctrcfg.Spec.ContainerRuntimeConfig.DefaultRuntime != "" - }, - - UpdateFunc: func(e event.UpdateEvent) bool { - if !validateUpdateEvent(e.ObjectOld, e.ObjectNew) { - return false - } - - ctrcfgOld := e.ObjectOld.(*mcov1.ContainerRuntimeConfig) - ctrcfgNew := e.ObjectNew.(*mcov1.ContainerRuntimeConfig) - return !reflect.DeepEqual(ctrcfgOld.Status.Conditions, ctrcfgNew.Status.Conditions) - }, - - DeleteFunc: func(e event.DeleteEvent) bool { - ctrcfg := e.Object.(*mcov1.ContainerRuntimeConfig) - return ctrcfg.Spec.ContainerRuntimeConfig.DefaultRuntime != "" - }, - } - return ctrl.NewControllerManagedBy(mgr). For(&performancev2.PerformanceProfile{}). Owns(&mcov1.MachineConfig{}, builder.WithPredicates(p)). @@ -166,9 +143,6 @@ func (r *PerformanceProfileReconciler) SetupWithManager(mgr ctrl.Manager) error handler.EnqueueRequestsFromMapFunc(r.tunedProfileToPerformanceProfile), builder.WithPredicates(tunedProfilePredicates), ). - Watches(&mcov1.ContainerRuntimeConfig{}, - handler.EnqueueRequestsFromMapFunc(r.ctrRuntimeConfToPerformanceProfile), - builder.WithPredicates(ctrcfgPredicates)). Complete(r) } @@ -245,30 +219,6 @@ func (r *PerformanceProfileReconciler) SetupWithManagerForHypershift(mgr ctrl.Ma }, } - containerRuntimeConfigPredicate := predicate.TypedFuncs[*corev1.ConfigMap]{ - UpdateFunc: func(ue event.TypedUpdateEvent[*corev1.ConfigMap]) bool { - if !validateUpdateEvent(ue.ObjectOld, ue.ObjectNew) { - klog.V(4).InfoS("UpdateEvent not valid", "objectName", ue.ObjectOld.GetName()) - return false - } - return validateLabels(ue.ObjectNew, operator.HypershiftControllerGeneratedContainerRuntimeConfig, "UpdateEvent") - }, - CreateFunc: func(ce event.TypedCreateEvent[*corev1.ConfigMap]) bool { - if ce.Object == nil { - klog.Error("Create event has no runtime object") - return false - } - return validateLabels(ce.Object, operator.HypershiftControllerGeneratedContainerRuntimeConfig, "CreateEvent") - }, - DeleteFunc: func(de event.TypedDeleteEvent[*corev1.ConfigMap]) bool { - if de.Object == nil { - klog.Error("Delete event has no runtime object") - return false - } - return validateLabels(de.Object, operator.HypershiftControllerGeneratedContainerRuntimeConfig, "DeleteEvent") - }, - } - return ctrl.NewControllerManagedBy(mgr). Named("performanceprofile_controller"). // we can't use For() and Owns(), because this calls are using the cache of the hosted cluster's client. @@ -277,10 +227,6 @@ func (r *PerformanceProfileReconciler) SetupWithManagerForHypershift(mgr ctrl.Ma &corev1.ConfigMap{}, &handler.TypedEnqueueRequestForObject[*corev1.ConfigMap]{}, performanceProfileConfigMapPredicate)). - WatchesRawSource(source.Kind(managementCluster.GetCache(), - &corev1.ConfigMap{}, - &handler.TypedEnqueueRequestForObject[*corev1.ConfigMap]{}, - containerRuntimeConfigPredicate)). WatchesRawSource(source.Kind(managementCluster.GetCache(), &corev1.ConfigMap{}, handler.TypedEnqueueRequestForOwner[*corev1.ConfigMap](r.ManagementClient.Scheme(), r.ManagementClient.RESTMapper(), &corev1.ConfigMap{}, handler.OnlyControllerOwner()), @@ -407,53 +353,6 @@ func (r *PerformanceProfileReconciler) tunedProfileToPerformanceProfile(ctx cont return requests } -func (r *PerformanceProfileReconciler) ctrRuntimeConfToPerformanceProfile(ctx context.Context, ctrRuntimeConfObj client.Object) []reconcile.Request { - ctrcfg := &mcov1.ContainerRuntimeConfig{} - - err := r.Get(ctx, client.ObjectKeyFromObject(ctrRuntimeConfObj), ctrcfg) - if err != nil { - klog.Errorf("failed to get container runtime config; name=%q err=%v", ctrRuntimeConfObj.GetName(), err) - return nil - } - klog.V(2).Infof("reconciling from ContainerRuntimeConfig %q", ctrcfg.Name) - - selector, err := metav1.LabelSelectorAsSelector(ctrcfg.Spec.MachineConfigPoolSelector) - if err != nil { - klog.Errorf("failed to parse the selector %v for container runtime config; name=%q err=%v", ctrcfg.Spec.MachineConfigPoolSelector, ctrRuntimeConfObj.GetName(), err) - return nil - } - - mcps := &mcov1.MachineConfigPoolList{} - opts := &client.ListOptions{ - LabelSelector: selector, - } - - err = r.List(ctx, mcps, opts) - if err != nil { - klog.Errorf("failed to list machine config pools; err=%v", err) - return nil - } - - klog.V(2).Infof("reconciling from ContainerRuntimeConfig %q selector %v: %d MCPs", ctrcfg.Name, ctrcfg.Spec.MachineConfigPoolSelector, len(mcps.Items)) - - profiles := &performancev2.PerformanceProfileList{} - err = r.List(ctx, profiles) - if err != nil { - klog.Errorf("failed to get performance profiles: %v", err) - return nil - } - - var allRequests []reconcile.Request - for i := 0; i < len(mcps.Items); i++ { - requests := mcpToPerformanceProfileReconcileRequests(profiles, &mcps.Items[i]) - if requests == nil { - return nil - } - allRequests = append(allRequests, requests...) - } - return allRequests -} - func getInfraPartitioningMode(ctx context.Context, client client.Client) (pinning apiconfigv1.CPUPartitioningMode, err error) { key := types.NamespacedName{ Name: "cluster", @@ -484,7 +383,7 @@ func validateUpdateEvent(old, new metav1.Object) bool { // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch // +kubebuilder:rbac:groups=performance.openshift.io,resources=performanceprofiles;performanceprofiles/status;performanceprofiles/finalizers,verbs=* // +kubebuilder:rbac:groups=machineconfiguration.openshift.io,resources=machineconfigs;kubeletconfigs,verbs=* -// +kubebuilder:rbac:groups=machineconfiguration.openshift.io,resources=machineconfigpools;containerruntimeconfigs,verbs=get;list;watch +// +kubebuilder:rbac:groups=machineconfiguration.openshift.io,resources=machineconfigpools,verbs=get;list;watch // +kubebuilder:rbac:groups=tuned.openshift.io,resources=tuneds;profiles,verbs=* // +kubebuilder:rbac:groups=node.k8s.io,resources=runtimeclasses,verbs=* // +kubebuilder:rbac:groups=config.openshift.io,resources=infrastructures,verbs=get;list;watch diff --git a/pkg/performanceprofile/controller/performanceprofile_controller_test.go b/pkg/performanceprofile/controller/performanceprofile_controller_test.go index 77cf062a5c..a59db8f3e1 100644 --- a/pkg/performanceprofile/controller/performanceprofile_controller_test.go +++ b/pkg/performanceprofile/controller/performanceprofile_controller_test.go @@ -68,7 +68,6 @@ var _ = Describe("Controller", func() { var profileMCP *mcov1.MachineConfigPool var infra *apiconfigv1.Infrastructure var clusterOperator *apiconfigv1.ClusterOperator - var ctrcfg *mcov1.ContainerRuntimeConfig BeforeEach(func() { profileMCP = testutils.NewProfileMCP() profile = testutils.NewPerformanceProfile(performanceProfileName) @@ -1063,52 +1062,6 @@ var _ = Describe("Controller", func() { }) }) - - Context("with ContainerRuntimeConfig enabling crun", func() { - BeforeEach(func() { - ctrcfg = testutils.NewContainerRuntimeConfig(mcov1.ContainerRuntimeDefaultRuntimeCrun, profile.Spec.MachineConfigPoolSelector) - }) - - It("should run high-performance runtimes class with crun as container-runtime", func() { - mc, err := machineconfig.New(profile, &components.MachineConfigOptions{PinningMode: &infra.Status.CPUPartitioning}) - Expect(err).ToNot(HaveOccurred()) - - r := newFakeReconciler(instance, profileMCP, infra, ctrcfg, clusterOperator) - Expect(reconcileTimes(r, request, 2)).To(Equal(reconcile.Result{})) - - By("Verifying MC update") - key := types.NamespacedName{ - Name: machineconfig.GetMachineConfigName(profile.Name), - Namespace: metav1.NamespaceNone, - } - mc = &mcov1.MachineConfig{} - err = r.ManagementClient.Get(context.TODO(), key, mc) - Expect(err).ToNot(HaveOccurred()) - - config := &igntypes.Config{} - err = json.Unmarshal(mc.Spec.Config.Raw, config) - Expect(err).ToNot(HaveOccurred()) - - mode := 420 - containFiles := []igntypes.File{ - { - Node: igntypes.Node{ - Path: "/etc/crio/crio.conf.d/99-runtimes.conf", - Group: &igntypes.NodeGroup{}, - User: &igntypes.NodeUser{}, - }, - FileEmbedded1: igntypes.FileEmbedded1{ - Contents: igntypes.FileContents{ - Verification: igntypes.Verification{}, - Source: "data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMC0zIgoKCgojIFdlIHNob3VsZCBjb3B5IHBhc3RlIHRoZSBkZWZhdWx0IHJ1bnRpbWUgYmVjYXVzZSB0aGlzIHNuaXBwZXQgd2lsbCBvdmVycmlkZSB0aGUgd2hvbGUgcnVudGltZXMgc2VjdGlvbgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLnJ1bmNdCnJ1bnRpbWVfcGF0aCA9ICIiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCgojIFRoZSBDUkktTyB3aWxsIGNoZWNrIHRoZSBhbGxvd2VkX2Fubm90YXRpb25zIHVuZGVyIHRoZSBydW50aW1lIGhhbmRsZXIgYW5kIGFwcGx5IGhpZ2gtcGVyZm9ybWFuY2UgaG9va3Mgd2hlbiBvbmUgb2YKIyBoaWdoLXBlcmZvcm1hbmNlIGFubm90YXRpb25zIHByZXNlbnRzIHVuZGVyIGl0LgojIFdlIHNob3VsZCBwcm92aWRlIHRoZSBydW50aW1lX3BhdGggYmVjYXVzZSB3ZSBuZWVkIHRvIGluZm9ybSB0aGF0IHdlIHdhbnQgdG8gcmUtdXNlIHJ1bmMgYmluYXJ5IGFuZCB3ZQojIGRvIG5vdCBoYXZlIGhpZ2gtcGVyZm9ybWFuY2UgYmluYXJ5IHVuZGVyIHRoZSAkUEFUSCB0aGF0IHdpbGwgcG9pbnQgdG8gaXQuCltjcmlvLnJ1bnRpbWUucnVudGltZXMuaGlnaC1wZXJmb3JtYW5jZV0KcnVudGltZV9wYXRoID0gIi91c3IvYmluL2NydW4iCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL2NydW4iCmFsbG93ZWRfYW5ub3RhdGlvbnMgPSBbImNwdS1sb2FkLWJhbGFuY2luZy5jcmlvLmlvIiwgImNwdS1xdW90YS5jcmlvLmlvIiwgImlycS1sb2FkLWJhbGFuY2luZy5jcmlvLmlvIiwgImNwdS1jLXN0YXRlcy5jcmlvLmlvIiwgImNwdS1mcmVxLWdvdmVybm9yLmNyaW8uaW8iXQo=", - }, - Mode: &mode, - }, - }, - } - Expect(config.Storage.Files).To(ContainElements(containFiles)) - }) - }) }, Entry("OpenShift Platform", newFakeReconciler, "foreground-deletion", false), Entry("HyperShift Platform", newFakeReconcilerForHypershift, "hypershift.openshift.io/foreground-deletion", true), @@ -1223,9 +1176,6 @@ func adaptObjectsForHypershift(instance client.Object, initObjects ...runtime.Ob klog.Fatal(err) } mngClusterObjects = append(mngClusterObjects, cm) - case *mcov1.ContainerRuntimeConfig: - cm := encapsulateContainerRuntimeConfigInConfigMap(obj.(*mcov1.ContainerRuntimeConfig)) - mngClusterObjects = append(mngClusterObjects, cm) case *apiconfigv1.Infrastructure, *apiconfigv1.ClusterOperator: hostedClusterObjects = append(hostedClusterObjects, obj) case *mcov1.MachineConfigPool: @@ -1257,26 +1207,6 @@ func encapsulateProfileInConfigMap(profile *performancev2.PerformanceProfile) *c } } -func encapsulateContainerRuntimeConfigInConfigMap(ctrcfg *mcov1.ContainerRuntimeConfig) *corev1.ConfigMap { - GinkgoHelper() - encodedObj, err := hypershift.EncodeManifest(ctrcfg, scheme.Scheme) - if err != nil { - Fail(fmt.Sprintf("failed to encode manifest %v", err)) - } - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ctrcfg-test-config", - Namespace: hostedControlPlaneNamespace, - Annotations: map[string]string{ - "hypershift.openshift.io/nodePool": "nodepool-test", - }, - }, - Data: map[string]string{ - "config": string(encodedObj), - }, - } -} - func skipForHypershift() { if _, ok := os.LookupEnv("HYPERSHIFT"); ok { Skip("This test is not applicable for hypershift, skipping...") diff --git a/test/e2e/performanceprofile/cluster-setup/container-runtime-crun/enable-crun-master.yaml b/test/e2e/performanceprofile/cluster-setup/container-runtime-crun/enable-crun-master.yaml deleted file mode 100644 index e4678c47a6..0000000000 --- a/test/e2e/performanceprofile/cluster-setup/container-runtime-crun/enable-crun-master.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: ContainerRuntimeConfig -metadata: - name: enable-crun-master -spec: - containerRuntimeConfig: - defaultRuntime: crun - machineConfigPoolSelector: - matchLabels: - pools.operator.machineconfiguration.openshift.io/master: "" -status: {} diff --git a/test/e2e/performanceprofile/functests-render-command/1_render_command/render_test.go b/test/e2e/performanceprofile/functests-render-command/1_render_command/render_test.go index f6c582417b..b538fb2c17 100644 --- a/test/e2e/performanceprofile/functests-render-command/1_render_command/render_test.go +++ b/test/e2e/performanceprofile/functests-render-command/1_render_command/render_test.go @@ -30,7 +30,6 @@ var ( snoLegacyPinnedDir string bootstrapPPDir string extraMCPDir string - containerRuntimeConfigDir string ) var _ = Describe("render command e2e test", func() { @@ -45,7 +44,6 @@ var _ = Describe("render command e2e test", func() { defaultPinnedDir = filepath.Join(workspaceDir, "test", "e2e", "performanceprofile", "cluster-setup", "pinned-cluster", "default") snoLegacyPinnedDir = filepath.Join(workspaceDir, "test", "e2e", "performanceprofile", "cluster-setup", "pinned-cluster", "single-node-legacy") testDataPath = filepath.Join(workspaceDir, "test", "e2e", "performanceprofile", "testdata") - containerRuntimeConfigDir = filepath.Join(workspaceDir, "test", "e2e", "performanceprofile", "cluster-setup", "container-runtime-crun") assetsInDirs = []string{assetsInDir, ppDir} assetsCpuFrequencyInDirs = []string{assetsInDir, ppCpuFrequencyDir} }) @@ -198,25 +196,6 @@ var _ = Describe("render command e2e test", func() { }) }) - Context("With performance profile and matching extra ContainerRuntimeConfig during bootstrap", func() { - It("should render ContainerRuntimeConfig", func() { - - bootstrapPPDirs := []string{bootstrapPPDir, defaultPinnedDir, containerRuntimeConfigDir} - - cmdline := []string{ - filepath.Join(binPath, "cluster-node-tuning-operator"), - "render", - "--asset-input-dir", strings.Join(bootstrapPPDirs, ","), - "--asset-output-dir", assetsOutDir, - } - fmt.Fprintf(GinkgoWriter, "running: %v\n", cmdline) - - cmd := exec.Command(cmdline[0], cmdline[1:]...) - runAndCompare(cmd, path.Join(bootstrapExpectedDir, "extra-ctrcfg")) - - }) - }) - AfterEach(func() { cleanArtifacts() }) diff --git a/test/e2e/performanceprofile/functests/2_performance_update/updating_profile.go b/test/e2e/performanceprofile/functests/2_performance_update/updating_profile.go index 89e7fd37bb..98c71fb6b2 100644 --- a/test/e2e/performanceprofile/functests/2_performance_update/updating_profile.go +++ b/test/e2e/performanceprofile/functests/2_performance_update/updating_profile.go @@ -29,6 +29,7 @@ import ( profilecomponent "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile" manifestsutil "github.com/openshift/cluster-node-tuning-operator/pkg/util" testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/runtime" testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/discovery" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" @@ -37,8 +38,10 @@ import ( "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/mcps" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodepools" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/pods" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profiles" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profilesupdate" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/tuned" hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" ) @@ -1154,64 +1157,76 @@ var _ = Describe("[rfe_id:28761][performance] Updating parameters in performance var ctrcfg *machineconfigv1.ContainerRuntimeConfig const ContainerRuntimeConfigName = "ctrcfg-test" mcp := &machineconfigv1.MachineConfigPool{} + var testpodTemplate *corev1.Pod BeforeAll(func() { key := types.NamespacedName{ Name: poolName, } - Expect(testclient.ControlPlaneClient.Get(context.TODO(), key, mcp)).ToNot(HaveOccurred(), "cannot get MCP %q", poolName) + Expect(testclient.Client.Get(context.TODO(), key, mcp)).ToNot(HaveOccurred(), "cannot get MCP %q", poolName) By("checking if ContainerRuntimeConfig object already exists") ctrcfg, err = getContainerRuntimeConfigFrom(context.TODO(), profile, mcp) Expect(err).ToNot(HaveOccurred(), "failed to get ContainerRuntimeConfig from profile %q mcp %q", profile.Name, mcp.Name) + if ctrcfg != nil { + Skip(fmt.Sprintf("ContainerRuntimeConfig %s exist in the cluster and not expected", ctrcfg.Name)) + } + testpodTemplate = pods.GetTestPod() + testpodTemplate.Namespace = testutils.NamespaceTesting + runtimeClass := components.GetComponentName(profile.Name, components.ComponentNamePrefix) + testpodTemplate.Spec.RuntimeClassName = &runtimeClass }) - - When("is not given", func() { - It("should run high-performance runtimes class with runc as container-runtime", func() { - if ctrcfg != nil { - Skip("runc is not the default runtime configuration") - } - cmd := []string{"cat", "/rootfs/etc/crio/crio.conf.d/99-runtimes.conf"} - for i := 0; i < len(workerRTNodes); i++ { - output, err := nodes.ExecCommand(context.TODO(), &workerRTNodes[i], cmd) - Expect(err).ToNot(HaveOccurred(), "cannot get 99-runtimes.conf from %q", workerRTNodes[i].Name) - out := testutils.ToString(output) - By(fmt.Sprintf("checking node: %q", workerRTNodes[i].Name)) - Expect(out).To(ContainSubstring("/bin/runc")) - Expect(out).To(ContainSubstring("/run/runc")) - } - }) - }) - When("updates the default runtime to crun", func() { - It("should run high-performance runtimes class with crun as container-runtime", func() { - if ctrcfg == nil { - testlog.Infof("ContainerRuntimeConfig not exist") + DescribeTable("verifies container runtime behavior", + func(withCTRCfg bool) { + var expectedRuntime string + if withCTRCfg { ctrcfg = newContainerRuntimeConfig(ContainerRuntimeConfigName, profile, mcp) By(fmt.Sprintf("creating ContainerRuntimeConfig %q", ctrcfg.Name)) Expect(testclient.Client.Create(context.TODO(), ctrcfg)).ToNot(HaveOccurred(), "failed to create ctrcfg %#v", ctrcfg) DeferCleanup(func() { - Expect(testclient.Client.Delete(context.TODO(), ctrcfg)).ToNot(HaveOccurred(), "failed to delete ctfcfg %#v", ctrcfg) - By(fmt.Sprintf("waiting for mcp %q transition to UPDATING state", poolName)) + Expect(testclient.Client.Delete(context.TODO(), ctrcfg)).ToNot(HaveOccurred(), "failed to delete ctrcfg %#v", ctrcfg) + By(fmt.Sprintf("waiting for MCP %q transition to UPDATING state", poolName)) mcps.WaitForConditionFunc(poolName, machineconfigv1.MachineConfigPoolUpdating, corev1.ConditionTrue, getMCPConditionStatus) - By(fmt.Sprintf("waiting for mcp %q transition to UPDATED state", poolName)) + By(fmt.Sprintf("waiting for MCP %q transition to UPDATED state", poolName)) mcps.WaitForConditionFunc(poolName, machineconfigv1.MachineConfigPoolUpdated, corev1.ConditionTrue, getMCPConditionStatus) }) - By(fmt.Sprintf("waiting for mcp %q transition to UPDATING state", poolName)) + By(fmt.Sprintf("waiting for MCP %q transition to UPDATING state", poolName)) mcps.WaitForConditionFunc(poolName, machineconfigv1.MachineConfigPoolUpdating, corev1.ConditionTrue, getMCPConditionStatus) - By(fmt.Sprintf("waiting for mcp %q transition to UPDATED state", poolName)) + By(fmt.Sprintf("waiting for MCP %q transition to UPDATED state", poolName)) mcps.WaitForConditionFunc(poolName, machineconfigv1.MachineConfigPoolUpdated, corev1.ConditionTrue, getMCPConditionStatus) } - Expect(ctrcfg.Spec.ContainerRuntimeConfig.DefaultRuntime == machineconfigv1.ContainerRuntimeDefaultRuntimeCrun).To(BeTrue()) - cmd := []string{"cat", "/rootfs/etc/crio/crio.conf.d/99-runtimes.conf"} + for i := 0; i < len(workerRTNodes); i++ { - out, err := nodes.ExecCommand(context.TODO(), &workerRTNodes[i], cmd) - Expect(err).ToNot(HaveOccurred(), "cannot get 99-runtimes.conf from %q", workerRTNodes[i].Name) - By(fmt.Sprintf("checking node: %q", workerRTNodes[i].Name)) - Expect(out).To(ContainSubstring("/usr/bin/crun")) - Expect(out).To(ContainSubstring("/run/crun")) + By("Determing the default container runtime used in the node") + tunedPod, err := tuned.GetPod(context.TODO(), &workerRTNodes[i]) + Expect(err).ToNot(HaveOccurred()) + expectedRuntime, err = runtime.GetContainerRuntimeTypeFor(context.TODO(), testclient.Client, tunedPod) + Expect(err).ToNot(HaveOccurred()) + testlog.Infof("Container runtime used for the node: %s", expectedRuntime) + + By("verifying pod using high-performance runtime class handled by the default container runtime aswell") + Expect(err).ToNot(HaveOccurred()) + testpod := testpodTemplate.DeepCopy() + testpod.Spec.NodeName = workerRTNodes[i].Name + testpod.Spec.NodeSelector = map[string]string{testutils.LabelHostname: workerRTNodes[i].Name} + By(fmt.Sprintf("creating a test pod using high-performance runtime class on node %s", workerRTNodes[i].Name)) + Expect(testclient.Client.Create(context.TODO(), testpod)).ToNot(HaveOccurred()) + DeferCleanup(func() { + By(fmt.Sprintf("deleting the test pod from node %s", workerRTNodes[i].Name)) + Expect(testclient.Client.Delete(context.TODO(), testpod)).ToNot(HaveOccurred()) + Expect(pods.WaitForDeletion(context.TODO(), testpod, pods.DefaultDeletionTimeout*time.Second)).ToNot(HaveOccurred()) + }) + testpod, err = pods.WaitForCondition(context.TODO(), client.ObjectKeyFromObject(testpod), corev1.PodReady, corev1.ConditionTrue, 10*time.Minute) + Expect(err).ToNot(HaveOccurred()) + runtimeType, err := runtime.GetContainerRuntimeTypeFor(context.TODO(), testclient.Client, testpod) + Expect(err).ToNot(HaveOccurred()) + testlog.Infof("Container runtime used for the test pod: %s", runtimeType) + Expect(runtimeType).To(Equal(expectedRuntime)) } - }) - }) + }, + Entry("test without ContainerRuntimeConfig", false), + Entry("create and test with ContainerRuntimeConfig", true), + ) }) }) @@ -1317,7 +1332,7 @@ func newContainerRuntimeConfig(name string, profile *performancev2.PerformancePr MatchLabels: profilecomponent.GetMachineConfigPoolSelector(profile, profileMCP), }, ContainerRuntimeConfig: &machineconfigv1.ContainerRuntimeConfiguration{ - DefaultRuntime: machineconfigv1.ContainerRuntimeDefaultRuntimeCrun, + DefaultRuntime: machineconfigv1.ContainerRuntimeDefaultRuntimeRunc, }, }, } diff --git a/test/e2e/performanceprofile/functests/utils/cgroup/runtime/runtime.go b/test/e2e/performanceprofile/functests/utils/cgroup/runtime/runtime.go index 17f0aa7c26..3d77f0cf0d 100644 --- a/test/e2e/performanceprofile/functests/utils/cgroup/runtime/runtime.go +++ b/test/e2e/performanceprofile/functests/utils/cgroup/runtime/runtime.go @@ -31,7 +31,7 @@ func GetContainerRuntimeTypeFor(ctx context.Context, c client.Client, pod *corev "/rootfs", "/bin/bash", "-c", - fmt.Sprintf("/bin/awk -F '\"' '/runtime_path.*/ { print $2 }' %s", CRIORuntimeConfigFile), + fmt.Sprintf("/bin/ps aux | grep '%s' | grep -oP '(?<=-r\\s)[^\\s]+'", pod.Name), } output, err := nodes.ExecCommand(ctx, node, cmd) if err != nil { diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/01_01-master-cpu-partitioning_workload_pinning_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/01_01-master-cpu-partitioning_workload_pinning_machineconfig.yaml deleted file mode 100644 index 5e899d0404..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/01_01-master-cpu-partitioning_workload_pinning_machineconfig.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - creationTimestamp: null - labels: - machineconfiguration.openshift.io/role: master - name: 01-master-cpu-partitioning -spec: - baseOSExtensionsContainerImage: "" - config: - ignition: - config: - replace: - verification: {} - proxy: {} - security: - tls: {} - timeouts: {} - version: 3.2.0 - passwd: {} - storage: - files: - - contents: - source: data:text/plain;charset=utf-8;base64,CnsKICAibWFuYWdlbWVudCI6IHsKICAgICJjcHVzZXQiOiAiIgogIH0KfQo= - verification: {} - group: {} - mode: 420 - path: /etc/kubernetes/openshift-workload-pinning - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWUud29ya2xvYWRzLm1hbmFnZW1lbnRdCmFjdGl2YXRpb25fYW5ub3RhdGlvbiA9ICJ0YXJnZXQud29ya2xvYWQub3BlbnNoaWZ0LmlvL21hbmFnZW1lbnQiCmFubm90YXRpb25fcHJlZml4ID0gInJlc291cmNlcy53b3JrbG9hZC5vcGVuc2hpZnQuaW8iCnJlc291cmNlcyA9IHsgImNwdXNoYXJlcyIgPSAwLCAiY3B1c2V0IiA9ICIiIH0K - verification: {} - group: {} - mode: 420 - path: /etc/crio/crio.conf.d/01-workload-pinning-default.conf - user: {} - systemd: {} - extensions: null - fips: false - kernelArguments: null - kernelType: "" - osImageURL: "" diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/01_01-worker-cpu-partitioning_workload_pinning_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/01_01-worker-cpu-partitioning_workload_pinning_machineconfig.yaml deleted file mode 100644 index 5ac333fb02..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/01_01-worker-cpu-partitioning_workload_pinning_machineconfig.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - creationTimestamp: null - labels: - machineconfiguration.openshift.io/role: worker - name: 01-worker-cpu-partitioning -spec: - baseOSExtensionsContainerImage: "" - config: - ignition: - config: - replace: - verification: {} - proxy: {} - security: - tls: {} - timeouts: {} - version: 3.2.0 - passwd: {} - storage: - files: - - contents: - source: data:text/plain;charset=utf-8;base64,CnsKICAibWFuYWdlbWVudCI6IHsKICAgICJjcHVzZXQiOiAiIgogIH0KfQo= - verification: {} - group: {} - mode: 420 - path: /etc/kubernetes/openshift-workload-pinning - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWUud29ya2xvYWRzLm1hbmFnZW1lbnRdCmFjdGl2YXRpb25fYW5ub3RhdGlvbiA9ICJ0YXJnZXQud29ya2xvYWQub3BlbnNoaWZ0LmlvL21hbmFnZW1lbnQiCmFubm90YXRpb25fcHJlZml4ID0gInJlc291cmNlcy53b3JrbG9hZC5vcGVuc2hpZnQuaW8iCnJlc291cmNlcyA9IHsgImNwdXNoYXJlcyIgPSAwLCAiY3B1c2V0IiA9ICIiIH0K - verification: {} - group: {} - mode: 420 - path: /etc/crio/crio.conf.d/01-workload-pinning-default.conf - user: {} - systemd: {} - extensions: null - fips: false - kernelArguments: null - kernelType: "" - osImageURL: "" diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_kubeletconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_kubeletconfig.yaml deleted file mode 100644 index bbe820ff1c..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_kubeletconfig.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: KubeletConfig -metadata: - creationTimestamp: null - labels: - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-master - name: performance-openshift-bootstrap-master -spec: - kubeletConfig: - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: {} - webhook: - cacheTTL: 0s - x509: {} - authorization: - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - containerRuntimeEndpoint: "" - cpuManagerPolicy: static - cpuManagerReconcilePeriod: 5s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - memory: 500Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - reservedSystemCPUs: 2-7 - runtimeRequestTimeout: 0s - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - memory: 500Mi - topologyManagerPolicy: best-effort - volumeStatsAggPeriod: 0s - machineConfigPoolSelector: - matchLabels: - pools.operator.machineconfiguration.openshift.io/master: "" -status: - conditions: null diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml deleted file mode 100644 index 4978822d0e..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml +++ /dev/null @@ -1,226 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - creationTimestamp: null - labels: - machineconfiguration.openshift.io/role: master - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-master - name: 50-performance-openshift-bootstrap-master -spec: - baseOSExtensionsContainerImage: "" - config: - ignition: - config: - replace: - verification: {} - proxy: {} - security: - tls: {} - timeouts: {} - version: 3.2.0 - passwd: {} - storage: - files: - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaAoKc2V0IC1ldW8gcGlwZWZhaWwKCm5vZGVzX3BhdGg9Ii9zeXMvZGV2aWNlcy9zeXN0ZW0vbm9kZSIKaHVnZXBhZ2VzX2ZpbGU9IiR7bm9kZXNfcGF0aH0vbm9kZSR7TlVNQV9OT0RFfS9odWdlcGFnZXMvaHVnZXBhZ2VzLSR7SFVHRVBBR0VTX1NJWkV9a0IvbnJfaHVnZXBhZ2VzIgoKaWYgWyAhIC1mICIke2h1Z2VwYWdlc19maWxlfSIgXTsgdGhlbgogIGVjaG8gIkVSUk9SOiAke2h1Z2VwYWdlc19maWxlfSBkb2VzIG5vdCBleGlzdCIKICBleGl0IDEKZmkKCnRpbWVvdXQ9NjAKc2FtcGxlPTEKY3VycmVudF90aW1lPTAKd2hpbGUgWyAiJChjYXQgIiR7aHVnZXBhZ2VzX2ZpbGV9IikiIC1uZSAiJHtIVUdFUEFHRVNfQ09VTlR9IiBdOyBkbwogIGVjaG8gIiR7SFVHRVBBR0VTX0NPVU5UfSIgPiIke2h1Z2VwYWdlc19maWxlfSIKCiAgY3VycmVudF90aW1lPSQoKGN1cnJlbnRfdGltZSArIHNhbXBsZSkpCiAgaWYgWyAkY3VycmVudF90aW1lIC1ndCAkdGltZW91dCBdOyB0aGVuCiAgICBlY2hvICJFUlJPUjogJHtodWdlcGFnZXNfZmlsZX0gZG9lcyBub3QgaGF2ZSB0aGUgZXhwZWN0ZWQgbnVtYmVyIG9mIGh1Z2VwYWdlcyAke0hVR0VQQUdFU19DT1VOVH0iCiAgICBleGl0IDEKICBmaQoKICBzbGVlcCAkc2FtcGxlCmRvbmUK - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/hugepages-allocation.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZnVuY3Rpb24gc2V0X3F1ZXVlX3Jwc19tYXNrKCkgewojIHJlcGxhY2UgeDJkIHdpdGggaHlwaGVuICgtKSB3aGljaCBpcyBhbiBlc2NhcGVkIGNoYXJhY3RlcgojIHRoYXQgd2FzIGFkZGVkIGJ5IHN5c3RlbWQtZXNjYXBlIGluIG9yZGVyIHRvIGVzY2FwZSB0aGUgc3lzdGVtZCB1bml0IG5hbWUgdGhhdCBpbnZva2VzIHRoaXMgc2NyaXB0CnBhdGg9JHtwYXRoL3gyZC8tfQojIHNldCBycHMgYWZmaW5pdHkgZm9yIHRoZSBxdWV1ZQplY2hvICIke21hc2t9IiAgMj4gL2Rldi9udWxsID4gIi9zeXMvJHtwYXRofS9ycHNfY3B1cyIKIyB3ZSByZXR1cm4gMCBiZWNhdXNlIHRoZSAnZWNobycgY29tbWFuZCBtaWdodCBmYWlsIGlmIHRoZSBkZXZpY2UgcGF0aCB0byB3aGljaCB0aGUgcXVldWUgYmVsb25ncyBoYXMgY2hhbmdlZC4KIyB0aGlzIGNhbiBoYXBwZW4gaW4gY2FzZSBvZiBTUkktT1YgZGV2aWNlcyByZW5hbWluZy4KcmV0dXJuIDAKfQoKZnVuY3Rpb24gc2V0X25ldF9kZXZfcnBzX21hc2soKSB7CiAgIyBpbiBjYXNlIG9mIGRldmljZSB3ZSB3YW50IHRvIGl0ZXJhdGUgdGhyb3VnaCBhbGwgcXVldWVzCmZvciBpIGluIC9zeXMvIiR7cGF0aH0iL3F1ZXVlcy9yeC0qOyBkbwogIGVjaG8gIiR7bWFza30iIDI+IC9kZXYvbnVsbCA+ICIke2l9L3Jwc19jcHVzIgpkb25lCiMgd2UgcmV0dXJuIDAgYmVjYXVzZSB0aGUgJ2VjaG8nIGNvbW1hbmQgbWlnaHQgZmFpbCBpZiB0aGUgZGV2aWNlIHBhdGggdG8gd2hpY2ggdGhlIHF1ZXVlIGJlbG9uZ3MgaGFzIGNoYW5nZWQuCiMgdGhpcyBjYW4gaGFwcGVuIGluIGNhc2Ugb2YgU1JJLU9WIGRldmljZXMgcmVuYW1pbmcuCnJldHVybiAwCiB9CgpwYXRoPSR7MX0KWyAtbiAiJHtwYXRofSIgXSB8fCB7IGVjaG8gIlRoZSBkZXZpY2UgcGF0aCBhcmd1bWVudCBpcyBtaXNzaW5nIiA+JjIgOyBleGl0IDE7IH0KCm1hc2s9JHsyfQpbIC1uICIke21hc2t9IiBdIHx8IHsgZWNobyAiVGhlIG1hc2sgYXJndW1lbnQgaXMgbWlzc2luZyIgPiYyIDsgZXhpdCAxOyB9CgppZiBbWyAiJHtwYXRofSIgPX4gInF1ZXVlcyIgXV07IHRoZW4KIHNldF9xdWV1ZV9ycHNfbWFzawplbHNlCiBzZXRfbmV0X2Rldl9ycHNfbWFzawpmaQo= - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/set-rps-mask.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9iYXNoCgpzZXQgLWV1byBwaXBlZmFpbAoKZm9yIGNwdSBpbiAke09GRkxJTkVfQ1BVUy8vLC8gfTsKICBkbwogICAgb25saW5lX2NwdV9maWxlPSIvc3lzL2RldmljZXMvc3lzdGVtL2NwdS9jcHUkY3B1L29ubGluZSIKICAgIGlmIFsgISAtZiAiJHtvbmxpbmVfY3B1X2ZpbGV9IiBdOyB0aGVuCiAgICAgIGVjaG8gIkVSUk9SOiAke29ubGluZV9jcHVfZmlsZX0gZG9lcyBub3QgZXhpc3QsIGFib3J0IHNjcmlwdCBleGVjdXRpb24iCiAgICAgIGV4aXQgMQogICAgZmkKICBkb25lCgplY2hvICJBbGwgY3B1cyBvZmZsaW5lZCBleGlzdHMsIHNldCB0aGVtIG9mZmxpbmUiCgpmb3IgY3B1IGluICR7T0ZGTElORV9DUFVTLy8sLyB9OwogIGRvCiAgICBvbmxpbmVfY3B1X2ZpbGU9Ii9zeXMvZGV2aWNlcy9zeXN0ZW0vY3B1L2NwdSRjcHUvb25saW5lIgogICAgZWNobyAwID4gIiR7b25saW5lX2NwdV9maWxlfSIKICAgIGVjaG8gIm9mZmxpbmUgY3B1IG51bSAkY3B1IgogIGRvbmUKCg== - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/set-cpus-offline.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/clear-irqbalance-banned-cpus.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgojIFdlIHNob3VsZCBjb3B5IHBhc3RlIHRoZSBkZWZhdWx0IHJ1bnRpbWUgYmVjYXVzZSB0aGlzIHNuaXBwZXQgd2lsbCBvdmVycmlkZSB0aGUgd2hvbGUgcnVudGltZXMgc2VjdGlvbgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLnJ1bmNdCnJ1bnRpbWVfcGF0aCA9ICIiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCgojIFRoZSBDUkktTyB3aWxsIGNoZWNrIHRoZSBhbGxvd2VkX2Fubm90YXRpb25zIHVuZGVyIHRoZSBydW50aW1lIGhhbmRsZXIgYW5kIGFwcGx5IGhpZ2gtcGVyZm9ybWFuY2UgaG9va3Mgd2hlbiBvbmUgb2YKIyBoaWdoLXBlcmZvcm1hbmNlIGFubm90YXRpb25zIHByZXNlbnRzIHVuZGVyIGl0LgojIFdlIHNob3VsZCBwcm92aWRlIHRoZSBydW50aW1lX3BhdGggYmVjYXVzZSB3ZSBuZWVkIHRvIGluZm9ybSB0aGF0IHdlIHdhbnQgdG8gcmUtdXNlIHJ1bmMgYmluYXJ5IGFuZCB3ZQojIGRvIG5vdCBoYXZlIGhpZ2gtcGVyZm9ybWFuY2UgYmluYXJ5IHVuZGVyIHRoZSAkUEFUSCB0aGF0IHdpbGwgcG9pbnQgdG8gaXQuCltjcmlvLnJ1bnRpbWUucnVudGltZXMuaGlnaC1wZXJmb3JtYW5jZV0KcnVudGltZV9wYXRoID0gIi91c3IvYmluL2NydW4iCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL2NydW4iCmFsbG93ZWRfYW5ub3RhdGlvbnMgPSBbImNwdS1sb2FkLWJhbGFuY2luZy5jcmlvLmlvIiwgImNwdS1xdW90YS5jcmlvLmlvIiwgImlycS1sb2FkLWJhbGFuY2luZy5jcmlvLmlvIiwgImNwdS1jLXN0YXRlcy5jcmlvLmlvIiwgImNwdS1mcmVxLWdvdmVybm9yLmNyaW8uaW8iXQo= - verification: {} - group: {} - mode: 420 - path: /etc/crio/crio.conf.d/99-runtimes.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyBBcHBseSB0aGUgUlBTIG1hc2sgb24gdGhlIHZpcnR1YWwgaW50ZXJmYWNlcyBvZiB0aGUgaG9zdCBieSBkZWZhdWx0LCBiZWNhc3VlCiMgZnJvbSB0aGUgY29udGFpbmVyIHBlcnNwZWN0aXZlIHRoZSBSUFMgbWFzayB0aGUgd2lsbCBiZSBjb25zdWx0ZWQsIGlzIHRoZSBvbmUgb24gdGhlIFJYIHNpZGUgb2YgdGhlIHZldGggaW4gdGhlIGhvc3QuCiMgQ29uc2lkZXIgdGhlIGZvbGxvd2luZyBkaWFncmFtOgojIFBvZCBBIDx2ZXRoMSAtIHZldGgyPiBob3N0IDx2ZXRoMyAtIHZldGg0PiBQb2QgQgojICB2ZXRoMidzIFJQUyBhZmZpbml0eSBpcyB0aGUgb25lIGRldGVybWluaW5nIHRoZSBDUFVzIHRoYXQgYXJlIGhhbmRsaW5nIHRoZSBwYWNrZXQgcHJvY2Vzc2luZyB3aGVuIHNlbmRpbmcgZGF0YSBmcm9tIFBvZCBBIHRvIHBvZCBCLgojIEFkZGl0aW9uYWwgY29tbW9uIHNjZW5hcmlvczoKIyAxLiBQb2QgQSA9IHNlbmRlciwgaG9zdCA9IHJlY2VpdmVyCiMgIFRoZSBSUFMgYWZmaW5pdHkgb2YgdGhlIGhvc3Qgc2lkZSBzaG91bGQgYmUgY29uc3VsdGVkIChiZWNhdXNlIGl04oCZcyB0aGUgcmVjZWl2ZXIpIGFuZCBpdCBzaG91bGQgYmUgc2V0IHRvIGNwdXMgbm90IHNlbnNpdGl2ZSB0byBwcmVlbXB0aW9uIChyZXNlcnZlZCBwb29sKS4KIyAyLiBQb2QgQSA9IHJlY2VpdmVyLCBob3N0ID0gc2VuZGVyCiMgIEluIGNhc2Ugb2Ygbm8gUlBTIG1hc2sgb24gdGhlIHJlY2VpdmVyIHNpZGUsIHRoZSBzZW5kZXIgbmVlZHMgdG8gcGF5IHRoZSBwcmljZSBhbmQgZG8gYWxsIHRoZSBwcm9jZXNzaW5nIG9uIGl0cyBjb3Jlcy4KbmV0LmNvcmUucnBzX2RlZmF1bHRfbWFzayA9IDAwMDAwMGZjCg== - verification: {} - group: {} - mode: 420 - path: /etc/sysctl.d/99-default-rps-mask.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,U1VCU1lTVEVNPT0icXVldWVzIiwgQUNUSU9OPT0iYWRkIiwgRU5We0RFVlBBVEh9PT0iL2RldmljZXMvcGNpKi9xdWV1ZXMvcngqIiwgVEFHKz0ic3lzdGVtZCIsIFBST0dSQU09Ii9iaW4vc3lzdGVtZC1lc2NhcGUgLS1wYXRoIC0tdGVtcGxhdGU9dXBkYXRlLXJwc0Auc2VydmljZSAkZW52e0RFVlBBVEh9IiwgRU5We1NZU1RFTURfV0FOVFN9PSIlYyIKCiMgU1ItSU9WIGRldmljZXMgYXJlIG1vdmVkIChyZW5hbWVkKSwgaGVuY2Ugd2Ugd2FudCB0byBjYXRjaCB0aGlzIGV2ZW50IGFzIHdlbGwKU1VCU1lTVEVNPT0ibmV0IiwgQUNUSU9OPT0ibW92ZSIsIEVOVntERVZQQVRIfSE9Ii9kZXZpY2VzL3ZpcnR1YWwvbmV0LyoiLCBUQUcrPSJzeXN0ZW1kIiwgUFJPR1JBTT0iL2Jpbi9zeXN0ZW1kLWVzY2FwZSAtLXBhdGggLS10ZW1wbGF0ZT11cGRhdGUtcnBzQC5zZXJ2aWNlICRlbnZ7REVWUEFUSH0iLCBFTlZ7U1lTVEVNRF9XQU5UU309IiVjIgo= - verification: {} - group: {} - mode: 420 - path: /etc/udev/rules.d/99-netdev-physical-rps.rules - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWUud29ya2xvYWRzLm1hbmFnZW1lbnRdCmFjdGl2YXRpb25fYW5ub3RhdGlvbiA9ICJ0YXJnZXQud29ya2xvYWQub3BlbnNoaWZ0LmlvL21hbmFnZW1lbnQiCmFubm90YXRpb25fcHJlZml4ID0gInJlc291cmNlcy53b3JrbG9hZC5vcGVuc2hpZnQuaW8iCnJlc291cmNlcyA9IHsgImNwdXNoYXJlcyIgPSAwLCAiY3B1c2V0IiA9ICIyLTciIH0K - verification: {} - group: {} - mode: 420 - path: /etc/crio/crio.conf.d/99-workload-pinning.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,CnsKICAibWFuYWdlbWVudCI6IHsKICAgICJjcHVzZXQiOiAiMi03IgogIH0KfQo= - verification: {} - group: {} - mode: 420 - path: /etc/kubernetes/openshift-workload-pinning - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2gKCiMgY3B1c2V0LWNvbmZpZ3VyZS5zaCBjb25maWd1cmVzIHRocmVlIGNwdXNldHMgaW4gcHJlcGFyYXRpb24gZm9yIGFsbG93aW5nIGNvbnRhaW5lcnMgdG8gaGF2ZSBjcHUgbG9hZCBiYWxhbmNpbmcgZGlzYWJsZWQuCiMgVG8gY29uZmlndXJlIGEgY3B1c2V0IHRvIGhhdmUgbG9hZCBiYWxhbmNlIGRpc2FibGVkIChvbiBjZ3JvdXAgdjEpLCBhIGNwdXNldCBjZ3JvdXAgbXVzdCBoYXZlIGBjcHVzZXQuc2NoZWRfbG9hZF9iYWxhbmNlYAojIHNldCB0byAwIChkaXNhYmxlKSwgYW5kIGFueSBjcHVzZXQgdGhhdCBjb250YWlucyB0aGUgc2FtZSBzZXQgYXMgYGNwdXNldC5jcHVzYCBtdXN0IGFsc28gaGF2ZSBgY3B1c2V0LnNjaGVkX2xvYWRfYmFsYW5jZWAgc2V0IHRvIGRpc2FibGVkLgoKc2V0IC1ldW8gcGlwZWZhaWwKCmlmIHRlc3QgIiQoc3RhdCAtZiAtYyVUIC9zeXMvZnMvY2dyb3VwKSIgPSAiY2dyb3VwMmZzIjsgdGhlbgoJZWNobyAiTm9kZSBpcyB1c2luZyBjZ3JvdXAgdjIsIG5vIGNvbmZpZ3VyYXRpb24gbmVlZGVkIgoJZXhpdCAwCmZpCgpyb290PS9zeXMvZnMvY2dyb3VwL2NwdXNldApzeXN0ZW09IiRyb290Ii9zeXN0ZW0uc2xpY2UKbWFjaGluZT0iJHJvb3QiL21hY2hpbmUuc2xpY2UKCm92c3NsaWNlPSIke3Jvb3R9L292cy5zbGljZSIKb3Zzc2xpY2Vfc3lzdGVtZD0iL3N5cy9mcy9jZ3JvdXAvcGlkcy9vdnMuc2xpY2UiCgojIEFzIHN1Y2gsIHRoZSByb290IGNncm91cCBuZWVkcyB0byBoYXZlIGNwdXNldC5zY2hlZF9sb2FkX2JhbGFuY2U9MC4gCmVjaG8gMCA+ICIkcm9vdCIvY3B1c2V0LnNjaGVkX2xvYWRfYmFsYW5jZQoKIyBIb3dldmVyLCB0aGlzIHdvdWxkIHByZXNlbnQgYSBwcm9ibGVtIGZvciBzeXN0ZW0gZGFlbW9ucywgd2hpY2ggc2hvdWxkIGhhdmUgbG9hZCBiYWxhbmNpbmcgZW5hYmxlZC4KIyBBcyBzdWNoLCBhIHNlY29uZCBjcHVzZXQgbXVzdCBiZSBjcmVhdGVkLCBoZXJlIGR1YmJlZCBgc3lzdGVtYCwgd2hpY2ggd2lsbCB0YWtlIGFsbCBzeXN0ZW0gZGFlbW9ucy4KIyBTaW5jZSBzeXN0ZW1kIHN0YXJ0cyBpdHMgY2hpbGRyZW4gd2l0aCB0aGUgY3B1c2V0IGl0IGlzIGluLCBtb3Zpbmcgc3lzdGVtZCB3aWxsIGVuc3VyZSBhbGwgcHJvY2Vzc2VzIHN5c3RlbWQgYmVnaW5zIHdpbGwgYmUgaW4gdGhlIGNvcnJlY3QgY2dyb3VwLgpta2RpciAtcCAiJHN5c3RlbSIKIyBjcHVzZXQubWVtcyBtdXN0IGJlIGluaXRpYWxpemVkIG9yIHByb2Nlc3NlcyB3aWxsIGZhaWwgdG8gYmUgbW92ZWQgaW50byBpdC4KY2F0ICIkcm9vdC9jcHVzZXQubWVtcyIgPiAiJHN5c3RlbSIvY3B1c2V0Lm1lbXMKIyBSZXRyaWV2ZSB0aGUgY3B1c2V0IG9mIHN5c3RlbWQsIGFuZCB3cml0ZSBpdCB0byBjcHVzZXQuY3B1cyBvZiB0aGUgc3lzdGVtIGNncm91cC4KcmVzZXJ2ZWRfc2V0PSQodGFza3NldCAtY3AgIDEgIHwgYXdrICdORnsgcHJpbnQgJE5GIH0nKQplY2hvICIkcmVzZXJ2ZWRfc2V0IiA+ICIkc3lzdGVtIi9jcHVzZXQuY3B1cwoKIyBBbmQgbW92ZSB0aGUgc3lzdGVtIHByb2Nlc3NlcyBpbnRvIGl0LgojIE5vdGUsIHNvbWUga2VybmVsIHRocmVhZHMgd2lsbCBmYWlsIHRvIGJlIG1vdmVkIHdpdGggIkludmFsaWQgQXJndW1lbnQiLiBUaGlzIHNob3VsZCBiZSBpZ25vcmVkLgpmb3IgcHJvY2VzcyBpbiAkKGNhdCAiJHJvb3QiL2Nncm91cC5wcm9jcyB8IHNvcnQgLXIpOyBkbwoJZWNobyAkcHJvY2VzcyA+ICIkc3lzdGVtIi9jZ3JvdXAucHJvY3MgMj4mMSB8IGdyZXAgLXYgIkludmFsaWQgQXJndW1lbnQiIHx8IHRydWU7CmRvbmUKCiMgRmluYWxseSwgYSB0aGUgYG1hY2hpbmUuc2xpY2VgIGNncm91cCBtdXN0IGJlIHByZWNvbmZpZ3VyZWQuIFBvZG1hbiB3aWxsIGNyZWF0ZSBjb250YWluZXJzIGFuZCBtb3ZlIHRoZW0gaW50byB0aGUgYG1hY2hpbmUuc2xpY2VgLCBidXQgdGhlcmUncwojIG5vIHdheSB0byB0ZWxsIHBvZG1hbiB0byB1cGRhdGUgbWFjaGluZS5zbGljZSB0byBub3QgaGF2ZSB0aGUgZnVsbCBzZXQgb2YgY3B1cy4gSW5zdGVhZCBvZiBkaXNhYmxpbmcgbG9hZCBiYWxhbmNpbmcgaW4gaXQsIHdlIGNhbiBwcmUtY3JlYXRlIGl0LgojIHdpdGggdGhlIHJlc2VydmVkIENQVXMgc2V0IGFoZWFkIG9mIHRpbWUsIHNvIHdoZW4gaXNvbGF0ZWQgcHJvY2Vzc2VzIGJlZ2luLCB0aGUgY2dyb3VwIGRvZXMgbm90IGhhdmUgYW4gb3ZlcmxhcHBpbmcgY3B1c2V0IGJldHdlZW4gbWFjaGluZS5zbGljZSBhbmQgaXNvbGF0ZWQgY29udGFpbmVycy4KbWtkaXIgLXAgIiRtYWNoaW5lIgoKIyBJdCdzIHVubGlrZWx5LCBidXQgcG9zc2libGUsIHRoYXQgdGhpcyBjcHVzZXQgYWxyZWFkeSBleGlzdGVkLiBJdGVyYXRlIGp1c3QgaW4gY2FzZS4KZm9yIGZpbGUgaW4gJChmaW5kICIkbWFjaGluZSIgLW5hbWUgY3B1c2V0LmNwdXMgfCBzb3J0IC1yKTsgZG8gZWNobyAiJHJlc2VydmVkX3NldCIgPiAiJGZpbGUiOyBkb25lCgojIE9WUyBpcyBydW5uaW5nIGluIGl0cyBvd24gc2xpY2UgdGhhdCBzcGFucyBhbGwgY3B1cy4gVGhlIHJlYWwgYWZmaW5pdHkgaXMgbWFuYWdlZCBieSBPVk4tSyBvdm5rdWJlLW5vZGUgZGFlbW9uc2V0CiMgTWFrZSBzdXJlIHRoaXMgc2xpY2Ugd2lsbCBub3QgZW5hYmxlIGNwdSBiYWxhbmNpbmcgZm9yIG90aGVyIHNsaWNlIGNvbmZpZ3VyZWQgYnkgdGhpcyBzY3JpcHQuCiMgVGhpcyBtaWdodCBzZWVtIGNvdW50ZXItaW50dWl0aXZlLCBidXQgdGhpcyB3aWxsIGFjdHVhbGx5IE5PVCBkaXNhYmxlIGNwdSBiYWxhbmNpbmcgZm9yIE9WUyBpdHNlbGYuCiMgLSBPVlMgaGFzIGFjY2VzcyB0byByZXNlcnZlZCBjcHVzLCBidXQgdGhvc2UgaGF2ZSBiYWxhbmNpbmcgZW5hYmxlZCB2aWEgdGhlIGBzeXN0ZW1gIGNncm91cCBjcmVhdGVkIGFib3ZlCiMgLSBPVlMgaGFzIGFjY2VzcyB0byBpc29sYXRlZCBjcHVzIHRoYXQgYXJlIGN1cnJlbnRseSBub3QgYXNzaWduZWQgdG8gcGlubmVkIHBvZHMuIFRob3NlIGhhdmUgYmFsYW5jaW5nIGVuYWJsZWQgYnkgdGhlCiMgICBwb2RzIHJ1bm5pbmcgdGhlcmUgKGJ1cnN0YWJsZSBhbmQgYmVzdC1lZmZvcnQgcG9kcyBoYXZlIGJhbGFuY2luZyBlbmFibGVkIGluIHRoZSBjb250YWluZXIgY2dyb3VwIGFuZCBhY2Nlc3MgdG8gYWxsCiMgICB1bnBpbm5lZCBjcHVzKS4KCiMgc3lzdGVtZCBkb2VzIG5vdCBtYW5hZ2UgdGhlIGNwdXNldCBjZ3JvdXAgY29udHJvbGxlciwgc28gbW92ZSBldmVyeXRoaW5nIGZyb20gdGhlIG1hbmFnZWQgcGlkcyBjb250cm9sbGVyJ3Mgb3ZzLnNsaWNlCiMgdG8gdGhlIGNwdXNldCBjb250cm9sbGVyLgoKIyBDcmVhdGUgdGhlIG92cy5zbGljZQpta2RpciAtcCAiJG92c3NsaWNlIgplY2hvIDAgPiAiJG92c3NsaWNlIi9jcHVzZXQuc2NoZWRfbG9hZF9iYWxhbmNlCmNhdCAiJHJvb3QiL2NwdXNldC5jcHVzID4gIiRvdnNzbGljZSIvY3B1c2V0LmNwdXMKY2F0ICIkcm9vdCIvY3B1c2V0Lm1lbXMgPiAiJG92c3NsaWNlIi9jcHVzZXQubWVtcwoKIyBNb3ZlIE9WUyBvdmVyCmZvciBwcm9jZXNzIGluICQoY2F0ICIkb3Zzc2xpY2Vfc3lzdGVtZCIvKi9jZ3JvdXAucHJvY3MgfCBzb3J0IC1yKTsgZG8KICAgICAgICBlY2hvICRwcm9jZXNzID4gIiRvdnNzbGljZSIvY2dyb3VwLnByb2NzIDI+JjEgfCBncmVwIC12ICJJbnZhbGlkIEFyZ3VtZW50IiB8fCB0cnVlOwpkb25lCg== - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/cpuset-configure.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,W1VuaXRdCkRlc2NyaXB0aW9uPVRvcCBsZXZlbCBzbGljZSB1c2VkIHRvIGdpdmUgb3BlbnZzd2l0Y2ggYWNjZXNzIHRvIGFuIHVucmVzdHJpY3RlZCBzZXQgb2YgY3B1cwoKW1NsaWNlXQo= - verification: {} - group: {} - mode: 420 - path: /etc/systemd/system/ovs.slice - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdClNsaWNlPW92cy5zbGljZQo= - verification: {} - group: {} - mode: 420 - path: /etc/systemd/system/openvswitch.service.d/01-use-ovs-slice.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdClNsaWNlPW92cy5zbGljZQo= - verification: {} - group: {} - mode: 420 - path: /etc/systemd/system/ovs-vswitchd.service.d/01-use-ovs-slice.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdClNsaWNlPW92cy5zbGljZQo= - verification: {} - group: {} - mode: 420 - path: /etc/systemd/system/ovsdb-server.service.d/01-use-ovs-slice.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyBUaGlzIGZpbGUgZW5hYmxlcyB0aGUgZHluYW1pYyBjcHUgYWZmaW5pdHkgbWFuYWdlbWVudCBvZiB0aGUgT1ZTIHNlcnZpY2VzCiMKIyBJdCBpcyByZWFkIGJ5IHRoZSBPVk4ncyBvdm5rdWJlLW5vZGUgRGFlbW9uU2V0IGNvbnRhaW5lciBhbmQgdGhlIGZlYXR1cmUKIyBpcyBlbmFibGVkIHdoZW4gdGhpcyBmaWxlIGV4aXN0cyBhbmQgaXMgbm90IGVtcHR5ICh0aGlzIGNvbW1lbnRhcnkgdGV4dAojIGVuc3VyZXMgdGhhdCkKIwojIEZvciBkaXNhYmxpbmcgdGhpcyBmZWF0dXJlIGluIGVtZXJnZW5jaWVzLCBlaXRoZXI6CiMgMSkgZGVsZXRlIHRoaXMgZmlsZSBhbmQgc2V0IHRoZSBjcHUgYWZmaW5pdHkgb2YgT1ZTIHNlcnZpY2VzIG1hbnVhbGx5CiMgMikgb3IgcmVwbGFjZSB0aGUgY29udGVudHMgb2YgdGhpcyBmaWxlIHdpdGggYW4gZW1wdHkgc3RyaW5nCiMgICAgdmlhIGEgTWFjaGluZUNvbmZpZwo= - verification: {} - group: {} - mode: 420 - path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity - user: {} - systemd: - units: - - contents: | - [Unit] - Description=Sets network devices RPS mask - - [Service] - Type=oneshot - ExecStart=/usr/local/bin/set-rps-mask.sh %I 0 - name: update-rps@.service - - contents: | - [Unit] - Description=Move services to reserved cpuset - Before=kubelet.service - After=crio.service - - [Service] - Type=oneshot - ExecStart=/usr/local/bin/cpuset-configure.sh - - [Install] - WantedBy=multi-user.target crio.service - enabled: true - name: cpuset-configure.service - - contents: | - [Unit] - Description=Clear the IRQBalance Banned CPU mask early in the boot - Before=kubelet.service - Before=irqbalance.service - - [Service] - Type=oneshot - RemainAfterExit=true - ExecStart=/usr/local/bin/clear-irqbalance-banned-cpus.sh - - [Install] - WantedBy=multi-user.target - enabled: true - name: clear-irqbalance-banned-cpus.service - - contents: | - [Unit] - Description=TuneD service from NTO image - After=firstboot-osupdate.target systemd-sysctl.service network.target polkit.service - # Requires is necessary to start this unit before kubelet-dependencies.target - Requires=kubelet-dependencies.target - Before=kubelet-dependencies.target - ConditionPathExists=/var/lib/ocp-tuned/image.env - - [Service] - # https://www.redhat.com/sysadmin/podman-shareable-systemd-services - # and also "podman systemd generate" uses "forking". However, this - # is strongly discouraged by "man systemd.service" and results in - # failed dependency for the kubelet service. - Type=oneshot - Restart=on-failure - # Time to wait between restart attempts. - RestartSec=5s - ExecReload=/bin/pkill --signal HUP --pidfile /run/tuned/tuned.pid - ExecStartPre=/bin/bash -c " \ - mkdir -p /run/tuned " - ExecStart=/usr/bin/podman run \ - --rm \ - --name openshift-tuned \ - --privileged \ - --authfile /var/lib/kubelet/config.json \ - --net=host \ - --pid=host \ - --security-opt label=disable \ - --log-driver=none \ - --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ - --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ - --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ - --volume /etc/sysconfig:/etc/sysconfig:rslave \ - --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ - --volume /etc/sysctl.conf:/etc/sysctl.conf:rslave,ro \ - --volume /etc/systemd:/etc/systemd:rslave \ - --volume /run/tuned:/run/tuned:rslave \ - --volume /run/systemd:/run/systemd:rslave \ - --volume /sys:/sys:rslave \ - --entrypoint '["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \ - $NTO_IMAGE - Environment=PODMAN_SYSTEMD_UNIT=%n - EnvironmentFile=/etc/mco/proxy.env - EnvironmentFile=-/var/lib/ocp-tuned/image.env - ExecStop=/usr/bin/podman stop -t 20 --ignore openshift-tuned - ExecStopPost=/usr/bin/podman rm -f --ignore openshift-tuned - - [Install] - # RequiredBy causes kubelet to depend on this service. - RequiredBy=kubelet-dependencies.target - enabled: true - name: ocp-tuned-one-shot.service - extensions: null - fips: false - kernelArguments: null - kernelType: default - osImageURL: "" diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_node.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_node.yaml deleted file mode 100644 index b13581ca54..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_node.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: config.openshift.io/v1 -kind: Node -metadata: - creationTimestamp: null - labels: - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-master - name: cluster -spec: - cgroupMode: v1 -status: {} diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_runtimeclass.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_runtimeclass.yaml deleted file mode 100644 index 28f8428057..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_runtimeclass.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: node.k8s.io/v1 -handler: high-performance -kind: RuntimeClass -metadata: - creationTimestamp: null - labels: - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-master - name: performance-openshift-bootstrap-master -scheduling: - nodeSelector: - node-role.kubernetes.io/master: "" diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_tuned.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_tuned.yaml deleted file mode 100644 index 35c69994ae..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_tuned.yaml +++ /dev/null @@ -1,148 +0,0 @@ -apiVersion: tuned.openshift.io/v1 -kind: Tuned -metadata: - creationTimestamp: null - labels: - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-master - name: openshift-node-performance-openshift-bootstrap-master - namespace: openshift-cluster-node-tuning-operator -spec: - profile: - - data: "[main]\nsummary=Openshift node optimized for deterministic performance - at the cost of increased power consumption, focused on low latency network performance. - Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\n\n# The final result of - the include depends on cpu vendor, cpu architecture, and whether the real time - kernel is enabled\n# The first line will be evaluated based on the CPU vendor - and architecture\n# This has three possible results:\n# include=openshift-node-performance-amd-x86;\n# - \ include=openshift-node-performance-arm-aarch64;\n# include=openshift-node-performance-intel-x86;\n# - The second line will be evaluated based on whether the real time kernel is enabled\n# - This has two possible results:\n# openshift-node,cpu-partitioning\n# openshift-node,cpu-partitioning,openshift-node-performance-rt-\ninclude=openshift-node,cpu-partitioning${f:regex_search_ternary:${f:exec:uname:-r}:rt:,openshift-node-performance-rt-openshift-bootstrap-master:};\n - \ openshift-node-performance-${f:lscpu_check:Vendor ID\\:\\s*GenuineIntel:intel:Vendor - ID\\:\\s*AuthenticAMD:amd:Vendor ID\\:\\s*ARM:arm}-${f:lscpu_check:Architecture\\:\\s*x86_64:x86:Architecture\\:\\s*aarch64:aarch64}-openshift-bootstrap-master\n\n# - Inheritance of base profiles legend:\n# cpu-partitioning -> network-latency - -> latency-performance\n# https://github.com/redhat-performance/tuned/blob/master/profiles/latency-performance/tuned.conf\n# - https://github.com/redhat-performance/tuned/blob/master/profiles/network-latency/tuned.conf\n# - https://github.com/redhat-performance/tuned/blob/master/profiles/cpu-partitioning/tuned.conf\n\n# - All values are mapped with a comment where a parent profile contains them.\n# - Different values will override the original values in parent profiles.\n\n[variables]\n#> - isolated_cores take a list of ranges; e.g. isolated_cores=2,4-7\n\nisolated_cores=0-1\n\n\nnot_isolated_cores_expanded=${f:cpulist_invert:${isolated_cores_expanded}}\n\n\n[cpu]\n#> - latency-performance\n#> (override)\nforce_latency=cstate.id:1|3\ngovernor=performance\nenergy_perf_bias=performance\nmin_perf_pct=100\n\n\n\n[service]\nservice.stalld=start,enable\n\n\n[vm]\n#> - network-latency\ntransparent_hugepages=never\n\n\n[irqbalance]\n# Disable the - plugin entirely, which was enabled by the parent profile `cpu-partitioning`.\n# - It can be racy if TuneD restarts for whatever reason.\n#> cpu-partitioning\nenabled=false\n\n\n[scheduler]\nruntime=0\ngroup.ksoftirqd=0:f:11:*:ksoftirqd.*\ngroup.rcuc=0:f:11:*:rcuc.*\ngroup.ktimers=0:f:11:*:ktimers.*\n\ndefault_irq_smp_affinity - = ignore\nirq_process=false\n\n\n[sysctl]\n\n#> cpu-partitioning #RealTimeHint\nkernel.hung_task_timeout_secs=600\n#> - cpu-partitioning #RealTimeHint\nkernel.nmi_watchdog=0\n#> RealTimeHint\nkernel.sched_rt_runtime_us=-1\n#> - cpu-partitioning #RealTimeHint\nvm.stat_interval=10\n\n# cpu-partitioning and - RealTimeHint for RHEL disable it (= 0)\n# OCP is too dynamic when partitioning - and needs to evacuate\n#> scheduled timers when starting a guaranteed workload - (= 1)\nkernel.timer_migration=1\n#> network-latency\nnet.ipv4.tcp_fastopen=3\n\n# - If a workload mostly uses anonymous memory and it hits this limit, the entire\n# - working set is buffered for I/O, and any more write buffering would require\n# - swapping, so it's time to throttle writes until I/O can catch up. Workloads\n# - that mostly use file mappings may be able to use even higher values.\n#\n# The - generator of dirty data starts writeback at this percentage (system default\n# - is 20%)\n#> latency-performance\nvm.dirty_ratio=10\n\n# Start background writeback - (via writeback threads) at this percentage (system\n# default is 10%)\n#> latency-performance\nvm.dirty_background_ratio=3\n\n# - The swappiness parameter controls the tendency of the kernel to move\n# processes - out of physical memory and onto the swap disk.\n# 0 tells the kernel to avoid - swapping processes out of physical memory\n# for as long as possible\n# 100 - tells the kernel to aggressively swap processes out of physical memory\n# and - move them to swap cache\n#> latency-performance\nvm.swappiness=10\n\n# also - configured via a sysctl.d file\n# placed here for documentation purposes and - commented out due\n# to a tuned logging bug complaining about duplicate sysctl:\n# - \ https://issues.redhat.com/browse/RHEL-18972\n#> rps configuration\n# net.core.rps_default_mask=${not_isolated_cpumask}\n\n\n[selinux]\n#> - Custom (atomic host)\navc_cache_threshold=8192\n\n\n[net]\nnf_conntrack_hashsize=131072\n\n\n[bootloader]\n# - !! The names are important for Intel and are referenced in openshift-node-performance-intel-x86\n\n# - set empty values to disable RHEL initrd setting in cpu-partitioning\ninitrd_remove_dir=\ninitrd_dst_img=\ninitrd_add_dir=\n\n# - overrides cpu-partitioning cmdline\ncmdline_cpu_part=+nohz=on rcu_nocbs=${isolated_cores} - tuned.non_isolcpus=${not_isolated_cpumask} systemd.cpu_affinity=${not_isolated_cores_expanded}\n\n# - No default value but will be composed conditionally based on platform\ncmdline_iommu=\n\n\ncmdline_isolation=+isolcpus=managed_irq,${isolated_cores}\n\n\n\ncmdline_realtime_nohzfull=+nohz_full=${isolated_cores}\ncmdline_realtime_nosoftlookup=+nosoftlockup\ncmdline_realtime_common=+skew_tick=1 - rcutree.kthread_prio=11\n\n\n\n\n\n\n \n\n\n\n[rtentsk]\n\n\n" - name: openshift-node-performance-openshift-bootstrap-master - - data: "[main]\nsummary=Real time profile to override unsupported settings\n\n[sysctl]\n#Real - time kernel doesn't support the following kernel parameters.\n#The openshift-node-performance - profile inherits these kernel parameters from the network-latency profile. \n#Therefore, - if the real time kernel is detected they will be dropped, meaning won't be applied.\ndrop=kernel.numa_balancing,net.core.busy_read,net.core.busy_poll\n" - name: openshift-node-performance-rt-openshift-bootstrap-master - - data: |+ - [main] - summary=Platform specific tuning for AMD x86 - - [bootloader] - cmdline_iommu_amd=iommu=pt - - - cmdline_pstate=amd_pstate=guided - - - - cmdline_realtime_amd=tsc=reliable nmi_watchdog=0 mce=off - - - - - - name: openshift-node-performance-amd-x86-openshift-bootstrap-master - - data: | - [main] - summary=Platform specific tuning for aarch64 - - [bootloader] - # No cstate for ARM - # No pstate args for ARM - - # aarch64 specific tuning options - cmdline_iommu_arm=iommu.passthrough=1 - name: openshift-node-performance-arm-aarch64-openshift-bootstrap-master - - data: |+ - [main] - summary=Platform specific tuning for Intel x86 - - [bootloader] - # DO NOT REMOVE THIS BLOCK - # It makes sure the kernel arguments for Intel are applied - # in the order compatible with OCP 4.17 which is important - # for preventing an extra reboot during upgrade - cmdline_cpu_part= - cmdline_iommu_intel= - cmdline_isolation= - cmdline_realtime_nohzfull= - cmdline_realtime_intel= - cmdline_realtime_nosoftlookup= - cmdline_realtime_intel_nmi= - cmdline_realtime_common= - cmdline_power_performance= - cmdline_power_performance_intel= - cmdline_idle_poll= - cmdline_idle_poll_intel= - cmdline_hugepages= - cmdline_pstate= - - # Here comes the Intel specific tuning - - cmdline_iommu_intel=intel_iommu=on iommu=pt - - - cmdline_realtime_intel=tsc=reliable - cmdline_realtime_intel_nmi=nmi_watchdog=0 mce=off - - - - - - - - cmdline_pstate=intel_pstate=${f:intel_recommended_pstate} - - - name: openshift-node-performance-intel-x86-openshift-bootstrap-master - recommend: - - machineConfigLabels: - machineconfiguration.openshift.io/role: master - operand: - tunedConfig: - reapply_sysctl: null - priority: 20 - profile: openshift-node-performance-openshift-bootstrap-master -status: {} diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_kubeletconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_kubeletconfig.yaml deleted file mode 100644 index dadc961c3b..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_kubeletconfig.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: KubeletConfig -metadata: - creationTimestamp: null - labels: - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-worker - name: performance-openshift-bootstrap-worker -spec: - kubeletConfig: - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: {} - webhook: - cacheTTL: 0s - x509: {} - authorization: - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - containerRuntimeEndpoint: "" - cpuManagerPolicy: static - cpuManagerReconcilePeriod: 5s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - memory: 500Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - reservedSystemCPUs: 2-3 - runtimeRequestTimeout: 0s - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - memory: 500Mi - topologyManagerPolicy: best-effort - volumeStatsAggPeriod: 0s - machineConfigPoolSelector: - matchLabels: - pools.operator.machineconfiguration.openshift.io/worker: "" -status: - conditions: null diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml deleted file mode 100644 index 67abea13eb..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml +++ /dev/null @@ -1,226 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - creationTimestamp: null - labels: - machineconfiguration.openshift.io/role: worker - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-worker - name: 50-performance-openshift-bootstrap-worker -spec: - baseOSExtensionsContainerImage: "" - config: - ignition: - config: - replace: - verification: {} - proxy: {} - security: - tls: {} - timeouts: {} - version: 3.2.0 - passwd: {} - storage: - files: - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaAoKc2V0IC1ldW8gcGlwZWZhaWwKCm5vZGVzX3BhdGg9Ii9zeXMvZGV2aWNlcy9zeXN0ZW0vbm9kZSIKaHVnZXBhZ2VzX2ZpbGU9IiR7bm9kZXNfcGF0aH0vbm9kZSR7TlVNQV9OT0RFfS9odWdlcGFnZXMvaHVnZXBhZ2VzLSR7SFVHRVBBR0VTX1NJWkV9a0IvbnJfaHVnZXBhZ2VzIgoKaWYgWyAhIC1mICIke2h1Z2VwYWdlc19maWxlfSIgXTsgdGhlbgogIGVjaG8gIkVSUk9SOiAke2h1Z2VwYWdlc19maWxlfSBkb2VzIG5vdCBleGlzdCIKICBleGl0IDEKZmkKCnRpbWVvdXQ9NjAKc2FtcGxlPTEKY3VycmVudF90aW1lPTAKd2hpbGUgWyAiJChjYXQgIiR7aHVnZXBhZ2VzX2ZpbGV9IikiIC1uZSAiJHtIVUdFUEFHRVNfQ09VTlR9IiBdOyBkbwogIGVjaG8gIiR7SFVHRVBBR0VTX0NPVU5UfSIgPiIke2h1Z2VwYWdlc19maWxlfSIKCiAgY3VycmVudF90aW1lPSQoKGN1cnJlbnRfdGltZSArIHNhbXBsZSkpCiAgaWYgWyAkY3VycmVudF90aW1lIC1ndCAkdGltZW91dCBdOyB0aGVuCiAgICBlY2hvICJFUlJPUjogJHtodWdlcGFnZXNfZmlsZX0gZG9lcyBub3QgaGF2ZSB0aGUgZXhwZWN0ZWQgbnVtYmVyIG9mIGh1Z2VwYWdlcyAke0hVR0VQQUdFU19DT1VOVH0iCiAgICBleGl0IDEKICBmaQoKICBzbGVlcCAkc2FtcGxlCmRvbmUK - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/hugepages-allocation.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZnVuY3Rpb24gc2V0X3F1ZXVlX3Jwc19tYXNrKCkgewojIHJlcGxhY2UgeDJkIHdpdGggaHlwaGVuICgtKSB3aGljaCBpcyBhbiBlc2NhcGVkIGNoYXJhY3RlcgojIHRoYXQgd2FzIGFkZGVkIGJ5IHN5c3RlbWQtZXNjYXBlIGluIG9yZGVyIHRvIGVzY2FwZSB0aGUgc3lzdGVtZCB1bml0IG5hbWUgdGhhdCBpbnZva2VzIHRoaXMgc2NyaXB0CnBhdGg9JHtwYXRoL3gyZC8tfQojIHNldCBycHMgYWZmaW5pdHkgZm9yIHRoZSBxdWV1ZQplY2hvICIke21hc2t9IiAgMj4gL2Rldi9udWxsID4gIi9zeXMvJHtwYXRofS9ycHNfY3B1cyIKIyB3ZSByZXR1cm4gMCBiZWNhdXNlIHRoZSAnZWNobycgY29tbWFuZCBtaWdodCBmYWlsIGlmIHRoZSBkZXZpY2UgcGF0aCB0byB3aGljaCB0aGUgcXVldWUgYmVsb25ncyBoYXMgY2hhbmdlZC4KIyB0aGlzIGNhbiBoYXBwZW4gaW4gY2FzZSBvZiBTUkktT1YgZGV2aWNlcyByZW5hbWluZy4KcmV0dXJuIDAKfQoKZnVuY3Rpb24gc2V0X25ldF9kZXZfcnBzX21hc2soKSB7CiAgIyBpbiBjYXNlIG9mIGRldmljZSB3ZSB3YW50IHRvIGl0ZXJhdGUgdGhyb3VnaCBhbGwgcXVldWVzCmZvciBpIGluIC9zeXMvIiR7cGF0aH0iL3F1ZXVlcy9yeC0qOyBkbwogIGVjaG8gIiR7bWFza30iIDI+IC9kZXYvbnVsbCA+ICIke2l9L3Jwc19jcHVzIgpkb25lCiMgd2UgcmV0dXJuIDAgYmVjYXVzZSB0aGUgJ2VjaG8nIGNvbW1hbmQgbWlnaHQgZmFpbCBpZiB0aGUgZGV2aWNlIHBhdGggdG8gd2hpY2ggdGhlIHF1ZXVlIGJlbG9uZ3MgaGFzIGNoYW5nZWQuCiMgdGhpcyBjYW4gaGFwcGVuIGluIGNhc2Ugb2YgU1JJLU9WIGRldmljZXMgcmVuYW1pbmcuCnJldHVybiAwCiB9CgpwYXRoPSR7MX0KWyAtbiAiJHtwYXRofSIgXSB8fCB7IGVjaG8gIlRoZSBkZXZpY2UgcGF0aCBhcmd1bWVudCBpcyBtaXNzaW5nIiA+JjIgOyBleGl0IDE7IH0KCm1hc2s9JHsyfQpbIC1uICIke21hc2t9IiBdIHx8IHsgZWNobyAiVGhlIG1hc2sgYXJndW1lbnQgaXMgbWlzc2luZyIgPiYyIDsgZXhpdCAxOyB9CgppZiBbWyAiJHtwYXRofSIgPX4gInF1ZXVlcyIgXV07IHRoZW4KIHNldF9xdWV1ZV9ycHNfbWFzawplbHNlCiBzZXRfbmV0X2Rldl9ycHNfbWFzawpmaQo= - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/set-rps-mask.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9iYXNoCgpzZXQgLWV1byBwaXBlZmFpbAoKZm9yIGNwdSBpbiAke09GRkxJTkVfQ1BVUy8vLC8gfTsKICBkbwogICAgb25saW5lX2NwdV9maWxlPSIvc3lzL2RldmljZXMvc3lzdGVtL2NwdS9jcHUkY3B1L29ubGluZSIKICAgIGlmIFsgISAtZiAiJHtvbmxpbmVfY3B1X2ZpbGV9IiBdOyB0aGVuCiAgICAgIGVjaG8gIkVSUk9SOiAke29ubGluZV9jcHVfZmlsZX0gZG9lcyBub3QgZXhpc3QsIGFib3J0IHNjcmlwdCBleGVjdXRpb24iCiAgICAgIGV4aXQgMQogICAgZmkKICBkb25lCgplY2hvICJBbGwgY3B1cyBvZmZsaW5lZCBleGlzdHMsIHNldCB0aGVtIG9mZmxpbmUiCgpmb3IgY3B1IGluICR7T0ZGTElORV9DUFVTLy8sLyB9OwogIGRvCiAgICBvbmxpbmVfY3B1X2ZpbGU9Ii9zeXMvZGV2aWNlcy9zeXN0ZW0vY3B1L2NwdSRjcHUvb25saW5lIgogICAgZWNobyAwID4gIiR7b25saW5lX2NwdV9maWxlfSIKICAgIGVjaG8gIm9mZmxpbmUgY3B1IG51bSAkY3B1IgogIGRvbmUKCg== - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/set-cpus-offline.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/clear-irqbalance-banned-cpus.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgojIFdlIHNob3VsZCBjb3B5IHBhc3RlIHRoZSBkZWZhdWx0IHJ1bnRpbWUgYmVjYXVzZSB0aGlzIHNuaXBwZXQgd2lsbCBvdmVycmlkZSB0aGUgd2hvbGUgcnVudGltZXMgc2VjdGlvbgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLnJ1bmNdCnJ1bnRpbWVfcGF0aCA9ICIiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCgojIFRoZSBDUkktTyB3aWxsIGNoZWNrIHRoZSBhbGxvd2VkX2Fubm90YXRpb25zIHVuZGVyIHRoZSBydW50aW1lIGhhbmRsZXIgYW5kIGFwcGx5IGhpZ2gtcGVyZm9ybWFuY2UgaG9va3Mgd2hlbiBvbmUgb2YKIyBoaWdoLXBlcmZvcm1hbmNlIGFubm90YXRpb25zIHByZXNlbnRzIHVuZGVyIGl0LgojIFdlIHNob3VsZCBwcm92aWRlIHRoZSBydW50aW1lX3BhdGggYmVjYXVzZSB3ZSBuZWVkIHRvIGluZm9ybSB0aGF0IHdlIHdhbnQgdG8gcmUtdXNlIHJ1bmMgYmluYXJ5IGFuZCB3ZQojIGRvIG5vdCBoYXZlIGhpZ2gtcGVyZm9ybWFuY2UgYmluYXJ5IHVuZGVyIHRoZSAkUEFUSCB0aGF0IHdpbGwgcG9pbnQgdG8gaXQuCltjcmlvLnJ1bnRpbWUucnVudGltZXMuaGlnaC1wZXJmb3JtYW5jZV0KcnVudGltZV9wYXRoID0gIi9iaW4vcnVuYyIKcnVudGltZV90eXBlID0gIm9jaSIKcnVudGltZV9yb290ID0gIi9ydW4vcnVuYyIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg== - verification: {} - group: {} - mode: 420 - path: /etc/crio/crio.conf.d/99-runtimes.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyBBcHBseSB0aGUgUlBTIG1hc2sgb24gdGhlIHZpcnR1YWwgaW50ZXJmYWNlcyBvZiB0aGUgaG9zdCBieSBkZWZhdWx0LCBiZWNhc3VlCiMgZnJvbSB0aGUgY29udGFpbmVyIHBlcnNwZWN0aXZlIHRoZSBSUFMgbWFzayB0aGUgd2lsbCBiZSBjb25zdWx0ZWQsIGlzIHRoZSBvbmUgb24gdGhlIFJYIHNpZGUgb2YgdGhlIHZldGggaW4gdGhlIGhvc3QuCiMgQ29uc2lkZXIgdGhlIGZvbGxvd2luZyBkaWFncmFtOgojIFBvZCBBIDx2ZXRoMSAtIHZldGgyPiBob3N0IDx2ZXRoMyAtIHZldGg0PiBQb2QgQgojICB2ZXRoMidzIFJQUyBhZmZpbml0eSBpcyB0aGUgb25lIGRldGVybWluaW5nIHRoZSBDUFVzIHRoYXQgYXJlIGhhbmRsaW5nIHRoZSBwYWNrZXQgcHJvY2Vzc2luZyB3aGVuIHNlbmRpbmcgZGF0YSBmcm9tIFBvZCBBIHRvIHBvZCBCLgojIEFkZGl0aW9uYWwgY29tbW9uIHNjZW5hcmlvczoKIyAxLiBQb2QgQSA9IHNlbmRlciwgaG9zdCA9IHJlY2VpdmVyCiMgIFRoZSBSUFMgYWZmaW5pdHkgb2YgdGhlIGhvc3Qgc2lkZSBzaG91bGQgYmUgY29uc3VsdGVkIChiZWNhdXNlIGl04oCZcyB0aGUgcmVjZWl2ZXIpIGFuZCBpdCBzaG91bGQgYmUgc2V0IHRvIGNwdXMgbm90IHNlbnNpdGl2ZSB0byBwcmVlbXB0aW9uIChyZXNlcnZlZCBwb29sKS4KIyAyLiBQb2QgQSA9IHJlY2VpdmVyLCBob3N0ID0gc2VuZGVyCiMgIEluIGNhc2Ugb2Ygbm8gUlBTIG1hc2sgb24gdGhlIHJlY2VpdmVyIHNpZGUsIHRoZSBzZW5kZXIgbmVlZHMgdG8gcGF5IHRoZSBwcmljZSBhbmQgZG8gYWxsIHRoZSBwcm9jZXNzaW5nIG9uIGl0cyBjb3Jlcy4KbmV0LmNvcmUucnBzX2RlZmF1bHRfbWFzayA9IDAwMDAwMDBjCg== - verification: {} - group: {} - mode: 420 - path: /etc/sysctl.d/99-default-rps-mask.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,U1VCU1lTVEVNPT0icXVldWVzIiwgQUNUSU9OPT0iYWRkIiwgRU5We0RFVlBBVEh9PT0iL2RldmljZXMvcGNpKi9xdWV1ZXMvcngqIiwgVEFHKz0ic3lzdGVtZCIsIFBST0dSQU09Ii9iaW4vc3lzdGVtZC1lc2NhcGUgLS1wYXRoIC0tdGVtcGxhdGU9dXBkYXRlLXJwc0Auc2VydmljZSAkZW52e0RFVlBBVEh9IiwgRU5We1NZU1RFTURfV0FOVFN9PSIlYyIKCiMgU1ItSU9WIGRldmljZXMgYXJlIG1vdmVkIChyZW5hbWVkKSwgaGVuY2Ugd2Ugd2FudCB0byBjYXRjaCB0aGlzIGV2ZW50IGFzIHdlbGwKU1VCU1lTVEVNPT0ibmV0IiwgQUNUSU9OPT0ibW92ZSIsIEVOVntERVZQQVRIfSE9Ii9kZXZpY2VzL3ZpcnR1YWwvbmV0LyoiLCBUQUcrPSJzeXN0ZW1kIiwgUFJPR1JBTT0iL2Jpbi9zeXN0ZW1kLWVzY2FwZSAtLXBhdGggLS10ZW1wbGF0ZT11cGRhdGUtcnBzQC5zZXJ2aWNlICRlbnZ7REVWUEFUSH0iLCBFTlZ7U1lTVEVNRF9XQU5UU309IiVjIgo= - verification: {} - group: {} - mode: 420 - path: /etc/udev/rules.d/99-netdev-physical-rps.rules - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWUud29ya2xvYWRzLm1hbmFnZW1lbnRdCmFjdGl2YXRpb25fYW5ub3RhdGlvbiA9ICJ0YXJnZXQud29ya2xvYWQub3BlbnNoaWZ0LmlvL21hbmFnZW1lbnQiCmFubm90YXRpb25fcHJlZml4ID0gInJlc291cmNlcy53b3JrbG9hZC5vcGVuc2hpZnQuaW8iCnJlc291cmNlcyA9IHsgImNwdXNoYXJlcyIgPSAwLCAiY3B1c2V0IiA9ICIyLTMiIH0K - verification: {} - group: {} - mode: 420 - path: /etc/crio/crio.conf.d/99-workload-pinning.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,CnsKICAibWFuYWdlbWVudCI6IHsKICAgICJjcHVzZXQiOiAiMi0zIgogIH0KfQo= - verification: {} - group: {} - mode: 420 - path: /etc/kubernetes/openshift-workload-pinning - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2gKCiMgY3B1c2V0LWNvbmZpZ3VyZS5zaCBjb25maWd1cmVzIHRocmVlIGNwdXNldHMgaW4gcHJlcGFyYXRpb24gZm9yIGFsbG93aW5nIGNvbnRhaW5lcnMgdG8gaGF2ZSBjcHUgbG9hZCBiYWxhbmNpbmcgZGlzYWJsZWQuCiMgVG8gY29uZmlndXJlIGEgY3B1c2V0IHRvIGhhdmUgbG9hZCBiYWxhbmNlIGRpc2FibGVkIChvbiBjZ3JvdXAgdjEpLCBhIGNwdXNldCBjZ3JvdXAgbXVzdCBoYXZlIGBjcHVzZXQuc2NoZWRfbG9hZF9iYWxhbmNlYAojIHNldCB0byAwIChkaXNhYmxlKSwgYW5kIGFueSBjcHVzZXQgdGhhdCBjb250YWlucyB0aGUgc2FtZSBzZXQgYXMgYGNwdXNldC5jcHVzYCBtdXN0IGFsc28gaGF2ZSBgY3B1c2V0LnNjaGVkX2xvYWRfYmFsYW5jZWAgc2V0IHRvIGRpc2FibGVkLgoKc2V0IC1ldW8gcGlwZWZhaWwKCmlmIHRlc3QgIiQoc3RhdCAtZiAtYyVUIC9zeXMvZnMvY2dyb3VwKSIgPSAiY2dyb3VwMmZzIjsgdGhlbgoJZWNobyAiTm9kZSBpcyB1c2luZyBjZ3JvdXAgdjIsIG5vIGNvbmZpZ3VyYXRpb24gbmVlZGVkIgoJZXhpdCAwCmZpCgpyb290PS9zeXMvZnMvY2dyb3VwL2NwdXNldApzeXN0ZW09IiRyb290Ii9zeXN0ZW0uc2xpY2UKbWFjaGluZT0iJHJvb3QiL21hY2hpbmUuc2xpY2UKCm92c3NsaWNlPSIke3Jvb3R9L292cy5zbGljZSIKb3Zzc2xpY2Vfc3lzdGVtZD0iL3N5cy9mcy9jZ3JvdXAvcGlkcy9vdnMuc2xpY2UiCgojIEFzIHN1Y2gsIHRoZSByb290IGNncm91cCBuZWVkcyB0byBoYXZlIGNwdXNldC5zY2hlZF9sb2FkX2JhbGFuY2U9MC4gCmVjaG8gMCA+ICIkcm9vdCIvY3B1c2V0LnNjaGVkX2xvYWRfYmFsYW5jZQoKIyBIb3dldmVyLCB0aGlzIHdvdWxkIHByZXNlbnQgYSBwcm9ibGVtIGZvciBzeXN0ZW0gZGFlbW9ucywgd2hpY2ggc2hvdWxkIGhhdmUgbG9hZCBiYWxhbmNpbmcgZW5hYmxlZC4KIyBBcyBzdWNoLCBhIHNlY29uZCBjcHVzZXQgbXVzdCBiZSBjcmVhdGVkLCBoZXJlIGR1YmJlZCBgc3lzdGVtYCwgd2hpY2ggd2lsbCB0YWtlIGFsbCBzeXN0ZW0gZGFlbW9ucy4KIyBTaW5jZSBzeXN0ZW1kIHN0YXJ0cyBpdHMgY2hpbGRyZW4gd2l0aCB0aGUgY3B1c2V0IGl0IGlzIGluLCBtb3Zpbmcgc3lzdGVtZCB3aWxsIGVuc3VyZSBhbGwgcHJvY2Vzc2VzIHN5c3RlbWQgYmVnaW5zIHdpbGwgYmUgaW4gdGhlIGNvcnJlY3QgY2dyb3VwLgpta2RpciAtcCAiJHN5c3RlbSIKIyBjcHVzZXQubWVtcyBtdXN0IGJlIGluaXRpYWxpemVkIG9yIHByb2Nlc3NlcyB3aWxsIGZhaWwgdG8gYmUgbW92ZWQgaW50byBpdC4KY2F0ICIkcm9vdC9jcHVzZXQubWVtcyIgPiAiJHN5c3RlbSIvY3B1c2V0Lm1lbXMKIyBSZXRyaWV2ZSB0aGUgY3B1c2V0IG9mIHN5c3RlbWQsIGFuZCB3cml0ZSBpdCB0byBjcHVzZXQuY3B1cyBvZiB0aGUgc3lzdGVtIGNncm91cC4KcmVzZXJ2ZWRfc2V0PSQodGFza3NldCAtY3AgIDEgIHwgYXdrICdORnsgcHJpbnQgJE5GIH0nKQplY2hvICIkcmVzZXJ2ZWRfc2V0IiA+ICIkc3lzdGVtIi9jcHVzZXQuY3B1cwoKIyBBbmQgbW92ZSB0aGUgc3lzdGVtIHByb2Nlc3NlcyBpbnRvIGl0LgojIE5vdGUsIHNvbWUga2VybmVsIHRocmVhZHMgd2lsbCBmYWlsIHRvIGJlIG1vdmVkIHdpdGggIkludmFsaWQgQXJndW1lbnQiLiBUaGlzIHNob3VsZCBiZSBpZ25vcmVkLgpmb3IgcHJvY2VzcyBpbiAkKGNhdCAiJHJvb3QiL2Nncm91cC5wcm9jcyB8IHNvcnQgLXIpOyBkbwoJZWNobyAkcHJvY2VzcyA+ICIkc3lzdGVtIi9jZ3JvdXAucHJvY3MgMj4mMSB8IGdyZXAgLXYgIkludmFsaWQgQXJndW1lbnQiIHx8IHRydWU7CmRvbmUKCiMgRmluYWxseSwgYSB0aGUgYG1hY2hpbmUuc2xpY2VgIGNncm91cCBtdXN0IGJlIHByZWNvbmZpZ3VyZWQuIFBvZG1hbiB3aWxsIGNyZWF0ZSBjb250YWluZXJzIGFuZCBtb3ZlIHRoZW0gaW50byB0aGUgYG1hY2hpbmUuc2xpY2VgLCBidXQgdGhlcmUncwojIG5vIHdheSB0byB0ZWxsIHBvZG1hbiB0byB1cGRhdGUgbWFjaGluZS5zbGljZSB0byBub3QgaGF2ZSB0aGUgZnVsbCBzZXQgb2YgY3B1cy4gSW5zdGVhZCBvZiBkaXNhYmxpbmcgbG9hZCBiYWxhbmNpbmcgaW4gaXQsIHdlIGNhbiBwcmUtY3JlYXRlIGl0LgojIHdpdGggdGhlIHJlc2VydmVkIENQVXMgc2V0IGFoZWFkIG9mIHRpbWUsIHNvIHdoZW4gaXNvbGF0ZWQgcHJvY2Vzc2VzIGJlZ2luLCB0aGUgY2dyb3VwIGRvZXMgbm90IGhhdmUgYW4gb3ZlcmxhcHBpbmcgY3B1c2V0IGJldHdlZW4gbWFjaGluZS5zbGljZSBhbmQgaXNvbGF0ZWQgY29udGFpbmVycy4KbWtkaXIgLXAgIiRtYWNoaW5lIgoKIyBJdCdzIHVubGlrZWx5LCBidXQgcG9zc2libGUsIHRoYXQgdGhpcyBjcHVzZXQgYWxyZWFkeSBleGlzdGVkLiBJdGVyYXRlIGp1c3QgaW4gY2FzZS4KZm9yIGZpbGUgaW4gJChmaW5kICIkbWFjaGluZSIgLW5hbWUgY3B1c2V0LmNwdXMgfCBzb3J0IC1yKTsgZG8gZWNobyAiJHJlc2VydmVkX3NldCIgPiAiJGZpbGUiOyBkb25lCgojIE9WUyBpcyBydW5uaW5nIGluIGl0cyBvd24gc2xpY2UgdGhhdCBzcGFucyBhbGwgY3B1cy4gVGhlIHJlYWwgYWZmaW5pdHkgaXMgbWFuYWdlZCBieSBPVk4tSyBvdm5rdWJlLW5vZGUgZGFlbW9uc2V0CiMgTWFrZSBzdXJlIHRoaXMgc2xpY2Ugd2lsbCBub3QgZW5hYmxlIGNwdSBiYWxhbmNpbmcgZm9yIG90aGVyIHNsaWNlIGNvbmZpZ3VyZWQgYnkgdGhpcyBzY3JpcHQuCiMgVGhpcyBtaWdodCBzZWVtIGNvdW50ZXItaW50dWl0aXZlLCBidXQgdGhpcyB3aWxsIGFjdHVhbGx5IE5PVCBkaXNhYmxlIGNwdSBiYWxhbmNpbmcgZm9yIE9WUyBpdHNlbGYuCiMgLSBPVlMgaGFzIGFjY2VzcyB0byByZXNlcnZlZCBjcHVzLCBidXQgdGhvc2UgaGF2ZSBiYWxhbmNpbmcgZW5hYmxlZCB2aWEgdGhlIGBzeXN0ZW1gIGNncm91cCBjcmVhdGVkIGFib3ZlCiMgLSBPVlMgaGFzIGFjY2VzcyB0byBpc29sYXRlZCBjcHVzIHRoYXQgYXJlIGN1cnJlbnRseSBub3QgYXNzaWduZWQgdG8gcGlubmVkIHBvZHMuIFRob3NlIGhhdmUgYmFsYW5jaW5nIGVuYWJsZWQgYnkgdGhlCiMgICBwb2RzIHJ1bm5pbmcgdGhlcmUgKGJ1cnN0YWJsZSBhbmQgYmVzdC1lZmZvcnQgcG9kcyBoYXZlIGJhbGFuY2luZyBlbmFibGVkIGluIHRoZSBjb250YWluZXIgY2dyb3VwIGFuZCBhY2Nlc3MgdG8gYWxsCiMgICB1bnBpbm5lZCBjcHVzKS4KCiMgc3lzdGVtZCBkb2VzIG5vdCBtYW5hZ2UgdGhlIGNwdXNldCBjZ3JvdXAgY29udHJvbGxlciwgc28gbW92ZSBldmVyeXRoaW5nIGZyb20gdGhlIG1hbmFnZWQgcGlkcyBjb250cm9sbGVyJ3Mgb3ZzLnNsaWNlCiMgdG8gdGhlIGNwdXNldCBjb250cm9sbGVyLgoKIyBDcmVhdGUgdGhlIG92cy5zbGljZQpta2RpciAtcCAiJG92c3NsaWNlIgplY2hvIDAgPiAiJG92c3NsaWNlIi9jcHVzZXQuc2NoZWRfbG9hZF9iYWxhbmNlCmNhdCAiJHJvb3QiL2NwdXNldC5jcHVzID4gIiRvdnNzbGljZSIvY3B1c2V0LmNwdXMKY2F0ICIkcm9vdCIvY3B1c2V0Lm1lbXMgPiAiJG92c3NsaWNlIi9jcHVzZXQubWVtcwoKIyBNb3ZlIE9WUyBvdmVyCmZvciBwcm9jZXNzIGluICQoY2F0ICIkb3Zzc2xpY2Vfc3lzdGVtZCIvKi9jZ3JvdXAucHJvY3MgfCBzb3J0IC1yKTsgZG8KICAgICAgICBlY2hvICRwcm9jZXNzID4gIiRvdnNzbGljZSIvY2dyb3VwLnByb2NzIDI+JjEgfCBncmVwIC12ICJJbnZhbGlkIEFyZ3VtZW50IiB8fCB0cnVlOwpkb25lCg== - verification: {} - group: {} - mode: 448 - path: /usr/local/bin/cpuset-configure.sh - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,W1VuaXRdCkRlc2NyaXB0aW9uPVRvcCBsZXZlbCBzbGljZSB1c2VkIHRvIGdpdmUgb3BlbnZzd2l0Y2ggYWNjZXNzIHRvIGFuIHVucmVzdHJpY3RlZCBzZXQgb2YgY3B1cwoKW1NsaWNlXQo= - verification: {} - group: {} - mode: 420 - path: /etc/systemd/system/ovs.slice - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdClNsaWNlPW92cy5zbGljZQo= - verification: {} - group: {} - mode: 420 - path: /etc/systemd/system/openvswitch.service.d/01-use-ovs-slice.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdClNsaWNlPW92cy5zbGljZQo= - verification: {} - group: {} - mode: 420 - path: /etc/systemd/system/ovs-vswitchd.service.d/01-use-ovs-slice.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdClNsaWNlPW92cy5zbGljZQo= - verification: {} - group: {} - mode: 420 - path: /etc/systemd/system/ovsdb-server.service.d/01-use-ovs-slice.conf - user: {} - - contents: - source: data:text/plain;charset=utf-8;base64,IyBUaGlzIGZpbGUgZW5hYmxlcyB0aGUgZHluYW1pYyBjcHUgYWZmaW5pdHkgbWFuYWdlbWVudCBvZiB0aGUgT1ZTIHNlcnZpY2VzCiMKIyBJdCBpcyByZWFkIGJ5IHRoZSBPVk4ncyBvdm5rdWJlLW5vZGUgRGFlbW9uU2V0IGNvbnRhaW5lciBhbmQgdGhlIGZlYXR1cmUKIyBpcyBlbmFibGVkIHdoZW4gdGhpcyBmaWxlIGV4aXN0cyBhbmQgaXMgbm90IGVtcHR5ICh0aGlzIGNvbW1lbnRhcnkgdGV4dAojIGVuc3VyZXMgdGhhdCkKIwojIEZvciBkaXNhYmxpbmcgdGhpcyBmZWF0dXJlIGluIGVtZXJnZW5jaWVzLCBlaXRoZXI6CiMgMSkgZGVsZXRlIHRoaXMgZmlsZSBhbmQgc2V0IHRoZSBjcHUgYWZmaW5pdHkgb2YgT1ZTIHNlcnZpY2VzIG1hbnVhbGx5CiMgMikgb3IgcmVwbGFjZSB0aGUgY29udGVudHMgb2YgdGhpcyBmaWxlIHdpdGggYW4gZW1wdHkgc3RyaW5nCiMgICAgdmlhIGEgTWFjaGluZUNvbmZpZwo= - verification: {} - group: {} - mode: 420 - path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity - user: {} - systemd: - units: - - contents: | - [Unit] - Description=Sets network devices RPS mask - - [Service] - Type=oneshot - ExecStart=/usr/local/bin/set-rps-mask.sh %I 0 - name: update-rps@.service - - contents: | - [Unit] - Description=Move services to reserved cpuset - Before=kubelet.service - After=crio.service - - [Service] - Type=oneshot - ExecStart=/usr/local/bin/cpuset-configure.sh - - [Install] - WantedBy=multi-user.target crio.service - enabled: true - name: cpuset-configure.service - - contents: | - [Unit] - Description=Clear the IRQBalance Banned CPU mask early in the boot - Before=kubelet.service - Before=irqbalance.service - - [Service] - Type=oneshot - RemainAfterExit=true - ExecStart=/usr/local/bin/clear-irqbalance-banned-cpus.sh - - [Install] - WantedBy=multi-user.target - enabled: true - name: clear-irqbalance-banned-cpus.service - - contents: | - [Unit] - Description=TuneD service from NTO image - After=firstboot-osupdate.target systemd-sysctl.service network.target polkit.service - # Requires is necessary to start this unit before kubelet-dependencies.target - Requires=kubelet-dependencies.target - Before=kubelet-dependencies.target - ConditionPathExists=/var/lib/ocp-tuned/image.env - - [Service] - # https://www.redhat.com/sysadmin/podman-shareable-systemd-services - # and also "podman systemd generate" uses "forking". However, this - # is strongly discouraged by "man systemd.service" and results in - # failed dependency for the kubelet service. - Type=oneshot - Restart=on-failure - # Time to wait between restart attempts. - RestartSec=5s - ExecReload=/bin/pkill --signal HUP --pidfile /run/tuned/tuned.pid - ExecStartPre=/bin/bash -c " \ - mkdir -p /run/tuned " - ExecStart=/usr/bin/podman run \ - --rm \ - --name openshift-tuned \ - --privileged \ - --authfile /var/lib/kubelet/config.json \ - --net=host \ - --pid=host \ - --security-opt label=disable \ - --log-driver=none \ - --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ - --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ - --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ - --volume /etc/sysconfig:/etc/sysconfig:rslave \ - --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ - --volume /etc/sysctl.conf:/etc/sysctl.conf:rslave,ro \ - --volume /etc/systemd:/etc/systemd:rslave \ - --volume /run/tuned:/run/tuned:rslave \ - --volume /run/systemd:/run/systemd:rslave \ - --volume /sys:/sys:rslave \ - --entrypoint '["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \ - $NTO_IMAGE - Environment=PODMAN_SYSTEMD_UNIT=%n - EnvironmentFile=/etc/mco/proxy.env - EnvironmentFile=-/var/lib/ocp-tuned/image.env - ExecStop=/usr/bin/podman stop -t 20 --ignore openshift-tuned - ExecStopPost=/usr/bin/podman rm -f --ignore openshift-tuned - - [Install] - # RequiredBy causes kubelet to depend on this service. - RequiredBy=kubelet-dependencies.target - enabled: true - name: ocp-tuned-one-shot.service - extensions: null - fips: false - kernelArguments: null - kernelType: default - osImageURL: "" diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_node.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_node.yaml deleted file mode 100644 index 82f3da2d62..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_node.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: config.openshift.io/v1 -kind: Node -metadata: - creationTimestamp: null - labels: - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-worker - name: cluster -spec: - cgroupMode: v1 -status: {} diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_runtimeclass.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_runtimeclass.yaml deleted file mode 100644 index bb397f9398..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_runtimeclass.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: node.k8s.io/v1 -handler: high-performance -kind: RuntimeClass -metadata: - creationTimestamp: null - labels: - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-worker - name: performance-openshift-bootstrap-worker -scheduling: - nodeSelector: - node-role.kubernetes.io/worker: "" diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_tuned.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_tuned.yaml deleted file mode 100644 index 83125780b7..0000000000 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_tuned.yaml +++ /dev/null @@ -1,148 +0,0 @@ -apiVersion: tuned.openshift.io/v1 -kind: Tuned -metadata: - creationTimestamp: null - labels: - performance.openshift.io/weak-owner-reference-name: openshift-bootstrap-worker - name: openshift-node-performance-openshift-bootstrap-worker - namespace: openshift-cluster-node-tuning-operator -spec: - profile: - - data: "[main]\nsummary=Openshift node optimized for deterministic performance - at the cost of increased power consumption, focused on low latency network performance. - Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\n\n# The final result of - the include depends on cpu vendor, cpu architecture, and whether the real time - kernel is enabled\n# The first line will be evaluated based on the CPU vendor - and architecture\n# This has three possible results:\n# include=openshift-node-performance-amd-x86;\n# - \ include=openshift-node-performance-arm-aarch64;\n# include=openshift-node-performance-intel-x86;\n# - The second line will be evaluated based on whether the real time kernel is enabled\n# - This has two possible results:\n# openshift-node,cpu-partitioning\n# openshift-node,cpu-partitioning,openshift-node-performance-rt-\ninclude=openshift-node,cpu-partitioning${f:regex_search_ternary:${f:exec:uname:-r}:rt:,openshift-node-performance-rt-openshift-bootstrap-worker:};\n - \ openshift-node-performance-${f:lscpu_check:Vendor ID\\:\\s*GenuineIntel:intel:Vendor - ID\\:\\s*AuthenticAMD:amd:Vendor ID\\:\\s*ARM:arm}-${f:lscpu_check:Architecture\\:\\s*x86_64:x86:Architecture\\:\\s*aarch64:aarch64}-openshift-bootstrap-worker\n\n# - Inheritance of base profiles legend:\n# cpu-partitioning -> network-latency - -> latency-performance\n# https://github.com/redhat-performance/tuned/blob/master/profiles/latency-performance/tuned.conf\n# - https://github.com/redhat-performance/tuned/blob/master/profiles/network-latency/tuned.conf\n# - https://github.com/redhat-performance/tuned/blob/master/profiles/cpu-partitioning/tuned.conf\n\n# - All values are mapped with a comment where a parent profile contains them.\n# - Different values will override the original values in parent profiles.\n\n[variables]\n#> - isolated_cores take a list of ranges; e.g. isolated_cores=2,4-7\n\nisolated_cores=0-1\n\n\nnot_isolated_cores_expanded=${f:cpulist_invert:${isolated_cores_expanded}}\n\n\n[cpu]\n#> - latency-performance\n#> (override)\nforce_latency=cstate.id:1|3\ngovernor=performance\nenergy_perf_bias=performance\nmin_perf_pct=100\n\n\n\n[service]\nservice.stalld=start,enable\n\n\n[vm]\n#> - network-latency\ntransparent_hugepages=never\n\n\n[irqbalance]\n# Disable the - plugin entirely, which was enabled by the parent profile `cpu-partitioning`.\n# - It can be racy if TuneD restarts for whatever reason.\n#> cpu-partitioning\nenabled=false\n\n\n[scheduler]\nruntime=0\ngroup.ksoftirqd=0:f:11:*:ksoftirqd.*\ngroup.rcuc=0:f:11:*:rcuc.*\ngroup.ktimers=0:f:11:*:ktimers.*\n\ndefault_irq_smp_affinity - = ignore\nirq_process=false\n\n\n[sysctl]\n\n#> cpu-partitioning #RealTimeHint\nkernel.hung_task_timeout_secs=600\n#> - cpu-partitioning #RealTimeHint\nkernel.nmi_watchdog=0\n#> RealTimeHint\nkernel.sched_rt_runtime_us=-1\n#> - cpu-partitioning #RealTimeHint\nvm.stat_interval=10\n\n# cpu-partitioning and - RealTimeHint for RHEL disable it (= 0)\n# OCP is too dynamic when partitioning - and needs to evacuate\n#> scheduled timers when starting a guaranteed workload - (= 1)\nkernel.timer_migration=1\n#> network-latency\nnet.ipv4.tcp_fastopen=3\n\n# - If a workload mostly uses anonymous memory and it hits this limit, the entire\n# - working set is buffered for I/O, and any more write buffering would require\n# - swapping, so it's time to throttle writes until I/O can catch up. Workloads\n# - that mostly use file mappings may be able to use even higher values.\n#\n# The - generator of dirty data starts writeback at this percentage (system default\n# - is 20%)\n#> latency-performance\nvm.dirty_ratio=10\n\n# Start background writeback - (via writeback threads) at this percentage (system\n# default is 10%)\n#> latency-performance\nvm.dirty_background_ratio=3\n\n# - The swappiness parameter controls the tendency of the kernel to move\n# processes - out of physical memory and onto the swap disk.\n# 0 tells the kernel to avoid - swapping processes out of physical memory\n# for as long as possible\n# 100 - tells the kernel to aggressively swap processes out of physical memory\n# and - move them to swap cache\n#> latency-performance\nvm.swappiness=10\n\n# also - configured via a sysctl.d file\n# placed here for documentation purposes and - commented out due\n# to a tuned logging bug complaining about duplicate sysctl:\n# - \ https://issues.redhat.com/browse/RHEL-18972\n#> rps configuration\n# net.core.rps_default_mask=${not_isolated_cpumask}\n\n\n[selinux]\n#> - Custom (atomic host)\navc_cache_threshold=8192\n\n\n[net]\nnf_conntrack_hashsize=131072\n\n\n[bootloader]\n# - !! The names are important for Intel and are referenced in openshift-node-performance-intel-x86\n\n# - set empty values to disable RHEL initrd setting in cpu-partitioning\ninitrd_remove_dir=\ninitrd_dst_img=\ninitrd_add_dir=\n\n# - overrides cpu-partitioning cmdline\ncmdline_cpu_part=+nohz=on rcu_nocbs=${isolated_cores} - tuned.non_isolcpus=${not_isolated_cpumask} systemd.cpu_affinity=${not_isolated_cores_expanded}\n\n# - No default value but will be composed conditionally based on platform\ncmdline_iommu=\n\n\ncmdline_isolation=+isolcpus=managed_irq,${isolated_cores}\n\n\n\ncmdline_realtime_nohzfull=+nohz_full=${isolated_cores}\ncmdline_realtime_nosoftlookup=+nosoftlockup\ncmdline_realtime_common=+skew_tick=1 - rcutree.kthread_prio=11\n\n\n\n\n\n\n \n\n\n\n[rtentsk]\n\n\n" - name: openshift-node-performance-openshift-bootstrap-worker - - data: "[main]\nsummary=Real time profile to override unsupported settings\n\n[sysctl]\n#Real - time kernel doesn't support the following kernel parameters.\n#The openshift-node-performance - profile inherits these kernel parameters from the network-latency profile. \n#Therefore, - if the real time kernel is detected they will be dropped, meaning won't be applied.\ndrop=kernel.numa_balancing,net.core.busy_read,net.core.busy_poll\n" - name: openshift-node-performance-rt-openshift-bootstrap-worker - - data: |+ - [main] - summary=Platform specific tuning for AMD x86 - - [bootloader] - cmdline_iommu_amd=iommu=pt - - - cmdline_pstate=amd_pstate=guided - - - - cmdline_realtime_amd=tsc=reliable nmi_watchdog=0 mce=off - - - - - - name: openshift-node-performance-amd-x86-openshift-bootstrap-worker - - data: | - [main] - summary=Platform specific tuning for aarch64 - - [bootloader] - # No cstate for ARM - # No pstate args for ARM - - # aarch64 specific tuning options - cmdline_iommu_arm=iommu.passthrough=1 - name: openshift-node-performance-arm-aarch64-openshift-bootstrap-worker - - data: |+ - [main] - summary=Platform specific tuning for Intel x86 - - [bootloader] - # DO NOT REMOVE THIS BLOCK - # It makes sure the kernel arguments for Intel are applied - # in the order compatible with OCP 4.17 which is important - # for preventing an extra reboot during upgrade - cmdline_cpu_part= - cmdline_iommu_intel= - cmdline_isolation= - cmdline_realtime_nohzfull= - cmdline_realtime_intel= - cmdline_realtime_nosoftlookup= - cmdline_realtime_intel_nmi= - cmdline_realtime_common= - cmdline_power_performance= - cmdline_power_performance_intel= - cmdline_idle_poll= - cmdline_idle_poll_intel= - cmdline_hugepages= - cmdline_pstate= - - # Here comes the Intel specific tuning - - cmdline_iommu_intel=intel_iommu=on iommu=pt - - - cmdline_realtime_intel=tsc=reliable - cmdline_realtime_intel_nmi=nmi_watchdog=0 mce=off - - - - - - - - cmdline_pstate=intel_pstate=${f:intel_recommended_pstate} - - - name: openshift-node-performance-intel-x86-openshift-bootstrap-worker - recommend: - - machineConfigLabels: - machineconfiguration.openshift.io/role: worker - operand: - tunedConfig: - reapply_sysctl: null - priority: 20 - profile: openshift-node-performance-openshift-bootstrap-worker -status: {} diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml index 8afae2ee87..8ab2a3d11a 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -50,7 +50,7 @@ spec: path: /usr/local/bin/clear-irqbalance-banned-cpus.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgojIFdlIHNob3VsZCBjb3B5IHBhc3RlIHRoZSBkZWZhdWx0IHJ1bnRpbWUgYmVjYXVzZSB0aGlzIHNuaXBwZXQgd2lsbCBvdmVycmlkZSB0aGUgd2hvbGUgcnVudGltZXMgc2VjdGlvbgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLnJ1bmNdCnJ1bnRpbWVfcGF0aCA9ICIiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCgojIFRoZSBDUkktTyB3aWxsIGNoZWNrIHRoZSBhbGxvd2VkX2Fubm90YXRpb25zIHVuZGVyIHRoZSBydW50aW1lIGhhbmRsZXIgYW5kIGFwcGx5IGhpZ2gtcGVyZm9ybWFuY2UgaG9va3Mgd2hlbiBvbmUgb2YKIyBoaWdoLXBlcmZvcm1hbmNlIGFubm90YXRpb25zIHByZXNlbnRzIHVuZGVyIGl0LgojIFdlIHNob3VsZCBwcm92aWRlIHRoZSBydW50aW1lX3BhdGggYmVjYXVzZSB3ZSBuZWVkIHRvIGluZm9ybSB0aGF0IHdlIHdhbnQgdG8gcmUtdXNlIHJ1bmMgYmluYXJ5IGFuZCB3ZQojIGRvIG5vdCBoYXZlIGhpZ2gtcGVyZm9ybWFuY2UgYmluYXJ5IHVuZGVyIHRoZSAkUEFUSCB0aGF0IHdpbGwgcG9pbnQgdG8gaXQuCltjcmlvLnJ1bnRpbWUucnVudGltZXMuaGlnaC1wZXJmb3JtYW5jZV0KcnVudGltZV9wYXRoID0gIi9iaW4vcnVuYyIKcnVudGltZV90eXBlID0gIm9jaSIKcnVudGltZV9yb290ID0gIi9ydW4vcnVuYyIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg== + source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K verification: {} group: {} mode: 420 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml index 67abea13eb..bb1c691342 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -50,7 +50,7 @@ spec: path: /usr/local/bin/clear-irqbalance-banned-cpus.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgojIFdlIHNob3VsZCBjb3B5IHBhc3RlIHRoZSBkZWZhdWx0IHJ1bnRpbWUgYmVjYXVzZSB0aGlzIHNuaXBwZXQgd2lsbCBvdmVycmlkZSB0aGUgd2hvbGUgcnVudGltZXMgc2VjdGlvbgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLnJ1bmNdCnJ1bnRpbWVfcGF0aCA9ICIiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCgojIFRoZSBDUkktTyB3aWxsIGNoZWNrIHRoZSBhbGxvd2VkX2Fubm90YXRpb25zIHVuZGVyIHRoZSBydW50aW1lIGhhbmRsZXIgYW5kIGFwcGx5IGhpZ2gtcGVyZm9ybWFuY2UgaG9va3Mgd2hlbiBvbmUgb2YKIyBoaWdoLXBlcmZvcm1hbmNlIGFubm90YXRpb25zIHByZXNlbnRzIHVuZGVyIGl0LgojIFdlIHNob3VsZCBwcm92aWRlIHRoZSBydW50aW1lX3BhdGggYmVjYXVzZSB3ZSBuZWVkIHRvIGluZm9ybSB0aGF0IHdlIHdhbnQgdG8gcmUtdXNlIHJ1bmMgYmluYXJ5IGFuZCB3ZQojIGRvIG5vdCBoYXZlIGhpZ2gtcGVyZm9ybWFuY2UgYmluYXJ5IHVuZGVyIHRoZSAkUEFUSCB0aGF0IHdpbGwgcG9pbnQgdG8gaXQuCltjcmlvLnJ1bnRpbWUucnVudGltZXMuaGlnaC1wZXJmb3JtYW5jZV0KcnVudGltZV9wYXRoID0gIi9iaW4vcnVuYyIKcnVudGltZV90eXBlID0gIm9jaSIKcnVudGltZV9yb290ID0gIi9ydW4vcnVuYyIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg== + source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K verification: {} group: {} mode: 420 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml index 8afae2ee87..8ab2a3d11a 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -50,7 +50,7 @@ spec: path: /usr/local/bin/clear-irqbalance-banned-cpus.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgojIFdlIHNob3VsZCBjb3B5IHBhc3RlIHRoZSBkZWZhdWx0IHJ1bnRpbWUgYmVjYXVzZSB0aGlzIHNuaXBwZXQgd2lsbCBvdmVycmlkZSB0aGUgd2hvbGUgcnVudGltZXMgc2VjdGlvbgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLnJ1bmNdCnJ1bnRpbWVfcGF0aCA9ICIiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCgojIFRoZSBDUkktTyB3aWxsIGNoZWNrIHRoZSBhbGxvd2VkX2Fubm90YXRpb25zIHVuZGVyIHRoZSBydW50aW1lIGhhbmRsZXIgYW5kIGFwcGx5IGhpZ2gtcGVyZm9ybWFuY2UgaG9va3Mgd2hlbiBvbmUgb2YKIyBoaWdoLXBlcmZvcm1hbmNlIGFubm90YXRpb25zIHByZXNlbnRzIHVuZGVyIGl0LgojIFdlIHNob3VsZCBwcm92aWRlIHRoZSBydW50aW1lX3BhdGggYmVjYXVzZSB3ZSBuZWVkIHRvIGluZm9ybSB0aGF0IHdlIHdhbnQgdG8gcmUtdXNlIHJ1bmMgYmluYXJ5IGFuZCB3ZQojIGRvIG5vdCBoYXZlIGhpZ2gtcGVyZm9ybWFuY2UgYmluYXJ5IHVuZGVyIHRoZSAkUEFUSCB0aGF0IHdpbGwgcG9pbnQgdG8gaXQuCltjcmlvLnJ1bnRpbWUucnVudGltZXMuaGlnaC1wZXJmb3JtYW5jZV0KcnVudGltZV9wYXRoID0gIi9iaW4vcnVuYyIKcnVudGltZV90eXBlID0gIm9jaSIKcnVudGltZV9yb290ID0gIi9ydW4vcnVuYyIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg== + source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K verification: {} group: {} mode: 420 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml index 67abea13eb..bb1c691342 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -50,7 +50,7 @@ spec: path: /usr/local/bin/clear-irqbalance-banned-cpus.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgojIFdlIHNob3VsZCBjb3B5IHBhc3RlIHRoZSBkZWZhdWx0IHJ1bnRpbWUgYmVjYXVzZSB0aGlzIHNuaXBwZXQgd2lsbCBvdmVycmlkZSB0aGUgd2hvbGUgcnVudGltZXMgc2VjdGlvbgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLnJ1bmNdCnJ1bnRpbWVfcGF0aCA9ICIiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCgojIFRoZSBDUkktTyB3aWxsIGNoZWNrIHRoZSBhbGxvd2VkX2Fubm90YXRpb25zIHVuZGVyIHRoZSBydW50aW1lIGhhbmRsZXIgYW5kIGFwcGx5IGhpZ2gtcGVyZm9ybWFuY2UgaG9va3Mgd2hlbiBvbmUgb2YKIyBoaWdoLXBlcmZvcm1hbmNlIGFubm90YXRpb25zIHByZXNlbnRzIHVuZGVyIGl0LgojIFdlIHNob3VsZCBwcm92aWRlIHRoZSBydW50aW1lX3BhdGggYmVjYXVzZSB3ZSBuZWVkIHRvIGluZm9ybSB0aGF0IHdlIHdhbnQgdG8gcmUtdXNlIHJ1bmMgYmluYXJ5IGFuZCB3ZQojIGRvIG5vdCBoYXZlIGhpZ2gtcGVyZm9ybWFuY2UgYmluYXJ5IHVuZGVyIHRoZSAkUEFUSCB0aGF0IHdpbGwgcG9pbnQgdG8gaXQuCltjcmlvLnJ1bnRpbWUucnVudGltZXMuaGlnaC1wZXJmb3JtYW5jZV0KcnVudGltZV9wYXRoID0gIi9iaW4vcnVuYyIKcnVudGltZV90eXBlID0gIm9jaSIKcnVudGltZV9yb290ID0gIi9ydW4vcnVuYyIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg== + source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K verification: {} group: {} mode: 420 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml index 50e3aca32e..89bff72034 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml @@ -49,7 +49,7 @@ spec: path: /usr/local/bin/clear-irqbalance-banned-cpus.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMC0xIgoKCgojIFdlIHNob3VsZCBjb3B5IHBhc3RlIHRoZSBkZWZhdWx0IHJ1bnRpbWUgYmVjYXVzZSB0aGlzIHNuaXBwZXQgd2lsbCBvdmVycmlkZSB0aGUgd2hvbGUgcnVudGltZXMgc2VjdGlvbgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLnJ1bmNdCnJ1bnRpbWVfcGF0aCA9ICIiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCgojIFRoZSBDUkktTyB3aWxsIGNoZWNrIHRoZSBhbGxvd2VkX2Fubm90YXRpb25zIHVuZGVyIHRoZSBydW50aW1lIGhhbmRsZXIgYW5kIGFwcGx5IGhpZ2gtcGVyZm9ybWFuY2UgaG9va3Mgd2hlbiBvbmUgb2YKIyBoaWdoLXBlcmZvcm1hbmNlIGFubm90YXRpb25zIHByZXNlbnRzIHVuZGVyIGl0LgojIFdlIHNob3VsZCBwcm92aWRlIHRoZSBydW50aW1lX3BhdGggYmVjYXVzZSB3ZSBuZWVkIHRvIGluZm9ybSB0aGF0IHdlIHdhbnQgdG8gcmUtdXNlIHJ1bmMgYmluYXJ5IGFuZCB3ZQojIGRvIG5vdCBoYXZlIGhpZ2gtcGVyZm9ybWFuY2UgYmluYXJ5IHVuZGVyIHRoZSAkUEFUSCB0aGF0IHdpbGwgcG9pbnQgdG8gaXQuCltjcmlvLnJ1bnRpbWUucnVudGltZXMuaGlnaC1wZXJmb3JtYW5jZV0KcnVudGltZV9wYXRoID0gIi9iaW4vcnVuYyIKcnVudGltZV90eXBlID0gIm9jaSIKcnVudGltZV9yb290ID0gIi9ydW4vcnVuYyIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg== + source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMC0xIgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K verification: {} group: {} mode: 420 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml index 2fd75bd352..957da40d43 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml @@ -50,7 +50,7 @@ spec: path: /usr/local/bin/clear-irqbalance-banned-cpus.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMCIKCgoKIyBXZSBzaG91bGQgY29weSBwYXN0ZSB0aGUgZGVmYXVsdCBydW50aW1lIGJlY2F1c2UgdGhpcyBzbmlwcGV0IHdpbGwgb3ZlcnJpZGUgdGhlIHdob2xlIHJ1bnRpbWVzIHNlY3Rpb24KW2NyaW8ucnVudGltZS5ydW50aW1lcy5ydW5jXQpydW50aW1lX3BhdGggPSAiIgpydW50aW1lX3R5cGUgPSAib2NpIgpydW50aW1lX3Jvb3QgPSAiL3J1bi9ydW5jIgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCnJ1bnRpbWVfcGF0aCA9ICIvYmluL3J1bmMiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCmFsbG93ZWRfYW5ub3RhdGlvbnMgPSBbImNwdS1sb2FkLWJhbGFuY2luZy5jcmlvLmlvIiwgImNwdS1xdW90YS5jcmlvLmlvIiwgImlycS1sb2FkLWJhbGFuY2luZy5jcmlvLmlvIiwgImNwdS1jLXN0YXRlcy5jcmlvLmlvIiwgImNwdS1mcmVxLWdvdmVybm9yLmNyaW8uaW8iXQo= + source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMCIKCgoKCiMgVGhlIENSSS1PIHdpbGwgY2hlY2sgdGhlIGFsbG93ZWRfYW5ub3RhdGlvbnMgdW5kZXIgdGhlIHJ1bnRpbWUgaGFuZGxlciBhbmQgYXBwbHkgaGlnaC1wZXJmb3JtYW5jZSBob29rcyB3aGVuIG9uZSBvZgojIGhpZ2gtcGVyZm9ybWFuY2UgYW5ub3RhdGlvbnMgcHJlc2VudHMgdW5kZXIgaXQuCiMgV2Ugc2hvdWxkIHByb3ZpZGUgdGhlIHJ1bnRpbWVfcGF0aCBiZWNhdXNlIHdlIG5lZWQgdG8gaW5mb3JtIHRoYXQgd2Ugd2FudCB0byByZS11c2UgcnVuYyBiaW5hcnkgYW5kIHdlCiMgZG8gbm90IGhhdmUgaGlnaC1wZXJmb3JtYW5jZSBiaW5hcnkgdW5kZXIgdGhlICRQQVRIIHRoYXQgd2lsbCBwb2ludCB0byBpdC4KW2NyaW8ucnVudGltZS5ydW50aW1lcy5oaWdoLXBlcmZvcm1hbmNlXQppbmhlcml0X2RlZmF1bHRfcnVudGltZSA9IHRydWUKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg== verification: {} group: {} mode: 420 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml index f0d86ce9c2..10b7f417e8 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml @@ -49,7 +49,7 @@ spec: path: /usr/local/bin/clear-irqbalance-banned-cpus.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMCIKCgoKIyBXZSBzaG91bGQgY29weSBwYXN0ZSB0aGUgZGVmYXVsdCBydW50aW1lIGJlY2F1c2UgdGhpcyBzbmlwcGV0IHdpbGwgb3ZlcnJpZGUgdGhlIHdob2xlIHJ1bnRpbWVzIHNlY3Rpb24KW2NyaW8ucnVudGltZS5ydW50aW1lcy5ydW5jXQpydW50aW1lX3BhdGggPSAiIgpydW50aW1lX3R5cGUgPSAib2NpIgpydW50aW1lX3Jvb3QgPSAiL3J1bi9ydW5jIgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCnJ1bnRpbWVfcGF0aCA9ICIvYmluL3J1bmMiCnJ1bnRpbWVfdHlwZSA9ICJvY2kiCnJ1bnRpbWVfcm9vdCA9ICIvcnVuL3J1bmMiCmFsbG93ZWRfYW5ub3RhdGlvbnMgPSBbImNwdS1sb2FkLWJhbGFuY2luZy5jcmlvLmlvIiwgImNwdS1xdW90YS5jcmlvLmlvIiwgImlycS1sb2FkLWJhbGFuY2luZy5jcmlvLmlvIiwgImNwdS1jLXN0YXRlcy5jcmlvLmlvIiwgImNwdS1mcmVxLWdvdmVybm9yLmNyaW8uaW8iXQo= + source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMCIKCgoKCiMgVGhlIENSSS1PIHdpbGwgY2hlY2sgdGhlIGFsbG93ZWRfYW5ub3RhdGlvbnMgdW5kZXIgdGhlIHJ1bnRpbWUgaGFuZGxlciBhbmQgYXBwbHkgaGlnaC1wZXJmb3JtYW5jZSBob29rcyB3aGVuIG9uZSBvZgojIGhpZ2gtcGVyZm9ybWFuY2UgYW5ub3RhdGlvbnMgcHJlc2VudHMgdW5kZXIgaXQuCiMgV2Ugc2hvdWxkIHByb3ZpZGUgdGhlIHJ1bnRpbWVfcGF0aCBiZWNhdXNlIHdlIG5lZWQgdG8gaW5mb3JtIHRoYXQgd2Ugd2FudCB0byByZS11c2UgcnVuYyBiaW5hcnkgYW5kIHdlCiMgZG8gbm90IGhhdmUgaGlnaC1wZXJmb3JtYW5jZSBiaW5hcnkgdW5kZXIgdGhlICRQQVRIIHRoYXQgd2lsbCBwb2ludCB0byBpdC4KW2NyaW8ucnVudGltZS5ydW50aW1lcy5oaWdoLXBlcmZvcm1hbmNlXQppbmhlcml0X2RlZmF1bHRfcnVudGltZSA9IHRydWUKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg== verification: {} group: {} mode: 420