Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions assets/performanceprofile/tuned/openshift-node-performance
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
[main]
summary=Openshift node optimized for deterministic performance at the cost of increased power consumption, focused on low latency network performance. Based on Tuned 2.11 and Cluster node tuning (oc 4.5)
include=openshift-node,cpu-partitioning

# In case real time kernel is enabled the following include section will be evaluated as:
# include=openshift-node,cpu-partitioning,openshift-node-performance-rt-<PerformanceProfile name>
# Otherwise:
# include=openshift-node,cpu-partitioning
include=openshift-node,cpu-partitioning${f:regex_search_ternary:${f:exec:uname:-r}:rt:,openshift-node-performance-rt-{{.PerformanceProfileName}}:}


# Inheritance of base profiles legend:
# cpu-partitioning -> network-latency -> latency-performance
Expand Down Expand Up @@ -78,10 +84,6 @@ vm.stat_interval=10
#> scheduled timers when starting a guaranteed workload (= 1)
kernel.timer_migration=1
#> network-latency
# TODO once rhbz#2120328 is solved: kernel.numa_balancing, net.core.busy_read and net.core.busy_poll do not exist on RT kernels
kernel.numa_balancing=0
net.core.busy_read=50
net.core.busy_poll=50
net.ipv4.tcp_fastopen=3

# If a workload mostly uses anonymous memory and it hits this limit, the entire
Expand Down
8 changes: 8 additions & 0 deletions assets/performanceprofile/tuned/openshift-node-performance-rt
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
[main]
summary=Real time profile to override unsupported settings

[sysctl]
#Real time kernel doesn't support the following kernel parameters.
#The openshift-node-performance profile inherits these kernel parameters from the network-latency profile.
#Therefore, if the real time kernel is detected they will be dropped, meaning won't be applied.
drop=kernel.numa_balancing,net.core.busy_read,net.core.busy_poll
4 changes: 3 additions & 1 deletion hack/render-sync.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,6 @@ function rendersync() {

rendersync manual-cluster/performance base/performance default
rendersync bootstrap-cluster/performance pinned-cluster/default bootstrap/no-mcp
rendersync bootstrap-cluster/performance pinned-cluster/default bootstrap-cluster/extra-mcp bootstrap/extra-mcp
rendersync bootstrap-cluster/performance pinned-cluster/default bootstrap-cluster/extra-mcp bootstrap/extra-mcp
rendersync --owner-ref none -- base/performance manual-cluster/performance no-ref
rendersync --owner-ref none -- base/performance manual-cluster/cpuFrequency default/cpuFrequency

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't we want owner reference in this case? it should be only no-ref without the reference (hence the name)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This syncs the manifests for this test https://github.com/openshift/cluster-node-tuning-operator/blob/master/test/e2e/performanceprofile/functests-render-command/1_render_command/render_test.go#L115

And it uses no owner references. Probably copy&paste. I believe we should default to having the references most of the time, but I do not think we should fix that as part of this PR.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fair enough

Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ const (
NamespaceNodeTuningOperator = "openshift-cluster-node-tuning-operator"
// ProfileNamePerformance defines the performance tuned profile name
ProfileNamePerformance = "openshift-node-performance"
// ProfileNamePerformanceRT defines the performance real time tuned profile name
ProfileNamePerformanceRT = "openshift-node-performance-rt"
)

const (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ const (
templateReservedCpuMaxFreq = "ReservedCpuMaxFreq"
templateIsolatedCpuList = "IsolatedCpuList"
templateReservedCpuList = "ReservedCpuList"
templatePerformanceProfileName = "PerformanceProfileName"
)

func new(name string, profiles []tunedv1.TunedProfile, recommends []tunedv1.TunedRecommend) *tunedv1.Tuned {
Expand All @@ -60,6 +61,8 @@ func new(name string, profiles []tunedv1.TunedProfile, recommends []tunedv1.Tune
func NewNodePerformance(profile *performancev2.PerformanceProfile) (*tunedv1.Tuned, error) {
templateArgs := make(map[string]interface{})

templateArgs[templatePerformanceProfileName] = profile.Name

if profile.Spec.CPU.Isolated != nil {
templateArgs[templateIsolatedCpus] = string(*profile.Spec.CPU.Isolated)
if profile.Spec.CPU.BalanceIsolated != nil && !*profile.Spec.CPU.BalanceIsolated {
Expand Down Expand Up @@ -215,12 +218,21 @@ func NewNodePerformance(profile *performancev2.PerformanceProfile) (*tunedv1.Tun
return nil, err
}

RealTimeKernelProfileData, err := getProfileData(filepath.Join("tuned", components.ProfileNamePerformanceRT), templateArgs)
if err != nil {
return nil, err
}
name := components.GetComponentName(profile.Name, components.ProfileNamePerformance)
RealTimeKernelProfileName := components.GetComponentName(profile.Name, components.ProfileNamePerformanceRT)
profiles := []tunedv1.TunedProfile{
{
Name: &name,
Data: &profileData,
},
{
Name: &RealTimeKernelProfileName,
Data: &RealTimeKernelProfileData,
},
}

priority := uint64(20)
Expand Down
25 changes: 25 additions & 0 deletions test/e2e/performanceprofile/functests/1_performance/performance.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,21 @@ var _ = Describe("[rfe_id:27368][performance]", Ordered, func() {
Expect(strings.TrimSpace(activeProfileName)).To(Equal(tunedExpectedName), "active profile name mismatch got %q expected %q", activeProfileName, tunedExpectedName)
}
})

It("Tuned profile shouldn't be degraded", func() {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should not be degraded after what? in this test you check a condition (this part LGTM) but there is no obvious link to how to get to this state. IOW, does this test depend on a specific order? which part of the system creates the conditions we're checking?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should not be degraded after tuned profile has been applied to the system.
The object will be degraded if error messages are observed by the tuned daemon when applying the TuneD daemon profile.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ffromani Any error in the tuned log causes the state to switch to degraded. This fix removes a typical case - a sysctl that does not exist for the given kernel.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure, and what in the test causes any change in the system which could lead to Degraded state?
After reading the surrounding tests (something that could have pointed out earlier) I see other tests do in this debatable way, so this is a more general issue which is out of the scope here.

for _, node := range workerRTNodes {
key := types.NamespacedName{
Name: node.Name,
Namespace: components.NamespaceNodeTuningOperator,
}
tunedProfile := &tunedv1.Profile{}
err := testclient.Client.Get(context.TODO(), key, tunedProfile)
Expect(err).ToNot(HaveOccurred(), "Failed to get the Tuned profile for node %s", node.Name)
degradedCondition := findCondition(tunedProfile.Status.Conditions, "Degraded")
Expect(degradedCondition).ToNot(BeNil(), "Degraded condition not found in Tuned profile status")
Expect(degradedCondition.Status).To(Equal(corev1.ConditionFalse), "Tuned profile is degraded")
}
})
})

Context("Pre boot tuning adjusted by tuned ", func() {
Expand Down Expand Up @@ -1345,3 +1360,13 @@ func makeDevRPSMap(content string) map[string]string {
}
return devRPSMap
}

// Helper function to find a condition in the status.conditions slice by type
func findCondition(conditions []tunedv1.ProfileStatusCondition, conditionType string) *tunedv1.ProfileStatusCondition {
for _, condition := range conditions {
if string(condition.Type) == conditionType {
return &condition
}
}
return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ spec:
profile:
- data: "[main]\nsummary=Openshift node optimized for deterministic performance
at the cost of increased power consumption, focused on low latency network performance.
Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\ninclude=openshift-node,cpu-partitioning\n\n#
Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\n\n# In case real time
kernel is enabled the following include section will be evaluated as:\n# include=openshift-node,cpu-partitioning,openshift-node-performance-rt-<PerformanceProfile
name>\n# Otherwise:\n# include=openshift-node,cpu-partitioning\ninclude=openshift-node,cpu-partitioning${f:regex_search_ternary:${f:exec:uname:-r}:rt:,openshift-node-performance-rt-openshift-bootstrap-master:}\n\n\n#
Inheritance of base profiles legend:\n# cpu-partitioning -> network-latency
-> latency-performance\n# https://github.com/redhat-performance/tuned/blob/master/profiles/latency-performance/tuned.conf\n#
https://github.com/redhat-performance/tuned/blob/master/profiles/network-latency/tuned.conf\n#
Expand All @@ -27,9 +29,7 @@ spec:
cpu-partitioning #RealTimeHint\nvm.stat_interval=10\n\n# cpu-partitioning and
RealTimeHint for RHEL disable it (= 0)\n# OCP is too dynamic when partitioning
and needs to evacuate\n#> scheduled timers when starting a guaranteed workload
(= 1)\nkernel.timer_migration=1\n#> network-latency\n# TODO once rhbz#2120328
is solved: kernel.numa_balancing, net.core.busy_read and net.core.busy_poll
do not exist on RT kernels\nkernel.numa_balancing=0\nnet.core.busy_read=50\nnet.core.busy_poll=50\nnet.ipv4.tcp_fastopen=3\n\n#
(= 1)\nkernel.timer_migration=1\n#> network-latency\nnet.ipv4.tcp_fastopen=3\n\n#
If a workload mostly uses anonymous memory and it hits this limit, the entire\n#
working set is buffered for I/O, and any more write buffering would require\n#
swapping, so it's time to throttle writes until I/O can catch up. Workloads\n#
Expand All @@ -53,6 +53,11 @@ spec:
tsc=reliable nosoftlockup nmi_watchdog=0 mce=off skew_tick=1 rcutree.kthread_prio=11\n\n\n\n\n\n\n
\n\n\n\n\ncmdline_pstate=+intel_pstate=disable\n\n\n[rtentsk]\n\n\n"
name: openshift-node-performance-openshift-bootstrap-master
- data: "[main]\nsummary=Real time profile to override unsupported settings\n\n[sysctl]\n#Real
time kernel doesn't support the following kernel parameters.\n#The openshift-node-performance
profile inherits these kernel parameters from the network-latency profile. \n#Therefore,
if the real time kernel is detected they will be dropped, meaning won't be applied.\ndrop=kernel.numa_balancing,net.core.busy_read,net.core.busy_poll\n"
name: openshift-node-performance-rt-openshift-bootstrap-master
recommend:
- machineConfigLabels:
machineconfiguration.openshift.io/role: master
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ spec:
profile:
- data: "[main]\nsummary=Openshift node optimized for deterministic performance
at the cost of increased power consumption, focused on low latency network performance.
Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\ninclude=openshift-node,cpu-partitioning\n\n#
Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\n\n# In case real time
kernel is enabled the following include section will be evaluated as:\n# include=openshift-node,cpu-partitioning,openshift-node-performance-rt-<PerformanceProfile
name>\n# Otherwise:\n# include=openshift-node,cpu-partitioning\ninclude=openshift-node,cpu-partitioning${f:regex_search_ternary:${f:exec:uname:-r}:rt:,openshift-node-performance-rt-openshift-bootstrap-worker:}\n\n\n#
Inheritance of base profiles legend:\n# cpu-partitioning -> network-latency
-> latency-performance\n# https://github.com/redhat-performance/tuned/blob/master/profiles/latency-performance/tuned.conf\n#
https://github.com/redhat-performance/tuned/blob/master/profiles/network-latency/tuned.conf\n#
Expand All @@ -27,9 +29,7 @@ spec:
cpu-partitioning #RealTimeHint\nvm.stat_interval=10\n\n# cpu-partitioning and
RealTimeHint for RHEL disable it (= 0)\n# OCP is too dynamic when partitioning
and needs to evacuate\n#> scheduled timers when starting a guaranteed workload
(= 1)\nkernel.timer_migration=1\n#> network-latency\n# TODO once rhbz#2120328
is solved: kernel.numa_balancing, net.core.busy_read and net.core.busy_poll
do not exist on RT kernels\nkernel.numa_balancing=0\nnet.core.busy_read=50\nnet.core.busy_poll=50\nnet.ipv4.tcp_fastopen=3\n\n#
(= 1)\nkernel.timer_migration=1\n#> network-latency\nnet.ipv4.tcp_fastopen=3\n\n#
If a workload mostly uses anonymous memory and it hits this limit, the entire\n#
working set is buffered for I/O, and any more write buffering would require\n#
swapping, so it's time to throttle writes until I/O can catch up. Workloads\n#
Expand All @@ -53,6 +53,11 @@ spec:
tsc=reliable nosoftlockup nmi_watchdog=0 mce=off skew_tick=1 rcutree.kthread_prio=11\n\n\n\n\n\n\n
\n\n\n\n\ncmdline_pstate=+intel_pstate=disable\n\n\n[rtentsk]\n\n\n"
name: openshift-node-performance-openshift-bootstrap-worker
- data: "[main]\nsummary=Real time profile to override unsupported settings\n\n[sysctl]\n#Real
time kernel doesn't support the following kernel parameters.\n#The openshift-node-performance
profile inherits these kernel parameters from the network-latency profile. \n#Therefore,
if the real time kernel is detected they will be dropped, meaning won't be applied.\ndrop=kernel.numa_balancing,net.core.busy_read,net.core.busy_poll\n"
name: openshift-node-performance-rt-openshift-bootstrap-worker
recommend:
- machineConfigLabels:
machineconfiguration.openshift.io/role: worker
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ spec:
profile:
- data: "[main]\nsummary=Openshift node optimized for deterministic performance
at the cost of increased power consumption, focused on low latency network performance.
Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\ninclude=openshift-node,cpu-partitioning\n\n#
Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\n\n# In case real time
kernel is enabled the following include section will be evaluated as:\n# include=openshift-node,cpu-partitioning,openshift-node-performance-rt-<PerformanceProfile
name>\n# Otherwise:\n# include=openshift-node,cpu-partitioning\ninclude=openshift-node,cpu-partitioning${f:regex_search_ternary:${f:exec:uname:-r}:rt:,openshift-node-performance-rt-openshift-bootstrap-master:}\n\n\n#
Inheritance of base profiles legend:\n# cpu-partitioning -> network-latency
-> latency-performance\n# https://github.com/redhat-performance/tuned/blob/master/profiles/latency-performance/tuned.conf\n#
https://github.com/redhat-performance/tuned/blob/master/profiles/network-latency/tuned.conf\n#
Expand All @@ -27,9 +29,7 @@ spec:
cpu-partitioning #RealTimeHint\nvm.stat_interval=10\n\n# cpu-partitioning and
RealTimeHint for RHEL disable it (= 0)\n# OCP is too dynamic when partitioning
and needs to evacuate\n#> scheduled timers when starting a guaranteed workload
(= 1)\nkernel.timer_migration=1\n#> network-latency\n# TODO once rhbz#2120328
is solved: kernel.numa_balancing, net.core.busy_read and net.core.busy_poll
do not exist on RT kernels\nkernel.numa_balancing=0\nnet.core.busy_read=50\nnet.core.busy_poll=50\nnet.ipv4.tcp_fastopen=3\n\n#
(= 1)\nkernel.timer_migration=1\n#> network-latency\nnet.ipv4.tcp_fastopen=3\n\n#
If a workload mostly uses anonymous memory and it hits this limit, the entire\n#
working set is buffered for I/O, and any more write buffering would require\n#
swapping, so it's time to throttle writes until I/O can catch up. Workloads\n#
Expand All @@ -53,6 +53,11 @@ spec:
tsc=reliable nosoftlockup nmi_watchdog=0 mce=off skew_tick=1 rcutree.kthread_prio=11\n\n\n\n\n\n\n
\n\n\n\n\ncmdline_pstate=+intel_pstate=disable\n\n\n[rtentsk]\n\n\n"
name: openshift-node-performance-openshift-bootstrap-master
- data: "[main]\nsummary=Real time profile to override unsupported settings\n\n[sysctl]\n#Real
time kernel doesn't support the following kernel parameters.\n#The openshift-node-performance
profile inherits these kernel parameters from the network-latency profile. \n#Therefore,
if the real time kernel is detected they will be dropped, meaning won't be applied.\ndrop=kernel.numa_balancing,net.core.busy_read,net.core.busy_poll\n"
name: openshift-node-performance-rt-openshift-bootstrap-master
recommend:
- machineConfigLabels:
machineconfiguration.openshift.io/role: master
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ spec:
profile:
- data: "[main]\nsummary=Openshift node optimized for deterministic performance
at the cost of increased power consumption, focused on low latency network performance.
Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\ninclude=openshift-node,cpu-partitioning\n\n#
Based on Tuned 2.11 and Cluster node tuning (oc 4.5)\n\n# In case real time
kernel is enabled the following include section will be evaluated as:\n# include=openshift-node,cpu-partitioning,openshift-node-performance-rt-<PerformanceProfile
name>\n# Otherwise:\n# include=openshift-node,cpu-partitioning\ninclude=openshift-node,cpu-partitioning${f:regex_search_ternary:${f:exec:uname:-r}:rt:,openshift-node-performance-rt-openshift-bootstrap-worker:}\n\n\n#
Inheritance of base profiles legend:\n# cpu-partitioning -> network-latency
-> latency-performance\n# https://github.com/redhat-performance/tuned/blob/master/profiles/latency-performance/tuned.conf\n#
https://github.com/redhat-performance/tuned/blob/master/profiles/network-latency/tuned.conf\n#
Expand All @@ -27,9 +29,7 @@ spec:
cpu-partitioning #RealTimeHint\nvm.stat_interval=10\n\n# cpu-partitioning and
RealTimeHint for RHEL disable it (= 0)\n# OCP is too dynamic when partitioning
and needs to evacuate\n#> scheduled timers when starting a guaranteed workload
(= 1)\nkernel.timer_migration=1\n#> network-latency\n# TODO once rhbz#2120328
is solved: kernel.numa_balancing, net.core.busy_read and net.core.busy_poll
do not exist on RT kernels\nkernel.numa_balancing=0\nnet.core.busy_read=50\nnet.core.busy_poll=50\nnet.ipv4.tcp_fastopen=3\n\n#
(= 1)\nkernel.timer_migration=1\n#> network-latency\nnet.ipv4.tcp_fastopen=3\n\n#
If a workload mostly uses anonymous memory and it hits this limit, the entire\n#
working set is buffered for I/O, and any more write buffering would require\n#
swapping, so it's time to throttle writes until I/O can catch up. Workloads\n#
Expand All @@ -53,6 +53,11 @@ spec:
tsc=reliable nosoftlockup nmi_watchdog=0 mce=off skew_tick=1 rcutree.kthread_prio=11\n\n\n\n\n\n\n
\n\n\n\n\ncmdline_pstate=+intel_pstate=disable\n\n\n[rtentsk]\n\n\n"
name: openshift-node-performance-openshift-bootstrap-worker
- data: "[main]\nsummary=Real time profile to override unsupported settings\n\n[sysctl]\n#Real
time kernel doesn't support the following kernel parameters.\n#The openshift-node-performance
profile inherits these kernel parameters from the network-latency profile. \n#Therefore,
if the real time kernel is detected they will be dropped, meaning won't be applied.\ndrop=kernel.numa_balancing,net.core.busy_read,net.core.busy_poll\n"
name: openshift-node-performance-rt-openshift-bootstrap-worker
recommend:
- machineConfigLabels:
machineconfiguration.openshift.io/role: worker
Expand Down
Loading