Codify performance manifests #24
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| non_iso_cpumask="" | ||
| cpu_affinity="" | ||
|
|
||
| get_reserved_cores() { | ||
| cores=() | ||
| while read part; do | ||
| if [[ $part =~ - ]]; then | ||
| cores+=($(seq ${part/-/ })) | ||
| elif [[ $part =~ , ]]; then | ||
| continue | ||
| else | ||
| cores+=($part) | ||
| fi | ||
| done < <( echo ${NON_ISOLATED_CPUS} | tr ',' '\n' ) | ||
| } | ||
|
|
||
| # $1 - 0 for irq balance banned cpus masking , 1 for non isolated cpus masking | ||
| get_cpu_mask() { | ||
| if [ "$1" = "1" ]; then | ||
| mask=( 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ) | ||
| else | ||
| mask=( 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ) | ||
| fi | ||
| get_reserved_cores | ||
| for core in ${cores[*]}; do | ||
| mask[$core]=$1 | ||
| done | ||
| cpumaskBinary=`echo ${mask[@]}| rev` | ||
| cpumaskBinary=${cpumaskBinary//[[:space:]]/} | ||
| non_iso_cpumask=`printf '%08x\n' "$((2#$cpumaskBinary))"` | ||
| } | ||
|
|
||
| get_cpu_affinity() { | ||
| cpu_affinity="" | ||
| get_reserved_cores | ||
| for core in ${cores[*]}; do | ||
| cpu_affinity+=" $core" | ||
| done | ||
| echo "CPU Affinity set to $cpu_affinity" | ||
| } | ||
|
|
||
| # TODO - find a more robust approach than keeping the last timestamp | ||
| RHCOS_OSTREE_PATH=$(ls -td /boot/ostree/*/ | head -1) | ||
| RHCOS_OSTREE_BOOTLOADER_PATH=${RHCOS_OSTREE_PATH#"/boot"} | ||
| INITRD_GENERATION_DIR="/root/initrd" | ||
| INITRD_NEW_IMAGE="${RHCOS_OSTREE_PATH}/iso_initrd.img" | ||
|
|
||
| # TODO: improve check for applied configuration | ||
| if [ -f ${INITRD_NEW_IMAGE} ] && grep -qsR "iso_initrd.img" "/boot/loader/entries/"; then | ||
| echo "Pre boot tuning configuration already applied" | ||
| echo "Setting kernel rcuo* threads to the housekeeping cpus" | ||
| get_cpu_mask 1 | ||
| pgrep rcuo* | while read line; do taskset -p $non_iso_cpumask $line || true; done | ||
| else | ||
| # Clean up | ||
| rm -rf ${INITRD_GENERATION_DIR} | ||
|
|
||
| # Create initrd image | ||
| mkdir ${INITRD_GENERATION_DIR} | ||
| mkdir -p ${INITRD_GENERATION_DIR}/usr/lib/dracut/hooks/pre-udev/ | ||
| mkdir -p ${INITRD_GENERATION_DIR}/etc/systemd/ | ||
| mkdir -p ${INITRD_GENERATION_DIR}/etc/sysconfig/ | ||
| touch ${INITRD_GENERATION_DIR}/etc/systemd/system.conf | ||
| touch ${INITRD_GENERATION_DIR}/etc/sysconfig/irqbalance | ||
| touch ${INITRD_GENERATION_DIR}/usr/lib/dracut/hooks/pre-udev/00-tuned-pre-udev.sh | ||
| chmod +x ${INITRD_GENERATION_DIR}/usr/lib/dracut/hooks/pre-udev/00-tuned-pre-udev.sh | ||
|
|
||
| get_cpu_mask 1 | ||
| echo '#!/bin/sh | ||
|
|
||
| type getargs >/dev/null 2>&1 || . /lib/dracut-lib.sh | ||
|
|
||
| #cpumask="$(getargs non_iso_cpumask)" | ||
| cpumask='$non_iso_cpumask' | ||
|
|
||
| log() | ||
| { | ||
| echo "tuned: $@" >> /dev/kmsg | ||
| } | ||
|
|
||
| if [ -n "$cpumask" ]; then | ||
| for file in /sys/devices/virtual/workqueue/cpumask /sys/bus/workqueue/devices/writeback/cpumask; do | ||
| log "setting $file CPU mask to $cpumask" | ||
| if ! echo $cpumask > $file 2>/dev/null; then | ||
| log "ERROR: could not write CPU mask for $file" | ||
| fi | ||
| done | ||
| fi' > ${INITRD_GENERATION_DIR}/usr/lib/dracut/hooks/pre-udev/00-tuned-pre-udev.sh | ||
|
|
||
| # Set CPU affinity according to NON_ISOLATED_CPUS | ||
| get_cpu_affinity | ||
| echo "[Manager]" >> ${INITRD_GENERATION_DIR}/etc/systemd/system.conf | ||
| echo "CPUAffinity=$cpu_affinity" >> ${INITRD_GENERATION_DIR}/etc/systemd/system.conf | ||
|
|
||
| # Set IRQ banned cpu according to NON_ISOLATED_CPUS | ||
| get_cpu_mask 0 | ||
| echo "IRQBALANCE_BANNED_CPUS=$non_iso_cpumask" >> ${INITRD_GENERATION_DIR}/etc/sysconfig/irqbalance | ||
|
|
||
| find ${INITRD_GENERATION_DIR} | cpio -co >${INITRD_NEW_IMAGE} | ||
|
|
||
| # Get current ostree config file according to the latest version | ||
| current_ver=1 | ||
| entry_file=$(ls -td /boot/loader/entries/* | head -1) | ||
| while read -r line ; do | ||
| ver=`awk '/version/ {print $2}' $line` | ||
| if [ "$ver" -gt "$current_ver" ]; then | ||
| current_ver=$ver | ||
| entry_file=$line | ||
| fi | ||
| done <<<$(egrep $(uname -r) -lr /boot/loader/entries/) | ||
|
|
||
| sed -i "s^initrd .*\$^& ${RHCOS_OSTREE_BOOTLOADER_PATH}iso_initrd.img^" $entry_file | ||
|
|
||
| #TODO - once RHCOS image contains the initrd content we can set parameters with rpm-ostree: | ||
| #rpm-ostree initramfs --enable --arg=-I --arg=/etc/systemd/system.conf | ||
| #rpm-ostree initramfs --enable --arg=-I --arg=/etc/sysconfig/irqbalance | ||
|
|
||
| touch /var/reboot | ||
| fi | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| if [[ -f /var/reboot ]]; then | ||
| rm -f /var/reboot | ||
| echo "File /var/reboot exists, initiate reboot" | ||
| systemctl reboot | ||
| fi |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| REPO_DIR="/etc/yum.repos.d" | ||
| RT_REPO="${REPO_DIR}/rt-kernel.repo" | ||
|
|
||
| # Enable yum repo | ||
| if [[ -f $RT_REPO ]] | ||
| then | ||
| # The env var might have been changed, so always create new rt repo | ||
| rm $RT_REPO | ||
| fi | ||
|
|
||
| mkdir -p $REPO_DIR | ||
| cat > $RT_REPO <<EOF | ||
| [rt] | ||
| baseurl=${RT_REPO_URL} | ||
| gpgcheck=0 | ||
| EOF | ||
|
|
||
| # update cache | ||
| rpm-ostree refresh-md -f | ||
|
|
||
| exit_handler () { | ||
| exit_code=$? | ||
| if [[ ${exit_code} -eq 77 ]]; then | ||
| echo "No update available, nothing to do"; | ||
| exit 0; | ||
| elif [[ ${exit_code} -eq 100 ]]; then | ||
| echo "Initiate reboot, touch /var/reboot" | ||
| touch /var/reboot | ||
| exit 0; | ||
| else | ||
| exit ${exit_code} | ||
| fi | ||
| } | ||
|
|
||
| trap exit_handler EXIT | ||
|
|
||
| # Swap to RT kernel | ||
| kernel=$(uname -a) | ||
| if [[ $kernel =~ "PREEMPT RT" ]] | ||
| then | ||
| echo "RT kernel already installed, checking for updates" | ||
| # if no upgrade is available the script will exit with code 77, and we will trap it | ||
| rpm-ostree upgrade --unchanged-exit-77 | ||
| echo "RT kernel updated" | ||
| exit 100 | ||
| else | ||
| echo "Installing RT kernel" | ||
| rpm-ostree override remove kernel{,-core,-modules,-modules-extra} --install kernel-rt-core --install kernel-rt-modules --install kernel-rt-modules-extra | ||
| echo "RT kernel installed" | ||
| exit 100 | ||
| fi |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| [main] | ||
| summary=Openshift node optimized for deterministic performance at the cost of increased power consumption, focused on low latency network performance | ||
| include=openshift-node | ||
|
|
||
| [cpu] | ||
| # https://github.com/redhat-performance/tuned/blob/master/profiles/latency-performance/tuned.conf | ||
| # https://github.com/redhat-performance/tuned/blob/master/profiles/network-latency/tuned.conf | ||
| force_latency=cstate.id:1|3 | ||
|
Member
There was a problem hiding this comment. there might be an issue with this format of setting latency (using the OR operand) , investigating this but worth checking if tuned pod doesn't have any errors with that here .
Contributor
Author
There was a problem hiding this comment. We can fix it under the separate PR |
||
| governor=performance | ||
| energy_perf_bias=performance | ||
| min_perf_pct=100 | ||
|
|
||
| [vm] | ||
| transparent_hugepages=never | ||
|
|
||
| [sysctl] | ||
| net.core.busy_read=50 | ||
| net.core.busy_poll=50 | ||
| net.ipv4.tcp_fastopen=3 | ||
| kernel.numa_balancing=0 | ||
|
|
||
| # ktune sysctl settings for rhel6 servers, maximizing i/o throughput | ||
| # | ||
| # Minimal preemption granularity for CPU-bound tasks: | ||
| # (default: 1 msec# (1 + ilog(ncpus)), units: nanoseconds) | ||
| kernel.sched_min_granularity_ns=10000000 | ||
|
|
||
| # If a workload mostly uses anonymous memory and it hits this limit, the entire | ||
| # working set is buffered for I/O, and any more write buffering would require | ||
| # swapping, so it's time to throttle writes until I/O can catch up. Workloads | ||
| # that mostly use file mappings may be able to use even higher values. | ||
| # | ||
| # The generator of dirty data starts writeback at this percentage (system default | ||
| # is 20%) | ||
| vm.dirty_ratio=10 | ||
|
|
||
| # Start background writeback (via writeback threads) at this percentage (system | ||
| # default is 10%) | ||
| vm.dirty_background_ratio=3 | ||
|
|
||
| # The swappiness parameter controls the tendency of the kernel to move | ||
| # processes out of physical memory and onto the swap disk. | ||
| # 0 tells the kernel to avoid swapping processes out of physical memory | ||
| # for as long as possible | ||
| # 100 tells the kernel to aggressively swap processes out of physical memory | ||
| # and move them to swap cache | ||
| vm.swappiness=10 | ||
|
|
||
| # The total time the scheduler will consider a migrated process | ||
| # "cache hot" and thus less likely to be re-migrated | ||
| # (system default is 500000, i.e. 0.5 ms) | ||
| kernel.sched_migration_cost_ns=5000000 | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| [main] | ||
| summary=Optimize nodes running real time kernel | ||
| include=openshift-node-network-latency | ||
|
|
||
| [selinux] | ||
| avc_cache_threshold=8192 | ||
|
|
||
| [net] | ||
| nf_conntrack_hashsize=131072 | ||
|
|
||
| [sysctl] | ||
| kernel.hung_task_timeout_secs = 600 | ||
| kernel.nmi_watchdog = 0 | ||
| kernel.sched_rt_runtime_us = -1 | ||
| vm.stat_interval = 10 | ||
| kernel.timer_migration = 0 | ||
|
|
||
| [sysfs] | ||
| /sys/devices/system/machinecheck/machinecheck*/ignore_ce = 1 | ||
|
|
||
| [scheduler] | ||
| isolated_cores={{.IsolatedCpus}} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We might actually be able to test that already with latest RHCOS changes , but ill do it in another PR