diff --git a/bindata/kube-proxy/kube-proxy.yaml b/bindata/kube-proxy/kube-proxy.yaml index 888b482ab0..6e144366a7 100644 --- a/bindata/kube-proxy/kube-proxy.yaml +++ b/bindata/kube-proxy/kube-proxy.yaml @@ -142,7 +142,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:{{.MetricsPort}} \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29102/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} diff --git a/bindata/network/multus-admission-controller/admission-controller.yaml b/bindata/network/multus-admission-controller/admission-controller.yaml index 57ad18f2e9..8b819b6111 100644 --- a/bindata/network/multus-admission-controller/admission-controller.yaml +++ b/bindata/network/multus-admission-controller/admission-controller.yaml @@ -179,7 +179,7 @@ spec: args: - --logtostderr - --secure-listen-address=:8443 - - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - --upstream=http://127.0.0.1:9091/ - --tls-private-key-file=/etc/webhook/tls.key - --tls-cert-file=/etc/webhook/tls.crt diff --git a/bindata/network/network-metrics/001-daemonset.yaml b/bindata/network/network-metrics/001-daemonset.yaml index 535bda9ff6..85e71200b1 100644 --- a/bindata/network/network-metrics/001-daemonset.yaml +++ b/bindata/network/network-metrics/001-daemonset.yaml @@ -62,7 +62,7 @@ spec: args: - --logtostderr - --secure-listen-address=:8443 - - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - --upstream=http://127.0.0.1:9091/ - --tls-private-key-file=/etc/metrics/tls.key - --tls-cert-file=/etc/metrics/tls.crt diff --git a/bindata/network/openshift-sdn/controller.yaml b/bindata/network/openshift-sdn/controller.yaml index 730348a673..af303326a6 100644 --- a/bindata/network/openshift-sdn/controller.yaml +++ b/bindata/network/openshift-sdn/controller.yaml @@ -99,7 +99,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:9106 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29100/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} diff --git a/bindata/network/openshift-sdn/sdn.yaml b/bindata/network/openshift-sdn/sdn.yaml index 54ac023329..fa38977842 100644 --- a/bindata/network/openshift-sdn/sdn.yaml +++ b/bindata/network/openshift-sdn/sdn.yaml @@ -293,7 +293,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:9101 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29101/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} diff --git a/bindata/network/ovn-kubernetes/common/008-script-lib.yaml b/bindata/network/ovn-kubernetes/common/008-script-lib.yaml new file mode 100644 index 0000000000..5965c85f8d --- /dev/null +++ b/bindata/network/ovn-kubernetes/common/008-script-lib.yaml @@ -0,0 +1,608 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ovnkube-script-lib + namespace: openshift-ovn-kubernetes + annotations: + kubernetes.io/description: | + This is a script used by the ovn-kubernetes daemonset + release.openshift.io/version: "{{.ReleaseVersion}}" +data: + ovnkube-lib.sh: |- + #!/bin/bash + set -x + + # Add node-specific overrides if the container has mounted any + K8S_NODE=${K8S_NODE:-} + if [[ -n "${K8S_NODE}" && -f "/env/${K8S_NODE}" ]]; then + set -o allexport + source "/env/${K8S_NODE}" + set +o allexport + fi + + northd_pidfile="/var/run/ovn/ovn-northd.pid" + controller_pidfile="/var/run/ovn/ovn-controller.pid" + controller_logfile="/var/log/ovn/acl-audit-log.log" + vswitch_dbsock="/var/run/openvswitch/db.sock" + nbdb_pidfile="/var/run/ovn/ovnnb_db.pid" + nbdb_sock="/var/run/ovn/ovnnb_db.sock" + nbdb_ctl="/var/run/ovn/ovnnb_db.ctl" + sbdb_pidfile="/var/run/ovn/ovnsb_db.pid" + sbdb_sock="/var/run/ovn/ovnsb_db.sock" + sbdb_ctl="/var/run/ovn/ovnsb_db.ctl" + + # start-ovn-controller() starts ovn-controller and does not return until + # ovn-controller exits + # + # Requires the following volume mounts: + # /run/openvswitch + # /run/ovn/ + # /etc/openvswitch + # /etc/ovn/ + # /var/lib/openvswitch + # /var/log/ovn/ + # /dev/log + start-ovn-controller() + { + local log_level=$1 + + if [[ $# -ne 1 ]]; then + echo "Expected one argument but got $#" + exit 1 + fi + + echo "$(date -Iseconds) - starting ovn-controller" + exec ovn-controller \ + unix:${vswitch_dbsock} \ + -vfile:off \ + --no-chdir \ + --pidfile=${controller_pidfile} \ + --syslog-method="{{.OVNPolicyAuditDestination}}" \ + --log-file=${controller_logfile} \ + -vFACILITY:"{{.OVNPolicyAuditSyslogFacility}}" \ + -vconsole:"${log_level}" \ + -vconsole:"acl_log:off" \ + -vPATTERN:console:"{{.OVN_LOG_PATTERN_CONSOLE}}" \ + -vsyslog:"acl_log:info" \ + -vfile:"acl_log:info" + } + + # quit-ovn-northd() will cleanly shut down ovn-northd. It is intended + # to be run from a bash 'trap' like so: + # + # trap quit-ovn-northd TERM INT + quit-ovn-northd() + { + echo "$(date -Iseconds) - stopping ovn-northd" + OVN_MANAGE_OVSDB=no /usr/share/ovn/scripts/ovn-ctl stop_northd + echo "$(date -Iseconds) - ovn-northd stopped" + rm -f ${northd_pidfile} + exit 0 + } + + # run-ovn-northd() starts ovn-northd and does not return until + # northd exits. + # + # Requires the following volume mounts: + # /etc/openvswitch/ + # /var/lib/openvswitch/ + # /run/openvswitch/ + # /run/ovn/ + # /var/log/ovn/ + start-ovn-northd() + { + local log_level=$1 + + if [[ $# -ne 1 ]]; then + echo "Expected one argument but got $#" + exit 1 + fi + + echo "$(date -Iseconds) - starting ovn-northd" + exec ovn-northd \ + --no-chdir \ + -vconsole:"${log_level}" \ + -vfile:off \ + -vPATTERN:console:"{{.OVN_LOG_PATTERN_CONSOLE}}" \ + --pidfile ${northd_pidfile} \ + --n-threads={{.NorthdThreads}} & + wait $! + } + + # start-audit-log-rotation() continuously watches ovn-controller's audit + # log directory and deletes old logs to ensure the total size of the logs + # does not exceed a given threshold. This function does not return. + # + # Requires the following volume mounts: + # /var/log/ovn/ + # /run/ovn/ + start-audit-log-rotation() + { + # Rotate audit log files when then get to max size (in bytes) + MAXFILESIZE=$(( "{{.OVNPolicyAuditMaxFileSize}}"*1000000 )) + MAXLOGFILES="{{.OVNPolicyAuditMaxLogFiles}}" + LOGDIR=$(dirname ${controller_logfile}) + + # wait a bit for ovn-controller to start + local retries=0 + while [[ 30 -gt "${retries}" ]]; do + (( retries += 1 )) + CONTROLLERPID=$(cat ${controller_pidfile}) + if [[ -n "${CONTROLLERPID}" ]]; then + break + fi + sleep 2 + done + if [[ -z "${CONTROLLERPID}" ]]; then + echo "Timed out waiting for ${controller_pidfile}" + return 1 + fi + + # Redirect err to null so no messages are shown upon rotation + tail -F ${controller_logfile} 2> /dev/null & + + while true + do + # Make sure ovn-controller's logfile exists, and get current size in bytes + if [ -f "${controller_logfile}" ]; then + file_size=`du -b ${controller_logfile} | tr -s '\t' ' ' | cut -d' ' -f1` + else + ovs-appctl -t /var/run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen + file_size=`du -b ${controller_logfile} | tr -s '\t' ' ' | cut -d' ' -f1` + fi + + if [ $file_size -gt $MAXFILESIZE ];then + echo "Rotating OVN ACL Log File" + timestamp=`date '+%Y-%m-%dT%H-%M-%S'` + mv ${controller_logfile} ${LOGDIR}/acl-audit-log.$timestamp.log + ovs-appctl -t /run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen + CONTROLLERPID=$(cat ${controller_pidfile}) + fi + + # Ensure total number of log files does not exceed the maximum configured from OVNPolicyAuditMaxLogFiles + num_files=$(ls -1 ${LOGDIR}/acl-audit-log* 2>/dev/null | wc -l) + if [ "$num_files" -gt "$MAXLOGFILES" ]; then + num_to_delete=$(( num_files - ${MAXLOGFILES} )) + ls -1t ${LOGDIR}/acl-audit-log* 2>/dev/null | tail -$num_to_delete | xargs -I {} rm {} + fi + + # sleep for 30 seconds to avoid wasting CPU + sleep 30 + done + } + + wait-for-certs() + { + local detail=$1 + local privkey=$2 + local clientcert=$3 + + if [[ $# -ne 3 ]]; then + echo "Expected three arguments but got $#" + exit 1 + fi + + retries=0 + TS=$(date +%s) + WARN_TS=$(( ${TS} + $(( 20 * 60)) )) + HAS_LOGGED_INFO=0 + while [[ ! -f "${privkey}" || ! -f "${clientcert}" ]] ; do + CUR_TS=$(date +%s) + if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then + echo "$(date -Iseconds) WARN: ${detail} certs not mounted after 20 minutes." + elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then + echo "$(date -Iseconds) INFO: ${detail} certs not mounted. Waiting one hour." + HAS_LOGGED_INFO=1 + fi + sleep 5 + done + } + + # start-rbac-proxy() starts the kube-rbac-proxy to expose ovnkube metrics to + # Prometheus on the given listen_port, proxying from upstream_port. This + # function does not return. + # + # Requires the following volume mounts: + # /etc/pki/tls/metrics-cert + start-rbac-proxy-node() + { + local detail=$1 + local listen_port=$2 + local upstream_port=$3 + local privkey=$4 + local clientcert=$5 + + if [[ $# -ne 5 ]]; then + echo "Expected five arguments but got $#" + exit 1 + fi + + # As the secret mount is optional we must wait for the files to be present. + # The service is created in monitor.yaml and this is created in sdn.yaml. + # If it isn't created there is probably an issue so we want to crashloop. + echo "$(date -Iseconds) INFO: waiting for ${detail} certs to be mounted" + wait-for-certs "${detail}" "${privkey}" "${clientcert}" + + echo "$(date -Iseconds) INFO: ${detail} certs mounted, starting kube-rbac-proxy" + exec /usr/bin/kube-rbac-proxy \ + --logtostderr \ + --secure-listen-address=:${listen_port} \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ + --upstream=http://127.0.0.1:${upstream_port}/ \ + --tls-private-key-file=${privkey} \ + --tls-cert-file=${clientcert} + } + + # quit-nbdb() will cleanly shut down the northbound dbserver. It is intended + # to be run from a bash 'trap' like so: + # + # trap quit-nbdb TERM INT + quit-nbdb() + { + echo "$(date -Iseconds) - stopping nbdb" + /usr/share/ovn/scripts/ovn-ctl stop_nb_ovsdb + echo "$(date -Iseconds) - nbdb stopped" + rm -f ${nbdb_pidfile} + exit 0 + } + + # start-nbdb() starts the OVN northbound database. This function does not + # return. + # + # Requires the following volume mounts: + # /etc/ovn + # /var/log/ovn + # /run/ovn/ + start-nbdb() + { + local log_level=$1 + + if [[ $# -ne 1 ]]; then + echo "Expected one argument but got $#" + exit 1 + fi + + exec /usr/share/ovn/scripts/ovn-ctl \ + --no-monitor \ + --db-nb-sock=${nbdb_sock} \ + --ovn-nb-log="-vconsole:${log_level} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ + run_nb_ovsdb & + wait $! + } + + # retry() an operation a number of times, sleeping 2 seconds between each try + retry() { + local tries=${1} + local desc=${2} + local cmd=${3} + + local retries=0 + while ! ${cmd}; do + (( retries += 1 )) + if [[ "${retries}" -gt ${tries} ]]; then + echo "$(date -Iseconds) - ERROR - ${desc} - too many failed attempts, giving up" + return 1 + fi + echo "$(date -Iseconds) - WARN - ${desc} - failed try ${retries}, retrying..." + sleep 2 + done + echo "$(date -Iseconds) - INFO - ${desc} - success" + return 0 + } + + # nbdb-post-start() tweaks nbdb database server settings and sets a number + # of options in NB_Globals to configure OVN global settings + nbdb-post-start() + { + local northd_probe_interval=${1:-10000} + + rm -f ${nbdb_pidfile} + + # set inactivity probe + if ! retry 60 "inactivity-probe" "ovn-nbctl -t 5 --inactivity-probe={{.OVN_NB_INACTIVITY_PROBE}} set-connection punix:${nbdb_sock}"; then + exit 1 + fi + # set trim-on-compaction + if ! retry 60 "trim-on-compaction" "ovn-appctl -t ${nbdb_ctl} --timeout=5 ovsdb-server/memory-trim-on-compaction on"; then + exit 1 + fi + + # set IC zone + echo "Setting the IC zone to ${K8S_NODE}" + IC_OPTION="name=\"${K8S_NODE}\" options:name=\"${K8S_NODE}\"" + + # northd probe interval + echo "Setting northd probe interval to ${northd_probe_interval} ms" + NORTHD_PROBE_OPTION="options:northd_probe_interval=${northd_probe_interval}" + + # let northd sleep so it takes less CPU + NORTHD_SLEEP_OPTION="options:northd-backoff-interval-ms={{.OVN_NORTHD_BACKOFF_MS}}" + + local ipsec=false + local ipsec_encapsulation=false +{{ if .OVNIPsecEnable }} + ipsec=true + # IBMCloud does not forward ESP (IP proto 50) + # Instead, force IBMCloud IPsec to always use NAT-T + if [ "{{.PlatformType}}" == "IBMCloud" ]; then + ipsec_encapsulation=true + fi +{{ end }} + IPSEC_OPTION="ipsec=${ipsec} options:ipsec_encapsulation=${ipsec_encapsulation}" + + # set all the NB_GLOBAL options + if ! retry 20 "nb-global options" "ovn-nbctl -t 5 set nb_global . ${IC_OPTION} ${NORTHD_PROBE_OPTION} ${NORTHD_SLEEP_OPTION} ${IPSEC_OPTION}"; then + exit 1 + fi + } + + # ovndb-readiness-probe() checks if the the database is in the active state + # and if not, exits with an error code. + ovndb-readiness-probe() + { + # dbname should be 'sb' or 'nb' + local dbname=$1 + + if [[ $# -ne 1 ]]; then + echo "Expected one argument but got $#" + exit 1 + fi + + local ctlfile + if [[ "${dbname}" = "nb" ]]; then + ctlfile=${nbdb_ctl} + elif [[ "${dbname}" = "sb" ]]; then + ctlfile=${sbdb_ctl} + else + echo "unknown DB name ${dbname}" + exit 1 + fi + + status=$(/usr/bin/ovn-appctl -t ${ctlfile} --timeout=3 ovsdb-server/sync-status 2>/dev/null | { grep "state: active" || false; }) + if [[ -z "${status}" ]]; then + echo "${dbname} DB is not running or active." + exit 1 + fi + } + + # quit-sbdb() will cleanly shut down the southbound dbserver. It is intended + # to be run from a bash 'trap' like so: + # + # trap quit-sbdb TERM INT + quit-sbdb() + { + echo "$(date -Iseconds) - stopping sbdb" + /usr/share/ovn/scripts/ovn-ctl stop_sb_ovsdb + echo "$(date -Iseconds) - sbdb stopped" + rm -f ${sbdb_pidfile} + exit 0 + } + + # start-sbdb() starts the OVN southbound database. This function does not + # return. + # + # Requires the following volume mounts: + # /etc/ovn + # /var/log/ovn + # /run/ovn/ + start-sbdb() + { + local log_level=$1 + + if [[ $# -ne 1 ]]; then + echo "Expected one argument but got $#" + exit 1 + fi + + exec /usr/share/ovn/scripts/ovn-ctl \ + --no-monitor \ + --db-sb-sock=${sbdb_sock} \ + --ovn-sb-log="-vconsole:${log_level} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ + run_sb_ovsdb & + wait $! + } + + # sbdb-post-start() tweaks sbdb database server settings + sbdb-post-start() + { + rm -f ${sbdb_pidfile} + + # set inactivity probe + if ! retry 60 "inactivity-probe" "ovn-sbctl -t 5 --inactivity-probe={{.OVN_CONTROLLER_INACTIVITY_PROBE}} set-connection punix:${sbdb_sock}"; then + exit 1 + fi + # set trim-on-compaction + if ! retry 60 "trim-on-compaction" "ovn-appctl -t ${sbdb_ctl} --timeout=5 ovsdb-server/memory-trim-on-compaction on"; then + exit 1 + fi + } + + function log() + { + echo "$(date --iso-8601=seconds) [{$1}] ${2}" + } + + # cni-bin-copy() detects the host OS and copies the correct shim binary to + # the CNI binary directory. + # + # Requires the following volume mounts: + # /host + # /cni-bin-dir + cni-bin-copy() + { + # collect host os information + . /host/etc/os-release + rhelmajor= + # detect which version we're using in order to copy the proper binaries + case "${ID}" in + rhcos|scos) + RHEL_VERSION=$(echo "${CPE_NAME}" | cut -f 5 -d :) + rhelmajor=$(echo $RHEL_VERSION | sed -E 's/([0-9]+)\.{1}[0-9]+(\.[0-9]+)?/\1/') + ;; + rhel) rhelmajor=$(echo "${VERSION_ID}" | cut -f 1 -d .) + ;; + fedora) + if [ "${VARIANT_ID}" == "coreos" ]; then + rhelmajor=8 + else + log "cnibincopy" "FATAL ERROR: Unsupported Fedora variant=${VARIANT_ID}" + exit 1 + fi + ;; + *) log "cnibincopy" "FATAL ERROR: Unsupported OS ID=${ID}"; exit 1 + ;; + esac + + # Set which directory we'll copy from, detect if it exists + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + 8) + sourcedir=/usr/libexec/cni/rhel8 + ;; + 9) + sourcedir=/usr/libexec/cni/rhel9 + ;; + *) + log "cnibincopy" "ERROR: RHEL Major Version Unsupported, rhelmajor=${rhelmajor}" + ;; + esac + + cp -f "$sourcedir/ovn-k8s-cni-overlay" /cni-bin-dir/ + } + + # start-ovnkube-node starts the ovnkube-node process. This function does not + # return. + start-ovnkube-node() + { + local log_level=$1 + local metrics_port=$2 + local ovn_metrics_port=$3 + + if [[ $# -ne 3 ]]; then + echo "Expected three arguments but got $#" + exit 1 + fi + + # copy the right CNI shim for the host OS + cni-bin-copy + + echo "I$(date "+%m%d %H:%M:%S.%N") - disable conntrack on geneve port" + iptables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK + + {{- if .OVNHybridOverlayVXLANPort}} + echo "I$(date "+%m%d %H:%M:%S.%N") - disable conntrack on hybrid overlay VXLAN port" + iptables -t raw -A PREROUTING -p udp --dport {{.OVNHybridOverlayVXLANPort}} -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport {{.OVNHybridOverlayVXLANPort}} -j NOTRACK + {{- end}} + + echo "I$(date "+%m%d %H:%M:%S.%N") - starting ovnkube-node" + + if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then + gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex" + elif [ "{{.OVN_GATEWAY_MODE}}" == "local" ]; then + gateway_mode_flags="--gateway-mode local --gateway-interface br-ex" + else + echo "Invalid OVN_GATEWAY_MODE: \"{{.OVN_GATEWAY_MODE}}\". Must be \"local\" or \"shared\"." + exit 1 + fi + + export_network_flows_flags= + if [[ -n "${NETFLOW_COLLECTORS}" ]] ; then + export_network_flows_flags="--netflow-targets ${NETFLOW_COLLECTORS}" + fi + if [[ -n "${SFLOW_COLLECTORS}" ]] ; then + export_network_flows_flags="$export_network_flows_flags --sflow-targets ${SFLOW_COLLECTORS}" + fi + if [[ -n "${IPFIX_COLLECTORS}" ]] ; then + export_network_flows_flags="$export_network_flows_flags --ipfix-targets ${IPFIX_COLLECTORS}" + fi + if [[ -n "${IPFIX_CACHE_MAX_FLOWS}" ]] ; then + export_network_flows_flags="$export_network_flows_flags --ipfix-cache-max-flows ${IPFIX_CACHE_MAX_FLOWS}" + fi + if [[ -n "${IPFIX_CACHE_ACTIVE_TIMEOUT}" ]] ; then + export_network_flows_flags="$export_network_flows_flags --ipfix-cache-active-timeout ${IPFIX_CACHE_ACTIVE_TIMEOUT}" + fi + if [[ -n "${IPFIX_SAMPLING}" ]] ; then + export_network_flows_flags="$export_network_flows_flags --ipfix-sampling ${IPFIX_SAMPLING}" + fi + gw_interface_flag= + # if br-ex1 is configured on the node, we want to use it for external gateway traffic + if [ -d /sys/class/net/br-ex1 ]; then + gw_interface_flag="--exgw-interface=br-ex1" + fi + + node_mgmt_port_netdev_flags= + if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then + node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}" + fi + if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then + node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" + fi + + multi_network_enabled_flag= + if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then + multi_network_enabled_flag="--enable-multi-network" + fi + + multi_network_policy_enabled_flag= + if [[ "{{.OVN_MULTI_NETWORK_POLICY_ENABLE}}" == "true" ]]; then + multi_network_policy_enabled_flag="--enable-multi-networkpolicy" + fi + + admin_network_policy_enabled_flag= + if [[ "{{.OVN_ADMIN_NETWORK_POLICY_ENABLE}}" == "true" ]]; then + admin_network_policy_enabled_flag="--enable-admin-network-policy" + fi + + # If IP Forwarding mode is global set it in the host here. + ip_forwarding_flag= + if [ "{{.IP_FORWARDING_MODE}}" == "Global" ]; then + sysctl -w net.ipv4.ip_forward=1 + sysctl -w net.ipv6.conf.all.forwarding=1 + else + ip_forwarding_flag="--disable-forwarding" + fi + + NETWORK_NODE_IDENTITY_ENABLE= + if [[ "{{.NETWORK_NODE_IDENTITY_ENABLE}}" == "true" ]]; then + NETWORK_NODE_IDENTITY_ENABLE=" + --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig + --cert-dir=/etc/ovn/ovnkube-node-certs + --cert-duration={{.NodeIdentityCertDuration}} + " + fi + + exec /usr/bin/ovnkube \ + --init-ovnkube-controller "${K8S_NODE}" \ + --init-node "${K8S_NODE}" \ + --config-file=/run/ovnkube-config/ovnkube.conf \ + --ovn-empty-lb-events \ + --loglevel "${log_level}" \ + --inactivity-probe="${OVN_CONTROLLER_INACTIVITY_PROBE}" \ + ${gateway_mode_flags} \ + ${node_mgmt_port_netdev_flags} \ +{{- if eq .OVN_NODE_MODE "dpu-host" }} + --ovnkube-node-mode dpu-host \ +{{- end }} + --metrics-bind-address "127.0.0.1:${metrics_port}" \ + --ovn-metrics-bind-address "127.0.0.1:${ovn_metrics_port}" \ + --metrics-enable-pprof \ + --metrics-enable-config-duration \ + --export-ovs-metrics \ + --disable-snat-multiple-gws \ + ${export_network_flows_flags} \ + ${multi_network_enabled_flag} \ + ${multi_network_policy_enabled_flag} \ + ${admin_network_policy_enabled_flag} \ + --enable-multicast \ + --zone ${K8S_NODE} \ + --enable-interconnect \ + --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}" \ + ${gw_interface_flag} \ + --enable-multi-external-gateway=true \ + ${ip_forwarding_flag} \ + ${NETWORK_NODE_IDENTITY_ENABLE} + } diff --git a/bindata/network/ovn-kubernetes/managed/multi-zone-interconnect/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/managed/multi-zone-interconnect/ovnkube-node.yaml index 5e9155e8f1..7c2038930d 100644 --- a/bindata/network/ovn-kubernetes/managed/multi-zone-interconnect/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/managed/multi-zone-interconnect/ovnkube-node.yaml @@ -8,7 +8,6 @@ metadata: {{ else }} name: ovnkube-node {{ end }} - name: ovnkube-node namespace: openshift-ovn-kubernetes annotations: kubernetes.io/description: | @@ -32,6 +31,7 @@ spec: metadata: annotations: target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + network.operator.openshift.io/ovnkube-script-lib-hash: "{{.OVNKubeConfigHash}}" labels: {{ if eq .OVN_NODE_MODE "dpu-host" }} app: ovnkube-node-dpu-host @@ -87,6 +87,7 @@ spec: # /run/openvswitch -> tmpfs - ovsdb sockets # /env -> configmap env-overrides - debug overrides containers: +{{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} # ovn-controller: programs the vswitch with flows from the sbdb - name: ovn-controller image: "{{.OvnImage}}" @@ -95,23 +96,8 @@ spec: - -c - | set -e - if [[ -f "/env/${K8S_NODE}" ]]; then - set -o allexport - source "/env/${K8S_NODE}" - set +o allexport - fi - - echo "$(date -Iseconds) - starting ovn-controller" - exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ - --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ - --syslog-method="{{.OVNPolicyAuditDestination}}" \ - --log-file=/var/log/ovn/acl-audit-log.log \ - -vFACILITY:"{{.OVNPolicyAuditSyslogFacility}}" \ - -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \ - -vconsole:"${OVN_LOG_LEVEL}" -vconsole:"acl_log:off" \ - -vPATTERN:console:"{{.OVN_LOG_PATTERN_CONSOLE}}" \ - -vsyslog:"acl_log:info" \ - -vfile:"acl_log:info" + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-ovn-controller ${OVN_LOG_LEVEL} securityContext: privileged: true env: @@ -122,6 +108,8 @@ spec: fieldRef: fieldPath: spec.nodeName volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - mountPath: /run/openvswitch name: run-openvswitch - mountPath: /run/ovn/ @@ -134,10 +122,6 @@ spec: name: var-lib-openvswitch - mountPath: /env name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - mountPath: /var/log/ovn/ name: node-log - mountPath: /dev/log @@ -154,55 +138,21 @@ spec: - -c - | set -euo pipefail - - # Rotate audit log files when then get to max size (in bytes) - MAXFILESIZE=$(( "{{.OVNPolicyAuditMaxFileSize}}"*1000000 )) - MAXLOGFILES="{{.OVNPolicyAuditMaxLogFiles}}" - LOGDIR=/var/log/ovn - LOGFILE=${LOGDIR}/acl-audit-log.log - CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid) - - # Redirect err to null so no messages are shown upon rotation - tail -F ${LOGFILE} 2> /dev/null & - - while true - do - # Make sure ovn-controller's logfile exists, and get current size in bytes - if [ -f "$LOGFILE" ]; then - file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1` - else - ovs-appctl -t /var/run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen - file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1` - fi - - if [ $file_size -gt $MAXFILESIZE ];then - echo "Rotating OVN ACL Log File" - timestamp=`date '+%Y-%m-%dT%H-%M-%S'` - mv ${LOGFILE} /var/log/ovn/acl-audit-log.$timestamp.log - ovs-appctl -t /run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen - CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid) - fi - - # Ensure total number of log files does not exceed the maximum configured from OVNPolicyAuditMaxLogFiles - num_files=$(ls -1 ${LOGDIR}/acl-audit-log* 2>/dev/null | wc -l) - if [ "$num_files" -gt "$MAXLOGFILES" ]; then - num_to_delete=$(( num_files - ${MAXLOGFILES} )) - ls -1t ${LOGDIR}/acl-audit-log* 2>/dev/null | tail -$num_to_delete | xargs -I {} rm {} - fi - - # sleep for 30 seconds to avoid wasting CPU - sleep 30 - done + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-audit-log-rotation resources: requests: cpu: 10m memory: 20Mi terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - mountPath: /var/log/ovn/ name: node-log - mountPath: /run/ovn/ name: run-ovn +{{ end }} - name: kube-rbac-proxy-node image: {{.KubeRBACProxyImage}} command: @@ -211,38 +161,8 @@ spec: - | #!/bin/bash set -euo pipefail - TLS_PK=/etc/pki/tls/metrics-cert/tls.key - TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt - # As the secret mount is optional we must wait for the files to be present. - # The service is created in monitor.yaml and this is created in sdn.yaml. - # If it isn't created there is probably an issue so we want to crashloop. - retries=0 - TS=$(date +%s) - WARN_TS=$(( ${TS} + $(( 20 * 60)) )) - HAS_LOGGED_INFO=0 - - log_missing_certs(){ - CUR_TS=$(date +%s) - if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then - echo $(date -Iseconds) WARN: ovn-node-metrics-cert not mounted after 20 minutes. - elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then - echo $(date -Iseconds) INFO: ovn-node-metrics-cert not mounted. Waiting one hour. - HAS_LOGGED_INFO=1 - fi - } - while [[ ! -f "${TLS_PK}" || ! -f "${TLS_CERT}" ]] ; do - log_missing_certs - sleep 5 - done - - echo $(date -Iseconds) INFO: ovn-node-metrics-certs mounted, starting kube-rbac-proxy - exec /usr/bin/kube-rbac-proxy \ - --logtostderr \ - --secure-listen-address=:9103 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ - --upstream=http://127.0.0.1:29103/ \ - --tls-private-key-file=${TLS_PK} \ - --tls-cert-file=${TLS_CERT} + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-rbac-proxy-node ovn-node-metrics 9103 29103 /etc/pki/tls/metrics-cert/tls.key /etc/pki/tls/metrics-cert/tls.crt ports: - containerPort: 9103 name: https @@ -252,6 +172,8 @@ spec: memory: 20Mi terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - name: ovn-node-metrics-cert mountPath: /etc/pki/tls/metrics-cert readOnly: True @@ -263,38 +185,8 @@ spec: - | #!/bin/bash set -euo pipefail - TLS_PK=/etc/pki/tls/metrics-cert/tls.key - TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt - # As the secret mount is optional we must wait for the files to be present. - # The service is created in monitor.yaml and this is created in sdn.yaml. - # If it isn't created there is probably an issue so we want to crashloop. - retries=0 - TS=$(date +%s) - WARN_TS=$(( ${TS} + $(( 20 * 60)) )) - HAS_LOGGED_INFO=0 - - log_missing_certs(){ - CUR_TS=$(date +%s) - if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then - echo $(date -Iseconds) WARN: ovn-node-metrics-cert not mounted after 20 minutes. - elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then - echo $(date -Iseconds) INFO: ovn-node-metrics-cert not mounted. Waiting one hour. - HAS_LOGGED_INFO=1 - fi - } - while [[ ! -f "${TLS_PK}" || ! -f "${TLS_CERT}" ]] ; do - log_missing_certs - sleep 5 - done - - echo $(date -Iseconds) INFO: ovn-node-metrics-certs mounted, starting kube-rbac-proxy - exec /usr/bin/kube-rbac-proxy \ - --logtostderr \ - --secure-listen-address=:9105 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ - --upstream=http://127.0.0.1:29105/ \ - --tls-private-key-file=${TLS_PK} \ - --tls-cert-file=${TLS_CERT} + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-rbac-proxy-node ovn-metrics 9105 29105 /etc/pki/tls/metrics-cert/tls.key /etc/pki/tls/metrics-cert/tls.crt ports: - containerPort: 9105 name: https @@ -304,6 +196,8 @@ spec: memory: 20Mi terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - name: ovn-node-metrics-cert mountPath: /etc/pki/tls/metrics-cert readOnly: True @@ -320,28 +214,16 @@ spec: source /env/_master set +o allexport fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 - quit() { - echo "$(date -Iseconds) - stopping ovn-northd" - OVN_MANAGE_OVSDB=no /usr/share/ovn/scripts/ovn-ctl stop_northd - echo "$(date -Iseconds) - ovn-northd stopped" - rm -f /var/run/ovn/ovn-northd.pid - exit 0 - } - # end of quit - trap quit TERM INT - - echo "$(date -Iseconds) - starting ovn-northd" - exec ovn-northd \ - --no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off "-vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - --pidfile /var/run/ovn/ovn-northd.pid \ - --n-threads={{.NorthdThreads}} & - - wait $! + trap quit-ovn-northd TERM INT + start-ovn-northd "${OVN_LOG_LEVEL}" env: - name: OVN_LOG_LEVEL value: info volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - mountPath: /etc/ovn name: etc-openvswitch - mountPath: /var/log/ovn @@ -350,10 +232,6 @@ spec: name: run-ovn - mountPath: /env name: env-overrides - - mountPath: /ovn-cert # not needed, but useful when exec'ing in to pod. - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca resources: requests: cpu: 10m @@ -373,26 +251,10 @@ spec: source /env/_master set +o allexport fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 - quit() { - echo "$(date -Iseconds) - stopping nbdb" - /usr/share/ovn/scripts/ovn-ctl stop_nb_ovsdb - echo "$(date -Iseconds) - nbdb stopped" - rm -f /var/run/ovn/ovnnb_db.pid - exit 0 - } - # end of quit - trap quit TERM INT - - bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } - - OVN_ARGS="--no-monitor" - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_nb_ovsdb & - - wait $! - + trap quit-nbdb TERM INT + start-nbdb ${OVN_LOG_LEVEL} lifecycle: postStart: exec: @@ -401,111 +263,12 @@ spec: - -c - | set -x - rm -f /var/run/ovn/ovnnb_db.pid - #configure northd_probe_interval - OVN_NB_CTL="ovn-nbctl" - - # retry an operation a number of times, sleeping 2 seconds between each try - retry() { - local tries=${1} - local desc=${2} - local cmd=${3} - - local retries=0 - while ! ${cmd}; do - (( retries += 1 )) - if [[ "${retries}" -gt ${tries} ]]; then - echo "$(date -Iseconds) - ERROR - nbdb ${desc} - too many failed attempts, giving up" - return 1 - fi - echo "$(date -Iseconds) - WARN - nbdb ${desc} - failed try ${retries}, retrying..." - sleep 2 - done - echo "$(date -Iseconds) - INFO - nbdb ${desc} - success" - return 0 - } - - # set the connection and inactivity probe - if ! retry 60 "inactivity-probe" "ovn-nbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_NB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe={{.OVN_NB_INACTIVITY_PROBE}}"; then - exit 1 - fi - - # Upgrade the db if required. - DB_SCHEMA="/usr/share/ovn/ovn-nb.ovsschema" - DB_SERVER="unix:/var/run/ovn/ovnnb_db.sock" - schema_name=$(ovsdb-tool schema-name $DB_SCHEMA) - db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" "$schema_name") - target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") - - if ovsdb-tool compare-versions "$db_version" == "$target_version"; then - : - elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then - echo "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?" - else - echo "Upgrading database $schema_name from schema version $db_version to $target_version" - ovsdb-client -t 30 convert "$DB_SERVER" "$DB_SCHEMA" - fi - - echo "Setting the IC zone to ${K8S_NODE}" - retries=0 - current_probe_interval=0 - while [[ "${retries}" -lt 10 ]]; do - current_probe_interval=$(${OVN_NB_CTL} set NB_Global . name="${K8S_NODE}" options:name="${K8S_NODE}") - if [[ $? == 0 ]]; then - current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"') - break - else - sleep 2 - (( retries += 1 )) - fi - done - - northd_probe_interval=${OVN_NORTHD_PROBE_INTERVAL:-10000} - echo "Setting northd probe interval to ${northd_probe_interval} ms" - retries=0 - current_probe_interval=0 - while [[ "${retries}" -lt 10 ]]; do - current_probe_interval=$(${OVN_NB_CTL} --if-exists get NB_GLOBAL . options:northd_probe_interval) - if [[ $? == 0 ]]; then - current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"') - break - else - sleep 2 - (( retries += 1 )) - fi - done - - if [[ "${current_probe_interval}" != "${northd_probe_interval}" ]]; then - retries=0 - while [[ "${retries}" -lt 10 ]]; do - ${OVN_NB_CTL} set NB_GLOBAL . options:northd_probe_interval=${northd_probe_interval} - if [[ $? != 0 ]]; then - echo "Failed to set northd probe interval to ${northd_probe_interval}. retrying....." - sleep 2 - (( retries += 1 )) - else - echo "Successfully set northd probe interval to ${northd_probe_interval} ms" - break - fi - done - fi - - # Enable/disable IPsec - {{ if .OVNIPsecEnable }} - ipsec=true - {{ else }} - ipsec=false - {{ end }} - if ! retry 20 "ipsec" "${OVN_NB_CTL} set nb_global . ipsec=${ipsec}"; then - exit 1 - fi - - # Tell northd to sleep a bit so it takes less CPU - if ! retry 20 "northd-backoff" "${OVN_NB_CTL} set nb_global . options:northd-backoff-interval-ms={{.OVN_NORTHD_BACKOFF_MS}}"; then - exit 1 - fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + nbdb-post-start {{.OVN_NORTHD_PROBE_INTERVAL}} readinessProbe: - initialDelaySeconds: 90 +{{ if not .IsSNO }} + initialDelaySeconds: 10 +{{ end }} timeoutSeconds: 5 exec: command: @@ -513,24 +276,19 @@ spec: - -c - | set -xeo pipefail - status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=3 ovsdb-server/sync-status 2>/dev/null | { grep "state: active" || false; }) - if [[ -z "${status}" ]]; then - echo "NB DB is not running or active." - exit 1 - fi - # set trim-on-compaction - /usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=5 ovsdb-server/memory-trim-on-compaction on 2>/dev/null + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + ovndb-readiness-probe "nb" env: - name: OVN_LOG_LEVEL value: info - - name: OVN_NORTHD_PROBE_INTERVAL - value: "{{.OVN_NORTHD_PROBE_INTERVAL}}" - name: K8S_NODE valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - - mountPath: /etc/ovn + - mountPath: /ovnkube-lib + name: ovnkube-script-lib + - mountPath: /etc/ovn/ name: etc-openvswitch - mountPath: /var/log/ovn name: node-log @@ -538,10 +296,6 @@ spec: name: run-ovn - mountPath: /env name: env-overrides - - mountPath: /ovn-cert # not needed, but useful when exec'ing in to pod. - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca resources: requests: cpu: 10m @@ -561,26 +315,10 @@ spec: source /env/_master set +o allexport fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 - quit() { - echo "$(date -Iseconds) - stopping sbdb" - /usr/share/ovn/scripts/ovn-ctl stop_sb_ovsdb - echo "$(date -Iseconds) - sbdb stopped" - rm -f /var/run/ovn/ovnsb_db.pid - exit 0 - } - # end of quit - trap quit TERM INT - - bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } - - OVN_ARGS="--no-monitor" - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_sb_ovsdb & - - wait $! - + trap quit-sbdb TERM INT + start-sbdb ${OVN_LOG_LEVEL} lifecycle: postStart: exec: @@ -589,56 +327,12 @@ spec: - -c - | set -x - rm -f /var/run/ovn/ovnsb_db.pid - - # retry an operation a number of times, sleeping 2 seconds between each try - retry() { - local tries=${1} - local desc=${2} - local cmd=${3} - - local retries=0 - while ! ${cmd}; do - (( retries += 1 )) - if [[ "${retries}" -gt ${tries} ]]; then - echo "$(date -Iseconds) - ERROR - sbdb ${desc} - too many failed attempts, giving up" - return 1 - fi - echo "$(date -Iseconds) - WARN - sbdb ${desc} - failed try ${retries}, retrying..." - sleep 2 - done - echo "$(date -Iseconds) - INFO - sbdb ${desc} - success" - return 0 - } - - # set the connection and inactivity probe - if ! retry 60 "inactivity-probe" "ovn-sbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_SB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe={{.OVN_CONTROLLER_INACTIVITY_PROBE}}"; then - exit 1 - fi - - # Upgrade the db if required. - DB_SCHEMA="/usr/share/ovn/ovn-sb.ovsschema" - DB_SERVER="unix:/var/run/ovn/ovnsb_db.sock" - schema_name=$(ovsdb-tool schema-name $DB_SCHEMA) - db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" "$schema_name") - target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") - - if ovsdb-tool compare-versions "$db_version" == "$target_version"; then - : - elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then - echo "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?" - else - echo "Upgrading database $schema_name from schema version $db_version to $target_version" - ovsdb-client -t 30 convert "$DB_SERVER" "$DB_SCHEMA" - fi - - # Kill some time while the cluster converges by checking IPsec status - OVN_SB_CTL="ovn-sbctl" - if ! retry 20 "ipsec" "${OVN_SB_CTL} get sb_global . ipsec"; then - exit 1 - fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + sbdb-post-start readinessProbe: - initialDelaySeconds: 90 +{{ if not .IsSNO }} + initialDelaySeconds: 10 +{{ end }} timeoutSeconds: 5 exec: command: @@ -646,30 +340,22 @@ spec: - -c - | set -xeo pipefail - status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=3 ovsdb-server/sync-status 2>/dev/null | { grep "state: active" || false; }) - if [[ -z "${status}" ]]; then - echo "SB DB is not running or active." - exit 1 - fi - # set trim-on-compaction - /usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=5 ovsdb-server/memory-trim-on-compaction on 2>/dev/null + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + ovndb-readiness-probe "sb" env: - name: OVN_LOG_LEVEL value: info volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - mountPath: /etc/ovn/ name: etc-openvswitch - mountPath: /run/ovn/ name: run-ovn - - mountPath: /env - name: env-overrides - - mountPath: /ovn-cert # not needed, but useful when exec'ing in to pod. - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - mountPath: /var/log/ovn name: node-log - + - mountPath: /env + name: env-overrides resources: requests: cpu: 10m @@ -684,166 +370,8 @@ spec: - -c - | set -xe - if [[ -f "/env/${K8S_NODE}" ]]; then - set -o allexport - source "/env/${K8S_NODE}" - set +o allexport - fi - - function log() - { - echo "$(date --iso-8601=seconds) [cnibincopy] ${1}" - } - # collect host os information - . /host/etc/os-release - rhelmajor= - # detect which version we're using in order to copy the proper binaries - case "${ID}" in - rhcos|scos) - RHEL_VERSION=$(echo "${CPE_NAME}" | cut -f 5 -d :) - rhelmajor=$(echo $RHEL_VERSION | sed -E 's/([0-9]+)\.{1}[0-9]+(\.[0-9]+)?/\1/') - ;; - rhel) rhelmajor=$(echo "${VERSION_ID}" | cut -f 1 -d .) - ;; - fedora) - if [ "${VARIANT_ID}" == "coreos" ]; then - rhelmajor=8 - else - log "FATAL ERROR: Unsupported Fedora variant=${VARIANT_ID}" - exit 1 - fi - ;; - *) log "FATAL ERROR: Unsupported OS ID=${ID}"; exit 1 - ;; - esac - - # Set which directory we'll copy from, detect if it exists - sourcedir=/usr/libexec/cni/ - case "${rhelmajor}" in - 8) - sourcedir=/usr/libexec/cni/rhel8 - ;; - 9) - sourcedir=/usr/libexec/cni/rhel9 - ;; - *) - log "ERROR: RHEL Major Version Unsupported, rhelmajor=${rhelmajor}" - ;; - esac - - cp -f "$sourcedir/ovn-k8s-cni-overlay" /cni-bin-dir/ - - ovn_config_namespace=openshift-ovn-kubernetes - echo "I$(date "+%m%d %H:%M:%S.%N") - disable conntrack on geneve port" - iptables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK - iptables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK - ip6tables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK - ip6tables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK - echo "I$(date "+%m%d %H:%M:%S.%N") - starting ovnkube-node" - - if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then - gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex" - elif [ "{{.OVN_GATEWAY_MODE}}" == "local" ]; then - gateway_mode_flags="--gateway-mode local --gateway-interface br-ex" - else - echo "Invalid OVN_GATEWAY_MODE: \"{{.OVN_GATEWAY_MODE}}\". Must be \"local\" or \"shared\"." - exit 1 - fi - - export_network_flows_flags= - if [[ -n "${NETFLOW_COLLECTORS}" ]] ; then - export_network_flows_flags="--netflow-targets ${NETFLOW_COLLECTORS}" - fi - if [[ -n "${SFLOW_COLLECTORS}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --sflow-targets ${SFLOW_COLLECTORS}" - fi - if [[ -n "${IPFIX_COLLECTORS}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --ipfix-targets ${IPFIX_COLLECTORS}" - fi - if [[ -n "${IPFIX_CACHE_MAX_FLOWS}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --ipfix-cache-max-flows ${IPFIX_CACHE_MAX_FLOWS}" - fi - if [[ -n "${IPFIX_CACHE_ACTIVE_TIMEOUT}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --ipfix-cache-active-timeout ${IPFIX_CACHE_ACTIVE_TIMEOUT}" - fi - if [[ -n "${IPFIX_SAMPLING}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --ipfix-sampling ${IPFIX_SAMPLING}" - fi - gw_interface_flag= - # if br-ex1 is configured on the node, we want to use it for external gateway traffic - if [ -d /sys/class/net/br-ex1 ]; then - gw_interface_flag="--exgw-interface=br-ex1" - fi - - node_mgmt_port_netdev_flags= - if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then - node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}" - fi - if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then - node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" - fi - - multi_network_enabled_flag= - if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then - multi_network_enabled_flag="--enable-multi-network" - fi - - multi_network_policy_enabled_flag= - if [[ "{{.OVN_MULTI_NETWORK_POLICY_ENABLE}}" == "true" ]]; then - multi_network_policy_enabled_flag="--enable-multi-networkpolicy" - fi - - admin_network_policy_enabled_flag= - if [[ "{{.OVN_ADMIN_NETWORK_POLICY_ENABLE}}" == "true" ]]; then - admin_network_policy_enabled_flag="--enable-admin-network-policy" - fi - - # If IP Forwarding mode is global set it in the host here. - ip_forwarding_flag= - if [ "{{.IP_FORWARDING_MODE}}" == "Global" ]; then - sysctl -w net.ipv4.ip_forward=1 - sysctl -w net.ipv6.conf.all.forwarding=1 - else - ip_forwarding_flag="--disable-forwarding" - fi - - NETWORK_NODE_IDENTITY_ENABLE= - if [[ "{{.NETWORK_NODE_IDENTITY_ENABLE}}" == "true" ]]; then - NETWORK_NODE_IDENTITY_ENABLE=" - --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig - --cert-dir=/etc/ovn/ovnkube-node-certs - --cert-duration={{.NodeIdentityCertDuration}} - " - fi - - exec /usr/bin/ovnkube --init-ovnkube-controller "${K8S_NODE}" --init-node "${K8S_NODE}" \ - --config-file=/run/ovnkube-config/ovnkube.conf \ - --ovn-empty-lb-events \ - --loglevel "${OVN_KUBE_LOG_LEVEL}" \ - --inactivity-probe="${OVN_CONTROLLER_INACTIVITY_PROBE}" \ - ${gateway_mode_flags} \ - ${node_mgmt_port_netdev_flags} \ - {{- if eq .OVN_NODE_MODE "dpu-host" }} - --ovnkube-node-mode dpu-host \ - {{- end }} - --metrics-bind-address "127.0.0.1:29103" \ - --ovn-metrics-bind-address "127.0.0.1:29105" \ - --metrics-enable-pprof \ - --metrics-enable-config-duration \ - --export-ovs-metrics \ - --disable-snat-multiple-gws \ - ${export_network_flows_flags} \ - ${multi_network_enabled_flag} \ - ${multi_network_policy_enabled_flag} \ - ${admin_network_policy_enabled_flag} \ - --enable-multicast \ - --zone ${K8S_NODE} \ - --enable-interconnect \ - --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}" \ - ${gw_interface_flag} \ - --enable-multi-external-gateway=true \ - ${ip_forwarding_flag} \ - ${NETWORK_NODE_IDENTITY_ENABLE} + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-ovnkube-node ${OVN_KUBE_LOG_LEVEL} 29103 29105 env: # for kubectl - name: KUBERNETES_SERVICE_PORT @@ -894,10 +422,15 @@ spec: ports: - name: metrics-port containerPort: 29103 + ports: + - name: ovnmetrics-port + containerPort: 29105 securityContext: privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib {{ if .NETWORK_NODE_IDENTITY_ENABLE }} - mountPath: /var/lib/kubelet name: host-kubelet @@ -945,10 +478,6 @@ spec: name: ovnkube-config - mountPath: /env name: env-overrides - - mountPath: /ovn-cert # not needed, but useful when exec'ing in to pod. - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca resources: requests: cpu: 10m @@ -1139,15 +668,13 @@ spec: configMap: name: env-overrides optional: true - - name: ovn-ca - configMap: - name: ovn-ca - - name: ovn-cert - secret: - secretName: ovn-cert - name: ovn-node-metrics-cert secret: secretName: ovn-node-metrics-cert optional: true + - name: ovnkube-script-lib + configMap: + name: ovnkube-script-lib + defaultMode: 0744 tolerations: - operator: "Exists" diff --git a/bindata/network/ovn-kubernetes/managed/single-zone-interconnect/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/managed/single-zone-interconnect/ovnkube-node.yaml index 5e168b7609..ee20a195bb 100644 --- a/bindata/network/ovn-kubernetes/managed/single-zone-interconnect/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/managed/single-zone-interconnect/ovnkube-node.yaml @@ -330,7 +330,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:9103 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29103/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} @@ -382,7 +382,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:9105 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29105/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} @@ -462,6 +462,11 @@ spec: iptables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK ip6tables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK ip6tables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK + {{- if .OVNHybridOverlayVXLANPort}} + echo "I$(date "+%m%d %H:%M:%S.%N") - disable conntrack on hybrid overlay VXLAN port" + iptables -t raw -A PREROUTING -p udp --dport {{.OVNHybridOverlayVXLANPort}} -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport {{.OVNHybridOverlayVXLANPort}} -j NOTRACK + {{- end}} retries=0 echo "I$(date "+%m%d %H:%M:%S.%N") - starting ovnkube-node db_ip ${db_ip}" diff --git a/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-control-plane.yaml b/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-control-plane.yaml index 8b0e96efc5..d2354656ca 100644 --- a/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-control-plane.yaml +++ b/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-control-plane.yaml @@ -71,7 +71,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:9108 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29108/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} diff --git a/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-node.yaml index 9ee98b2e8a..6dbe32017b 100644 --- a/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-node.yaml @@ -31,6 +31,7 @@ spec: metadata: annotations: target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + network.operator.openshift.io/ovnkube-script-lib-hash: "{{.OVNKubeConfigHash}}" labels: {{ if eq .OVN_NODE_MODE "dpu-host" }} app: ovnkube-node-dpu-host @@ -120,7 +121,7 @@ spec: # /run/openvswitch -> tmpfs - ovsdb sockets # /env -> configmap env-overrides - debug overrides containers: - {{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} +{{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} # ovn-controller: programs the vswitch with flows from the sbdb - name: ovn-controller image: "{{.OvnImage}}" @@ -129,23 +130,8 @@ spec: - -c - | set -e - if [[ -f "/env/${K8S_NODE}" ]]; then - set -o allexport - source "/env/${K8S_NODE}" - set +o allexport - fi - - echo "$(date -Iseconds) - starting ovn-controller" - exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ - --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ - --syslog-method="{{.OVNPolicyAuditDestination}}" \ - --log-file=/var/log/ovn/acl-audit-log.log \ - -vFACILITY:"{{.OVNPolicyAuditSyslogFacility}}" \ - -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \ - -vconsole:"${OVN_LOG_LEVEL}" -vconsole:"acl_log:off" \ - -vPATTERN:console:"{{.OVN_LOG_PATTERN_CONSOLE}}" \ - -vsyslog:"acl_log:info" \ - -vfile:"acl_log:info" + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-ovn-controller ${OVN_LOG_LEVEL} securityContext: privileged: true env: @@ -156,6 +142,8 @@ spec: fieldRef: fieldPath: spec.nodeName volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - mountPath: /run/openvswitch name: run-openvswitch - mountPath: /run/ovn/ @@ -168,10 +156,6 @@ spec: name: var-lib-openvswitch - mountPath: /env name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - mountPath: /var/log/ovn/ name: node-log - mountPath: /dev/log @@ -188,56 +172,21 @@ spec: - -c - | set -euo pipefail - - # Rotate audit log files when then get to max size (in bytes) - MAXFILESIZE=$(( "{{.OVNPolicyAuditMaxFileSize}}"*1000000 )) - MAXLOGFILES="{{.OVNPolicyAuditMaxLogFiles}}" - LOGDIR=/var/log/ovn - LOGFILE=${LOGDIR}/acl-audit-log.log - CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid) - - # Redirect err to null so no messages are shown upon rotation - tail -F ${LOGFILE} 2> /dev/null & - - while true - do - # Make sure ovn-controller's logfile exists, and get current size in bytes - if [ -f "$LOGFILE" ]; then - file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1` - else - ovs-appctl -t /var/run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen - file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1` - fi - - if [ $file_size -gt $MAXFILESIZE ];then - echo "Rotating OVN ACL Log File" - timestamp=`date '+%Y-%m-%dT%H-%M-%S'` - mv ${LOGFILE} /var/log/ovn/acl-audit-log.$timestamp.log - ovs-appctl -t /run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen - CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid) - fi - - # Ensure total number of log files does not exceed the maximum configured from OVNPolicyAuditMaxLogFiles - num_files=$(ls -1 ${LOGDIR}/acl-audit-log* 2>/dev/null | wc -l) - if [ "$num_files" -gt "$MAXLOGFILES" ]; then - num_to_delete=$(( num_files - ${MAXLOGFILES} )) - ls -1t ${LOGDIR}/acl-audit-log* 2>/dev/null | tail -$num_to_delete | xargs -I {} rm {} - fi - - # sleep for 30 seconds to avoid wasting CPU - sleep 30 - done + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-audit-log-rotation resources: requests: cpu: 10m memory: 20Mi terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - mountPath: /var/log/ovn/ name: node-log - mountPath: /run/ovn/ name: run-ovn - {{ end }} +{{ end }} - name: kube-rbac-proxy-node image: {{.KubeRBACProxyImage}} command: @@ -246,39 +195,8 @@ spec: - | #!/bin/bash set -euo pipefail - - TLS_PK=/etc/pki/tls/metrics-cert/tls.key - TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt - # As the secret mount is optional we must wait for the files to be present. - # The service is created in monitor.yaml and this is created in sdn.yaml. - # If it isn't created there is probably an issue so we want to crashloop. - retries=0 - TS=$(date +%s) - WARN_TS=$(( ${TS} + $(( 20 * 60)) )) - HAS_LOGGED_INFO=0 - - log_missing_certs(){ - CUR_TS=$(date +%s) - if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then - echo $(date -Iseconds) WARN: ovn-node-metrics-cert not mounted after 20 minutes. - elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then - echo $(date -Iseconds) INFO: ovn-node-metrics-cert not mounted. Waiting one hour. - HAS_LOGGED_INFO=1 - fi - } - while [[ ! -f "${TLS_PK}" || ! -f "${TLS_CERT}" ]] ; do - log_missing_certs - sleep 5 - done - - echo $(date -Iseconds) INFO: ovn-node-metrics-certs mounted, starting kube-rbac-proxy - exec /usr/bin/kube-rbac-proxy \ - --logtostderr \ - --secure-listen-address=:9103 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ - --upstream=http://127.0.0.1:29103/ \ - --tls-private-key-file=${TLS_PK} \ - --tls-cert-file=${TLS_CERT} + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-rbac-proxy-node ovn-node-metrics 9103 29103 /etc/pki/tls/metrics-cert/tls.key /etc/pki/tls/metrics-cert/tls.crt ports: - containerPort: 9103 name: https @@ -288,6 +206,8 @@ spec: memory: 20Mi terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - name: ovn-node-metrics-cert mountPath: /etc/pki/tls/metrics-cert readOnly: True @@ -299,39 +219,8 @@ spec: - | #!/bin/bash set -euo pipefail - - TLS_PK=/etc/pki/tls/metrics-cert/tls.key - TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt - # As the secret mount is optional we must wait for the files to be present. - # The service is created in monitor.yaml and this is created in sdn.yaml. - # If it isn't created there is probably an issue so we want to crashloop. - retries=0 - TS=$(date +%s) - WARN_TS=$(( ${TS} + $(( 20 * 60)) )) - HAS_LOGGED_INFO=0 - - log_missing_certs(){ - CUR_TS=$(date +%s) - if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then - echo $(date -Iseconds) WARN: ovn-node-metrics-cert not mounted after 20 minutes. - elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then - echo $(date -Iseconds) INFO: ovn-node-metrics-cert not mounted. Waiting one hour. - HAS_LOGGED_INFO=1 - fi - } - while [[ ! -f "${TLS_PK}" || ! -f "${TLS_CERT}" ]] ; do - log_missing_certs - sleep 5 - done - - echo $(date -Iseconds) INFO: ovn-node-metrics-certs mounted, starting kube-rbac-proxy - exec /usr/bin/kube-rbac-proxy \ - --logtostderr \ - --secure-listen-address=:9105 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ - --upstream=http://127.0.0.1:29105/ \ - --tls-private-key-file=${TLS_PK} \ - --tls-cert-file=${TLS_CERT} + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-rbac-proxy-node ovn-metrics 9105 29105 /etc/pki/tls/metrics-cert/tls.key /etc/pki/tls/metrics-cert/tls.crt ports: - containerPort: 9105 name: https @@ -341,6 +230,8 @@ spec: memory: 20Mi terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - name: ovn-node-metrics-cert mountPath: /etc/pki/tls/metrics-cert readOnly: True @@ -357,41 +248,24 @@ spec: source /env/_master set +o allexport fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 - quit() { - echo "$(date -Iseconds) - stopping ovn-northd" - OVN_MANAGE_OVSDB=no /usr/share/ovn/scripts/ovn-ctl stop_northd - echo "$(date -Iseconds) - ovn-northd stopped" - rm -f /var/run/ovn/ovn-northd.pid - exit 0 - } - # end of quit - trap quit TERM INT - - echo "$(date -Iseconds) - starting ovn-northd" - exec ovn-northd \ - --no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off "-vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - --pidfile /var/run/ovn/ovn-northd.pid \ - --n-threads={{.NorthdThreads}} & - wait $! + trap quit-ovn-northd TERM INT + start-ovn-northd "${OVN_LOG_LEVEL}" env: - name: OVN_LOG_LEVEL value: info volumeMounts: - - mountPath: /etc/openvswitch/ + - mountPath: /ovnkube-lib + name: ovnkube-script-lib + - mountPath: /etc/ovn name: etc-openvswitch - - mountPath: /var/lib/openvswitch/ - name: var-lib-openvswitch - - mountPath: /run/openvswitch/ - name: run-openvswitch + - mountPath: /var/log/ovn + name: node-log - mountPath: /run/ovn/ name: run-ovn - mountPath: /env name: env-overrides - - mountPath: /ovn-cert # not needed, but useful when exec'ing in to pod. - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca resources: requests: cpu: 10m @@ -411,26 +285,10 @@ spec: source /env/_master set +o allexport fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 - quit() { - echo "$(date -Iseconds) - stopping nbdb" - /usr/share/ovn/scripts/ovn-ctl stop_nb_ovsdb - echo "$(date -Iseconds) - nbdb stopped" - rm -f /var/run/ovn/ovnnb_db.pid - exit 0 - } - # end of quit - trap quit TERM INT - - bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } - - OVN_ARGS="--no-monitor" - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_nb_ovsdb & - - wait $! - + trap quit-nbdb TERM INT + start-nbdb ${OVN_LOG_LEVEL} lifecycle: postStart: exec: @@ -439,120 +297,11 @@ spec: - -c - | set -x - rm -f /var/run/ovn/ovnnb_db.pid - #configure northd_probe_interval - OVN_NB_CTL="ovn-nbctl" - - # retry an operation a number of times, sleeping 2 seconds between each try - retry() { - local tries=${1} - local desc=${2} - local cmd=${3} - - local retries=0 - while ! ${cmd}; do - (( retries += 1 )) - if [[ "${retries}" -gt ${tries} ]]; then - echo "$(date -Iseconds) - ERROR - nbdb ${desc} - too many failed attempts, giving up" - return 1 - fi - echo "$(date -Iseconds) - WARN - nbdb ${desc} - failed try ${retries}, retrying..." - sleep 2 - done - echo "$(date -Iseconds) - INFO - nbdb ${desc} - success" - return 0 - } - - # set the connection and inactivity probe - if ! retry 60 "inactivity-probe" "ovn-nbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_NB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe={{.OVN_NB_INACTIVITY_PROBE}}"; then - exit 1 - fi - - # Upgrade the db if required. - DB_SCHEMA="/usr/share/ovn/ovn-nb.ovsschema" - DB_SERVER="unix:/var/run/ovn/ovnnb_db.sock" - schema_name=$(ovsdb-tool schema-name $DB_SCHEMA) - db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" "$schema_name") - target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") - - if ovsdb-tool compare-versions "$db_version" == "$target_version"; then - : - elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then - echo "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?" - else - echo "Upgrading database $schema_name from schema version $db_version to $target_version" - ovsdb-client -t 30 convert "$DB_SERVER" "$DB_SCHEMA" - fi - - echo "Setting the IC zone to ${K8S_NODE}" - retries=0 - current_probe_interval=0 - while [[ "${retries}" -lt 10 ]]; do - current_probe_interval=$(${OVN_NB_CTL} set NB_Global . name="${K8S_NODE}" options:name="${K8S_NODE}") - if [[ $? == 0 ]]; then - current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"') - break - else - sleep 2 - (( retries += 1 )) - fi - done - - northd_probe_interval=${OVN_NORTHD_PROBE_INTERVAL:-10000} - echo "Setting northd probe interval to ${northd_probe_interval} ms" - retries=0 - current_probe_interval=0 - while [[ "${retries}" -lt 10 ]]; do - current_probe_interval=$(${OVN_NB_CTL} --if-exists get NB_GLOBAL . options:northd_probe_interval) - if [[ $? == 0 ]]; then - current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"') - break - else - sleep 2 - (( retries += 1 )) - fi - done - - if [[ "${current_probe_interval}" != "${northd_probe_interval}" ]]; then - retries=0 - while [[ "${retries}" -lt 10 ]]; do - ${OVN_NB_CTL} set NB_GLOBAL . options:northd_probe_interval=${northd_probe_interval} - if [[ $? != 0 ]]; then - echo "Failed to set northd probe interval to ${northd_probe_interval}. retrying..." - sleep 2 - (( retries += 1 )) - else - echo "Successfully set northd probe interval to ${northd_probe_interval} ms" - break - fi - done - fi - - # Enable/disable IPsec - {{ if .OVNIPsecEnable }} - ipsec=true - {{ else }} - ipsec=false - {{ end }} - ipsec_encapsulation=false - if [ "${ipsec}" == "true" ]; then - # IBMCloud does not forward ESP (IP proto 50) - # Instead, force IBMCloud IPsec to always use NAT-T - if [ "{{.PlatformType}}" == "IBMCloud" ]; then - ipsec_encapsulation=true - fi - fi - if ! retry 20 "ipsec" "${OVN_NB_CTL} set nb_global . ipsec=${ipsec} options:ipsec_encapsulation=${ipsec_encapsulation}"; then - exit 1 - fi - - # Tell northd to sleep a bit so it takes less CPU - if ! retry 20 "northd-backoff" "${OVN_NB_CTL} set nb_global . options:northd-backoff-interval-ms={{.OVN_NORTHD_BACKOFF_MS}}"; then - exit 1 - fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + nbdb-post-start {{.OVN_NORTHD_PROBE_INTERVAL}} readinessProbe: {{ if not .IsSNO }} - initialDelaySeconds: 90 + initialDelaySeconds: 10 {{ end }} timeoutSeconds: 5 exec: @@ -561,40 +310,26 @@ spec: - -c - | set -xeo pipefail - status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=3 ovsdb-server/sync-status 2>/dev/null | { grep "state: active" || false; }) - if [[ -z "${status}" ]]; then - echo "NB DB is not running or active." - exit 1 - fi - - # set trim-on-compaction - /usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=5 ovsdb-server/memory-trim-on-compaction on 2>/dev/null + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + ovndb-readiness-probe "nb" env: - name: OVN_LOG_LEVEL value: info - - name: OVN_NORTHD_PROBE_INTERVAL - value: "{{.OVN_NORTHD_PROBE_INTERVAL}}" - name: K8S_NODE valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - - mountPath: /etc/openvswitch/ - name: etc-openvswitch + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - mountPath: /etc/ovn/ name: etc-openvswitch - - mountPath: /var/lib/openvswitch/ - name: var-lib-openvswitch - - mountPath: /run/openvswitch/ - name: run-openvswitch + - mountPath: /var/log/ovn + name: node-log - mountPath: /run/ovn/ name: run-ovn - mountPath: /env name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca resources: requests: cpu: 10m @@ -614,26 +349,10 @@ spec: source /env/_master set +o allexport fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 - quit() { - echo "$(date -Iseconds) - stopping sbdb" - /usr/share/ovn/scripts/ovn-ctl stop_sb_ovsdb - echo "$(date -Iseconds) - sbdb stopped" - rm -f /var/run/ovn/ovnsb_db.pid - exit 0 - } - # end of quit - trap quit TERM INT - - bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } - - OVN_ARGS="--no-monitor" - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_sb_ovsdb & - - wait $! - + trap quit-sbdb TERM INT + start-sbdb ${OVN_LOG_LEVEL} lifecycle: postStart: exec: @@ -642,56 +361,11 @@ spec: - -c - | set -x - rm -f /var/run/ovn/ovnsb_db.pid - - # retry an operation a number of times, sleeping 2 seconds between each try - retry() { - local tries=${1} - local desc=${2} - local cmd=${3} - - local retries=0 - while ! ${cmd}; do - (( retries += 1 )) - if [[ "${retries}" -gt ${tries} ]]; then - echo "$(date -Iseconds) - ERROR - sbdb ${desc} - too many failed attempts, giving up" - return 1 - fi - echo "$(date -Iseconds) - WARN - sbdb ${desc} - failed try ${retries}, retrying..." - sleep 2 - done - echo "$(date -Iseconds) - INFO - sbdb ${desc} - success" - return 0 - } - - # set the connection and inactivity probe - if ! retry 60 "inactivity-probe" "ovn-sbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_SB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe={{.OVN_CONTROLLER_INACTIVITY_PROBE}}"; then - exit 1 - fi - - # Upgrade the db if required. - DB_SCHEMA="/usr/share/ovn/ovn-sb.ovsschema" - DB_SERVER="unix:/var/run/ovn/ovnsb_db.sock" - schema_name=$(ovsdb-tool schema-name $DB_SCHEMA) - db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" "$schema_name") - target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") - if ovsdb-tool compare-versions "$db_version" == "$target_version"; then - : - elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then - echo "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?" - else - echo "Upgrading database $schema_name from schema version $db_version to $target_version" - ovsdb-client -t 30 convert "$DB_SERVER" "$DB_SCHEMA" - fi - - # Kill some time while the cluster converges by checking IPsec status - OVN_SB_CTL="ovn-sbctl" - if ! retry 20 "ipsec" "${OVN_SB_CTL} get sb_global . ipsec"; then - exit 1 - fi + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + sbdb-post-start readinessProbe: {{ if not .IsSNO }} - initialDelaySeconds: 90 + initialDelaySeconds: 10 {{ end }} timeoutSeconds: 5 exec: @@ -700,33 +374,22 @@ spec: - -c - | set -xeo pipefail - status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=3 ovsdb-server/sync-status 2>/dev/null | { grep "state: active" || false; }) - if [[ -z "${status}" ]]; then - echo "SB DB is not running or active." - exit 1 - fi - # set trim-on-compaction - /usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=5 ovsdb-server/memory-trim-on-compaction on 2>/dev/null + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + ovndb-readiness-probe "sb" env: - name: OVN_LOG_LEVEL value: info volumeMounts: - - mountPath: /etc/openvswitch/ - name: etc-openvswitch + - mountPath: /ovnkube-lib + name: ovnkube-script-lib - mountPath: /etc/ovn/ name: etc-openvswitch - - mountPath: /var/lib/openvswitch/ - name: var-lib-openvswitch - - mountPath: /run/openvswitch/ - name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /var/log/ovn + name: node-log - mountPath: /env name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca resources: requests: cpu: 10m @@ -741,167 +404,8 @@ spec: - -c - | set -xe - if [[ -f "/env/${K8S_NODE}" ]]; then - set -o allexport - source "/env/${K8S_NODE}" - set +o allexport - fi - - function log() - { - echo "$(date --iso-8601=seconds) [cnibincopy] ${1}" - } - - # collect host os information - . /host/etc/os-release - rhelmajor= - # detect which version we're using in order to copy the proper binaries - case "${ID}" in - rhcos|scos) - RHEL_VERSION=$(echo "${CPE_NAME}" | cut -f 5 -d :) - rhelmajor=$(echo $RHEL_VERSION | sed -E 's/([0-9]+)\.{1}[0-9]+(\.[0-9]+)?/\1/') - ;; - rhel) rhelmajor=$(echo "${VERSION_ID}" | cut -f 1 -d .) - ;; - fedora) - if [ "${VARIANT_ID}" == "coreos" ]; then - rhelmajor=8 - else - log "FATAL ERROR: Unsupported Fedora variant=${VARIANT_ID}" - exit 1 - fi - ;; - *) log "FATAL ERROR: Unsupported OS ID=${ID}"; exit 1 - ;; - esac - - # Set which directory we'll copy from, detect if it exists - sourcedir=/usr/libexec/cni/ - case "${rhelmajor}" in - 8) - sourcedir=/usr/libexec/cni/rhel8 - ;; - 9) - sourcedir=/usr/libexec/cni/rhel9 - ;; - *) - log "ERROR: RHEL Major Version Unsupported, rhelmajor=${rhelmajor}" - ;; - esac - - cp -f "$sourcedir/ovn-k8s-cni-overlay" /cni-bin-dir/ - - ovn_config_namespace=openshift-ovn-kubernetes - echo "I$(date "+%m%d %H:%M:%S.%N") - disable conntrack on geneve port" - iptables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK - iptables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK - ip6tables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK - ip6tables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK - echo "I$(date "+%m%d %H:%M:%S.%N") - starting ovnkube-node" - - if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then - gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex" - elif [ "{{.OVN_GATEWAY_MODE}}" == "local" ]; then - gateway_mode_flags="--gateway-mode local --gateway-interface br-ex" - else - echo "Invalid OVN_GATEWAY_MODE: \"{{.OVN_GATEWAY_MODE}}\". Must be \"local\" or \"shared\"." - exit 1 - fi - - export_network_flows_flags= - if [[ -n "${NETFLOW_COLLECTORS}" ]] ; then - export_network_flows_flags="--netflow-targets ${NETFLOW_COLLECTORS}" - fi - if [[ -n "${SFLOW_COLLECTORS}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --sflow-targets ${SFLOW_COLLECTORS}" - fi - if [[ -n "${IPFIX_COLLECTORS}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --ipfix-targets ${IPFIX_COLLECTORS}" - fi - if [[ -n "${IPFIX_CACHE_MAX_FLOWS}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --ipfix-cache-max-flows ${IPFIX_CACHE_MAX_FLOWS}" - fi - if [[ -n "${IPFIX_CACHE_ACTIVE_TIMEOUT}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --ipfix-cache-active-timeout ${IPFIX_CACHE_ACTIVE_TIMEOUT}" - fi - if [[ -n "${IPFIX_SAMPLING}" ]] ; then - export_network_flows_flags="$export_network_flows_flags --ipfix-sampling ${IPFIX_SAMPLING}" - fi - gw_interface_flag= - # if br-ex1 is configured on the node, we want to use it for external gateway traffic - if [ -d /sys/class/net/br-ex1 ]; then - gw_interface_flag="--exgw-interface=br-ex1" - fi - - node_mgmt_port_netdev_flags= - if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then - node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}" - fi - if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then - node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" - fi - - multi_network_enabled_flag= - if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then - multi_network_enabled_flag="--enable-multi-network" - fi - - multi_network_policy_enabled_flag= - if [[ "{{.OVN_MULTI_NETWORK_POLICY_ENABLE}}" == "true" ]]; then - multi_network_policy_enabled_flag="--enable-multi-networkpolicy" - fi - - admin_network_policy_enabled_flag= - if [[ "{{.OVN_ADMIN_NETWORK_POLICY_ENABLE}}" == "true" ]]; then - admin_network_policy_enabled_flag="--enable-admin-network-policy" - fi - - # If IP Forwarding mode is global set it in the host here. - ip_forwarding_flag= - if [ "{{.IP_FORWARDING_MODE}}" == "Global" ]; then - sysctl -w net.ipv4.ip_forward=1 - sysctl -w net.ipv6.conf.all.forwarding=1 - else - ip_forwarding_flag="--disable-forwarding" - fi - - NETWORK_NODE_IDENTITY_ENABLE= - if [[ "{{.NETWORK_NODE_IDENTITY_ENABLE}}" == "true" ]]; then - NETWORK_NODE_IDENTITY_ENABLE=" - --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig - --cert-dir=/etc/ovn/ovnkube-node-certs - --cert-duration={{.NodeIdentityCertDuration}} - " - fi - - exec /usr/bin/ovnkube --init-ovnkube-controller "${K8S_NODE}" --init-node "${K8S_NODE}" \ - --config-file=/run/ovnkube-config/ovnkube.conf \ - --ovn-empty-lb-events \ - --loglevel "${OVN_KUBE_LOG_LEVEL}" \ - --inactivity-probe="${OVN_CONTROLLER_INACTIVITY_PROBE}" \ - ${gateway_mode_flags} \ - ${node_mgmt_port_netdev_flags} \ - {{- if eq .OVN_NODE_MODE "dpu-host" }} - --ovnkube-node-mode dpu-host \ - {{- end }} - --metrics-bind-address "127.0.0.1:29103" \ - --ovn-metrics-bind-address "127.0.0.1:29105" \ - --metrics-enable-pprof \ - --metrics-enable-config-duration \ - --export-ovs-metrics \ - --disable-snat-multiple-gws \ - ${export_network_flows_flags} \ - ${multi_network_enabled_flag} \ - ${multi_network_policy_enabled_flag} \ - ${admin_network_policy_enabled_flag} \ - --enable-multicast \ - --zone ${K8S_NODE} \ - --enable-interconnect \ - --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}" \ - ${gw_interface_flag} \ - --enable-multi-external-gateway=true \ - ${ip_forwarding_flag} \ - ${NETWORK_NODE_IDENTITY_ENABLE} + . /ovnkube-lib/ovnkube-lib.sh || exit 1 + start-ovnkube-node ${OVN_KUBE_LOG_LEVEL} 29103 29105 env: # for kubectl - name: KUBERNETES_SERVICE_PORT @@ -952,10 +456,15 @@ spec: ports: - name: metrics-port containerPort: 29103 + ports: + - name: ovnmetrics-port + containerPort: 29105 securityContext: privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /ovnkube-lib + name: ovnkube-script-lib {{ if .NETWORK_NODE_IDENTITY_ENABLE }} - mountPath: /var/lib/kubelet name: host-kubelet @@ -1003,10 +512,6 @@ spec: name: ovnkube-config - mountPath: /env name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca resources: requests: cpu: 10m @@ -1174,15 +679,13 @@ spec: configMap: name: env-overrides optional: true - - name: ovn-ca - configMap: - name: ovn-ca - - name: ovn-cert - secret: - secretName: ovn-cert - name: ovn-node-metrics-cert secret: secretName: ovn-node-metrics-cert optional: true + - name: ovnkube-script-lib + configMap: + name: ovnkube-script-lib + defaultMode: 0744 tolerations: - operator: "Exists" diff --git a/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-master.yaml b/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-master.yaml index 5a240fb8b7..ec0b2e0816 100644 --- a/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-master.yaml +++ b/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-master.yaml @@ -501,7 +501,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:9102 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29102/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} diff --git a/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-node.yaml index 3b53710830..7207ad312f 100644 --- a/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-node.yaml @@ -240,7 +240,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:9103 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29103/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} @@ -292,7 +292,7 @@ spec: exec /usr/bin/kube-rbac-proxy \ --logtostderr \ --secure-listen-address=:9105 \ - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 \ --upstream=http://127.0.0.1:29105/ \ --tls-private-key-file=${TLS_PK} \ --tls-cert-file=${TLS_CERT} @@ -371,6 +371,11 @@ spec: iptables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK ip6tables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK ip6tables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK + {{- if .OVNHybridOverlayVXLANPort}} + echo "I$(date "+%m%d %H:%M:%S.%N") - disable conntrack on hybrid overlay VXLAN port" + iptables -t raw -A PREROUTING -p udp --dport {{.OVNHybridOverlayVXLANPort}} -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport {{.OVNHybridOverlayVXLANPort}} -j NOTRACK + {{- end}} echo "I$(date "+%m%d %H:%M:%S.%N") - starting ovnkube-node" if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go index 3beaf1a79d..07bac64cbc 100644 --- a/pkg/network/ovn_kubernetes.go +++ b/pkg/network/ovn_kubernetes.go @@ -2,6 +2,8 @@ package network import ( "context" + "crypto/sha1" + "encoding/hex" "encoding/json" "fmt" "log" @@ -407,20 +409,49 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo data.Data["OVN_MULTI_NETWORK_POLICY_ENABLE"] = true } - var manifestSubDir string + commonManifestDir := filepath.Join(manifestDir, "network/ovn-kubernetes/common") + + cmPaths := []string{ + filepath.Join(commonManifestDir, "008-script-lib.yaml"), + } + + // Many ovnkube config options are stored in ConfigMaps; the ovnkube + // daemonsets need to know when those ConfigMaps change so they can + // restart with the new options. Render those ConfigMaps first and + // embed a hash of their data into the ovnkube-node daemonsets. + h := sha1.New() + for _, path := range cmPaths { + manifests, err := render.RenderTemplate(path, &data) + if err != nil { + return nil, progressing, errors.Wrapf(err, "failed to render ConfigMap template %q", path) + } + + // Hash each rendered ConfigMap object's data + for _, m := range manifests { + bytes, err := json.Marshal(m) + if err != nil { + return nil, progressing, errors.Wrapf(err, "failed to marshal ConfigMap %q manifest", path) + } + if _, err := h.Write(bytes); err != nil { + return nil, progressing, errors.Wrapf(err, "failed to hash ConfigMap %q data", path) + } + } + } + data.Data["OVNKubeConfigHash"] = hex.EncodeToString(h.Sum(nil)) + manifestDirs := make([]string, 0, 2) - manifestDirs = append(manifestDirs, filepath.Join(manifestDir, "network/ovn-kubernetes/common")) + manifestDirs = append(manifestDirs, commonManifestDir) productFlavor := "self-hosted" if bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.Enabled { productFlavor = "managed" } - manifestSubDirBasePath := filepath.Join("network/ovn-kubernetes", productFlavor) + manifestSubDirBasePath := filepath.Join(manifestDir, "network/ovn-kubernetes", productFlavor) - manifestDirs = append(manifestDirs, filepath.Join(manifestDir, manifestSubDirBasePath, "common")) + manifestDirs = append(manifestDirs, filepath.Join(manifestSubDirBasePath, "common")) // choose the YAMLs based on the target zone mode (4.14 only) TODO: starting from 4.15, support multizone only - manifestSubDir = filepath.Join(manifestSubDirBasePath, "multi-zone-interconnect") // default is multizone + manifestSubDir := filepath.Join(manifestSubDirBasePath, "multi-zone-interconnect") // default is multizone if targetZoneMode.zoneMode == zoneModeSingleZone { // non-default, internal use only; this is selected in the first phase of an upgrade from a // non-interconnect version (< 4.14) to an interconnect version (>= 4.14) @@ -432,7 +463,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo manifestSubDir = filepath.Join(manifestSubDirBasePath, "multi-zone-interconnect-tmp") } klog.Infof("render YAMLs from %s folder", manifestSubDir) - manifestDirs = append(manifestDirs, filepath.Join(manifestDir, manifestSubDir)) + manifestDirs = append(manifestDirs, manifestSubDir) manifests, err := render.RenderDirs(manifestDirs, &data) if err != nil { @@ -447,7 +478,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo if len(bootstrapResult.OVN.OVNKubernetesConfig.SmartNicModeNodes) > 0 { data.Data["OVN_NODE_MODE"] = OVN_NODE_MODE_SMART_NIC - manifests, err = render.RenderTemplate(filepath.Join(manifestDir, manifestSubDir+"/ovnkube-node.yaml"), &data) + manifests, err = render.RenderTemplate(filepath.Join(manifestSubDir, "ovnkube-node.yaml"), &data) if err != nil { return nil, progressing, errors.Wrap(err, "failed to render manifests for smart-nic") } @@ -456,7 +487,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo if len(bootstrapResult.OVN.OVNKubernetesConfig.DpuHostModeNodes) > 0 { data.Data["OVN_NODE_MODE"] = OVN_NODE_MODE_DPU_HOST - manifests, err = render.RenderTemplate(filepath.Join(manifestDir, manifestSubDir+"/ovnkube-node.yaml"), &data) + manifests, err = render.RenderTemplate(filepath.Join(manifestSubDir, "ovnkube-node.yaml"), &data) if err != nil { return nil, progressing, errors.Wrap(err, "failed to render manifests for dpu-host") } @@ -467,7 +498,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo // "OVN_NODE_MODE" not set when render.RenderDir() called above, // so render just the error-cni.yaml with "OVN_NODE_MODE" set. data.Data["OVN_NODE_MODE"] = OVN_NODE_MODE_DPU - manifests, err = render.RenderTemplate(filepath.Join(manifestDir, "network/ovn-kubernetes/common/error-cni.yaml"), &data) + manifests, err = render.RenderTemplate(filepath.Join(commonManifestDir, "error-cni.yaml"), &data) if err != nil { return nil, progressing, errors.Wrap(err, "failed to render manifests for dpu") } @@ -2249,7 +2280,7 @@ func isInterConnectEnabledOnDaemonset(ds *appsv1.DaemonSet, containerName string for _, container := range ds.Spec.Template.Spec.Containers { if container.Name == containerName { for _, c := range container.Command { - if strings.Contains(c, "--enable-interconnect") { + if strings.Contains(c, "--enable-interconnect") || strings.Contains(c, "ovnkube-lib.sh") { return true } } diff --git a/pkg/network/ovn_kubernetes_test.go b/pkg/network/ovn_kubernetes_test.go index 0e5c5700a9..fdefb27aa6 100644 --- a/pkg/network/ovn_kubernetes_test.go +++ b/pkg/network/ovn_kubernetes_test.go @@ -876,7 +876,7 @@ func checkOVNKubernetesPostStart(objects []*uns.Unstructured) error { return fmt.Errorf("could not find nbdb postStart script in daemonset %s", ovnkubeNode.GetName()) } - expectedScriptSubStr := "Successfully set northd probe interval" + expectedScriptSubStr := "nbdb-post-start" if !strings.Contains(strings.Join(script, " "), expectedScriptSubStr) { return fmt.Errorf("postStart script in daemonset %s does not contain %s: %s", ovnkubeNode.GetName(), expectedScriptSubStr, script) }