[DNM] WIP merge

contrib/kind.sh

@@ -579,6 +579,7 @@ create_ovn_kube_manifests() {
     --ovn-loglevel-controller="${OVN_LOG_LEVEL_CONTROLLER}" \
     --egress-ip-enable=true \
     --egress-firewall-enable=true \
+    --egress-qos-enable=true \
     --v4-join-subnet="${JOIN_SUBNET_IPV4}" \
     --v6-join-subnet="${JOIN_SUBNET_IPV6}" \
     --ex-gw-network-interface="${OVN_EX_GW_NETWORK_INTERFACE}"
@@ -606,6 +607,7 @@ install_ovn() {
 
   run_kubectl apply -f k8s.ovn.org_egressfirewalls.yaml
   run_kubectl apply -f k8s.ovn.org_egressips.yaml
+  run_kubectl apply -f k8s.ovn.org_egressqoses.yaml
   run_kubectl apply -f ovn-setup.yaml
   MASTER_NODES=$(kind get nodes --name "${KIND_CLUSTER_NAME}" | sort | head -n "${KIND_NUM_MASTER}")
   # We want OVN HA not Kubernetes HA

dist/images/daemonset.sh

@@ -60,6 +60,7 @@ OVN_EMPTY_LB_EVENTS=""
 OVN_MULTICAST_ENABLE=""
 OVN_EGRESSIP_ENABLE=
 OVN_EGRESSFIREWALL_ENABLE=
+OVN_EGRESSQOS_ENABLE=
 OVN_DISABLE_OVN_IFACE_ID_VER="false"
 OVN_V4_JOIN_SUBNET=""
 OVN_V6_JOIN_SUBNET=""
@@ -204,6 +205,9 @@ while [ "$1" != "" ]; do
   --egress-firewall-enable)
     OVN_EGRESSFIREWALL_ENABLE=$VALUE
     ;;
+  --egress-qos-enable)
+    OVN_EGRESSQOS_ENABLE=$VALUE
+    ;;
   --v4-join-subnet)
     OVN_V4_JOIN_SUBNET=$VALUE
     ;;
@@ -302,6 +306,8 @@ ovn_egress_ip_enable=${OVN_EGRESSIP_ENABLE}
 echo "ovn_egress_ip_enable: ${ovn_egress_ip_enable}"
 ovn_egress_firewall_enable=${OVN_EGRESSFIREWALL_ENABLE}
 echo "ovn_egress_firewall_enable: ${ovn_egress_firewall_enable}"
+ovn_egress_qos_enable=${OVN_EGRESSQOS_ENABLE}
+echo "ovn_egress_qos_enable: ${ovn_egress_qos_enable}"
 ovn_disable_ovn_iface_id_ver=${OVN_DISABLE_OVN_IFACE_ID_VER}
 echo "ovn_disable_ovn_iface_id_ver: ${ovn_disable_ovn_iface_id_ver}"
 ovn_hybrid_overlay_net_cidr=${OVN_HYBRID_OVERLAY_NET_CIDR}
@@ -448,6 +454,7 @@ ovn_image=${image} \
   ovn_multicast_enable=${ovn_multicast_enable} \
   ovn_egress_ip_enable=${ovn_egress_ip_enable} \
   ovn_egress_firewall_enable=${ovn_egress_firewall_enable} \
+  ovn_egress_qos_enable=${ovn_egress_qos_enable} \
   ovn_ssl_en=${ovn_ssl_en} \
   ovn_master_count=${ovn_master_count} \
   ovn_gateway_mode=${ovn_gateway_mode} \
@@ -506,5 +513,6 @@ net_cidr=${net_cidr} svc_cidr=${svc_cidr} \
 cp ../templates/ovnkube-monitor.yaml.j2 ${output_dir}/ovnkube-monitor.yaml
 cp ../templates/k8s.ovn.org_egressfirewalls.yaml.j2 ${output_dir}/k8s.ovn.org_egressfirewalls.yaml
 cp ../templates/k8s.ovn.org_egressips.yaml.j2 ${output_dir}/k8s.ovn.org_egressips.yaml
+cp ../templates/k8s.ovn.org_egressqoses.yaml.j2 ${output_dir}/k8s.ovn.org_egressqoses.yaml
 
 exit 0

dist/images/ovnkube.sh

@@ -73,6 +73,7 @@ fi
 # OVN_LFLOW_CACHE_LIMIT_KB - maximum size of the logical flow cache of ovn-controller
 # OVN_EGRESSIP_ENABLE - enable egress IP for ovn-kubernetes
 # OVN_EGRESSFIREWALL_ENABLE - enable egressFirewall for ovn-kubernetes
+# OVN_EGRESSQOS_ENABLE - enable egress QoS for ovn-kubernetes
 # OVN_UNPRIVILEGED_MODE - execute CNI ovs/netns commands from host (default no)
 # OVNKUBE_NODE_MODE - ovnkube node mode of operation, one of: full, dpu, dpu-host (default: full)
 # OVNKUBE_NODE_MGMT_PORT_NETDEV - ovnkube node management port netdev. valid when ovnkube node mode is: dpu, dpu-host
@@ -209,6 +210,8 @@ ovn_multicast_enable=${OVN_MULTICAST_ENABLE:-}
 ovn_egressip_enable=${OVN_EGRESSIP_ENABLE:-false}
 #OVN_EGRESSFIREWALL_ENABLE - enable egressFirewall for ovn-kubernetes
 ovn_egressfirewall_enable=${OVN_EGRESSFIREWALL_ENABLE:-false}
+#OVN_EGRESSQOS_ENABLE - enable egress QoS for ovn-kubernetes
+ovn_egressqos_enable=${OVN_EGRESSQOS_ENABLE:-false}
 #OVN_DISABLE_OVN_IFACE_ID_VER - disable usage of the OVN iface-id-ver option
 ovn_disable_ovn_iface_id_ver=${OVN_DISABLE_OVN_IFACE_ID_VER:-false}
 ovn_acl_logging_rate_limit=${OVN_ACL_LOGGING_RATE_LIMIT:-"20"}
@@ -946,8 +949,19 @@ ovn-master() {
 	  egressfirewall_enabled_flag="--enable-egress-firewall"
   fi
   echo "egressfirewall_enabled_flag=${egressfirewall_enabled_flag}"
+  egressqos_enabled_flag=
+  if [[ ${ovn_egressqos_enable} == "true" ]]; then
+	  egressqos_enabled_flag="--enable-egress-qos"
+  fi
 
   ovnkube_master_metrics_bind_address="${metrics_endpoint_ip}:9409"
+  local ovnkube_metrics_tls_opts=""
+  if [[ ${OVNKUBE_METRICS_PK} != "" && ${OVNKUBE_METRICS_CERT} != "" ]]; then
+    ovnkube_metrics_tls_opts="
+        --node-server-privkey ${OVNKUBE_METRICS_PK}
+        --node-server-cert ${OVNKUBE_METRICS_CERT}
+      "
+  fi
 
   echo "=============== ovn-master ========== MASTER ONLY"
   /usr/bin/ovnkube \
@@ -967,10 +981,12 @@ ovn-master() {
     --pidfile ${OVN_RUNDIR}/ovnkube-master.pid \
     --logfile /var/log/ovn-kubernetes/ovnkube-master.log \
     ${ovn_master_ssl_opts} \
+    ${ovnkube_metrics_tls_opts} \
     ${multicast_enabled_flag} \
     ${ovn_acl_logging_rate_limit_flag} \
     ${egressip_enabled_flag} \
     ${egressfirewall_enabled_flag} \
+    ${egressqos_enabled_flag} \
     --metrics-bind-address ${ovnkube_master_metrics_bind_address} \
     --host-network-namespace ${ovn_host_network_namespace} &
 
@@ -1187,6 +1203,14 @@ ovn-node() {
   ovn_metrics_bind_address="${metrics_endpoint_ip}:9476"
   ovnkube_node_metrics_bind_address="${metrics_endpoint_ip}:9410"
 
+  local ovnkube_metrics_tls_opts=""
+  if [[ ${OVNKUBE_METRICS_PK} != "" && ${OVNKUBE_METRICS_CERT} != "" ]]; then
+    ovnkube_metrics_tls_opts="
+        --node-server-privkey ${OVNKUBE_METRICS_PK}
+        --node-server-cert ${OVNKUBE_METRICS_CERT}
+      "
+  fi
+
   echo "=============== ovn-node   --init-node"
   /usr/bin/ovnkube --init-node ${K8S_NODE} \
     --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \
@@ -1208,6 +1232,7 @@ ovn-node() {
     --pidfile ${OVN_RUNDIR}/ovnkube.pid \
     --logfile /var/log/ovn-kubernetes/ovnkube.log \
     ${ovn_node_ssl_opts} \
+    ${ovnkube_metrics_tls_opts} \
     --inactivity-probe=${ovn_remote_probe_interval} \
     ${monitor_all} \
     ${enable_lflow_cache} \

dist/templates/k8s.ovn.org_egressqoses.yaml.j2

@@ -0,0 +1,128 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+  creationTimestamp: null
+  name: egressqoses.k8s.ovn.org
+spec:
+  group: k8s.ovn.org
+  names:
+    kind: EgressQoS
+    listKind: EgressQoSList
+    plural: egressqoses
+    singular: egressqos
+  scope: Namespaced
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        description: EgressQoS is a CRD that allows the user to define a DSCP value
+          for pods egress traffic on its namespace to specified CIDRs. Traffic from
+          these pods will be checked against each EgressQoSRule in the namespace's
+          EgressQoS, and if there is a match the traffic is marked with the relevant
+          DSCP value.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+            properties:
+              name:
+                type: string
+                pattern: ^default$
+          spec:
+            description: EgressQoSSpec defines the desired state of EgressQoS
+            properties:
+              egress:
+                description: a collection of Egress QoS rule objects
+                items:
+                  properties:
+                    dscp:
+                      description: DSCP marking value for matching pods' traffic.
+                      maximum: 63
+                      minimum: 0
+                      type: integer
+                    dstCIDR:
+                      description: DstCIDR specifies the destination's CIDR. Only
+                        traffic heading to this CIDR will be marked with the DSCP
+                        value. This field is optional, and in case it is not set the
+                        rule is applied to all egress traffic regardless of the destination.
+                      type: string
+                    podSelector:
+                      description: PodSelector applies the QoS rule only to the pods
+                        in the namespace whose label matches this definition. This
+                        field is optional, and in case it is not set results in the
+                        rule being applied to all pods in the namespace.
+                      properties:
+                        matchExpressions:
+                          description: matchExpressions is a list of label selector
+                            requirements. The requirements are ANDed.
+                          items:
+                            description: A label selector requirement is a selector
+                              that contains values, a key, and an operator that relates
+                              the key and values.
+                            properties:
+                              key:
+                                description: key is the label key that the selector
+                                  applies to.
+                                type: string
+                              operator:
+                                description: operator represents a key's relationship
+                                  to a set of values. Valid operators are In, NotIn,
+                                  Exists and DoesNotExist.
+                                type: string
+                              values:
+                                description: values is an array of string values.
+                                  If the operator is In or NotIn, the values array
+                                  must be non-empty. If the operator is Exists or
+                                  DoesNotExist, the values array must be empty. This
+                                  array is replaced during a strategic merge patch.
+                                items:
+                                  type: string
+                                type: array
+                            required:
+                            - key
+                            - operator
+                            type: object
+                          type: array
+                        matchLabels:
+                          additionalProperties:
+                            type: string
+                          description: matchLabels is a map of {key,value} pairs.
+                            A single {key,value} in the matchLabels map is equivalent
+                            to an element of matchExpressions, whose key field is
+                            "key", the operator is "In", and the values array contains
+                            only "value". The requirements are ANDed.
+                          type: object
+                      type: object
+                  required:
+                  - dscp
+                  type: object
+                type: array
+            required:
+            - egress
+            type: object
+          status:
+            description: EgressQoSStatus defines the observed state of EgressQoS
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

dist/templates/ovn-setup.yaml.j2

@@ -81,6 +81,7 @@ rules:
   resources:
   - egressfirewalls
   - egressips
+  - egressqoses
   verbs: ["list", "get", "watch", "update", "patch"]
 - apiGroups:
   - apiextensions.k8s.io

dist/templates/ovnkube-master.yaml.j2

@@ -195,6 +195,8 @@ spec:
           value: "{{ ovn_egress_ip_enable }}"
         - name: OVN_EGRESSFIREWALL_ENABLE
           value: "{{ ovn_egress_firewall_enable }}"
+        - name: OVN_EGRESSQOS_ENABLE
+          value: "{{ ovn_egress_qos_enable }}"
         - name: OVN_HYBRID_OVERLAY_NET_CIDR
           value: "{{ ovn_hybrid_overlay_net_cidr }}"
         - name: OVN_DISABLE_SNAT_MULTIPLE_GWS

go-controller/cmd/ovnkube/ovnkube.go

@@ -279,7 +279,8 @@ func runOvnKube(ctx *cli.Context) error {
 	// now that ovnkube master/node are running, lets expose the metrics HTTP endpoint if configured
 	// start the prometheus server to serve OVN K8s Metrics (default master port: 9409, node port: 9410)
 	if config.Metrics.BindAddress != "" {
-		metrics.StartMetricsServer(config.Metrics.BindAddress, config.Metrics.EnablePprof)
+		metrics.StartMetricsServer(config.Metrics.BindAddress, config.Metrics.EnablePprof,
+			config.Metrics.NodeServerCert, config.Metrics.NodeServerPrivKey)
 	}
 
 	// start the prometheus server to serve OVS and OVN Metrics (default port: 9476)
@@ -289,7 +290,8 @@ func runOvnKube(ctx *cli.Context) error {
 			metrics.RegisterOvsMetricsWithOvnMetrics()
 		}
 		metrics.RegisterOvnMetrics(ovnClientset.KubeClient, node)
-		metrics.StartOVNMetricsServer(config.Metrics.OVNMetricsBindAddress)
+		metrics.StartOVNMetricsServer(config.Metrics.OVNMetricsBindAddress,
+			config.Metrics.NodeServerCert, config.Metrics.NodeServerPrivKey)
 	}
 
 	// run until cancelled

go-controller/hack/boilerplate.go.txt

@@ -0,0 +1,15 @@
+/*
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/

go-controller/hack/update-codegen.sh

@@ -40,13 +40,15 @@ fi
 for crd in ${crds}; do
   echo "Generating deepcopy funcs for $crd"
   deepcopy-gen \
+    --go-header-file hack/boilerplate.go.txt \
     --input-dirs github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1 \
     -O zz_generated.deepcopy \
     --bounding-dirs github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd
 
 
   echo "Generating clientset for $crd"
   client-gen \
+    --go-header-file hack/boilerplate.go.txt \
     --clientset-name "${CLIENTSET_NAME_VERSIONED:-versioned}" \
     --input-base "" \
     --input github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1 \
@@ -55,12 +57,14 @@ for crd in ${crds}; do
 
   echo "Generating listers for $crd"
   lister-gen \
+    --go-header-file hack/boilerplate.go.txt \
     --input-dirs github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1 \
     --output-package github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1/apis/listers \
     "$@"
 
   echo "Generating informers for $crd"
   informer-gen \
+    --go-header-file hack/boilerplate.go.txt \
     --input-dirs github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1 \
     --versioned-clientset-package github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1/apis/clientset/versioned \
     --listers-package  github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1/apis/listers \
@@ -82,3 +86,8 @@ sed -i -e':begin;$!N;s/.*metadata:\n.*type: object/&\n            properties:\n
 ## adding validation to objects only to the fields
 sed -i -e ':begin;$!N;s/                          type: string\n.*type: object/&\n                      minProperties: 1\n                      maxProperties: 1/;P;D' \
 	_output/crds/k8s.ovn.org_egressfirewalls.yaml
+
+echo "Editing EgressQoS CRD"
+## We desire that only EgressQoS with the name "default" are accepted by the apiserver.
+sed -i -e':begin;$!N;s/.*metadata:\n.*type: object/&\n            properties:\n              name:\n                type: string\n                pattern: ^default$/;P;D' \
+	_output/crds/k8s.ovn.org_egressqoses.yaml