[wip] June 1st downstream merge part 2

.github/workflows/test.yml

@@ -378,7 +378,7 @@ jobs:
           - {"target": "control-plane",     "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "separate-cluster-manager": "false"}
           - {"target": "multi-homing",      "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv4",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "separate-cluster-manager": "false"}
           - {"target": "node-ip-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "separate-cluster-manager": "true"}
-          - {"target": "node-ip-migration", "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv4",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "separate-cluster-manager": "false"}
+          - {"target": "compact-mode",      "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "separate-cluster-manager": "false"}
     needs: [ build-pr ]
     env:
       JOB_NAME: "${{ matrix.target }}-${{ matrix.ha }}-${{ matrix.gateway-mode }}-${{ matrix.ipfamily }}-${{ matrix.disable-snat-multiple-gws }}-${{ matrix.second-bridge }}"
@@ -394,6 +394,7 @@ jobs:
       KIND_IPV6_SUPPORT: "${{ matrix.ipfamily == 'IPv6' || matrix.ipfamily == 'dualstack' }}"
       ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' }}"
       OVN_SEPARATE_CLUSTER_MANAGER: "${{ matrix.separate-cluster-manager == 'true' }}"
+      OVN_COMPACT_MODE: "${{ matrix.target == 'compact-mode' }}"
     steps:
 
     - name: Free up disk space
@@ -448,6 +449,8 @@ jobs:
           make -C test control-plane WHAT="Multi Homing"
         elif [ "${{ matrix.target }}" == "node-ip-migration" ]; then
           make -C test control-plane WHAT="Node IP address migration"
+        elif [ "${{ matrix.target }}" == "compact-mode" ]; then
+          SINGLE_NODE_CLUSTER="true" make -C test shard-network
         else
           make -C test ${{ matrix.target }}
         fi

README.md

@@ -140,6 +140,8 @@ that can be brought up locally and within a few minutes.
 
 [Debugging OVN](./docs/debugging.md)
 
+[Exposed metrics](./docs/metrics.md)
+
 The golang based [ovn kubernetes go-controller](./go-controller/README.md) is a reliable way to
 deploy the OVN SDN using kubernetes clients and watchers based on golang. Contains `ovnkube` and
 `ovn-k8s-cni-overlay` build and usage instructions.
@@ -207,6 +209,8 @@ cluster network.
 [ovnkube-trace](./docs/ovnkube-trace.md) a tool to trace packet simulations between points in an 
 ovn-kubernetes driven cluster.
 
+[ACLs used by ovn-k and their priorities](./docs/acls.md)
+
 # OVN Kubernetes Basics
 A good resource to get started with understanding `ovn-kubernetes` is the following recording and slides, which run through the basic architecture and functionality of the system.
 [slides](https://docs.google.com/presentation/d/1ZtwP3t6uNAU0g4S7IbqSxPg2bmQW-pPGyMW2ZNj9Nrg/edit?usp=sharing)

contrib/kind.sh

@@ -101,7 +101,7 @@ usage() {
     echo "                 [-nl |--node-loglevel <num>] [-ml|--master-loglevel <num>]"
     echo "                 [-dbl|--dbchecker-loglevel <num>] [-ndl|--ovn-loglevel-northd <loglevel>]"
     echo "                 [-nbl|--ovn-loglevel-nb <loglevel>] [-sbl|--ovn-loglevel-sb <loglevel>]"
-    echo "                 [-cl |--ovn-loglevel-controller <loglevel>]"
+    echo "                 [-cl |--ovn-loglevel-controller <loglevel>] [-me|--multicast-enabled]"
     echo "                 [-ep |--experimental-provider <name>] |"
     echo "                 [-eb |--egress-gw-separate-bridge] |"
     echo "                 [-lr |--local-kind-registry |"
@@ -110,6 +110,7 @@ usage() {
     echo "                 [-cn | --cluster-name |"
     echo "                 [-ehp|--egress-ip-healthcheck-port <num>]"
     echo "                 [-is | --ipsec]"
+    echo "                 [-cm | --compact-mode]"
     echo "                 [--isolated]"
     echo "                 [-h]]"
     echo ""
@@ -118,6 +119,8 @@ usage() {
     echo "-kt  | --keep-taint                 Do not remove taint components."
     echo "                                    DEFAULT: Remove taint components."
     echo "-ha  | --ha-enabled                 Enable high availability. DEFAULT: HA Disabled."
+    echo "-scm | --separate-cluster-manager   Separate cluster manager from ovnkube-master and run as a separate container within ovnkube-master deployment."
+    echo "-me  | --multicast-enabled          Enable multicast. DEFAULT: Disabled."
     echo "-ho  | --hybrid-enabled             Enable hybrid overlay. DEFAULT: Disabled."
     echo "-ds  | --disable-snat-multiple-gws  Disable SNAT for multiple gws. DEFAULT: Disabled."
     echo "-dp  | --disable-pkt-mtu-check      Disable checking packet size greater than MTU. Default: Disabled"
@@ -157,6 +160,7 @@ usage() {
     echo "-ehp | --egress-ip-healthcheck-port TCP port used for gRPC session by egress IP node check. DEFAULT: 9107 (Use "0" for legacy dial to port 9)."
     echo "-is  | --ipsec                      Enable IPsec encryption (spawns ovn-ipsec pods)"
     echo "-sm  | --scale-metrics              Enable scale metrics"
+    echo "-cm  | --compact-mode               Enable compact mode, ovnkube master and node run in the same process."
     echo "--isolated                          Deploy with an isolated environment (no default gateway)"
     echo "--delete                            Delete current cluster"
     echo "--deploy                            Deploy ovn kubernetes without restarting kind"
@@ -307,6 +311,8 @@ parse_args() {
                                                 ;;
            -sm  | --scale-metrics )             OVN_METRICS_SCALE_ENABLE=true
                                                 ;;
+           -cm  | --compact-mode )              OVN_COMPACT_MODE=true
+                                                ;;
             --isolated )                        OVN_ISOLATED=true
                                                 ;;
             -mne | --multi-network-enable )     shift
@@ -537,6 +543,10 @@ set_default_params() {
   fi
   ENABLE_MULTI_NET=${ENABLE_MULTI_NET:-false}
   OVN_SEPARATE_CLUSTER_MANAGER=${OVN_SEPARATE_CLUSTER_MANAGER:-false}
+  OVN_COMPACT_MODE=${OVN_COMPACT_MODE:-false}
+  if [ "$OVN_COMPACT_MODE" == true ]; then
+    KIND_NUM_WORKER=0
+  fi
 }
 
 detect_apiserver_url() {
@@ -791,7 +801,8 @@ create_ovn_kube_manifests() {
     --v6-join-subnet="${JOIN_SUBNET_IPV6}" \
     --ex-gw-network-interface="${OVN_EX_GW_NETWORK_INTERFACE}" \
     --multi-network-enable="${ENABLE_MULTI_NET}" \
-    --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}"
+    --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}" \
+    --compact-mode="${OVN_COMPACT_MODE}"
   popd
 }
 
@@ -875,6 +886,9 @@ install_metallb() {
   fi
   git clone https://github.com/metallb/metallb.git
   pushd metallb
+  # temporary fix for metallb issue
+  # https://github.com/metallb/metallb/commit/fdf92741c7fac20eedf3caa0aa922f9ff0f0e7dd#r110009241
+  git reset --hard f5ba918
   pip install -r dev-env/requirements.txt
   inv dev-env -n ovn -b frr -p bgp
   docker network create --driver bridge clientnet

dist/images/daemonset.sh

@@ -284,6 +284,9 @@ while [ "$1" != "" ]; do
   --stateless-netpol-enable)
     OVN_STATELESS_NETPOL_ENABLE=$VALUE
     ;;
+  --compact-mode)
+    COMPACT_MODE=$VALUE
+    ;;
 
   *)
     echo "WARNING: unknown parameter \"$PARAM\""
@@ -432,8 +435,11 @@ ovnkube_metrics_scale_enable=${OVNKUBE_METRICS_SCALE_ENABLE}
 echo "ovnkube_metrics_scale_enable: ${ovnkube_metrics_scale_enable}"
 ovn_stateless_netpol_enable=${OVN_STATELESS_NETPOL_ENABLE}
 echo "ovn_stateless_netpol_enable: ${ovn_stateless_netpol_enable}"
+ovnkube_compact_mode_enable=${COMPACT_MODE:-"false"}
+echo "ovnkube_compact_mode_enable: ${ovnkube_compact_mode_enable}"
 
 ovn_image=${ovnkube_image} \
+  ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
   ovn_image_pull_policy=${image_pull_policy} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   ovn_gateway_mode=${ovn_gateway_mode} \
@@ -476,6 +482,7 @@ ovn_image=${ovnkube_image} \
 # ovnkube node for dpu-host daemonset
 # TODO: we probably dont need all of these when running on dpu host
 ovn_image=${image} \
+  ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
   ovn_image_pull_policy=${image_pull_policy} \
   kind=${KIND} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
@@ -536,6 +543,8 @@ ovn_image=${ovnkube_image} \
   ovn_gateway_mode=${ovn_gateway_mode} \
   ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \
   ovn_stateless_netpol_enable=${ovn_netpol_acl_enable} \
+  ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
+  ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   j2 ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml
 
 ovn_image=${image} \

dist/images/ovnkube.sh

@@ -249,6 +249,8 @@ ovnkube_metrics_scale_enable=${OVNKUBE_METRICS_SCALE_ENABLE:-false}
 ovn_encap_ip=${OVN_ENCAP_IP:-}
 
 ovn_ex_gw_network_interface=${OVN_EX_GW_NETWORK_INTERFACE:-}
+# OVNKUBE_COMPACT_MODE_ENABLE indicate if ovnkube run master and node in one process
+ovnkube_compact_mode_enable=${OVNKUBE_COMPACT_MODE_ENABLE:-false}
 
 # Determine the ovn rundir.
 if [[ -f /usr/bin/ovn-appctl ]]; then
@@ -1020,9 +1022,18 @@ ovn-master() {
   fi
   echo "ovn_stateless_netpol_enable_flag: ${ovn_stateless_netpol_enable_flag}"
 
-  echo "=============== ovn-master ========== MASTER ONLY"
+  init_node_flags=
+  if [[ ${ovnkube_compact_mode_enable} == "true" ]]; then
+    init_node_flags="--init-node ${K8S_NODE} --nodeport"
+    echo "init_node_flags: ${init_node_flags}"
+    echo "=============== ovn-master ========== MASTER and NODE"
+  else
+    echo "=============== ovn-master ========== MASTER ONLY"
+  fi
+
   /usr/bin/ovnkube \
     --init-master ${K8S_NODE} \
+    ${init_node_flags} \
     --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \
     --nb-address=${ovn_nbdb} --sb-address=${ovn_sbdb} \
     --gateway-mode=${ovn_gateway_mode} \
@@ -1055,6 +1066,9 @@ ovn-master() {
 
   echo "=============== ovn-master ========== running"
   wait_for_event attempts=3 process_ready ovnkube-master
+  if [[ ${ovnkube_compact_mode_enable} == "true" ]] && [[ ${ovnkube_node_mode} != "dpu" ]]; then
+    setup_cni
+  fi
 
   process_healthy ovnkube-master
   exit 9

dist/templates/ovnkube-master.yaml.j2

@@ -37,7 +37,9 @@ spec:
       serviceAccountName: ovn
       hostNetwork: true
       dnsPolicy: Default
-
+      {% if ovnkube_compact_mode_enable=="true" and ovn_unprivileged_mode=="no" %}
+      hostPID: true
+      {% endif %}
       # required to be scheduled on a linux node with node-role.kubernetes.io/control-plane label and
       # only one instance of ovnkube-master pod per node
       affinity:
@@ -125,12 +127,22 @@ spec:
       - name: ovnkube-master
         image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}"
         imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}"
-
+        {% if ovnkube_compact_mode_enable=="true" %}
         command: ["/root/ovnkube.sh", "ovn-master"]
-
         securityContext:
           runAsUser: 0
-
+          {% if ovn_unprivileged_mode=="no" -%}
+          privileged: true
+          {% else %}
+          capabilities:
+            add:
+            - NET_ADMIN
+          {% endif %}
+        {% else %}
+        command: ["/root/ovnkube.sh", "ovn-master"]
+        securityContext:
+          runAsUser: 0
+        {% endif %}
         terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         # Run directories where we need to be able to access sockets
@@ -146,7 +158,29 @@ spec:
         - mountPath: /ovn-cert
           name: host-ovn-cert
           readOnly: true
-
+        {% if ovnkube_compact_mode_enable=="true" %}
+        # Common mounts
+        # for the iptables wrapper
+        - mountPath: /host
+          name: host-slash
+          readOnly: true
+        # CNI related mounts which we take over
+        - mountPath: /opt/cni/bin
+          name: host-opt-cni-bin
+        - mountPath: /etc/cni/net.d
+          name: host-etc-cni-netd
+        - mountPath: /var/run/netns
+          name: host-netns
+          mountPropagation: HostToContainer
+        - mountPath: /etc/openvswitch/
+          name: host-var-lib-ovs
+          readOnly: true
+        - mountPath: /etc/ovn/
+          name: host-var-lib-ovs
+          readOnly: true
+        - mountPath: /var/run/ovn-kubernetes
+          name: host-var-run-ovn-kubernetes
+        {% endif %}
         resources:
           requests:
             cpu: 100m
@@ -166,6 +200,8 @@ spec:
           value: "{{ ovnkube_config_duration_enable }}"
         - name: OVNKUBE_METRICS_SCALE_ENABLE
           value: "{{ ovnkube_metrics_scale_enable }}"
+        - name: OVNKUBE_COMPACT_MODE_ENABLE
+          value: "{{ ovnkube_compact_mode_enable }}"
         - name: OVN_NET_CIDR
           valueFrom:
             configMapKeyRef:
@@ -252,5 +288,28 @@ spec:
         hostPath:
           path: /etc/ovn
           type: DirectoryOrCreate
+      {% if ovnkube_compact_mode_enable=="true" %}
+      - name: host-slash
+        hostPath:
+          path: /
+      - name: host-opt-cni-bin
+        hostPath:
+          path: /opt/cni/bin
+      - name: host-etc-cni-netd
+        hostPath:
+          path: /etc/cni/net.d
+      - name: host-netns
+        hostPath:
+          path: /var/run/netns
+      - name: host-run-ovs
+        hostPath:
+          path: /run/openvswitch
+      - name: host-var-lib-ovs
+        hostPath:
+          path: /var/lib/openvswitch
+      - name: host-var-run-ovn-kubernetes
+        hostPath:
+          path: /var/run/ovn-kubernetes
+      {% endif %}
       tolerations:
       - operator: "Exists"

dist/templates/ovnkube-node.yaml.j2

@@ -35,7 +35,7 @@ spec:
       dnsPolicy: Default
       {{ "hostPID: true" if ovn_unprivileged_mode=="no" }}
       containers:
-
+      {% if ovnkube_compact_mode_enable=="false" %}
       - name: ovnkube-node
         image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}"
         imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}"
@@ -228,7 +228,7 @@ spec:
           initialDelaySeconds: 30
           timeoutSeconds: 30
           periodSeconds: 60
-
+      {% endif %}
       {% if ovnkube_app_name=="ovnkube-node" -%}
       - name: ovn-controller
         image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}"