[wip] InterConnect Testing

.github/workflows/docker.yml

@@ -4,6 +4,9 @@ on:
   push:
     branches: [ master ]
 
+permissions:
+  contents: read
+
 env:
   GO_VERSION: 1.18.4
   REGISTRY: ghcr.io 

.github/workflows/test.yml

@@ -1,11 +1,15 @@
 name: ovn-ci
 
 on:
+  merge_group:
   pull_request:
     branches: [ master ]
   schedule:
     - cron: '0 */12 * * *'
 
+permissions:
+  contents: read
+
 env:
   GO_VERSION: "1.18.4"
   K8S_VERSION: v1.24.0
@@ -395,6 +399,7 @@ jobs:
       ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' }}"
       OVN_SEPARATE_CLUSTER_MANAGER: "${{ matrix.separate-cluster-manager == 'true' }}"
       OVN_COMPACT_MODE: "${{ matrix.target == 'compact-mode' }}"
+      OVN_DUMMY_GATEWAY_BRIDGE: "${{ matrix.target == 'compact-mode' }}"
     steps:
 
     - name: Free up disk space

.github/workflows/test_periodic.yml

@@ -5,6 +5,9 @@ on:
     - cron:  '0 6 * * *'
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 env:
   GO_VERSION: "1.18.4"
   K8S_VERSION: v1.24.0

CODEOWNERS

@@ -0,0 +1 @@
+* @trozet @dcbw @girishmg @jcaamano

FAQ.md

@@ -21,7 +21,7 @@ need to be made aware of it by following the instructions below.
 OVN CNI requires several TCP and UDP ports to be opened on each of the node
 that is part of the K8s cluster.
 
- 1. The node on which ovnkube-master or ovnkube-network-controller-manager runs, open following ports:
+ 1. The node on which ovnkube-master or ovnkube-controller runs, open following ports:
     ```text
     TCP:
       port 9409 (prometheus port to export ovnkube-master metrics)

README.md

@@ -69,6 +69,8 @@ kubectl create -f $HOME/work/src/github.com/ovn-org/ovn-kubernetes/dist/yaml/ovn
 kubectl create -f $HOME/work/src/github.com/ovn-org/ovn-kubernetes/dist/yaml/k8s.ovn.org_egressips.yaml
 # create egressfirewalls.k8s.ovn.org CRD
 kubectl create -f $HOME/work/src/github.com/ovn-org/ovn-kubernetes/dist/yaml/k8s.ovn.org_egressfirewalls.yaml
+# create adminpolicybasedexternalroute.k8s.ovn.org CRD
+kubectl create -f $HOME/work/src/github.com/ovn-org/ovn-kubernetes/dist/yaml/k8s.ovn.org_adminpolicybasedexternalroutes.yaml
 
 # Run ovnkube-db deployment.
 kubectl create -f $HOME/work/src/github.com/ovn-org/ovn-kubernetes/dist/yaml/ovnkube-db.yaml

contrib/kind.sh

@@ -249,6 +249,8 @@ parse_args() {
                                                 fi
                                                 OVN_GATEWAY_MODE=$1
                                                 ;;
+            -dgb | --dummy-gateway-bridge)      OVN_DUMMY_GATEWAY_BRIDGE=true
+                                                ;;
             -ov | --ovn-image )           	    shift
                                           	    OVN_IMAGE=$1
                                           	    ;;
@@ -319,8 +321,7 @@ parse_args() {
                                                 ;;
             --isolated )                        OVN_ISOLATED=true
                                                 ;;
-            -mne | --multi-network-enable )     shift
-                                                ENABLE_MULTI_NET=true
+            -mne | --multi-network-enable )     ENABLE_MULTI_NET=true
                                                 ;;
             --delete )                          delete
                                                 exit
@@ -361,6 +362,7 @@ print_params() {
      echo "KIND_ALLOW_SYSTEM_WRITES = $KIND_ALLOW_SYSTEM_WRITES"
      echo "KIND_EXPERIMENTAL_PROVIDER = $KIND_EXPERIMENTAL_PROVIDER"
      echo "OVN_GATEWAY_MODE = $OVN_GATEWAY_MODE"
+     echo "OVN_DUMMY_GATEWAY_BRIDGE = $OVN_DUMMY_GATEWAY_BRIDGE"
      echo "OVN_HYBRID_OVERLAY_ENABLE = $OVN_HYBRID_OVERLAY_ENABLE"
      echo "OVN_DISABLE_SNAT_MULTIPLE_GWS = $OVN_DISABLE_SNAT_MULTIPLE_GWS"
      echo "OVN_DISABLE_FORWARDING = $OVN_DISABLE_FORWARDING"
@@ -543,10 +545,14 @@ set_default_params() {
   OVN_DEPLOY_PODS=${OVN_DEPLOY_PODS:-"ovnkube-master ovnkube-node"}
   OVN_METRICS_SCALE_ENABLE=${OVN_METRICS_SCALE_ENABLE:-false}
   OVN_ISOLATED=${OVN_ISOLATED:-false}
-  OVN_GATEWAY_OPTS=""
+  OVN_GATEWAY_OPTS=${OVN_GATEWAY_OPTS:-""}
   if [ "$OVN_ISOLATED" == true ]; then
     OVN_GATEWAY_OPTS="--gateway-interface=eth0"
   fi
+  OVN_DUMMY_GATEWAY_BRIDGE=${OVN_DUMMY_GATEWAY_BRIDGE:-false}
+  if [ "$OVN_DUMMY_GATEWAY_BRIDGE" == true ]; then
+    OVN_GATEWAY_OPTS="--allow-no-uplink --gateway-interface=br-ex"
+  fi
   ENABLE_MULTI_NET=${ENABLE_MULTI_NET:-false}
   OVN_SEPARATE_CLUSTER_MANAGER=${OVN_SEPARATE_CLUSTER_MANAGER:-false}
   OVN_COMPACT_MODE=${OVN_COMPACT_MODE:-false}
@@ -780,6 +786,7 @@ create_ovn_kube_manifests() {
     --net-cidr="${NET_CIDR}" \
     --svc-cidr="${SVC_CIDR}" \
     --gateway-mode="${OVN_GATEWAY_MODE}" \
+    --dummy-gateway-bridge="${OVN_DUMMY_GATEWAY_BRIDGE}" \
     --gateway-options="${OVN_GATEWAY_OPTS}" \
     --enable-ipsec="${ENABLE_IPSEC}" \
     --hybrid-enabled="${OVN_HYBRID_OVERLAY_ENABLE}" \
@@ -803,6 +810,7 @@ create_ovn_kube_manifests() {
     --egress-ip-healthcheck-port="${OVN_EGRESSIP_HEALTHCHECK_PORT}" \
     --egress-firewall-enable=true \
     --egress-qos-enable=true \
+    --egress-service-enable=true \
     --v4-join-subnet="${JOIN_SUBNET_IPV4}" \
     --v6-join-subnet="${JOIN_SUBNET_IPV6}" \
     --ex-gw-network-interface="${OVN_EX_GW_NETWORK_INTERFACE}" \
@@ -834,6 +842,8 @@ install_ovn() {
   run_kubectl apply -f k8s.ovn.org_egressfirewalls.yaml
   run_kubectl apply -f k8s.ovn.org_egressips.yaml
   run_kubectl apply -f k8s.ovn.org_egressqoses.yaml
+  run_kubectl apply -f k8s.ovn.org_egressservices.yaml
+  run_kubectl apply -f k8s.ovn.org_adminpolicybasedexternalroutes.yaml
   run_kubectl apply -f ovn-setup.yaml
   MASTER_NODES=$(kind get nodes --name "${KIND_CLUSTER_NAME}" | sort | head -n "${KIND_NUM_MASTER}")
   # We want OVN HA not Kubernetes HA
@@ -1055,11 +1065,25 @@ docker_create_second_interface() {
   echo "adding second interfaces to nodes"
 
   # Create the network as dual stack, regardless of the type of the deployment. Ignore if already exists.
-  docker network create --ipv6 --driver=bridge kindexgw --subnet=172.19.0.0/16 --subnet=fc00:f853:ccd:e798::/64 || true
+  "$OCI_BIN" network create --ipv6 --driver=bridge kindexgw --subnet=172.19.0.0/16 --subnet=fc00:f853:ccd:e798::/64 || true
+
+  KIND_NODES=$(kind get nodes --name "${KIND_CLUSTER_NAME}")
+  for n in $KIND_NODES; do
+    "$OCI_BIN" network connect kindexgw "$n"
+  done
+}
+
+docker_create_second_disconnected_interface() {
+  echo "adding second interfaces to nodes"
+  local bridge_name="${1:-kindexgw}"
+  echo "bridge: $bridge_name"
+
+  # Create the network without subnets; ignore if already exists.
+  "$OCI_BIN" network create --internal --driver=bridge "$bridge_name" || true
 
   KIND_NODES=$(kind get nodes --name "${KIND_CLUSTER_NAME}")
   for n in $KIND_NODES; do
-    docker network connect kindexgw "$n"
+    "$OCI_BIN" network connect "$bridge_name" "$n"
   done
 }
 
@@ -1158,6 +1182,7 @@ fi
 if [ "$ENABLE_MULTI_NET" == true ]; then
   install_multus
   install_mpolicy_crd
+  docker_create_second_disconnected_interface "underlay"  # localnet scenarios require an extra interface
 fi
 kubectl_wait_pods
 sleep_until_pods_settle

dist/images/daemonset.sh

@@ -31,6 +31,7 @@ OVN_SVC_CIDR=""
 OVN_K8S_APISERVER=""
 OVN_GATEWAY_MODE=""
 OVN_GATEWAY_OPTS=""
+OVN_DUMMY_GATEWAY_BRIDGE=""
 OVN_DB_REPLICAS=""
 OVN_MTU=""
 OVN_SSL_ENABLE=""
@@ -64,6 +65,7 @@ OVN_EGRESSIP_ENABLE=
 OVN_EGRESSIP_HEALTHCHECK_PORT=
 OVN_EGRESSFIREWALL_ENABLE=
 OVN_EGRESSQOS_ENABLE=
+OVN_EGRESSSERVICE_ENABLE=
 OVN_DISABLE_OVN_IFACE_ID_VER="false"
 OVN_MULTI_NETWORK_ENABLE=
 OVN_V4_JOIN_SUBNET=""
@@ -107,6 +109,9 @@ while [ "$1" != "" ]; do
   --gateway-options)
     OVN_GATEWAY_OPTS=$VALUE
     ;;
+  --dummy-gateway-bridge)
+    OVN_DUMMY_GATEWAY_BRIDGE=$VALUE
+    ;;
   --enable-ipsec)
     ENABLE_IPSEC=$VALUE
     ;;
@@ -236,6 +241,9 @@ while [ "$1" != "" ]; do
   --multi-network-enable)
     OVN_MULTI_NETWORK_ENABLE=$VALUE
     ;;
+  --egress-service-enable)
+    OVN_EGRESSSERVICE_ENABLE=$VALUE
+    ;;
   --v4-join-subnet)
     OVN_V4_JOIN_SUBNET=$VALUE
     ;;
@@ -324,6 +332,9 @@ echo "ovn_gateway_mode: ${ovn_gateway_mode}"
 ovn_gateway_opts=${OVN_GATEWAY_OPTS}
 echo "ovn_gateway_opts: ${ovn_gateway_opts}"
 
+ovn_dummy_gateway_bridge=${OVN_DUMMY_GATEWAY_BRIDGE}
+echo "ovn_dummy_gateway_bridge: ${ovn_dummy_gateway_bridge}"
+
 enable_ipsec=${ENABLE_IPSEC:-false}
 echo "enable_ipsec: ${enable_ipsec}"
 
@@ -363,6 +374,8 @@ ovn_egress_firewall_enable=${OVN_EGRESSFIREWALL_ENABLE}
 echo "ovn_egress_firewall_enable: ${ovn_egress_firewall_enable}"
 ovn_egress_qos_enable=${OVN_EGRESSQOS_ENABLE}
 echo "ovn_egress_qos_enable: ${ovn_egress_qos_enable}"
+ovn_egress_service_enable=${OVN_EGRESSSERVICE_ENABLE}
+echo "ovn_egress_service_enable: ${ovn_egress_service_enable}"
 ovn_disable_ovn_iface_id_ver=${OVN_DISABLE_OVN_IFACE_ID_VER}
 echo "ovn_disable_ovn_iface_id_ver: ${ovn_disable_ovn_iface_id_ver}"
 ovn_multi_network_enable=${OVN_MULTI_NETWORK_ENABLE}
@@ -444,6 +457,7 @@ ovn_image=${ovnkube_image} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   ovn_gateway_mode=${ovn_gateway_mode} \
   ovn_gateway_opts=${ovn_gateway_opts} \
+  ovn_dummy_gateway_bridge=${ovn_dummy_gateway_bridge} \
   ovnkube_node_loglevel=${node_loglevel} \
   ovn_loglevel_controller=${ovn_loglevel_controller} \
   ovnkube_logfile_maxsize=${ovnkube_logfile_maxsize} \
@@ -460,6 +474,7 @@ ovn_image=${ovnkube_image} \
   ovn_egress_ip_enable=${ovn_egress_ip_enable} \
   ovn_egress_ip_healthcheck_port=${ovn_egress_ip_healthcheck_port} \
   ovn_multi_network_enable=${ovn_multi_network_enable} \
+  ovn_egress_service_enable=${ovn_egress_service_enable} \
   ovn_ssl_en=${ovn_ssl_en} \
   ovn_remote_probe_interval=${ovn_remote_probe_interval} \
   ovn_monitor_all=${ovn_monitor_all} \
@@ -488,6 +503,7 @@ ovn_image=${image} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   ovn_gateway_mode=${ovn_gateway_mode} \
   ovn_gateway_opts=${ovn_gateway_opts} \
+  ovn_dummy_gateway_bridge=${ovn_dummy_gateway_bridge} \
   ovnkube_node_loglevel=${node_loglevel} \
   ovn_loglevel_controller=${ovn_loglevel_controller} \
   ovnkube_logfile_maxsize=${ovnkube_logfile_maxsize} \
@@ -503,6 +519,7 @@ ovn_image=${image} \
   ovn_multicast_enable=${ovn_multicast_enable} \
   ovn_egress_ip_enable=${ovn_egress_ip_enable} \
   ovn_egress_ip_healthcheck_port=${ovn_egress_ip_healthcheck_port} \
+  ovn_egress_service_enable=${ovn_egress_service_enable} \
   ovn_netflow_targets=${ovn_netflow_targets} \
   ovn_sflow_targets=${ovn_sflow_targets} \
   ovn_ipfix_targets=${ovn_ipfix_targets} \
@@ -538,9 +555,12 @@ ovn_image=${ovnkube_image} \
   ovn_egress_firewall_enable=${ovn_egress_firewall_enable} \
   ovn_egress_qos_enable=${ovn_egress_qos_enable} \
   ovn_multi_network_enable=${ovn_multi_network_enable} \
+  ovn_egress_service_enable=${ovn_egress_service_enable} \
   ovn_ssl_en=${ovn_ssl_en} \
   ovn_master_count=${ovn_master_count} \
   ovn_gateway_mode=${ovn_gateway_mode} \
+  ovn_gateway_opts=${ovn_gateway_opts} \
+  ovn_dummy_gateway_bridge=${ovn_dummy_gateway_bridge} \
   ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \
   ovn_stateless_netpol_enable=${ovn_netpol_acl_enable} \
   ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
@@ -569,6 +589,7 @@ ovn_image=${image} \
   ovn_egress_firewall_enable=${ovn_egress_firewall_enable} \
   ovn_egress_qos_enable=${ovn_egress_qos_enable} \
   ovn_multi_network_enable=${ovn_multi_network_enable} \
+  ovn_egress_service_enable=${ovn_egress_service_enable} \
   ovn_ssl_en=${ovn_ssl_en} \
   ovn_master_count=${ovn_master_count} \
   ovn_gateway_mode=${ovn_gateway_mode} \
@@ -638,5 +659,7 @@ cp ../templates/ovnkube-monitor.yaml.j2 ${output_dir}/ovnkube-monitor.yaml
 cp ../templates/k8s.ovn.org_egressfirewalls.yaml.j2 ${output_dir}/k8s.ovn.org_egressfirewalls.yaml
 cp ../templates/k8s.ovn.org_egressips.yaml.j2 ${output_dir}/k8s.ovn.org_egressips.yaml
 cp ../templates/k8s.ovn.org_egressqoses.yaml.j2 ${output_dir}/k8s.ovn.org_egressqoses.yaml
+cp ../templates/k8s.ovn.org_egressservices.yaml.j2 ${output_dir}/k8s.ovn.org_egressservices.yaml
+cp ../templates/k8s.ovn.org_adminpolicybasedexternalroutes.yaml.j2 ${output_dir}/k8s.ovn.org_adminpolicybasedexternalroutes.yaml
 
 exit 0