Handle multiple default routes correctly

scripts/build-toybox.sh

@@ -5,7 +5,8 @@ set -exu
 # Toybox route is "pending" but the TODOs are in add/del;
 # we're using it for display only so we should be good.
 # Toybox sh has more TODOs in multiple areas so don't use it.
-toys="base64 mktemp mv route timeout"
+# Command mkdir is used for tests only.
+toys="base64 mkdir mktemp mv route sort timeout"
 
 cd /toybox-*/
 

scripts/install-cni.sh

@@ -311,6 +311,7 @@ if [ "$POPULATE_IP6TABLES" == "true" ] ; then
 
   if [ "${ENABLE_CALICO_NETWORK_POLICY}" == "true" ]; then
     echo "Enabling IPv6 forwarding..."
+    # IPV6_FORWARDING_CONF override only to be used in tests.
     echo 1 > "${IPV6_FORWARDING_CONF:-/proc/sys/net/ipv6/conf/all/forwarding}"
   fi
 fi
@@ -329,13 +330,17 @@ MTU_SOURCE="<default>"
 # last field, which is the interface name. We stick to using grep to avoid
 # introducing too many new dependencies.
 
-default_nic=$(route -n | grep -E '^0\.0\.0\.0\s+\S+\s+0\.0\.0\.0' | grep -oE '\S+$')
+# In the case where there are multiple default routes (e.g. multi-networking
+# with route manager), try each one starting from the lowest metric value.
+readarray -t default_nics < <(route -n | grep -E '^0\.0\.0\.0\s+\S+\s+0\.0\.0\.0' | sort -n -k5,5 | grep -oE '\S+$')
 
 # cilium_wg0 is the interface for node-to-node encryption. If it's available /
 # in use, it has a lower MTU than the default NIC (eth0) due to encryption headers.
-for nic in cilium_wg0 "$default_nic"; do
-  if [ -f "/sys/class/net/$nic/mtu" ]; then
-    MTU=$(cat "/sys/class/net/$nic/mtu")
+for nic in cilium_wg0 "${default_nics[@]}"; do
+  # SYS_CLASS_NET override only to be used in tests.
+  mtu_file=${SYS_CLASS_NET:-/sys/class/net}/$nic/mtu
+  if [[ -f "$mtu_file" ]]; then
+    MTU=$(cat "$mtu_file")
     MTU_SOURCE=$nic
     break
   fi

scripts/shell-test.sh

@@ -59,7 +59,10 @@ run_test inotify_cmd
 inotify / '' /bin/cat /dev/null >/dev/null && pass || fail
 
 run_test default_nic_mtu
-[[ -f "/sys/class/net/$(route -n | grep -E '^0\.0\.0\.0\s+\S+\s+0\.0\.0\.0' | grep -oE '\S+$')/mtu" ]] && pass || fail
+[[ -f "/sys/class/net/$(route -n | grep -E '^0\.0\.0\.0\s+\S+\s+0\.0\.0\.0' | sort -n -k5,5 | grep -oE '\S+$')/mtu" ]] && pass || fail
+
+run_test sort_cmd
+[[ "$(echo $'A 11\nB 2' | sort -n -k2,2)" == $'B 2\nA 11' ]] && pass || fail
 
 run_test mktemp_cmd
 [[ -f "$(mktemp /tmp.XXXXXX)" ]] && pass || fail

scripts/testcase/testcase-mtu.sh

@@ -0,0 +1,83 @@
+export KUBERNETES_SERVICE_HOST=kubernetes.default.svc
+export KUBERNETES_SERVICE_PORT=443
+
+export ENABLE_CALICO_NETWORK_POLICY=false
+export ENABLE_BANDWIDTH_PLUGIN=false
+export ENABLE_CILIUM_PLUGIN=false
+export ENABLE_MASQUERADE=false
+export ENABLE_IPV6=false
+export SYS_CLASS_NET=/tmp/mock-sys_class_net
+
+CNI_SPEC_TEMPLATE=$(cat testdata/spec-template.json)
+export CNI_SPEC_TEMPLATE
+
+function before_test() {
+
+  function curl() {
+    # shellcheck disable=SC2317
+    case "$*" in
+      *http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0*)
+        echo '{"ipv6s": ["2600:1900:4000:318:0:7:0:0"]}'
+        ;;
+      *https://kubernetes.default.svc:443/api/v1/nodes/*)
+        echo '{
+                "metadata": {
+                  "labels": {
+                  },
+                  "creationTimestamp": "2024-01-03T11:54:01Z",
+                  "name": "gke-my-cluster-default-pool-128bc25d-9c94",
+                  "resourceVersion": "891003",
+                  "uid": "f2353a2f-ca8c-4ca0-8dd3-ad1f964a54f0"
+                },
+                "spec": {
+                  "podCIDR": "10.52.1.0/24",
+                  "podCIDRs": [
+                    "10.52.1.0/24"
+                  ],
+                  "providerID": "gce://my-gke-project/us-central1-c/gke-my-cluster-default-pool-128bc25d-9c94"
+                }
+              }'
+        ;;
+      *)
+        #unsupported
+        exit 1
+    esac
+  }
+  export -f curl
+
+  function route() {
+    # shellcheck disable=SC2317
+    echo 'Kernel IP routing table
+Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
+0.0.0.0         172.17.0.1      0.0.0.0         UG    1000   0        0 ens1
+0.0.0.0         172.17.0.1      0.0.0.0         UG    999    0        0 ens2
+0.0.0.0         172.17.0.1      255.0.0.0       UG    10     0        0 ens3'
+  }
+  export -f route
+
+  rm -rf "${SYS_CLASS_NET}"
+  mkdir -p "${SYS_CLASS_NET}/ens1"
+  mkdir -p "${SYS_CLASS_NET}/ens2"
+  mkdir -p "${SYS_CLASS_NET}/ens3"
+  echo 1461 >"${SYS_CLASS_NET}/ens1/mtu"
+  echo 1462 >"${SYS_CLASS_NET}/ens2/mtu"
+  echo 1463 >"${SYS_CLASS_NET}/ens3/mtu"
+
+}
+
+function verify() {
+  local expected
+  local actual
+
+  expected=$(jq -S . <"testdata/expected-mtu.json")
+  actual=$(jq -S . <"/host/etc/cni/net.d/${CNI_SPEC_NAME}")
+
+  if [ "$expected" != "$actual" ] ; then
+    echo "Expected cni_spec value:"
+    echo "$expected"
+    echo "but actual was"
+    echo "$actual"
+    return 1
+  fi
+
+}

scripts/testdata/expected-mtu.json

@@ -0,0 +1,25 @@
+{
+  "name": "gke-pod-network",
+  "cniVersion": "0.3.1",
+  "plugins": [
+    {
+      "type": "ptp",
+      "mtu": 1462,
+      "ipam": {
+        "type": "host-local",
+        "ranges": [
+          [{"subnet": "10.52.1.0/24"}]
+        ],
+        "routes": [
+          {"dst": "0.0.0.0/0"}
+        ]
+      }
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+        "portMappings": true
+      }
+    }
+  ]
+}