diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 077a193c77..11ada543f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -439,6 +439,7 @@ jobs: - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-disabled"} - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} + - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "traffic-flow-test-only","ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "traffic-flow-tests": "1-24"} - {"target": "tools", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} needs: [ build-pr ] diff --git a/contrib/kind-common b/contrib/kind-common index 8fb327d936..0384e5237c 100644 --- a/contrib/kind-common +++ b/contrib/kind-common @@ -358,7 +358,7 @@ install_kubevirt() { local kubevirt_stable_release_url=$(get_kubevirt_release_url "stable") local passt_binding_image="quay.io/kubevirt/network-passt-binding:${kubevirt_stable_release_url##*/}" - kubectl -n kubevirt patch kubevirt kubevirt --type=json --patch '[{"op":"add","path":"/spec/configuration/network","value":{}},{"op":"add","path":"/spec/configuration/network/binding","value":{"passt":{"computeResourceOverhead":{"requests":{"memory":"500Mi"}},"migration":{"method":"link-refresh"},"networkAttachmentDefinition":"default/primary-udn-kubevirt-binding","sidecarImage":"'"${passt_binding_image}"'"},"managedTap":{"domainAttachmentType":"managedTap","migration":{}}}}]' + kubectl -n kubevirt patch kubevirt kubevirt --type=json --patch '[{"op":"add","path":"/spec/configuration/network","value":{}},{"op":"add","path":"/spec/configuration/network/binding","value":{"passt":{"computeResourceOverhead":{"requests":{"memory":"500Mi"}},"migration":{"method":"link-refresh"},"networkAttachmentDefinition":"default/primary-udn-kubevirt-binding","sidecarImage":"'"${passt_binding_image}"'"},"l2bridge":{"domainAttachmentType":"managedTap","migration":{}}}}]' if [ ! -d "./bin" ] then diff --git a/contrib/kind.sh b/contrib/kind.sh index 6a997d10c9..e5fc378074 100755 --- a/contrib/kind.sh +++ b/contrib/kind.sh @@ -618,6 +618,10 @@ set_default_params() { echo "Route advertisements requires multi-network to be enabled (-mne)" exit 1 fi + if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ] && [ "$OVN_ENABLE_INTERCONNECT" != true ]; then + echo "Route advertisements requires interconnect to be enabled (-ic)" + exit 1 + fi OVN_COMPACT_MODE=${OVN_COMPACT_MODE:-false} if [ "$OVN_COMPACT_MODE" == true ]; then KIND_NUM_WORKER=0 diff --git a/dist/images/Dockerfile.ubuntu.arm64 b/dist/images/Dockerfile.ubuntu.arm64 index b33df55add..48a408b036 100644 --- a/dist/images/Dockerfile.ubuntu.arm64 +++ b/dist/images/Dockerfile.ubuntu.arm64 @@ -12,7 +12,7 @@ FROM ubuntu:24.10 USER root -RUN apt-get update && apt-get install -y iproute2 curl software-properties-common util-linux +RUN apt-get update && apt-get install -y iproute2 curl software-properties-common util-linux nftables RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh index baa6d631c9..99947ba4c0 100755 --- a/dist/images/daemonset.sh +++ b/dist/images/daemonset.sh @@ -1037,10 +1037,12 @@ ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \ ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \ +ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ jinjanate ../templates/rbac-ovnkube-cluster-manager.yaml.j2 -o ${output_dir}/rbac-ovnkube-cluster-manager.yaml ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \ +ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ jinjanate ../templates/rbac-ovnkube-master.yaml.j2 -o ${output_dir}/rbac-ovnkube-master.yaml cp ../templates/rbac-ovnkube-identity.yaml.j2 ${output_dir}/rbac-ovnkube-identity.yaml diff --git a/dist/templates/k8s.ovn.org_routeadvertisements.yaml.j2 b/dist/templates/k8s.ovn.org_routeadvertisements.yaml.j2 index 49a6cb8657..e5b4a07100 100644 --- a/dist/templates/k8s.ovn.org_routeadvertisements.yaml.j2 +++ b/dist/templates/k8s.ovn.org_routeadvertisements.yaml.j2 @@ -50,6 +50,9 @@ spec: description: advertisements determines what is advertised. items: description: AdvertisementType determines the type of advertisement. + enum: + - PodNetwork + - EgressIP type: string maxItems: 2 minItems: 1 diff --git a/dist/templates/rbac-ovnkube-cluster-manager.yaml.j2 b/dist/templates/rbac-ovnkube-cluster-manager.yaml.j2 index b658630b28..3b347dd91c 100644 --- a/dist/templates/rbac-ovnkube-cluster-manager.yaml.j2 +++ b/dist/templates/rbac-ovnkube-cluster-manager.yaml.j2 @@ -127,3 +127,9 @@ rules: - dnsnameresolvers verbs: [ "create", "delete", "list", "patch", "update", "watch" ] {%- endif %} + {% if ovn_route_advertisements_enable == "true" -%} + - apiGroups: ["frrk8s.metallb.io"] + resources: + - frrconfigurations + verbs: [ "create", "delete", "list", "patch", "update", "watch" ] + {%- endif %} diff --git a/dist/templates/rbac-ovnkube-master.yaml.j2 b/dist/templates/rbac-ovnkube-master.yaml.j2 index 119f085657..b4c5f64816 100644 --- a/dist/templates/rbac-ovnkube-master.yaml.j2 +++ b/dist/templates/rbac-ovnkube-master.yaml.j2 @@ -85,7 +85,6 @@ rules: - adminpolicybasedexternalroutes - userdefinednetworks - clusteruserdefinednetworks - - routeadvertisements verbs: [ "get", "list", "watch" ] - apiGroups: ["k8s.cni.cncf.io"] resources: @@ -120,7 +119,6 @@ rules: - clusteruserdefinednetworks - clusteruserdefinednetworks/status - clusteruserdefinednetworks/finalizers - - routeadvertisements/status verbs: [ "patch", "update" ] - apiGroups: [""] resources: diff --git a/go-controller/go.mod b/go-controller/go.mod index a5b34faa02..1d11e0fe68 100644 --- a/go-controller/go.mod +++ b/go-controller/go.mod @@ -27,6 +27,7 @@ require ( github.com/k8snetworkplumbingwg/sriovnet v1.2.1-0.20230427090635-4929697df2dc github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 github.com/mdlayher/ndp v1.0.1 + github.com/metallb/frr-k8s v0.0.15 github.com/miekg/dns v1.1.31 github.com/mitchellh/copystructure v1.2.0 github.com/onsi/ginkgo/v2 v2.19.0 @@ -41,11 +42,11 @@ require ( github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.2 github.com/vishvananda/netlink v1.2.1-beta.2.0.20231024175852-77df5d35f725 - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.21.0 - golang.org/x/time v0.3.0 + golang.org/x/time v0.5.0 google.golang.org/grpc v1.65.0 google.golang.org/grpc/security/advancedtls v0.0.0-20240425232638-1e8b9b7fc655 google.golang.org/protobuf v1.34.2 @@ -76,12 +77,12 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -91,7 +92,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect github.com/gorilla/websocket v1.5.0 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -122,11 +123,11 @@ require ( github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.24.0 // indirect - golang.org/x/mod v0.17.0 // indirect + golang.org/x/mod v0.18.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.22.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect @@ -136,7 +137,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.31.1 // indirect k8s.io/component-base v0.31.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect kubevirt.io/containerized-data-importer-api v1.55.0 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go-controller/go.sum b/go-controller/go.sum index ec0da37bb6..38956566c7 100644 --- a/go-controller/go.sum +++ b/go-controller/go.sum @@ -266,8 +266,8 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -319,15 +319,15 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= @@ -335,9 +335,8 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -477,8 +476,8 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -565,6 +564,8 @@ github.com/mdlayher/packet v1.0.0 h1:InhZJbdShQYt6XV2GPj5XHxChzOfhJJOMbvnGAmOfQ8 github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU= github.com/mdlayher/socket v0.2.1 h1:F2aaOwb53VsBE+ebRS9bLd7yPOfYUMC8lOODdCBDY6w= github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= +github.com/metallb/frr-k8s v0.0.15 h1:6M3UGhovX1EFoaSGjrRD7djUAx3w2I+g81FH8OVtHkM= +github.com/metallb/frr-k8s v0.0.15/go.mod h1:TjrGoAf+v00hYGlI8jUdyDxY5udMAOs2GWwrvLWnA4E= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= @@ -868,8 +869,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -895,8 +896,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1090,8 +1091,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1153,8 +1154,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1389,8 +1390,8 @@ k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a h1:zD1uj3Jf+mD4zmA7W+goE5TxDkI7OGJjBNBzq5fJtLA= +k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/kubernetes v1.31.1 h1:1fcYJe8SAhtannpChbmnzHLwAV9Je99PrGaFtBvCxms= k8s.io/kubernetes v1.31.1/go.mod h1:/YGPL//Fb9mdv5vukvAQ7Xon+Bqwry52bmjTdORAw+Q= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= diff --git a/go-controller/hack/test-go.sh b/go-controller/hack/test-go.sh index 1cb02e3416..1cec25324e 100755 --- a/go-controller/hack/test-go.sh +++ b/go-controller/hack/test-go.sh @@ -48,7 +48,7 @@ function testrun { go_test="sudo ${testfile}" fi fi - if [[ -n "$gingko_focus" ]]; then + if [[ -n "$ginkgo_focus" ]]; then local ginkgoargs=${ginkgo_focus:-} fi local path=${pkg#github.com/ovn-org/ovn-kubernetes/go-controller} diff --git a/go-controller/pkg/clustermanager/clustermanager.go b/go-controller/pkg/clustermanager/clustermanager.go index fcb03bfb93..09d83d11fa 100644 --- a/go-controller/pkg/clustermanager/clustermanager.go +++ b/go-controller/pkg/clustermanager/clustermanager.go @@ -13,6 +13,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/dnsnameresolver" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/egressservice" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/endpointslicemirror" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/routeadvertisements" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/status_manager" udncontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork" udntemplate "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork/template" @@ -58,6 +59,8 @@ type ClusterManager struct { // networkManager creates and deletes network controllers networkManager networkmanager.Controller + + raController *routeadvertisements.Controller } // NewClusterManager creates a new cluster manager to manage the cluster nodes. @@ -83,17 +86,15 @@ func NewClusterManager(ovnClient *util.OVNClusterManagerClientset, wf *factory.W cm.networkManager = networkmanager.Default() if config.OVNKubernetesFeature.EnableMultiNetwork { - cm.secondaryNetClusterManager, err = newSecondaryNetworkClusterManager(ovnClient, wf, cm.networkManager.Interface(), recorder) + cm.networkManager, err = networkmanager.NewForCluster(cm, wf, ovnClient, recorder) if err != nil { return nil, err } - cm.networkManager, err = networkmanager.NewForCluster(cm.secondaryNetClusterManager, wf, recorder) + cm.secondaryNetClusterManager, err = newSecondaryNetworkClusterManager(ovnClient, wf, cm.networkManager.Interface(), recorder) if err != nil { return nil, err } - cm.secondaryNetClusterManager.networkManager = cm.networkManager.Interface() - } if config.OVNKubernetesFeature.EnableEgressIP { @@ -156,6 +157,10 @@ func NewClusterManager(ovnClient *util.OVNClusterManagerClientset, wf *factory.W } } + if util.IsRouteAdvertisementsEnabled() { + cm.raController = routeadvertisements.NewController(cm.networkManager.Interface(), wf, ovnClient) + } + return cm, nil } @@ -173,12 +178,6 @@ func (cm *ClusterManager) Start(ctx context.Context) error { return err } - if cm.secondaryNetClusterManager != nil { - if err := cm.secondaryNetClusterManager.Start(); err != nil { - return err - } - } - if err := cm.defaultNetClusterController.Start(ctx); err != nil { return err } @@ -219,6 +218,14 @@ func (cm *ClusterManager) Start(ctx context.Context) error { return err } } + + if cm.raController != nil { + err := cm.raController.Start() + if err != nil { + return err + } + } + return nil } @@ -227,9 +234,7 @@ func (cm *ClusterManager) Stop() { klog.Info("Stopping the cluster manager") cm.defaultNetClusterController.Stop() cm.zoneClusterController.Stop() - if config.OVNKubernetesFeature.EnableMultiNetwork { - cm.secondaryNetClusterManager.Stop() - } + if config.OVNKubernetesFeature.EnableEgressIP { cm.eIPC.Stop() } @@ -246,4 +251,27 @@ func (cm *ClusterManager) Stop() { if util.IsNetworkSegmentationSupportEnabled() { cm.userDefinedNetworkController.Shutdown() } + if cm.raController != nil { + cm.raController.Stop() + cm.raController = nil + } +} + +func (cm *ClusterManager) NewNetworkController(netInfo util.NetInfo) (networkmanager.NetworkController, error) { + return cm.secondaryNetClusterManager.NewNetworkController(netInfo) +} + +func (cm *ClusterManager) GetDefaultNetworkController() networkmanager.ReconcilableNetworkController { + return cm.defaultNetClusterController +} + +func (cm *ClusterManager) CleanupStaleNetworks(validNetworks ...util.NetInfo) error { + return cm.secondaryNetClusterManager.CleanupStaleNetworks(validNetworks...) +} + +func (cm *ClusterManager) Reconcile(name string, old, new util.NetInfo) error { + if cm.raController != nil { + cm.raController.ReconcileNetwork(name, old, new) + } + return nil } diff --git a/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller_test.go b/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller_test.go index 953c33bfd1..78c8a30a90 100644 --- a/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller_test.go +++ b/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller_test.go @@ -39,7 +39,7 @@ var _ = ginkgo.Describe("Cluster manager EndpointSlice mirror controller", func( fakeClient = util.GetOVNClientset(objects...).GetClusterManagerClientset() wf, err := factory.NewClusterManagerWatchFactory(fakeClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - networkManager, err = networkmanager.NewForCluster(&testnm.FakeControllerManager{}, wf, nil) + networkManager, err = networkmanager.NewForCluster(&testnm.FakeControllerManager{}, wf, fakeClient, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) controller, err = NewController(fakeClient, wf, networkManager.Interface()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/go-controller/pkg/clustermanager/network_cluster_controller.go b/go-controller/pkg/clustermanager/network_cluster_controller.go index 3c5b8491a6..6dd8fdb4c1 100644 --- a/go-controller/pkg/clustermanager/network_cluster_controller.go +++ b/go-controller/pkg/clustermanager/network_cluster_controller.go @@ -60,7 +60,6 @@ type networkClusterController struct { tunnelIDAllocator id.Allocator podAllocator *pod.PodAllocator nodeAllocator *node.NodeAllocator - networkIDAllocator id.NamedAllocator ipamClaimReconciler *persistentips.IPAMClaimReconciler subnetAllocator subnet.Allocator @@ -82,7 +81,6 @@ type networkClusterController struct { } func newNetworkClusterController( - networkIDAllocator id.NamedAllocator, netInfo util.NetInfo, ovnClient *util.OVNClusterManagerClientset, wf *factory.WatchFactory, @@ -105,7 +103,6 @@ func newNetworkClusterController( kube: kube, stopChan: make(chan struct{}), wg: wg, - networkIDAllocator: networkIDAllocator, recorder: recorder, networkManager: networkManager, statusReporter: errorReporter, @@ -126,8 +123,7 @@ func newDefaultNetworkClusterController(netInfo util.NetInfo, ovnClient *util.OV panic(fmt.Errorf("could not reserve default network ID: %w", err)) } - namedIDAllocator := networkIDAllocator.ForName(types.DefaultNetworkName) - return newNetworkClusterController(namedIDAllocator, netInfo, ovnClient, wf, recorder, networkmanager.Default().Interface(), nil) + return newNetworkClusterController(netInfo, ovnClient, wf, recorder, networkmanager.Default().Interface(), nil) } func (ncc *networkClusterController) hasPodAllocation() bool { @@ -171,11 +167,9 @@ func (ncc *networkClusterController) init() error { return fmt.Errorf("failed to reset network status: %w", err) } - networkID, err := ncc.networkIDAllocator.AllocateID() - if err != nil { - return err - } + networkID := ncc.GetNetworkID() + var err error if util.DoesNetworkRequireTunnelIDs(ncc.GetNetInfo()) { ncc.tunnelIDAllocator = id.NewIDAllocator(ncc.GetNetworkName(), types.MaxLogicalPortTunnelKey) // Reserve the id 0. We don't want to assign this id to any of the pods or nodes. @@ -438,7 +432,6 @@ func (ncc *networkClusterController) Cleanup() error { if err != nil { return err } - ncc.networkIDAllocator.ReleaseID() } return nil diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller.go b/go-controller/pkg/clustermanager/routeadvertisements/controller.go new file mode 100644 index 0000000000..f4e7715769 --- /dev/null +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller.go @@ -0,0 +1,1145 @@ +package routeadvertisements + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "slices" + "strings" + "time" + + nadtypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + nadclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" + nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" + frrtypes "github.com/metallb/frr-k8s/api/v1beta1" + frrclientset "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" + frrlisters "github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1" + core "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + metaapply "k8s.io/client-go/applyconfigurations/meta/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + controllerutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + eiptypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1" + egressiplisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/listers/egressip/v1" + ratypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1" + raapply "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/applyconfiguration/routeadvertisements/v1" + raclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/clientset/versioned" + ralisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/listers/routeadvertisements/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +const ( + generateName = "ovnk-generated-" + fieldManager = "clustermanager-routeadvertisements-controller" +) + +var ( + errConfig = errors.New("configuration error") + errPending = errors.New("configuration pending") +) + +// Controller reconciles RouteAdvertisements +type Controller struct { + eipLister egressiplisters.EgressIPLister + frrLister frrlisters.FRRConfigurationLister + nadLister nadlisters.NetworkAttachmentDefinitionLister + nodeLister corelisters.NodeLister + raLister ralisters.RouteAdvertisementsLister + + frrClient frrclientset.Interface + nadClient nadclientset.Interface + raClient raclientset.Interface + + eipController controllerutil.Controller + frrController controllerutil.Controller + nadController controllerutil.Controller + nodeController controllerutil.Controller + raController controllerutil.Controller + + nm networkmanager.Interface +} + +// NewController builds a controller that reconciles RouteAdvertisements +func NewController( + nm networkmanager.Interface, + wf *factory.WatchFactory, + ovnClient *util.OVNClusterManagerClientset, +) *Controller { + c := &Controller{ + eipLister: wf.EgressIPInformer().Lister(), + frrLister: wf.FRRConfigurationsInformer().Lister(), + nadLister: wf.NADInformer().Lister(), + nodeLister: wf.NodeCoreInformer().Lister(), + raLister: wf.RouteAdvertisementsInformer().Lister(), + frrClient: ovnClient.FRRClient, + nadClient: ovnClient.NetworkAttchDefClient, + raClient: ovnClient.RouteAdvertisementsClient, + nm: nm, + } + + handleError := func(key string, errorstatus error) error { + ra, err := c.raLister.Get(key) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("cannot get RouteAdvertisements %q to report error %v in status: %v", + key, + errorstatus, + err, + ) + } + + return c.updateRAStatus(ra, false, err) + } + + raConfig := &controllerutil.ControllerConfig[ratypes.RouteAdvertisements]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: c.reconcile, + Threadiness: 1, + Informer: wf.RouteAdvertisementsInformer().Informer(), + Lister: wf.RouteAdvertisementsInformer().Lister().List, + ObjNeedsUpdate: raNeedsUpdate, + HandleError: handleError, + } + c.raController = controllerutil.NewController("clustermanager routeadvertisements controller", raConfig) + + frrConfig := &controllerutil.ControllerConfig[frrtypes.FRRConfiguration]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: c.reconcileFRRConfiguration, + Threadiness: 1, + Informer: wf.FRRConfigurationsInformer().Informer(), + Lister: wf.FRRConfigurationsInformer().Lister().List, + ObjNeedsUpdate: frrConfigurationNeedsUpdate, + } + c.frrController = controllerutil.NewController("clustermanager routeadvertisements frrconfiguration controller", frrConfig) + + nadConfig := &controllerutil.ControllerConfig[nadtypes.NetworkAttachmentDefinition]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: c.reconcileNAD, + Threadiness: 1, + Informer: wf.NADInformer().Informer(), + Lister: wf.NADInformer().Lister().List, + ObjNeedsUpdate: nadNeedsUpdate, + } + c.nadController = controllerutil.NewController("clustermanager routeadvertisements nad controller", nadConfig) + + nodeConfig := &controllerutil.ControllerConfig[core.Node]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: func(key string) error { c.raController.ReconcileAll(); return nil }, + Threadiness: 1, + Informer: wf.NodeCoreInformer().Informer(), + Lister: wf.NodeCoreInformer().Lister().List, + ObjNeedsUpdate: nodeNeedsUpdate, + } + c.nodeController = controllerutil.NewController("clustermanager routeadvertisements node controller", nodeConfig) + + eipConfig := &controllerutil.ControllerConfig[eiptypes.EgressIP]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: c.reconcileEgressIP, + Threadiness: 1, + Informer: wf.EgressIPInformer().Informer(), + Lister: wf.EgressIPInformer().Lister().List, + ObjNeedsUpdate: egressIPNeedsUpdate, + } + c.eipController = controllerutil.NewController("clustermanager routeadvertisements egressip controller", eipConfig) + + return c +} + +func (c *Controller) Start() error { + defer klog.Infof("Cluster manager routeadvertisements started") + return controllerutil.Start( + c.eipController, + c.frrController, + c.nadController, + c.nodeController, + c.raController, + ) +} + +func (c *Controller) Stop() { + controllerutil.Stop( + c.eipController, + c.frrController, + c.nadController, + c.nodeController, + c.raController, + ) + klog.Infof("Cluster manager routeadvertisements stoppedu") +} + +func (c *Controller) ReconcileNetwork(name string, old, new util.NetInfo) { + // This controller already listens on NAD events however we skip NADs + // pointing to networks that network manager is still not aware of; so we + // only need to signal the reconciliation of new networks. Reconcile one of + // the NADs of the network to do s + if new == nil || old != nil { + return + } + c.nadController.Reconcile(new.GetNADs()[0]) +} + +// Reconcile RouteAdvertisements. For each selected FRRConfiguration and node, +// another FRRConfiguration might be generated: +// +// - If pod network advertisements are enabled, the generated FRRConfiguration +// will announce from the node the selected network prefixes for that node on +// the matching target VRFs. +// +// - If EgressIP advertisements are enabled, the generated FRRConfiguration will +// announce from the node the EgressIPs allocated to it on the matching target +// VRFs. +// +// - If pod network advertisements are enabled, the generated FRRConfiguration +// will import the target VRFs on the selected networks as required. +// +// - The generated FRRConfiguration will be labeled with the RouteAdvertisements +// name and annotated with an internal key to facilitate updating it when +// needed. +// +// The controller will also annotate the NADs of the selected networks with the +// RouteAdvertisements that select them to facilitate processing for downstream +// zone/node controllers. +// +// Finally, it will update the status of the RouteAdvertisements. +// +// The controller processes selected events of RouteAdvertisements, +// FRRConfigurations, Nodes, EgressIPs and NADs. +func (c *Controller) reconcile(name string) error { + startTime := time.Now() + klog.V(5).Infof("Syncing routeadvertisements %q", name) + defer func() { + klog.V(4).Infof("Finished syncing routeadvertisements %q, took %v", name, time.Since(startTime)) + }() + + ra, err := c.raLister.Get(name) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get RouteAdvertisements %q: %w", name, err) + } + + hadUpdates, err := c.reconcileRouteAdvertisements(name, ra) + if err != nil && !errors.Is(err, errConfig) && !errors.Is(err, errPending) { + return fmt.Errorf("failed to reconcile RouteAdvertisements %q: %w", name, err) + } + + return c.updateRAStatus(ra, hadUpdates, err) +} + +func (c *Controller) reconcileRouteAdvertisements(name string, ra *ratypes.RouteAdvertisements) (bool, error) { + // generate FRRConfigurations + frrConfigs, nads, cfgErr := c.generateFRRConfigurations(ra) + if cfgErr != nil && !errors.Is(cfgErr, errPending) { + return false, cfgErr + } + + // update them + hadFRRConfigUpdates, err := c.updateFRRConfigurations(name, frrConfigs) + if err != nil { + return false, fmt.Errorf("failed updating FRRConfigurations for RouteAdvertisements %q: %w", name, err) + } + + // annotate NADs + hadNADUpdates, err := c.updateNADs(name, nads) + if err != nil { + return false, fmt.Errorf("failed annotating NADs for RouteAdvertisements %q: %w", name, err) + } + + return hadFRRConfigUpdates || hadNADUpdates, cfgErr +} + +// selectedNetworks is a helper struct that stores information about networks +// that have been selected by a RouteAdvertisements. It is important that prefix +// lists are ordered to generate consistent FRRConfigurations. +type selectedNetworks struct { + // networks is an ordered list of selected network names + networks []string + // vrfs is an ordered list of selected networks VRF's + vrfs []string + // networkVRFs is a mapping of VRF to corresponding network + networkVRFs map[string]string + // subnets is an ordered list of all selected network subnets + subnets []string + // hostSubnets is an ordered list of all selected network subnets specific to a node + hostSubnets []string + // networkSubnets is a map of selected network names to their ordered network subnets + networkSubnets map[string][]string + // hostNetworkSubnets is a map of selected network names to their ordered network subnets specific for a node + hostNetworkSubnets map[string][]string + // prefixLength is a map of selected network to their prefix length + prefixLength map[string]uint32 +} + +// generateFRRConfigurations generates FRRConfigurations for the route +// advertisements. Also returns the selected network NADs. +func (c *Controller) generateFRRConfigurations(ra *ratypes.RouteAdvertisements) ([]*frrtypes.FRRConfiguration, []*nadtypes.NetworkAttachmentDefinition, error) { + if ra == nil { + return nil, nil, nil + } + + // if we are matching on the well known default network label, create an + // internal nad for it if it doesn't exist + if matchesDefaultNetworkLabel(ra.Spec.NetworkSelector) { + _, err := c.getOrCreateDefaultNetworkNAD() + if err != nil { + return nil, nil, fmt.Errorf("failed to get/create default network NAD: %w", err) + } + } + + // gather selected networks + var nads []*nadtypes.NetworkAttachmentDefinition + nadSelector, err := metav1.LabelSelectorAsSelector(&ra.Spec.NetworkSelector) + if err != nil { + return nil, nil, err + } + nads, err = c.nadLister.List(nadSelector) + if err != nil { + return nil, nil, err + } + if len(nads) == 0 { + return nil, nil, fmt.Errorf("%w: no networks selected", errPending) + } + + // validate and gather information about the networks + networkSet := sets.New[string]() + selectedNetworks := &selectedNetworks{ + networkVRFs: map[string]string{}, + networkSubnets: map[string][]string{}, + prefixLength: map[string]uint32{}, + } + for _, nad := range nads { + networkName := util.GetAnnotatedNetworkName(nad) + network := c.nm.GetNetwork(networkName) + if network == nil { + // network not yet known by network manager, skip + continue + } + if networkSet.Has(networkName) { + continue + } + if !network.IsDefault() && !network.IsPrimaryNetwork() { + return nil, nil, fmt.Errorf("%w: selected network %q is not the default nor a primary network", errConfig, networkName) + } + if network.TopologyType() != types.Layer3Topology { + // TODO don't really know what to do with layer 2 topologies yet + return nil, nil, fmt.Errorf("%w: selected network %q has unsupported topology %q", errConfig, networkName, network.TopologyType()) + } + vrf := util.GetNetworkVRFName(network) + if vfrNet, hasVFR := selectedNetworks.networkVRFs[vrf]; hasVFR && vfrNet != networkName { + return nil, nil, fmt.Errorf("%w: vrf %q found to be mapped to multiple networks %v", errConfig, vrf, []string{vfrNet, networkName}) + } + networkSet.Insert(networkName) + selectedNetworks.vrfs = append(selectedNetworks.vrfs, vrf) + selectedNetworks.networkVRFs[vrf] = networkName + // TODO check overlaps? + for _, cidr := range network.Subnets() { + subnet := cidr.CIDR.String() + len := uint32(cidr.HostSubnetLength) + selectedNetworks.networkSubnets[networkName] = append(selectedNetworks.networkSubnets[networkName], subnet) + selectedNetworks.subnets = append(selectedNetworks.subnets, subnet) + selectedNetworks.prefixLength[subnet] = len + } + // ordered + slices.Sort(selectedNetworks.networkSubnets[networkName]) + } + // ordered + slices.Sort(selectedNetworks.vrfs) + slices.Sort(selectedNetworks.subnets) + selectedNetworks.networks = sets.List(networkSet) + + // gather selected nodes + nodeSelector, err := metav1.LabelSelectorAsSelector(&ra.Spec.NodeSelector) + if err != nil { + return nil, nil, err + } + advertisements := sets.New(ra.Spec.Advertisements...) + if !nodeSelector.Empty() && advertisements.Has(ratypes.PodNetwork) { + return nil, nil, fmt.Errorf("%w: node selector cannot be specified if pod network is advertised", errConfig) + } + nodes, err := c.nodeLister.List(nodeSelector) + if err != nil { + return nil, nil, err + } + if len(nodes) == 0 { + return nil, nil, fmt.Errorf("%w: no nodes selected", errPending) + } + // prepare a map of selected nodes to the FRRConfigurations that apply to + // them + nodeToFRRConfig := map[string][]*frrtypes.FRRConfiguration{} + for _, node := range nodes { + nodeToFRRConfig[node.Name] = nil + } + + // gather selected FRRConfigurations, map them to the selected nodes + frrSelector, err := metav1.LabelSelectorAsSelector(&ra.Spec.FRRConfigurationSelector) + if err != nil { + return nil, nil, err + } + frrConfigs, err := c.frrLister.List(frrSelector) + if err != nil { + return nil, nil, err + } + if len(frrConfigs) == 0 { + return nil, nil, fmt.Errorf("%w: no FRRConfigurations selected", errPending) + } + for _, frrConfig := range frrConfigs { + if strings.HasPrefix(frrConfig.Name, generateName) { + klog.V(4).Infof("Skipping FRRConfiguration %q selected by RouteAdvertisements %q as it was generated by ovn-kubernetes", frrConfig.Name, ra.Name) + continue + } + nodeSelector, err := metav1.LabelSelectorAsSelector(&frrConfig.Spec.NodeSelector) + if err != nil { + return nil, nil, err + } + nodes, err := c.nodeLister.List(nodeSelector) + if err != nil { + return nil, nil, err + } + for _, node := range nodes { + if _, selected := nodeToFRRConfig[node.Name]; !selected { + // this RouteAdvertisements does not select this node, skip + continue + } + nodeToFRRConfig[node.Name] = append(nodeToFRRConfig[node.Name], frrConfig) + } + } + + // helper to gather host subnets and cache during reconcile + // TODO perhaps cache across reconciles as well + hostSubnets := map[string]map[string][]string{} + getHostSubnets := func(nodeName string, network string) ([]string, error) { + if _, parsed := hostSubnets[nodeName]; !parsed { + node, err := c.nodeLister.Get(nodeName) + if err != nil { + return nil, err + } + subnets, err := util.ParseNodeHostSubnetsAnnotation(node) + if err != nil { + return nil, fmt.Errorf("%w: waiting for subnet annotation to be set for node %q: %w", errConfig, nodeName, err) + } + hostSubnets[nodeName] = make(map[string][]string, len(subnets)) + for network, subnet := range subnets { + hostSubnets[nodeName][network] = util.StringSlice(subnet) + } + } + return hostSubnets[nodeName][network], nil + } + + // helper to gather egress ips and cache during reconcile + // TODO perhaps cache across reconciles as well + var nodeEgressIPs map[string][]string + getEgressIPs := func(nodeName string) ([]string, error) { + if nodeEgressIPs == nil { + nodeEgressIPs, err = c.getEgressIPsByNode() + if err != nil { + return nil, err + } + } + return nodeEgressIPs[nodeName], nil + } + + // helper to gather host subnets and egress ips as prefixes + getPrefixes := func(nodeName string, network string) ([]string, error) { + // gather host subnets + var subnets []string + if advertisements.Has(ratypes.PodNetwork) { + subnets, err = getHostSubnets(nodeName, network) + if err != nil || len(subnets) == 0 { + return nil, fmt.Errorf("%w: will wait for subnet annotation to be set for node %q and network %q: %w", errConfig, nodeName, network, err) + } + + } + // gather EgressIPs + var eips []string + if advertisements.Has(ratypes.EgressIP) { + if network != types.DefaultNetworkName { + return nil, fmt.Errorf("%w: can't advertise EgressIP in selected non default network %q: %w", errConfig, network, err) + } + eips, err = getEgressIPs(nodeName) + if err != nil { + return nil, err + } + } + + prefixes := make([]string, 0, len(subnets)+len(eips)) + prefixes = append(prefixes, subnets...) + prefixes = append(prefixes, eips...) + return prefixes, nil + } + + generated := []*frrtypes.FRRConfiguration{} + for nodeName, frrConfigs := range nodeToFRRConfig { + // reset node specific information + selectedNetworks.hostNetworkSubnets = map[string][]string{} + selectedNetworks.hostSubnets = []string{} + + // gather node specific information + for _, network := range selectedNetworks.networks { + selectedNetworks.hostNetworkSubnets[network], err = getPrefixes(nodeName, network) + if err != nil { + return nil, nil, err + } + selectedNetworks.hostSubnets = append(selectedNetworks.hostSubnets, selectedNetworks.hostNetworkSubnets[network]...) + // ordered + slices.Sort(selectedNetworks.hostNetworkSubnets[network]) + } + // ordered + slices.Sort(selectedNetworks.hostSubnets) + + matchedNetworks := sets.New[string]() + for _, frrConfig := range frrConfigs { + // generate FRRConfiguration for each source FRRConfiguration/node combination + new := c.generateFRRConfiguration( + ra, + frrConfig, + nodeName, + selectedNetworks, + matchedNetworks, + ) + if new == nil { + // if we got nil, we didn't match any VRF + return nil, nil, fmt.Errorf("%w: FRRConfiguration %q selected for node %q has no VRF matching the RouteAdvertisements target VRF or any selected network", + errConfig, frrConfig.Name, nodeName) + } + generated = append(generated, new) + } + // check that we matched all the selected networks on 'auto' + if ra.Spec.TargetVRF == "auto" && !matchedNetworks.HasAll(selectedNetworks.networks...) { + return nil, nil, fmt.Errorf("%w: selected FRRConfigurations for node %q don't match all selected networks with target VRF 'auto'", errConfig, nodeName) + } + } + + return generated, nads, nil +} + +// generateFRRConfiguration generates a FRRConfiguration from a source for a +// specific node. Also fills matchedNetworks with the networks that have a VRF +// that matched any router VRF of the FRRConfiguration. +func (c *Controller) generateFRRConfiguration( + ra *ratypes.RouteAdvertisements, + source *frrtypes.FRRConfiguration, + nodeName string, + selectedNetworks *selectedNetworks, + matchedNetworks sets.Set[string], +) *frrtypes.FRRConfiguration { + routers := []frrtypes.Router{} + advertisements := sets.New(ra.Spec.Advertisements...) + + // go over the source routers + for i, router := range source.Spec.BGP.Routers { + + targetVRF := ra.Spec.TargetVRF + var matchedVRF, matchedNetwork string + var receivePrefixes, advertisePrefixes []string + + // We will use the router if: + // - the router VRF matches the target VRF + // - if the target VRF is 'auto', the router VRF is that of a selected network + // Prepare each scenario with a switch statement and check after that + switch { + case targetVRF == "auto" && router.VRF == "": + // match on default network/VRF, advertise node prefixes and receive + // any prefix of default network. + matchedVRF = "" + matchedNetwork = types.DefaultNetworkName + advertisePrefixes = selectedNetworks.hostNetworkSubnets[matchedNetwork] + receivePrefixes = selectedNetworks.networkSubnets[matchedNetwork] + case targetVRF == "auto": + // match router.VRF to network.VRF, advertise node prefixes and + // receive any prefix of the matched network + matchedVRF = router.VRF + matchedNetwork = selectedNetworks.networkVRFs[matchedVRF] + advertisePrefixes = selectedNetworks.hostNetworkSubnets[matchedNetwork] + receivePrefixes = selectedNetworks.networkSubnets[matchedNetwork] + case targetVRF == "": + // match on default network/VRF, advertise node prefixes and + // receive any prefix of selected networks + matchedVRF = "" + matchedNetwork = types.DefaultNetworkName + advertisePrefixes = selectedNetworks.hostSubnets + receivePrefixes = selectedNetworks.subnets + default: + // match router.VRF to network.VRF, advertise node prefixes and + // receive any prefix of selected networks + matchedVRF = targetVRF + matchedNetwork = selectedNetworks.networkVRFs[matchedVRF] + advertisePrefixes = selectedNetworks.hostSubnets + receivePrefixes = selectedNetworks.subnets + } + if matchedVRF != router.VRF || len(advertisePrefixes) == 0 { + // either this router VRF does not match the target VRF or we don't + // have prefixes for it (which might be due to this RA not selecting + // this network, but not just) + continue + } + matchedNetworks.Insert(matchedNetwork) + + // if this router's VRF matches the target VRF, copy it and set the + // prefixes as appropriate + targetRouter := router + targetRouter.Prefixes = advertisePrefixes + targetRouter.Neighbors = make([]frrtypes.Neighbor, 0, len(source.Spec.BGP.Routers[i].Neighbors)) + for _, neighbor := range source.Spec.BGP.Routers[i].Neighbors { + advertisePrefixes := advertisePrefixes + receivePrefixes := receivePrefixes + if neighbor.DisableMP { + isIPV6 := utilnet.IsIPv6String(neighbor.Address) + advertisePrefixes = util.MatchAllIPNetsStringFamily(isIPV6, advertisePrefixes) + receivePrefixes = util.MatchAllIPNetsStringFamily(isIPV6, receivePrefixes) + } + if len(advertisePrefixes) == 0 { + continue + } + neighbor.ToAdvertise = frrtypes.Advertise{ + Allowed: frrtypes.AllowedOutPrefixes{ + Mode: frrtypes.AllowRestricted, + Prefixes: advertisePrefixes, + }, + } + neighbor.ToReceive = frrtypes.Receive{ + Allowed: frrtypes.AllowedInPrefixes{ + Mode: frrtypes.AllowRestricted, + }, + } + if advertisements.Has(ratypes.PodNetwork) { + for _, prefix := range receivePrefixes { + neighbor.ToReceive.Allowed.Prefixes = append(neighbor.ToReceive.Allowed.Prefixes, + frrtypes.PrefixSelector{ + Prefix: prefix, + LE: selectedNetworks.prefixLength[prefix], + GE: selectedNetworks.prefixLength[prefix], + }, + ) + } + } + targetRouter.Neighbors = append(targetRouter.Neighbors, neighbor) + } + if len(targetRouter.Neighbors) == 0 { + // we ended up with no neighbor + continue + } + + // append this router to the list of routers we will include in the + // generated FRR config and track its index as we might need to add + // imports to it + routers = append(routers, targetRouter) + targetRouterIndex := len(routers) - 1 + + // VRFs are isolated in "auto" so no need to handle imports + if targetVRF == "auto" { + continue + } + + // we won't do imports if the pod network is not advertised + if !advertisements.Has(ratypes.PodNetwork) { + continue + } + + // before handling imports, lets normalize the VRF for the default + // network: when doing imports, the default VRF is is referred to as + // "default" instead of "" + if matchedVRF == "" { + matchedVRF = types.DefaultNetworkName + } + + // handle imports: when the target VRF is not "auto" we need to leak + // between the target VRF and the selected networks, reciprocally + // importing from each + for _, vrf := range selectedNetworks.vrfs { // ordered + // skip self + if vrf == matchedVRF { + continue + } + + // import all other selected networks into this router's network. + routers[targetRouterIndex].Imports = append(routers[targetRouterIndex].Imports, frrtypes.Import{VRF: vrf}) + + // add an additional router to import the target VRF into selected + // network + importRouter := frrtypes.Router{ + ASN: router.ASN, + ID: router.ID, + Imports: []frrtypes.Import{{VRF: matchedVRF}}, + } + if vrf != types.DefaultNetworkName { + importRouter.VRF = vrf + } + routers = append(routers, importRouter) + } + } + if len(routers) == 0 { + // we ended up with no routers, bail out + return nil + } + + new := &frrtypes.FRRConfiguration{} + new.GenerateName = generateName + new.Namespace = source.Namespace + // label the FRRConfigurations with the RA name, we use this to find the + // existing set of FRRConfigurations that need to be reconciled for a given + // RA + new.Labels = map[string]string{ + types.OvnRouteAdvertisementsKey: ra.Name, + } + // annotate each generated FRRConfiguration with a unique key + // (ra/source/node) which is used in the reconciliation to know whether an + // existing FRRConfiguration should be deleted or not. + new.Annotations = map[string]string{ + types.OvnRouteAdvertisementsKey: fmt.Sprintf("%s/%s/%s", ra.Name, source.Name, nodeName), + } + new.Spec = source.Spec + new.Spec.BGP.Routers = routers + new.Spec.NodeSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kubernetes.io/hostname": nodeName, + }, + } + + return new +} + +// updateFRRConfigurations updates the FRRConfigurations that apply for a +// RouteAdvertisements. It fetches existing FRRConfigurations by label and +// indexes them by the annotated key. Then compares this state with desired +// state and creates, updates or deletes the FRRConfigurations accordingly. +func (c *Controller) updateFRRConfigurations(ra string, frrConfigurations []*frrtypes.FRRConfiguration) (bool, error) { + var hadUpdates bool + + // fetch the currently existing FRRConfigurations for this + // RouteAdvertisements + selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: map[string]string{types.OvnRouteAdvertisementsKey: ra}, + }) + if err != nil { + return hadUpdates, err + } + frrConfigs, err := c.frrLister.List(selector) + if err != nil { + return hadUpdates, err + } + // map them by our internal unique key + existing := make(map[string][]*frrtypes.FRRConfiguration, len(frrConfigs)) + for _, frrConfig := range frrConfigs { + key := frrConfig.Annotations[types.OvnRouteAdvertisementsKey] + if key == "" { + continue + } + existing[key] = append(existing[key], frrConfig) + } + + // go through the FRRConfigurations that should exist for this + // RouteAdvertisements + for _, newFRRConfig := range frrConfigurations { + key := newFRRConfig.Annotations[types.OvnRouteAdvertisementsKey] + oldFRRConfigs := existing[key] + + if len(oldFRRConfigs) == 0 { + // does not exist, create + _, err := c.frrClient.ApiV1beta1().FRRConfigurations(newFRRConfig.Namespace).Create( + context.Background(), + newFRRConfig, + metav1.CreateOptions{ + FieldManager: fieldManager, + }, + ) + if err != nil { + return hadUpdates, err + } + hadUpdates = true + continue + } + + // If it already exists, update. Unexpected user actions can lead us to + // have multiple FRRConfigurations with the same key, in that case we + // pick one to update and delete the rest. + oldFRRConfig := oldFRRConfigs[len(oldFRRConfigs)-1] + existing[key] = oldFRRConfigs[:len(oldFRRConfigs)-1] + + // no changes needed so skip + if reflect.DeepEqual(newFRRConfig.Spec, oldFRRConfig.Spec) { + continue + } + + // otherwise update + newFRRConfig.Name = oldFRRConfig.Name + newFRRConfig.ResourceVersion = oldFRRConfig.ResourceVersion + _, err := c.frrClient.ApiV1beta1().FRRConfigurations(newFRRConfig.Namespace).Update( + context.Background(), + newFRRConfig, + metav1.UpdateOptions{ + FieldManager: fieldManager, + }, + ) + if err != nil { + return hadUpdates, err + } + hadUpdates = true + } + + // delete FRRConfigurations that should not exist + for _, obsoleteFRRConfigs := range existing { + for _, obsoleteFRRConfig := range obsoleteFRRConfigs { + err := c.frrClient.ApiV1beta1().FRRConfigurations(obsoleteFRRConfig.Namespace).Delete( + context.Background(), + obsoleteFRRConfig.Name, + metav1.DeleteOptions{}, + ) + if err != nil && !apierrors.IsNotFound(err) { + return hadUpdates, err + } + hadUpdates = true + } + } + + return hadUpdates, nil +} + +// updateNADs updates the annotation of the NADs that apply for a +// RouteAdvertisements. It iterates all the existing NADs updating the +// annotation accordingly, adding or removing the RouteAdvertisements reference +// as needed. +func (c *Controller) updateNADs(ra string, nads []*nadtypes.NetworkAttachmentDefinition) (bool, error) { + var hadUpdates bool + selected := sets.New[string]() + for _, nad := range nads { + selected.Insert(nad.Namespace + "/" + nad.Name) + } + + nads, err := c.nadLister.List(labels.Everything()) + if err != nil { + return hadUpdates, err + } + + k := kube.KubeOVN{ + NADClient: c.nadClient, + } + + // go through all the NADs and update the annotation adding or removing the + // reference to this RouteAdvertisements as required + for _, nad := range nads { + var ras []string + + if nad.Annotations[types.OvnRouteAdvertisementsKey] != "" { + err := json.Unmarshal([]byte(nad.Annotations[types.OvnRouteAdvertisementsKey]), &ras) + if err != nil { + return hadUpdates, err + } + } + + raSet := sets.New(ras...) + nadName := nad.Namespace + "/" + nad.Name + if selected.Has(nadName) { + raSet.Insert(ra) + selected.Delete(nadName) + } else { + raSet.Delete(ra) + } + + if len(ras) == raSet.Len() { + continue + } + + nadRAjson, err := json.Marshal(raSet.UnsortedList()) + if err != nil { + return hadUpdates, err + } + + err = k.SetAnnotationsOnNAD( + nad.Namespace, + nad.Name, + map[string]string{ + types.OvnRouteAdvertisementsKey: string(nadRAjson), + }, + fieldManager, + ) + if err != nil { + return hadUpdates, fmt.Errorf("failed to annotate NAD %q: %w", nad.Name, err) + } + + hadUpdates = true + } + if selected.Len() != 0 { + return hadUpdates, fmt.Errorf("failed to annotate NADs that were not found %v", selected.UnsortedList()) + } + + return hadUpdates, nil +} + +// updateRAStatus update the RouteAdvertisements 'Accepted' status according to +// the error provided +func (c *Controller) updateRAStatus(ra *ratypes.RouteAdvertisements, hadUpdates bool, err error) error { + if ra == nil { + return nil + } + + condition := meta.FindStatusCondition(ra.Status.Conditions, "Accepted") + updateStatus := hadUpdates || condition == nil || condition.ObservedGeneration != ra.Generation + updateStatus = updateStatus || err != nil + + if !updateStatus { + return nil + } + + status := "Accepted" + cstatus := metav1.ConditionTrue + reason := "Accepted" + msg := "ovn-kubernetes cluster-manager validated the resource and requested the necessary configuration changes" + if err != nil { + status = fmt.Sprintf("Not Accepted: %v", err) + cstatus = metav1.ConditionFalse + msg = err.Error() + switch { + case errors.Is(err, errConfig): + reason = "ConfigurationError" + case errors.Is(err, errPending): + reason = "ConfigurationPending" + default: + reason = "InternalError" + } + } + + _, err = c.raClient.K8sV1().RouteAdvertisements().ApplyStatus( + context.Background(), + raapply.RouteAdvertisements(ra.Name).WithStatus( + raapply.RouteAdvertisementsStatus().WithStatus(status).WithConditions( + metaapply.Condition(). + WithType("Accepted"). + WithStatus(cstatus). + WithLastTransitionTime(metav1.NewTime(time.Now())). + WithReason(reason). + WithMessage(msg). + WithObservedGeneration(ra.Generation), + ), + ), + metav1.ApplyOptions{ + FieldManager: fieldManager, + }, + ) + if err != nil { + return fmt.Errorf("failed to apply status for RouteAdvertisements %q: %w", ra.Name, err) + } + + return nil +} + +// getOrCreateDefaultNetworkNAD ensure that a well-known NAD exists for the +// default network in ovn-k namespace. +func (c *Controller) getOrCreateDefaultNetworkNAD() (*nadtypes.NetworkAttachmentDefinition, error) { + nad, err := c.nadLister.NetworkAttachmentDefinitions(config.Kubernetes.OVNConfigNamespace).Get(types.DefaultNetworkName) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + if nad != nil { + return nad, nil + } + return c.nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(config.Kubernetes.OVNConfigNamespace).Create( + context.Background(), + &nadtypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: types.DefaultNetworkName, + Namespace: config.Kubernetes.OVNConfigNamespace, + Labels: map[string]string{types.DefaultNetworkLabelSelector: ""}, + }, + Spec: nadtypes.NetworkAttachmentDefinitionSpec{ + Config: fmt.Sprintf("{\"cniVersion\": \"0.4.0\", \"name\": \"ovn-kubernetes\", \"type\": \"%s\"}", config.CNI.Plugin), + }, + }, + // note we don't set ourselves as field manager for this create as we + // want to process the resulting event that would otherwise be filtered + // out in nadNeedsUpdate + metav1.CreateOptions{}, + ) +} + +// getEgressIPsByNode iterates all existing egress IPs and returns them indexed +// by node +func (c *Controller) getEgressIPsByNode() (map[string][]string, error) { + eips, err := c.eipLister.List(labels.Everything()) + if err != nil { + return nil, err + } + + eipsByNode := map[string][]string{} + for _, eip := range eips { + for _, item := range eip.Status.Items { + if item.EgressIP == "" { + continue + } + ip := item.EgressIP + util.GetIPFullMaskString(item.EgressIP) + eipsByNode[item.Node] = append(eipsByNode[item.Node], ip) + } + } + + return eipsByNode, nil +} + +// isOwnUpdate checks if an object was updated by us last, as indicated by its +// managed fields. Used to avoid reconciling an update that we made ourselves. +func isOwnUpdate(managedFields []metav1.ManagedFieldsEntry) bool { + return util.IsLastUpdatedByManager(fieldManager, managedFields) +} + +func raNeedsUpdate(oldObj, newObj *ratypes.RouteAdvertisements) bool { + return oldObj == nil || newObj == nil || oldObj.Generation != newObj.Generation +} + +func frrConfigurationNeedsUpdate(oldObj, newObj *frrtypes.FRRConfiguration) bool { + // ignore if it was created or updated by ourselves + if newObj != nil && isOwnUpdate(newObj.ManagedFields) { + return false + } + return oldObj == nil || newObj == nil || oldObj.Generation != newObj.Generation || + !reflect.DeepEqual(oldObj.Labels, newObj.Labels) || + oldObj.Annotations[types.OvnRouteAdvertisementsKey] != newObj.Annotations[types.OvnRouteAdvertisementsKey] +} + +func nadNeedsUpdate(oldObj, newObj *nadtypes.NetworkAttachmentDefinition) bool { + // ignore if it updated by ourselves + if newObj != nil && isOwnUpdate(newObj.ManagedFields) { + return false + } + nadSupported := func(nad *nadtypes.NetworkAttachmentDefinition) bool { + if nad == nil { + return false + } + network, err := util.ParseNADInfo(newObj) + if err != nil { + return true + } + return network.IsDefault() || (network.IsPrimaryNetwork() && network.TopologyType() == types.Layer3Topology) + } + // ignore if we don't support this NAD + if !nadSupported(oldObj) && !nadSupported(newObj) { + return false + } + + return oldObj == nil || newObj == nil || + !reflect.DeepEqual(oldObj.Labels, newObj.Labels) || + oldObj.Annotations[types.OvnRouteAdvertisementsKey] != newObj.Annotations[types.OvnRouteAdvertisementsKey] +} + +func nodeNeedsUpdate(oldObj, newObj *core.Node) bool { + return oldObj == nil || newObj == nil || + !reflect.DeepEqual(oldObj.Labels, newObj.Labels) || + util.NodeSubnetAnnotationChanged(oldObj, newObj) +} + +func egressIPNeedsUpdate(oldObj, newObj *eiptypes.EgressIP) bool { + if oldObj != nil && newObj != nil && reflect.DeepEqual(oldObj.Status, newObj.Status) { + return false + } + if oldObj != nil && len(oldObj.Status.Items) > 0 { + return true + } + if newObj != nil && len(newObj.Status.Items) > 0 { + return true + } + return false +} + +func (c *Controller) reconcileFRRConfiguration(key string) error { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + klog.Errorf("Failed spliting FRFConfiguration reconcile key %q: %v", key, err) + return nil + } + + frrConfig, err := c.frrLister.FRRConfigurations(namespace).Get(name) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + + // safest approach is to reconcile all existing RouteAdvertisements (could + // be potentially avoided with additional caching but let's hold it until we + // know we need it) + c.raController.ReconcileAll() + + // on startup, we might be syncing a FRRConfiguration generated by us for a + // RouteAdvertisements that does not exist, so make sure to reconcile it so + // that the FRRConfiguration is deleted if needed + if frrConfig != nil && frrConfig.Labels[types.OvnRouteAdvertisementsKey] != "" { + c.raController.Reconcile(frrConfig.Labels[types.OvnRouteAdvertisementsKey]) + } + + return nil +} + +func (c *Controller) reconcileNAD(key string) error { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + klog.Errorf("Failed spliting NAD reconcile key %q: %v", key, err) + return nil + } + + nad, err := c.nadLister.NetworkAttachmentDefinitions(namespace).Get(name) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + + // safest approach is to reconcile all existing RouteAdvertisements (could be potentially + // avoided with additional caching but let's hold it until we know we need + // it) + c.raController.ReconcileAll() + + // on startup, we might be syncing a NAD annotated by us with a + // RouteAdvertisements that does not longer exist, so make sure to reconcile + // annotated RouteAdvertisements so that the annotation is updated + // accordingly + if nad != nil && nad.Annotations[types.OvnRouteAdvertisementsKey] != "" { + var ras []string + err := json.Unmarshal([]byte(nad.Annotations[types.OvnRouteAdvertisementsKey]), &ras) + if err != nil { + return err + } + for _, ra := range ras { + c.raController.Reconcile(ra) + } + } + + return nil +} + +func (c *Controller) reconcileEgressIP(eipName string) error { + // reconcile RAs that advertise EIPs + ras, err := c.raLister.List(labels.Everything()) + if err != nil { + return err + } + + for _, ra := range ras { + if sets.New(ra.Spec.Advertisements...).Has(ratypes.EgressIP) { + c.raController.Reconcile(ra.Name) + } + } + + return nil +} + +func matchesDefaultNetworkLabel(selector metav1.LabelSelector) bool { + _, matchesLabel := selector.MatchLabels[types.DefaultNetworkLabelSelector] + if matchesLabel { + return true + } + for _, expr := range selector.MatchExpressions { + if expr.Key == types.DefaultNetworkLabelSelector { + return true + } + } + return false +} diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go new file mode 100644 index 0000000000..e09cad5ec7 --- /dev/null +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go @@ -0,0 +1,1259 @@ +package routeadvertisements + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + + nadtypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + frrapi "github.com/metallb/frr-k8s/api/v1beta1" + corev1 "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + frrfake "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + controllerutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + eiptypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1" + ratypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + nmtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +type testRA struct { + Name string + TargetVRF string + NetworkSelector map[string]string + NodeSelector map[string]string + FRRConfigurationSelector map[string]string + AdvertisePods bool + AdvertiseEgressIPs bool +} + +func (tra testRA) RouteAdvertisements() *ratypes.RouteAdvertisements { + ra := &ratypes.RouteAdvertisements{ + ObjectMeta: metav1.ObjectMeta{ + Name: tra.Name, + }, + Spec: ratypes.RouteAdvertisementsSpec{ + TargetVRF: tra.TargetVRF, + Advertisements: []ratypes.AdvertisementType{}, + }, + } + if tra.AdvertisePods { + ra.Spec.Advertisements = append(ra.Spec.Advertisements, ratypes.PodNetwork) + } + if tra.AdvertiseEgressIPs { + ra.Spec.Advertisements = append(ra.Spec.Advertisements, ratypes.EgressIP) + } + if tra.NetworkSelector != nil { + ra.Spec.NetworkSelector = metav1.LabelSelector{ + MatchLabels: tra.NetworkSelector, + } + } + if tra.NodeSelector != nil { + ra.Spec.NodeSelector = metav1.LabelSelector{ + MatchLabels: tra.NodeSelector, + } + } + if tra.FRRConfigurationSelector != nil { + ra.Spec.FRRConfigurationSelector = metav1.LabelSelector{ + MatchLabels: tra.FRRConfigurationSelector, + } + } + return ra +} + +type testNode struct { + Name string + Generation int + Labels map[string]string + SubnetsAnnotation string +} + +func (tn testNode) Node() *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: tn.Name, + Labels: tn.Labels, + Generation: int64(tn.Generation), + Annotations: map[string]string{ + "k8s.ovn.org/node-subnets": tn.SubnetsAnnotation, + }, + }, + } +} + +type testNeighbor struct { + ASN uint32 + Address string + Receive []string + Advertise []string +} + +func (tn testNeighbor) Neighbor() frrapi.Neighbor { + n := frrapi.Neighbor{ + ASN: tn.ASN, + Address: tn.Address, + ToReceive: frrapi.Receive{ + Allowed: frrapi.AllowedInPrefixes{ + Mode: frrapi.AllowRestricted, + }, + }, + ToAdvertise: frrapi.Advertise{ + Allowed: frrapi.AllowedOutPrefixes{ + Mode: frrapi.AllowRestricted, + Prefixes: tn.Advertise, + }, + }, + } + for _, receive := range tn.Receive { + sep := strings.LastIndex(receive, "/") + if sep == -1 { + continue + } + first := receive[:sep] + last := receive[sep+1:] + len := ovntest.MustAtoi(last) + n.ToReceive.Allowed.Prefixes = append(n.ToReceive.Allowed.Prefixes, + frrapi.PrefixSelector{ + Prefix: first, + GE: uint32(len), + LE: uint32(len), + }, + ) + } + + return n +} + +type testRouter struct { + ASN uint32 + VRF string + Prefixes []string + Neighbors []*testNeighbor + Imports []string +} + +func (tr testRouter) Router() frrapi.Router { + r := frrapi.Router{ + ASN: tr.ASN, + VRF: tr.VRF, + Prefixes: tr.Prefixes, + } + for _, n := range tr.Neighbors { + r.Neighbors = append(r.Neighbors, n.Neighbor()) + } + for _, vrf := range tr.Imports { + r.Imports = append(r.Imports, frrapi.Import{VRF: vrf}) + } + return r +} + +type testFRRConfig struct { + Name string + Namespace string + Generation int + Labels map[string]string + Annotations map[string]string + Routers []*testRouter + NodeSelector map[string]string + OwnUpdate bool +} + +func (tf testFRRConfig) FRRConfiguration() *frrapi.FRRConfiguration { + f := &frrapi.FRRConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: tf.Name, + Namespace: tf.Namespace, + Labels: tf.Labels, + Annotations: tf.Annotations, + Generation: int64(tf.Generation), + }, + Spec: frrapi.FRRConfigurationSpec{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: tf.NodeSelector, + }, + }, + } + for _, r := range tf.Routers { + f.Spec.BGP.Routers = append(f.Spec.BGP.Routers, r.Router()) + } + if tf.OwnUpdate { + f.ManagedFields = append(f.ManagedFields, metav1.ManagedFieldsEntry{ + Manager: fieldManager, + Time: &metav1.Time{Time: time.Now()}, + }) + } + return f +} + +type testEIP struct { + Name string + Generation int + EIPs map[string]string +} + +func (te testEIP) EgressIP() *eiptypes.EgressIP { + eip := eiptypes.EgressIP{ + ObjectMeta: metav1.ObjectMeta{ + Name: te.Name, + Generation: int64(te.Generation), + }, + Status: eiptypes.EgressIPStatus{ + Items: []eiptypes.EgressIPStatusItem{}, + }, + } + for node, ip := range te.EIPs { + eip.Status.Items = append(eip.Status.Items, eiptypes.EgressIPStatusItem{Node: node, EgressIP: ip}) + } + return &eip +} + +type testNAD struct { + Name string + Namespace string + Network string + Subnet string + Labels map[string]string + Annotations map[string]string + IsSecondary bool + Topology string + OwnUpdate bool +} + +func (tn testNAD) NAD() *nadtypes.NetworkAttachmentDefinition { + if tn.Annotations == nil { + tn.Annotations = map[string]string{} + } + tn.Annotations[types.OvnNetworkNameAnnotation] = tn.Network + nad := &nadtypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: tn.Name, + Namespace: tn.Namespace, + Labels: tn.Labels, + Annotations: tn.Annotations, + }, + } + topology := tn.Topology + if topology == "" { + topology = "layer3" + } + switch { + case tn.IsSecondary: + nad.Spec.Config = fmt.Sprintf("{\"cniVersion\": \"0.4.0\", \"name\": \"%s\", \"type\": \"%s\", \"topology\": \"%s\", \"netAttachDefName\": \"%s\", \"subnets\": \"%s\"}", + tn.Network, + config.CNI.Plugin, + topology, + tn.Namespace+"/"+tn.Name, + tn.Subnet, + ) + case tn.Topology != "": + nad.Spec.Config = fmt.Sprintf("{\"cniVersion\": \"0.4.0\", \"name\": \"%s\", \"type\": \"%s\", \"topology\": \"%s\", \"netAttachDefName\": \"%s\", \"role\": \"primary\", \"subnets\": \"%s\"}", + tn.Network, + config.CNI.Plugin, + topology, + tn.Namespace+"/"+tn.Name, + tn.Subnet, + ) + default: + nad.Spec.Config = fmt.Sprintf("{\"cniVersion\": \"0.4.0\", \"name\": \"%s\", \"type\": \"%s\"}", tn.Network, config.CNI.Plugin) + } + if tn.OwnUpdate { + nad.ManagedFields = append(nad.ManagedFields, metav1.ManagedFieldsEntry{ + Manager: fieldManager, + Time: &metav1.Time{Time: time.Now()}, + }) + } + return nad +} + +type Fake interface { + PrependReactor(verb, resource string, reaction ctesting.ReactionFunc) +} + +var count = uint32(0) + +// source +// https://stackoverflow.com/questions/68794562/kubernetes-fake-client-doesnt-handle-generatename-in-objectmeta/68794563#68794563 +func addGenerateNameReactor[T Fake](client any) { + fake := client.(Fake) + fake.PrependReactor( + "create", + "*", + func(action ctesting.Action) (handled bool, ret runtime.Object, err error) { + ret = action.(ctesting.CreateAction).GetObject() + meta, ok := ret.(metav1.Object) + if !ok { + return + } + + if meta.GetName() == "" && meta.GetGenerateName() != "" { + meta.SetName(meta.GetGenerateName() + fmt.Sprintf("%d", atomic.AddUint32(&count, 1))) + } + + return + }, + ) +} + +func init() { + // set this once at the beginning to avoid races that happen because we + // cannot stop the NAD informer properly (the api we use was generated with + // an old codegen and the informer has no shutdown method) + config.IPv4Mode = true +} + +func TestController_reconcile(t *testing.T) { + frrNamespace := "frrNamespace" + tests := []struct { + name string + ra *testRA + frrConfigs []*testFRRConfig + nads []*testNAD + nodes []*testNode + eips []*testEIP + reconcile string + wantErr bool + expectAcceptedStatus metav1.ConditionStatus + expectFRRConfigs []*testFRRConfig + expectNADAnnotations map[string]map[string]string + }{ + { + name: "reconciles pod+eip RouteAdvertisement for a single FRR config, node and default network and target VRF", + ra: &testRA{Name: "ra", AdvertisePods: true, AdvertiseEgressIPs: true}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}"}}, + eips: []*testEIP{{Name: "eip", EIPs: map[string]string{"node": "1.0.1.1"}}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.0.1.1/32", "1.1.0.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.0.1.1/32", "1.1.0.0/24"}, Receive: []string{"1.1.0.0/16/24"}}, + }}, + }}, + }, + expectNADAnnotations: map[string]map[string]string{"default": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, + { + name: "reconciles pod RouteAdvertisement for a single FRR config, node, non default networks and default target VRF", + ra: &testRA{Name: "ra", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.1.1.0/24"}}, + }}, + }, + }, + }, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: "cluster.udn.red", Topology: "layer3", Subnet: "1.2.0.0/16", Labels: map[string]string{"selected": "true"}}, + {Name: "blue", Namespace: "blue", Network: "cluster.udn.blue", Topology: "layer3", Subnet: "1.3.0.0/16", Labels: map[string]string{"selected": "true"}}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\", \"cluster.udn.red\":\"1.2.0.0/24\", \"cluster.udn.blue\":\"1.3.0.0/24\"}"}}, + eips: []*testEIP{{Name: "eip", EIPs: map[string]string{"node": "1.0.1.1"}}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.2.0.0/24", "1.3.0.0/24"}, Imports: []string{"blue", "red"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.2.0.0/24", "1.3.0.0/24"}, Receive: []string{"1.2.0.0/16/24", "1.3.0.0/16/24"}}, + }}, + {ASN: 1, VRF: "blue", Imports: []string{"default"}}, + {ASN: 1, VRF: "red", Imports: []string{"default"}}, + }}, + }, + expectNADAnnotations: map[string]map[string]string{"red": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}, "blue": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, + { + name: "reconciles eip RouteAdvertisement for a single FRR config, node, default network and non default target VRF", + ra: &testRA{Name: "ra", TargetVRF: "red", AdvertiseEgressIPs: true}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, VRF: "red", Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}"}}, + eips: []*testEIP{{Name: "eip", EIPs: map[string]string{"node": "1.0.1.1"}}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 1, VRF: "red", Prefixes: []string{"1.0.1.1/32"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.0.1.1/32"}}, + }}, + }}, + }, + expectNADAnnotations: map[string]map[string]string{"default": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, + { + name: "reconciles a RouteAdvertisement updating the generated FRRConfigurations if needed", + ra: &testRA{Name: "ra", AdvertisePods: true, AdvertiseEgressIPs: true}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + { + Name: "generated", + Namespace: frrNamespace, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"2.0.1.1", "2.1.0.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}"}}, + eips: []*testEIP{{Name: "eip", EIPs: map[string]string{"node": "1.0.1.1"}}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.0.1.1/32", "1.1.0.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.0.1.1/32", "1.1.0.0/24"}, Receive: []string{"1.1.0.0/16/24"}}, + }}, + }, + }, + }, + expectNADAnnotations: map[string]map[string]string{"default": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, + { + name: "reconciles a deleted RouteAdvertisement", + frrConfigs: []*testFRRConfig{ + { + Name: "generated", + Namespace: frrNamespace, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/default/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.0.1.1"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + reconcile: "ra", + }, + { + name: "reconciles a RouteAdvertisement for multiple selected FRR configs, nodes and networks on auto target VRF", + ra: &testRA{ + Name: "ra", + AdvertisePods: true, + TargetVRF: "auto", + FRRConfigurationSelector: map[string]string{"selected": "true"}, + NetworkSelector: map[string]string{"selected": "true"}, + }, + nads: []*testNAD{ + {Name: "default", Namespace: "ovn-kubernetes", Network: "default", Labels: map[string]string{"selected": "true"}}, + {Name: "red", Namespace: "red", Network: "cluster.udn.red", Topology: "layer3", Subnet: "1.2.0.0/16", Labels: map[string]string{"selected": "true"}}, + {Name: "blue", Namespace: "blue", Network: "cluster.udn.blue", Topology: "layer3"}, // not selected + }, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig-node1", + Namespace: frrNamespace, + Labels: map[string]string{"selected": "true"}, + NodeSelector: map[string]string{"node": "node1"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.0.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + {ASN: 1, VRF: "red", Prefixes: []string{"1.0.2.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + { + Name: "frrConfig-node2", + Namespace: frrNamespace, + Labels: map[string]string{"selected": "true"}, + NodeSelector: map[string]string{"node": "node2"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.0.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + { + Name: "another-frrConfig-node2", + Namespace: frrNamespace, + Labels: map[string]string{"selected": "true"}, + NodeSelector: map[string]string{"node": "node2"}, + Routers: []*testRouter{ + {ASN: 1, VRF: "red", Prefixes: []string{"1.0.2.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + { // not selected + Name: "another-frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 3, VRF: "blue", Prefixes: []string{"3.0.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 3, Address: "3.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{ + {Name: "node1", Labels: map[string]string{"selected": "true", "node": "node1"}, SubnetsAnnotation: "{\"default\":\"1.1.1.0/24\", \"cluster.udn.red\":\"1.2.1.0/24\", \"cluster.udn.blue\":\"1.3.1.0/24\"}"}, + {Name: "node2", Labels: map[string]string{"selected": "true", "node": "node2"}, SubnetsAnnotation: "{\"default\":\"1.1.2.0/24\", \"cluster.udn.red\":\"1.2.2.0/24\", \"cluster.udn.blue\":\"1.3.2.0/24\"}"}, + }, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig-node1/node1"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node1"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.1.1.0/24"}, Receive: []string{"1.1.0.0/16/24"}}, + }}, + {ASN: 1, VRF: "red", Prefixes: []string{"1.2.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.2.1.0/24"}, Receive: []string{"1.2.0.0/16/24"}}, + }}, + }, + }, + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig-node2/node2"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node2"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.2.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.1.2.0/24"}, Receive: []string{"1.1.0.0/16/24"}}, + }}, + }, + }, + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/another-frrConfig-node2/node2"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node2"}, + Routers: []*testRouter{ + {ASN: 1, VRF: "red", Prefixes: []string{"1.2.2.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.2.2.0/24"}, Receive: []string{"1.2.0.0/16/24"}}, + }}, + }, + }, + }, + expectNADAnnotations: map[string]map[string]string{"default": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}, "red": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, + { + name: "fails to reconcile a secondary network", + ra: &testRA{Name: "ra", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: "red", IsSecondary: true, Labels: map[string]string{"selected": "true"}}, + }, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile an unsupported topology", + ra: &testRA{Name: "ra", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: "red", Topology: "layer2", Subnet: "1.2.0.0/16", Labels: map[string]string{"selected": "true"}}, + }, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile pod network if node selector is not empty", + ra: &testRA{Name: "ra", AdvertisePods: true, NodeSelector: map[string]string{"selected": "true"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile if no FRRConfiguration is selected for selected node", + ra: &testRA{Name: "ra", AdvertisePods: true, NodeSelector: map[string]string{"selected-by": "RouteAdvertisements"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + NodeSelector: map[string]string{"selected-by": "FRRConfiguration"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{ + {Name: "node1", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}", Labels: map[string]string{"selected-by": "FRRConfiguration"}}, + {Name: "node2", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}", Labels: map[string]string{"selected-by": "RouteAdvertisements"}}, + }, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile when subnet annotation is missing from node", + ra: &testRA{Name: "ra", AdvertisePods: true}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile when subnet annotation is missing for network", + ra: &testRA{Name: "ra", AdvertisePods: true}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"red\":\"1.1.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile if egress IPs are advertised for non-default network", + ra: &testRA{Name: "ra", AdvertiseEgressIPs: true, NetworkSelector: map[string]string{"selected": "true"}}, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: "red", Topology: "layer3", Labels: map[string]string{"selected": "true"}}, + }, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"red\":\"1.1.0.0/24\"}"}}, + eips: []*testEIP{{Name: "eip", EIPs: map[string]string{"node": "1.0.1.1"}}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile if a selectd FRRConfiguration has no matching VRF", + ra: &testRA{Name: "ra", TargetVRF: "red", AdvertisePods: true}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile if not all VRFs were matched on auto", + ra: &testRA{Name: "ra", TargetVRF: "auto", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: "red", Labels: map[string]string{"selected": "true"}}, + {Name: "blue", Namespace: "blue", Network: "blue", Labels: map[string]string{"selected": "true"}}, + }, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, VRF: "red", Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"red\":\"1.1.0.0/24\", \"blue\":\"1.2.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile if network names are too long to fit as a VFR name", + ra: &testRA{Name: "ra", TargetVRF: "auto", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: "cluster.udn.red.name.too.long", Labels: map[string]string{"selected": "true"}}, + }, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, VRF: "red", Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"cluster.udn.red.name.too.long\":\"1.1.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile if network is not a cluster UDN", + ra: &testRA{Name: "ra", TargetVRF: "auto", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: "red", Labels: map[string]string{"selected": "true"}}, + }, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, VRF: "red", Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"red\":\"1.1.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + gMaxLength := format.MaxLength + format.MaxLength = 0 + defer func() { format.MaxLength = gMaxLength }() + + config.Default.ClusterSubnets = []config.CIDRNetworkEntry{ + { + CIDR: ovntest.MustParseIPNet("1.1.0.0/16"), + HostSubnetLength: 24, + }, + } + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableRouteAdvertisements = true + config.OVNKubernetesFeature.EnableEgressIP = true + + fakeClientset := util.GetOVNClientset().GetClusterManagerClientset() + addGenerateNameReactor[*frrfake.Clientset](fakeClientset.FRRClient) + + // create test objects (we could initialize these objects with the + // clients but at least for the NADs iit doesn't work) + if tt.ra != nil { + _, err := fakeClientset.RouteAdvertisementsClient.K8sV1().RouteAdvertisements().Create(context.Background(), tt.ra.RouteAdvertisements(), metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + } + + for _, frrConfig := range tt.frrConfigs { + _, err := fakeClientset.FRRClient.ApiV1beta1().FRRConfigurations(frrConfig.Namespace).Create(context.Background(), frrConfig.FRRConfiguration(), metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + } + + var defaultNAD *nadtypes.NetworkAttachmentDefinition + for _, nad := range tt.nads { + n, err := fakeClientset.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(nad.Namespace).Create(context.Background(), nad.NAD(), metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + if nad.Name == types.DefaultNetworkName && nad.Namespace == config.Kubernetes.OVNConfigNamespace { + defaultNAD = n + } + } + + for _, node := range tt.nodes { + _, err := fakeClientset.KubeClient.CoreV1().Nodes().Create(context.Background(), node.Node(), metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + } + + for _, eip := range tt.eips { + _, err := fakeClientset.EgressIPClient.K8sV1().EgressIPs().Create(context.Background(), eip.EgressIP(), metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + } + + wf, err := factory.NewClusterManagerWatchFactory(fakeClientset) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + nm, err := networkmanager.NewForCluster(&nmtest.FakeControllerManager{}, wf, fakeClientset, nil) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + c := NewController(nm.Interface(), wf, fakeClientset) + + // prime the default network NAD + if defaultNAD == nil { + defaultNAD, err = c.getOrCreateDefaultNetworkNAD() + g.Expect(err).ToNot(gomega.HaveOccurred()) + } + + // update it with the annotation that network manager would set + defaultNAD.Annotations = map[string]string{types.OvnNetworkNameAnnotation: types.DefaultNetworkName} + _, err = fakeClientset.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(defaultNAD.Namespace).Update(context.Background(), defaultNAD, metav1.UpdateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + err = wf.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer wf.Shutdown() + + // wait for caches to sync + cache.WaitForCacheSync( + context.Background().Done(), + wf.RouteAdvertisementsInformer().Informer().HasSynced, + wf.FRRConfigurationsInformer().Informer().HasSynced, + wf.NADInformer().Informer().HasSynced, + wf.NodeCoreInformer().Informer().HasSynced, + wf.EgressIPInformer().Informer().HasSynced, + ) + + err = nm.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + // we just need the inital sync + nm.Stop() + + if err := c.reconcile(tt.reconcile); (err != nil) != tt.wantErr { + t.Fatalf("Controller.reconcile() error = %v, wantErr %v", err, tt.wantErr) + } + + // verify RA status is set as expected + if tt.ra != nil { + ra, err := fakeClientset.RouteAdvertisementsClient.K8sV1().RouteAdvertisements().Get(context.Background(), tt.reconcile, metav1.GetOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + accepted := meta.FindStatusCondition(ra.Status.Conditions, "Accepted") + g.Expect(accepted).NotTo(gomega.BeNil()) + g.Expect(accepted.Status).To(gomega.Equal(tt.expectAcceptedStatus)) + } + + // verify FRRConfigurations have been created/updated/deleted as expected + actualFRRConfigs, err := fakeClientset.FRRClient.ApiV1beta1().FRRConfigurations(frrNamespace).List(context.Background(), metav1.ListOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + var actualFRRConfigKeys []string + actualFRRConfigLabels := map[string]map[string]string{} + actualFRRConfigSpecs := map[string]*frrapi.FRRConfigurationSpec{} + for _, frrConfig := range actualFRRConfigs.Items { + if _, generated := frrConfig.Annotations[types.OvnRouteAdvertisementsKey]; generated { + actualFRRConfigKeys = append(actualFRRConfigKeys, frrConfig.Annotations[types.OvnRouteAdvertisementsKey]) + actualFRRConfigLabels[frrConfig.Annotations[types.OvnRouteAdvertisementsKey]] = frrConfig.Labels + actualFRRConfigSpecs[frrConfig.Annotations[types.OvnRouteAdvertisementsKey]] = &frrConfig.Spec + } + } + + var expectedRRConfigKeys []string + expectedFRRConfigLabels := map[string]map[string]string{} + expectedFRRConfigSpecs := map[string]*frrapi.FRRConfigurationSpec{} + for _, frrConfig := range tt.expectFRRConfigs { + expectedFRRConfig := frrConfig.FRRConfiguration() + expectedRRConfigKeys = append(expectedRRConfigKeys, expectedFRRConfig.Annotations[types.OvnRouteAdvertisementsKey]) + expectedFRRConfigLabels[expectedFRRConfig.Annotations[types.OvnRouteAdvertisementsKey]] = expectedFRRConfig.Labels + expectedFRRConfigSpecs[expectedFRRConfig.Annotations[types.OvnRouteAdvertisementsKey]] = &expectedFRRConfig.Spec + } + + g.Expect(actualFRRConfigKeys).To(gomega.ConsistOf(expectedRRConfigKeys)) + g.Expect(actualFRRConfigLabels).To(gomega.Equal(expectedFRRConfigLabels)) + g.Expect(actualFRRConfigSpecs).To(gomega.Equal(expectedFRRConfigSpecs)) + + // verify NADs have been annotated as expected + actualNADs, err := fakeClientset.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions("").List(context.Background(), metav1.ListOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + actualNADAnnotations := map[string]map[string]string{} + for _, actualNAD := range actualNADs.Items { + if len(actualNAD.Annotations) != 0 { + actualNADAnnotations[actualNAD.Name] = actualNAD.Annotations + } + } + for nad, annotations := range tt.expectNADAnnotations { + for k, v := range annotations { + g.Expect(actualNADAnnotations[nad]).To(gomega.HaveKeyWithValue(k, v)) + } + } + }) + } +} + +func TestUpdates(t *testing.T) { + testRAs := []*testRA{ + { + Name: "ra1", + FRRConfigurationSelector: map[string]string{"select": "1"}, + NetworkSelector: map[string]string{"select": "1"}, + AdvertiseEgressIPs: true, + AdvertisePods: true, + }, + { + Name: "ra2", + FRRConfigurationSelector: map[string]string{"select": "2"}, + NetworkSelector: map[string]string{"select": "2"}, + NodeSelector: map[string]string{"select": "2"}, + }, + { + Name: "ra3", + AdvertiseEgressIPs: true, + FRRConfigurationSelector: map[string]string{"select": "3"}, + NetworkSelector: map[string]string{"select": "3"}, + NodeSelector: map[string]string{"select": "3"}, + }, + } + + tests := []struct { + name string + oldObject any + newObject any + expectedReconcile []string + }{ + { + name: "reconciles all RAs when an FRRConfig gets created", + newObject: &testFRRConfig{Labels: map[string]string{"select": "1"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs when an FRRConfig gets deleted", + oldObject: &testFRRConfig{Labels: map[string]string{"select": "1"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs when an FRRConfig labels get updated", + oldObject: &testFRRConfig{Labels: map[string]string{"select": "1"}}, + newObject: &testFRRConfig{Labels: map[string]string{"select": "2"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs when an FRRConfig annotation changes", + oldObject: &testFRRConfig{Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "A"}}, + newObject: &testFRRConfig{}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs when an FRRConfig spec changes", + oldObject: &testFRRConfig{Generation: 1}, + newObject: &testFRRConfig{Generation: 2}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles a deleted RA referenced from FRRConfig", + newObject: &testFRRConfig{Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra4"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3", "ra4"}, + }, + { + name: "does not reconcile an irrelevant update of FRRConfig", + oldObject: &testFRRConfig{Annotations: map[string]string{"irrelevant": "irrelevant"}}, + newObject: &testFRRConfig{Annotations: map[string]string{"irrelevant": "still-irrelevant"}}, + }, + { + name: "does not reconcile own update of FRRConfig", + oldObject: &testFRRConfig{Generation: 1}, + newObject: &testFRRConfig{Generation: 2, OwnUpdate: true}, + }, + { + name: "does not reconcile own update of FRRConfig", + oldObject: &testFRRConfig{Generation: 1}, + newObject: &testFRRConfig{Generation: 2, OwnUpdate: true}, + }, + { + name: "reconciles all RAs on new NAD", + newObject: &testNAD{Name: "net", Namespace: "net", Labels: map[string]string{"select": "2"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs on deleted NAD", + oldObject: &testNAD{Name: "net", Namespace: "net", Labels: map[string]string{"select": "2"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs when NAD labels change", + oldObject: &testNAD{Name: "net", Namespace: "net", Labels: map[string]string{"select": "2"}}, + newObject: &testNAD{Name: "net", Namespace: "net", Labels: map[string]string{"select": "1"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs when NAD annotation changes", + oldObject: &testNAD{Name: "net", Namespace: "net", OwnUpdate: true, Labels: map[string]string{"select": "2"}, Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "[\"ra2\"]"}}, + newObject: &testNAD{Name: "net", Namespace: "net", Labels: map[string]string{"select": "1"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles a deleted RA referenced from NAD", + newObject: &testNAD{Name: "net", Namespace: "net", Network: "net", Topology: "layer3", Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "[\"ra4\"]"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3", "ra4"}, + }, + { + name: "does not reconcile own update of NAD", + oldObject: &testNAD{Name: "net", Namespace: "net", Labels: map[string]string{"select": "2"}}, + newObject: &testNAD{Name: "net", Namespace: "net", OwnUpdate: true, Labels: map[string]string{"select": "2"}, Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "[\"ra2\"]"}}, + }, + { + name: "does not reconcile a new unsupported (secondary) NAD", + newObject: &testNAD{Name: "net", Namespace: "net", Network: "net", IsSecondary: true, Topology: "layer3", Labels: map[string]string{"select": "2"}}, + }, + { + name: "does not reconcile a new unsupported (layer2 primary) NAD", + newObject: &testNAD{Name: "net", Namespace: "net", Network: "net", Topology: "layer2", Subnet: "1.2.0.0/16", Labels: map[string]string{"select": "2"}}, + }, + { + name: "does not reconcile an updated unsupported NAD", + oldObject: &testNAD{Name: "net", Namespace: "net", Network: "net", Topology: "layer2", Subnet: "1.2.0.0/16", Labels: map[string]string{"select": "2"}}, + newObject: &testNAD{Name: "net", Namespace: "net", Network: "net", Topology: "layer2", Subnet: "1.2.0.0/16", Labels: map[string]string{"select": "1"}}, + }, + { + // TODO shouldn't happen but needs FIX in controller utility which + // does not call filter predicate on deletes + name: "reconciles all RAs on deleted unsupported NAD", + oldObject: &testNAD{Name: "net", Namespace: "net", Network: "net", Topology: "layer2", Subnet: "1.2.0.0/16", Labels: map[string]string{"select": "2"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs that advertise EIPs on new EIP with status", + newObject: &testEIP{Name: "eip", EIPs: map[string]string{"node": "ip"}}, + expectedReconcile: []string{"ra1", "ra3"}, + }, + { + name: "reconciles all RAs that advertise EIPs on deleted EIP with status", + oldObject: &testEIP{Name: "eip", EIPs: map[string]string{"node": "ip"}}, + expectedReconcile: []string{"ra1", "ra3"}, + }, + { + name: "reconciles all RAs that advertise EIPs on updated EIP status", + oldObject: &testEIP{Name: "eip", EIPs: map[string]string{"node": "ip"}}, + newObject: &testEIP{Name: "eip", EIPs: map[string]string{"node": "ip2"}}, + expectedReconcile: []string{"ra1", "ra3"}, + }, + { + name: "does not reconcile RAs on new EIP with no status", + newObject: &testEIP{Name: "eip"}, + }, + { + // TODO shouldn't happen but needs FIX in controller utility which + // does not call filter predicate on deletes + name: "reconciles all RAs that advertise EIPs on deleted EIP", + oldObject: &testEIP{Name: "eip"}, + expectedReconcile: []string{"ra1", "ra3"}, + }, + { + name: "does not reconcile RAs on updated EIP with no status update", + oldObject: &testEIP{Name: "eip", Generation: 1, EIPs: map[string]string{"node": "ip"}}, + newObject: &testEIP{Name: "eip", Generation: 2, EIPs: map[string]string{"node": "ip"}}, + }, + { + name: "reconciles all RAs on new Node", + newObject: &testNode{Name: "eip"}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs on deleted Node", + oldObject: &testNode{Name: "eip"}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs on updated Node labels", + oldObject: &testNode{Name: "eip"}, + newObject: &testNode{Name: "eip", Labels: map[string]string{"select": "1"}}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs on updated Node subnet annotation", + oldObject: &testNode{Name: "eip"}, + newObject: &testNode{Name: "eip", SubnetsAnnotation: "subnets"}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "does not reconcile RAs on node irrelevant change", + oldObject: &testNode{Name: "eip", Generation: 1}, + newObject: &testNode{Name: "eip", Generation: 2}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + gMaxLength := format.MaxLength + format.MaxLength = 0 + defer func() { format.MaxLength = gMaxLength }() + + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableRouteAdvertisements = true + config.OVNKubernetesFeature.EnableEgressIP = true + + fakeClientset := util.GetOVNClientset().GetClusterManagerClientset() + + wf, err := factory.NewClusterManagerWatchFactory(fakeClientset) + g.Expect(err).ToNot(gomega.HaveOccurred()) + err = wf.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer wf.Shutdown() + + reconciled := []string{} + reconciledMutex := sync.Mutex{} + reconcile := func(ra string) error { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + reconciled = append(reconciled, ra) + return nil + } + matchReconciledRAs := func(g gomega.Gomega, expected []string) { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + g.Expect(reconciled).To(gomega.ConsistOf(expected)) + } + resetReconciles := func() { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + reconciled = []string{} + } + + c := NewController(networkmanager.Default().Interface(), wf, fakeClientset) + config := &controllerutil.ControllerConfig[ratypes.RouteAdvertisements]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: reconcile, + Threadiness: 1, + Informer: wf.RouteAdvertisementsInformer().Informer(), + Lister: wf.RouteAdvertisementsInformer().Lister().List, + ObjNeedsUpdate: raNeedsUpdate, + } + c.raController = controllerutil.NewController("", config) + + err = c.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer c.Stop() + + createObj := func(obj any) error { + var err error + switch t := obj.(type) { + case *testFRRConfig: + _, err = fakeClientset.FRRClient.ApiV1beta1().FRRConfigurations(t.Namespace).Create(context.Background(), t.FRRConfiguration(), metav1.CreateOptions{}) + case *testNAD: + _, err = fakeClientset.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(t.Namespace).Create(context.Background(), t.NAD(), metav1.CreateOptions{}) + case *testEIP: + _, err = fakeClientset.EgressIPClient.K8sV1().EgressIPs().Create(context.Background(), t.EgressIP(), metav1.CreateOptions{}) + case *testNode: + _, err = fakeClientset.KubeClient.CoreV1().Nodes().Create(context.Background(), t.Node(), metav1.CreateOptions{}) + } + return err + } + updateObj := func(obj any) error { + var err error + switch t := obj.(type) { + case *testFRRConfig: + _, err = fakeClientset.FRRClient.ApiV1beta1().FRRConfigurations(t.Namespace).Update(context.Background(), t.FRRConfiguration(), metav1.UpdateOptions{}) + case *testNAD: + _, err = fakeClientset.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(t.Namespace).Update(context.Background(), t.NAD(), metav1.UpdateOptions{}) + case *testEIP: + _, err = fakeClientset.EgressIPClient.K8sV1().EgressIPs().Update(context.Background(), t.EgressIP(), metav1.UpdateOptions{}) + case *testNode: + _, err = fakeClientset.KubeClient.CoreV1().Nodes().Update(context.Background(), t.Node(), metav1.UpdateOptions{}) + } + return err + } + deleteObj := func(obj any) error { + var err error + switch t := obj.(type) { + case *testFRRConfig: + err = fakeClientset.FRRClient.ApiV1beta1().FRRConfigurations(t.Namespace).Delete(context.Background(), t.Name, metav1.DeleteOptions{}) + case *testNAD: + err = fakeClientset.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(t.Namespace).Delete(context.Background(), t.Name, metav1.DeleteOptions{}) + case *testEIP: + err = fakeClientset.EgressIPClient.K8sV1().EgressIPs().Delete(context.Background(), t.Name, metav1.DeleteOptions{}) + case *testNode: + err = fakeClientset.KubeClient.CoreV1().Nodes().Delete(context.Background(), t.Name, metav1.DeleteOptions{}) + } + return err + } + + if tt.oldObject != nil { + err = createObj(tt.oldObject) + g.Expect(err).ToNot(gomega.HaveOccurred()) + } + + // since we haven't created the RAs yet, this should not reconcile anything + g.Consistently(matchReconciledRAs).WithArguments([]string{}).Should(gomega.Succeed()) + + var raNames []string + for _, t := range testRAs { + raNames = append(raNames, t.Name) + _, err = fakeClientset.RouteAdvertisementsClient.K8sV1().RouteAdvertisements().Create(context.Background(), t.RouteAdvertisements(), metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + } + + // creating the testRAs, should reconcile them + g.Eventually(matchReconciledRAs).WithArguments(raNames).Should(gomega.Succeed()) + g.Consistently(matchReconciledRAs).WithArguments(raNames).Should(gomega.Succeed()) + // reset for the actual test + resetReconciles() + + switch { + case tt.newObject != nil && tt.oldObject == nil: + err = createObj(tt.newObject) + case tt.newObject != nil: + err = updateObj(tt.newObject) + default: + err = deleteObj(tt.oldObject) + } + g.Expect(err).ToNot(gomega.HaveOccurred()) + + g.Eventually(matchReconciledRAs).WithArguments(tt.expectedReconcile).Should(gomega.Succeed()) + g.Consistently(matchReconciledRAs).WithArguments(tt.expectedReconcile).Should(gomega.Succeed()) + }) + } +} diff --git a/go-controller/pkg/clustermanager/secondary_network_cluster_manager.go b/go-controller/pkg/clustermanager/secondary_network_cluster_manager.go index 5bc7448905..37022b77e8 100644 --- a/go-controller/pkg/clustermanager/secondary_network_cluster_manager.go +++ b/go-controller/pkg/clustermanager/secondary_network_cluster_manager.go @@ -1,13 +1,10 @@ package clustermanager import ( - "fmt" - "github.com/containernetworking/cni/pkg/types" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" @@ -16,11 +13,6 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) -const ( - // Maximum secondary network IDs that can be generated. An arbitrary value is chosen. - maxSecondaryNetworkIDs = 4096 -) - // secondaryNetworkClusterManager object manages the multi net-attach-def controllers. // It implements networkmanager.ControllerManager interface and can be used // by network manager to create and delete network controllers. @@ -29,8 +21,6 @@ type secondaryNetworkClusterManager struct { networkManager networkmanager.Interface ovnClient *util.OVNClusterManagerClientset watchFactory *factory.WatchFactory - // networkIDAllocator is used to allocate a unique ID for each secondary layer3 network - networkIDAllocator id.Allocator // event recorder used to post events to k8s recorder record.EventRecorder @@ -45,17 +35,11 @@ func newSecondaryNetworkClusterManager( recorder record.EventRecorder, ) (*secondaryNetworkClusterManager, error) { klog.Infof("Creating secondary network cluster manager") - networkIDAllocator := id.NewIDAllocator("NetworkIDs", maxSecondaryNetworkIDs) - // Reserve the id 0 for the default network. - if err := networkIDAllocator.ReserveID(ovntypes.DefaultNetworkName, defaultNetworkID); err != nil { - return nil, fmt.Errorf("idAllocator failed to reserve defaultNetworkID %d", defaultNetworkID) - } sncm := &secondaryNetworkClusterManager{ - ovnClient: ovnClient, - watchFactory: wf, - networkIDAllocator: networkIDAllocator, - networkManager: networkManager, - recorder: recorder, + ovnClient: ovnClient, + watchFactory: wf, + networkManager: networkManager, + recorder: recorder, } return sncm, nil } @@ -64,41 +48,6 @@ func (sncm *secondaryNetworkClusterManager) SetNetworkStatusReporter(errorReport sncm.errorReporter = errorReporter } -// Start the secondary network controller, handles all events and creates all -// needed logical entities -func (sncm *secondaryNetworkClusterManager) Start() error { - klog.Infof("Starting secondary network cluster manager") - return sncm.init() -} - -func (sncm *secondaryNetworkClusterManager) init() error { - // Reserve the network ids in the id allocator for the existing secondary layer3 networks. - nodes, err := sncm.watchFactory.GetNodes() - if err != nil { - return fmt.Errorf("error getting the nodes from the watch factory : err - %v", err) - } - - for _, n := range nodes { - networkIdsMap, err := util.GetNodeNetworkIDsAnnotationNetworkIDs(n) - if err == nil { - for networkName, id := range networkIdsMap { - // Reserver the id for the network name. We can safely - // ignore any errors if there are duplicate ids or if - // two networks have the same id. We will resync the node - // annotations correctly when the network controller - // is created. - _ = sncm.networkIDAllocator.ReserveID(networkName, id) - } - } - } - - return nil -} - -func (sncm *secondaryNetworkClusterManager) Stop() { - klog.Infof("Stopping secondary network cluster manager") -} - func (sncm *secondaryNetworkClusterManager) GetDefaultNetworkController() networkmanager.ReconcilableNetworkController { return nil } @@ -112,9 +61,7 @@ func (sncm *secondaryNetworkClusterManager) NewNetworkController(nInfo util.NetI klog.Infof("Creating new network controller for network %s of topology %s", nInfo.GetNetworkName(), nInfo.TopologyType()) - namedIDAllocator := sncm.networkIDAllocator.ForName(nInfo.GetNetworkName()) sncc := newNetworkClusterController( - namedIDAllocator, nInfo, sncm.ovnClient, sncm.watchFactory, @@ -201,9 +148,7 @@ func (sncm *secondaryNetworkClusterManager) CleanupStaleNetworks(validNetworks . // newDummyNetworkController creates a dummy network controller used to clean up specific network func (sncm *secondaryNetworkClusterManager) newDummyLayer3NetworkController(netName string) (networkmanager.NetworkController, error) { netInfo, _ := util.NewNetInfo(&ovncnitypes.NetConf{NetConf: types.NetConf{Name: netName}, Topology: ovntypes.Layer3Topology}) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) nc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, diff --git a/go-controller/pkg/clustermanager/secondary_network_unit_test.go b/go-controller/pkg/clustermanager/secondary_network_unit_test.go index 21a80a26a0..45a06ba284 100644 --- a/go-controller/pkg/clustermanager/secondary_network_unit_test.go +++ b/go-controller/pkg/clustermanager/secondary_network_unit_test.go @@ -185,9 +185,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) nc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, @@ -384,9 +382,6 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = sncm.init() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Create a fake nad controller for blue network so that the red network gets cleared // when CleanupDeletedNetworks is called. If we don't pass any controller to // CleanupDeletedNetworks it will try to cleanup both blue and red networks and @@ -396,9 +391,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { netInfo, err := util.NewNetInfo(&ovncnitypes.NetConf{NetConf: types.NetConf{Name: "blue"}, Topology: ovntypes.Layer3Topology}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) oc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, @@ -416,8 +409,6 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { expectBlueCleanup = false expectRedCleanup = true gomega.Eventually(checkNodeAnnotations).ShouldNot(gomega.HaveOccurred()) - err = sncm.networkIDAllocator.ReserveID("was_red_network_id_released", 2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Now call CleanupDeletedNetworks() with empty nad controllers. // Blue network should also be cleared. @@ -427,8 +418,6 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { expectBlueCleanup = true expectRedCleanup = true gomega.Eventually(checkNodeAnnotations).ShouldNot(gomega.HaveOccurred()) - err = sncm.networkIDAllocator.ReserveID("was_blue_network_id_released", 1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) return nil } @@ -491,9 +480,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) nc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, @@ -546,9 +533,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) nc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, @@ -597,9 +582,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) nc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, @@ -668,9 +651,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) nc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, @@ -742,9 +723,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) nc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, @@ -810,9 +789,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) nc := newNetworkClusterController( - namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller.go index 8aa06120a5..bc90b1ac0e 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller.go @@ -41,6 +41,8 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) +const conditionTypeNetworkCreated = "NetworkCreated" + type RenderNetAttachDefManifest func(obj client.Object, targetNamespace string) (*netv1.NetworkAttachmentDefinition, error) type networkInUseError struct { @@ -417,9 +419,9 @@ func (c *Controller) updateUserDefinedNetworkStatus(udn *userdefinednetworkv1.Us return nil } - networkReadyCondition := newNetworkReadyCondition(nad, syncError) + networkCreatedCondition := newNetworkCreatedCondition(nad, syncError) - conditions, updated := updateCondition(udn.Status.Conditions, networkReadyCondition) + conditions, updated := updateCondition(udn.Status.Conditions, networkCreatedCondition) if updated { var err error @@ -450,28 +452,28 @@ func (c *Controller) updateUserDefinedNetworkStatus(udn *userdefinednetworkv1.Us return nil } -func newNetworkReadyCondition(nad *netv1.NetworkAttachmentDefinition, syncError error) *metav1.Condition { +func newNetworkCreatedCondition(nad *netv1.NetworkAttachmentDefinition, syncError error) *metav1.Condition { now := metav1.Now() - networkReadyCondition := &metav1.Condition{ - Type: "NetworkReady", + networkCreatedCondition := &metav1.Condition{ + Type: conditionTypeNetworkCreated, Status: metav1.ConditionTrue, - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created", LastTransitionTime: now, } if nad != nil && !nad.DeletionTimestamp.IsZero() { - networkReadyCondition.Status = metav1.ConditionFalse - networkReadyCondition.Reason = "NetworkAttachmentDefinitionDeleted" - networkReadyCondition.Message = "NetworkAttachmentDefinition is being deleted" + networkCreatedCondition.Status = metav1.ConditionFalse + networkCreatedCondition.Reason = "NetworkAttachmentDefinitionDeleted" + networkCreatedCondition.Message = "NetworkAttachmentDefinition is being deleted" } if syncError != nil { - networkReadyCondition.Status = metav1.ConditionFalse - networkReadyCondition.Reason = "SyncError" - networkReadyCondition.Message = syncError.Error() + networkCreatedCondition.Status = metav1.ConditionFalse + networkCreatedCondition.Reason = "SyncError" + networkCreatedCondition.Message = syncError.Error() } - return networkReadyCondition + return networkCreatedCondition } func updateCondition(conditions []metav1.Condition, cond *metav1.Condition) ([]metav1.Condition, bool) { @@ -635,9 +637,9 @@ func (c *Controller) updateClusterUDNStatus(cudn *userdefinednetworkv1.ClusterUs return strings.Compare(a.Namespace, b.Namespace) }) - networkReadyCondition := newClusterNetworkReadyCondition(nads, syncError) + networkCreatedCondition := newClusterNetworCreatedCondition(nads, syncError) - conditions, updated := updateCondition(cudn.Status.Conditions, networkReadyCondition) + conditions, updated := updateCondition(cudn.Status.Conditions, networkCreatedCondition) if !updated { return nil } @@ -669,7 +671,7 @@ func (c *Controller) updateClusterUDNStatus(cudn *userdefinednetworkv1.ClusterUs return nil } -func newClusterNetworkReadyCondition(nads []netv1.NetworkAttachmentDefinition, syncError error) *metav1.Condition { +func newClusterNetworCreatedCondition(nads []netv1.NetworkAttachmentDefinition, syncError error) *metav1.Condition { var namespaces []string for _, nad := range nads { namespaces = append(namespaces, nad.Namespace) @@ -678,9 +680,9 @@ func newClusterNetworkReadyCondition(nads []netv1.NetworkAttachmentDefinition, s now := metav1.Now() condition := &metav1.Condition{ - Type: "NetworkReady", + Type: conditionTypeNetworkCreated, Status: metav1.ConditionTrue, - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: fmt.Sprintf("NetworkAttachmentDefinition has been created in following namespaces: [%s]", affectedNamespaces), LastTransitionTime: now, } diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go index bd3fb29d75..7d2dc34979 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go @@ -84,9 +84,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(udn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created", }})) @@ -107,7 +107,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(udn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: "failed to generate NetworkAttachmentDefinition: " + renderErr.Error(), @@ -132,7 +132,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(udn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: "failed to create NetworkAttachmentDefinition: create NAD error", @@ -154,7 +154,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(udn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: "foreign NetworkAttachmentDefinition with the desired name already exist [test/test]", @@ -171,9 +171,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(udn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created", }})) @@ -210,9 +210,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(udn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created", }})) actualNAD, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(udn.Namespace).Get(context.Background(), udn.Name, metav1.GetOptions{}) @@ -229,7 +229,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(udn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: "failed to update NetworkAttachmentDefinition: " + expectedErr.Error(), @@ -256,7 +256,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(updatedUDN.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: `primary network already exist in namespace "test": "primary-net-1"`, @@ -278,7 +278,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(updatedUDN.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: `failed to validate no primary network exist: unmarshal failed [test/another-primary-net]: invalid character '!' looking for beginning of value`, @@ -314,7 +314,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(updatedUDN.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: `failed to add finalizer to UserDefinedNetwork: ` + expectedErr.Error(), @@ -384,9 +384,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(cudn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: [blue, red]", }}), "status should reflect NAD exist in test namespaces") for testNamespace, expectedNAD := range expectedNsNADs { @@ -441,9 +441,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(cudn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: [green, yellow]", }}), "status should report NAD created in test labeled namespaces") for _, nsName := range connectedNsNames { @@ -500,9 +500,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(cudn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: [black, gray, green, yellow]", }}), "status should report NAD exist in existing and new labeled namespaces") for _, nsName := range append(connectedNsNames, newNsNames...) { @@ -529,9 +529,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(cudn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: []", }})) for _, nsName := range connectedNsNames { @@ -621,9 +621,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(cudn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: [black, gray, green, yellow]", }}), "status should report NAD created in existing and new test namespaces") for _, nsName := range append(connectedNsNames, newNsNames...) { @@ -647,9 +647,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(cudn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: [blue, green, red, yellow]", }}), "status should report NAD created in existing and new test namespaces") for _, nsName := range append(connectedNsNames, disconnectedNsNames...) { @@ -676,9 +676,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(cudn.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: [" + connectedNsName + "]", }}), "status should report NAD created in label namespace only") @@ -728,9 +728,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return normalizeConditions(cudn.Status.Conditions) }, 50*time.Millisecond).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: [green, yellow]", }}), "status should report NAD created in test labeled namespaces") @@ -922,9 +922,9 @@ var _ = Describe("User Defined Network Controller", func() { &udnv1.UserDefinedNetworkStatus{ Conditions: []metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created", }, }, @@ -936,7 +936,7 @@ var _ = Describe("User Defined Network Controller", func() { &udnv1.UserDefinedNetworkStatus{ Conditions: []metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "NetworkAttachmentDefinitionDeleted", Message: "NetworkAttachmentDefinition is being deleted", @@ -950,7 +950,7 @@ var _ = Describe("User Defined Network Controller", func() { &udnv1.UserDefinedNetworkStatus{ Conditions: []metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: "sync error", @@ -971,7 +971,7 @@ var _ = Describe("User Defined Network Controller", func() { expectedStatus := &udnv1.UserDefinedNetworkStatus{ Conditions: []metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: syncErr.Error(), @@ -986,7 +986,7 @@ var _ = Describe("User Defined Network Controller", func() { expectedUpdatedStatus := &udnv1.UserDefinedNetworkStatus{ Conditions: []metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: anotherSyncErr.Error(), @@ -1153,9 +1153,9 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).ToNot(HaveOccurred()) Expect(normalizeConditions(cudn.Status.Conditions)).To(ConsistOf([]metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "True", - Reason: "NetworkAttachmentDefinitionReady", + Reason: "NetworkAttachmentDefinitionCreated", Message: "NetworkAttachmentDefinition has been created in following namespaces: [green, red]", }, })) @@ -1179,7 +1179,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).ToNot(HaveOccurred()) Expect(normalizeConditions(cudn.Status.Conditions)).To(ConsistOf([]metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "NetworkAttachmentDefinitionDeleted", Message: "NetworkAttachmentDefinition are being deleted: [green/test]", @@ -1204,7 +1204,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).ToNot(HaveOccurred()) Expect(normalizeConditions(cudn.Status.Conditions)).To(ConsistOf([]metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "NetworkAttachmentDefinitionSyncError", Message: "test sync NAD error", @@ -1225,7 +1225,7 @@ func assertConditionReportNetworkInUse(conditions []metav1.Condition, messageNAD } c := conditions[0] - if c.Type != "NetworkReady" || + if c.Type != "NetworkCreated" || c.Status != metav1.ConditionFalse || c.Reason != "NetworkAttachmentDefinitionSyncError" { @@ -1273,7 +1273,7 @@ func assertFinalizersPresent( Expect(err).NotTo(HaveOccurred()) return normalizeConditions(updatedUDN.Status.Conditions) }).Should(Equal([]metav1.Condition{{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: expectedConditionMsg, diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index dab8145de3..3a4f521826 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -232,9 +232,11 @@ type DefaultConfig struct { // EncapType value defines the encapsulation protocol to use to transmit packets between // hypervisors. By default the value is 'geneve' EncapType string `gcfg:"encap-type"` - // The IP address of the encapsulation endpoint. If not specified, the IP address the - // NodeName resolves to will be used + // Configured IP address of the encapsulation endpoint. EncapIP string `gcfg:"encap-ip"` + // Effective encap IP. It may be different from EncapIP if EncapIP meant to be + // the node's primary IP which can be updated when node's primary IP changes. + EffectiveEncapIP string // The UDP Port of the encapsulation endpoint. If not specified, the IP default port // of 6081 will be used EncapPort uint `gcfg:"encap-port"` diff --git a/go-controller/pkg/controller/controller.go b/go-controller/pkg/controller/controller.go index 9570c8c7c0..df5547c3db 100644 --- a/go-controller/pkg/controller/controller.go +++ b/go-controller/pkg/controller/controller.go @@ -2,6 +2,7 @@ package controller import ( "fmt" + "math" "sync" "time" @@ -17,7 +18,10 @@ import ( "k8s.io/klog/v2" ) -const maxRetries = 15 +const ( + DefaultMaxAttempts = 15 + InfiniteAttempts = math.MaxInt +) // Reconciler is a basic level-driven controller that is fed externally of items // to reconcile through its Reconcile method @@ -41,6 +45,12 @@ type ReconcilerConfig struct { Reconcile func(key string) error // How many workers should be started for this reconciler. Threadiness int + // MaxAttempts is the number of times a key will be reconciled before giving + // up on error. If 0, will use DefaultMaxAttempts. Set to InfiniteAttempts + // to retry forever. + MaxAttempts int + // HandleError when MaxRetries has been reached + HandleError func(key string, err error) error } type ControllerConfig[T any] struct { @@ -48,8 +58,15 @@ type ControllerConfig[T any] struct { Reconcile func(key string) error // How many workers should be started for this controller. Threadiness int - Informer cache.SharedIndexInformer - Lister func(selector labels.Selector) (ret []*T, err error) + // MaxAttempts is the number of times a key will be reconciled before giving + // up on error. If 0, will use DefaultMaxAttempts. Set to InfiniteAttempts to + // retry forever. + MaxAttempts int + // HandleError when MaxRetries has been reached + HandleError func(key string, err error) error + + Informer cache.SharedIndexInformer + Lister func(selector labels.Selector) (ret []*T, err error) // ObjNeedsUpdate tells if object should be reconciled. // May be called with oldObj = nil on Add, won't be called on Delete. ObjNeedsUpdate func(oldObj, newObj *T) bool @@ -74,6 +91,8 @@ func NewReconciler(name string, config *ReconcilerConfig) Reconciler { RateLimiter: config.RateLimiter, Reconcile: config.Reconcile, Threadiness: config.Threadiness, + MaxAttempts: config.MaxAttempts, + HandleError: config.HandleError, } return NewController(name, controllerConfig) } @@ -81,7 +100,7 @@ func NewReconciler(name string, config *ReconcilerConfig) Reconciler { // NewController creates a new level-driven controller. It should be started and // stopped using Start/StartWithInitialSync/Stop functions. func NewController[T any](name string, config *ControllerConfig[T]) Controller { - return &controller[T]{ + c := &controller[T]{ name: name, config: config, queue: workqueue.NewTypedRateLimitingQueueWithConfig( @@ -93,6 +112,16 @@ func NewController[T any](name string, config *ControllerConfig[T]) Controller { stopChan: make(chan struct{}), wg: &sync.WaitGroup{}, } + if c.config.MaxAttempts == 0 { + c.config.MaxAttempts = DefaultMaxAttempts + } + if c.config.HandleError == nil { + c.config.HandleError = func(key string, err error) error { + utilruntime.HandleError(err) + return nil + } + } + return c } func (c *controller[T]) addHandler() error { @@ -223,13 +252,17 @@ func (c *controller[T]) processNextQueueItem() bool { err := c.config.Reconcile(key) if err != nil { - if c.queue.NumRequeues(key) < maxRetries { + retry := c.config.MaxAttempts == InfiniteAttempts || c.queue.NumRequeues(key) < c.config.MaxAttempts + if retry { klog.Infof("Controller %s: error found while processing %s: %v", c.name, key, err) c.queue.AddRateLimited(key) return true } klog.Warningf("Controller %s: dropping %s out of the queue: %v", c.name, key, err) - utilruntime.HandleError(err) + err = c.config.HandleError(key, err) + if err != nil { + utilruntime.HandleError(err) + } } c.queue.Forget(key) return true diff --git a/go-controller/pkg/controller/controller_test.go b/go-controller/pkg/controller/controller_test.go index d1c7f7be70..9580a87165 100644 --- a/go-controller/pkg/controller/controller_test.go +++ b/go-controller/pkg/controller/controller_test.go @@ -137,19 +137,19 @@ var _ = Describe("Level-driven controller", func() { failureCounter.Add(1) return fmt.Errorf("failure") } - config.RateLimiter = workqueue.NewTypedItemFastSlowRateLimiter[string](100*time.Millisecond, 1*time.Second, maxRetries) + config.RateLimiter = workqueue.NewTypedItemFastSlowRateLimiter[string](100*time.Millisecond, 1*time.Second, DefaultMaxAttempts) startController(config, nil, namespace, pod) - Eventually(failureCounter.Load, (maxRetries+1)*100*time.Millisecond).Should(BeEquivalentTo(maxRetries)) + Eventually(failureCounter.Load, (DefaultMaxAttempts+1)*100*time.Millisecond).Should(BeEquivalentTo(DefaultMaxAttempts)) time.Sleep(1 * time.Second) - Expect(failureCounter.Load()).To(BeEquivalentTo(maxRetries + 1)) + Expect(failureCounter.Load()).To(BeEquivalentTo(DefaultMaxAttempts + 1)) // trigger update, check it is handled pod.Labels = map[string]string{"key": "value"} _, err := fakeClient.KubeClient.CoreV1().Pods(namespace.Name).Update(context.TODO(), pod, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - Eventually(func() bool { return failureCounter.Load() > maxRetries+1 }).Should(BeTrue()) + Eventually(func() bool { return failureCounter.Load() > DefaultMaxAttempts+1 }).Should(BeTrue()) }) It("ignores events when ObjNeedsUpdate returns false", func() { namespace := util.NewNamespace(namespace1Name) diff --git a/go-controller/pkg/controllermanager/controller_manager.go b/go-controller/pkg/controllermanager/controller_manager.go index a5c4153ccf..be156d2eef 100644 --- a/go-controller/pkg/controllermanager/controller_manager.go +++ b/go-controller/pkg/controllermanager/controller_manager.go @@ -22,6 +22,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/udnenabledsvc" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/routeimport" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -56,7 +57,8 @@ type ControllerManager struct { defaultNetworkController networkmanager.BaseNetworkController // networkManager creates and deletes network controllers - networkManager networkmanager.Controller + networkManager networkmanager.Controller + routeImportManager routeimport.Controller // eIPController programs OVN to support EgressIP eIPController *ovn.EgressIPController @@ -72,7 +74,7 @@ func (cm *ControllerManager) NewNetworkController(nInfo util.NetInfo) (networkma case ovntypes.Layer3Topology: return ovn.NewSecondaryLayer3NetworkController(cnci, nInfo, cm.networkManager.Interface(), cm.eIPController, cm.portCache) case ovntypes.Layer2Topology: - return ovn.NewSecondaryLayer2NetworkController(cnci, nInfo, cm.networkManager.Interface()) + return ovn.NewSecondaryLayer2NetworkController(cnci, nInfo, cm.networkManager.Interface(), cm.eIPController, cm.portCache) case ovntypes.LocalnetTopology: return ovn.NewSecondaryLocalnetNetworkController(cnci, nInfo, cm.networkManager.Interface()), nil } @@ -90,7 +92,7 @@ func (cm *ControllerManager) newDummyNetworkController(topoType, netName string) case ovntypes.Layer3Topology: return ovn.NewSecondaryLayer3NetworkController(cnci, netInfo, cm.networkManager.Interface(), cm.eIPController, cm.portCache) case ovntypes.Layer2Topology: - return ovn.NewSecondaryLayer2NetworkController(cnci, netInfo, cm.networkManager.Interface()) + return ovn.NewSecondaryLayer2NetworkController(cnci, netInfo, cm.networkManager.Interface(), cm.eIPController, cm.portCache) case ovntypes.LocalnetTopology: return ovn.NewSecondaryLocalnetNetworkController(cnci, netInfo, cm.networkManager.Interface()), nil } @@ -224,6 +226,13 @@ func NewControllerManager(ovnClient *util.OVNClientset, wf *factory.WatchFactory } } + if util.IsRouteAdvertisementsEnabled() { + if !config.OVNKubernetesFeature.EnableInterconnect { + return nil, fmt.Errorf("RouteAdvertisements can only be used if Interconnect is enabled") + } + cm.routeImportManager = routeimport.New(config.Default.Zone, cm.nbClient) + } + return cm, nil } @@ -303,7 +312,7 @@ func (cm *ControllerManager) initDefaultNetworkController(observManager *observa if err != nil { return fmt.Errorf("failed to create common network controller info: %w", err) } - defaultController, err := ovn.NewDefaultNetworkController(cnci, observManager, cm.networkManager.Interface(), cm.eIPController, cm.portCache) + defaultController, err := ovn.NewDefaultNetworkController(cnci, observManager, cm.networkManager.Interface(), cm.routeImportManager, cm.eIPController, cm.portCache) if err != nil { return err } @@ -442,6 +451,13 @@ func (cm *ControllerManager) Start(ctx context.Context) error { } } + if cm.routeImportManager != nil { + err = cm.routeImportManager.Start() + if err != nil { + return fmt.Errorf("failed to start route import: %v", err) + } + } + err = cm.defaultNetworkController.Start(ctx) if err != nil { return fmt.Errorf("failed to start default network controller: %v", err) @@ -464,4 +480,12 @@ func (cm *ControllerManager) Stop() { if cm.networkManager != nil { cm.networkManager.Stop() } + + if cm.routeImportManager != nil { + cm.routeImportManager.Stop() + } +} + +func (cm *ControllerManager) Reconcile(name string, old, new util.NetInfo) error { + return nil } diff --git a/go-controller/pkg/controllermanager/node_controller_manager.go b/go-controller/pkg/controllermanager/node_controller_manager.go index ef89c90874..11bb6c5891 100644 --- a/go-controller/pkg/controllermanager/node_controller_manager.go +++ b/go-controller/pkg/controllermanager/node_controller_manager.go @@ -73,28 +73,11 @@ func (ncm *NodeControllerManager) CleanupStaleNetworks(validNetworks ...util.Net if !network.IsPrimaryNetwork() { continue } - networkID, err := ncm.getNetworkID(network) - if err != nil { - klog.Errorf("Failed to get network identifier for network %s, error: %s", network.GetNetworkName(), err) - continue - } - validVRFDevices.Insert(util.GetVRFDeviceNameForUDN(networkID)) + validVRFDevices.Insert(util.GetNetworkVRFName(network)) } return ncm.vrfManager.Repair(validVRFDevices) } -func (ncm *NodeControllerManager) getNetworkID(network util.NetInfo) (int, error) { - nodes, err := ncm.watchFactory.GetNodes() - if err != nil { - return util.InvalidID, err - } - networkID, err := util.GetNetworkID(nodes, network) - if err != nil { - return util.InvalidID, err - } - return networkID, nil -} - // newCommonNetworkControllerInfo creates and returns the base node network controller info func (ncm *NodeControllerManager) newCommonNetworkControllerInfo() *node.CommonNodeNetworkControllerInfo { return node.NewCommonNodeNetworkControllerInfo(ncm.ovnNodeClient.KubeClient, ncm.ovnNodeClient.AdminPolicyRouteClient, ncm.watchFactory, ncm.recorder, ncm.name, ncm.routeManager) @@ -389,3 +372,7 @@ func checkForStaleOVSInternalPorts() { stderr, err) } } + +func (ncm *NodeControllerManager) Reconcile(name string, old, new util.NetInfo) error { + return nil +} diff --git a/go-controller/pkg/controllermanager/node_controller_manager_test.go b/go-controller/pkg/controllermanager/node_controller_manager_test.go index d414833300..3854c28d7e 100644 --- a/go-controller/pkg/controllermanager/node_controller_manager_test.go +++ b/go-controller/pkg/controllermanager/node_controller_manager_test.go @@ -14,8 +14,11 @@ import ( ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" nadinformermocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1" nadlistermocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" + coreinformermocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/client-go/informers/core/v1" + corelistermocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/client-go/listers/core/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/stretchr/testify/mock" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -176,12 +179,12 @@ var _ = Describe("Healthcheck tests", func() { Context("verify cleanup of deleted networks", func() { var ( - staleNetID uint = 100 + staleNetID uint = 1000 nodeName string = "worker1" nad = ovntest.GenerateNAD("bluenet", "rednad", "greenamespace", types.Layer3Topology, "100.128.0.0/16", types.NetworkRolePrimary) netName = "bluenet" - netID = 3 + netID = 1003 v4NodeSubnet = "10.128.0.0/24" v6NodeSubnet = "ae70::66/112" testNS ns.NetNS @@ -211,7 +214,8 @@ var _ = Describe("Healthcheck tests", func() { config.OVNKubernetesFeature.EnableMultiNetwork = true factoryMock := factoryMocks.NodeWatchFactory{} - NetInfo, err := util.ParseNADInfo(nad) + netInfo, err := util.ParseNADInfo(nad) + mutableNetInfo := util.NewMutableNetInfo(netInfo) Expect(err).NotTo(HaveOccurred()) node := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -232,6 +236,11 @@ var _ = Describe("Healthcheck tests", func() { nadInformerMock.On("Lister").Return(nadListerMock) nadInformerMock.On("Informer").Return(nil) factoryMock.On("NADInformer").Return(nadInformerMock) + nodeListerMock := &corelistermocks.NodeLister{} + nodeListerMock.On("List", mock.Anything).Return(nodeList, nil) + nodeInformerMock := &coreinformermocks.NodeInformer{} + nodeInformerMock.On("Lister").Return(nodeListerMock) + factoryMock.On("NodeCoreInformer").Return(nodeInformerMock) ncm, err := NewNodeControllerManager(fakeClient, &factoryMock, nodeName, &sync.WaitGroup{}, nil, routeManager) Expect(err).NotTo(HaveOccurred()) @@ -239,17 +248,19 @@ var _ = Describe("Healthcheck tests", func() { err = testNS.Do(func(ns.NetNS) error { defer GinkgoRecover() - staleVrfDevice := util.GetVRFDeviceNameForUDN(int(staleNetID)) + mutableNetInfo.SetNetworkID(int(staleNetID)) + staleVrfDevice := util.GetNetworkVRFName(mutableNetInfo) ovntest.AddVRFLink(staleVrfDevice, uint32(staleNetID)) _, err = util.GetNetLinkOps().LinkByName(staleVrfDevice) Expect(err).NotTo(HaveOccurred()) - validVrfDevice := util.GetVRFDeviceNameForUDN(int(netID)) + mutableNetInfo.SetNetworkID(int(int(netID))) + validVrfDevice := util.GetNetworkVRFName(mutableNetInfo) ovntest.AddVRFLink(validVrfDevice, uint32(netID)) _, err = util.GetNetLinkOps().LinkByName(validVrfDevice) Expect(err).NotTo(HaveOccurred()) - err = ncm.CleanupStaleNetworks(NetInfo) + err = ncm.CleanupStaleNetworks(mutableNetInfo) Expect(err).NotTo(HaveOccurred()) // Verify CleanupDeletedNetworks cleans up VRF configuration for diff --git a/go-controller/pkg/crd/routeadvertisements/v1/types.go b/go-controller/pkg/crd/routeadvertisements/v1/types.go index 70c545bb25..2fc2ccf59a 100644 --- a/go-controller/pkg/crd/routeadvertisements/v1/types.go +++ b/go-controller/pkg/crd/routeadvertisements/v1/types.go @@ -51,6 +51,7 @@ type RouteAdvertisementsSpec struct { } // AdvertisementType determines the type of advertisement. +// +kubebuilder:validation:Enum=PodNetwork;EgressIP type AdvertisementType string const ( diff --git a/go-controller/pkg/factory/factory.go b/go-controller/pkg/factory/factory.go index 354bca1cc0..845dff4a24 100644 --- a/go-controller/pkg/factory/factory.go +++ b/go-controller/pkg/factory/factory.go @@ -80,6 +80,11 @@ import ( routeadvertisementsinformerfactory "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/informers/externalversions" routeadvertisementsinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/informers/externalversions/routeadvertisements/v1" + frrapi "github.com/metallb/frr-k8s/api/v1beta1" + frrscheme "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme" + frrinformerfactory "github.com/metallb/frr-k8s/pkg/client/informers/externalversions" + frrinformer "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1" + kapi "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" knet "k8s.io/api/networking/v1" @@ -120,6 +125,7 @@ type WatchFactory struct { nadFactory nadinformerfactory.SharedInformerFactory udnFactory userdefinednetworkapiinformerfactory.SharedInformerFactory raFactory routeadvertisementsinformerfactory.SharedInformerFactory + frrFactory frrinformerfactory.SharedInformerFactory informers map[reflect.Type]*informer stopChan chan struct{} @@ -564,6 +570,15 @@ func (wf *WatchFactory) Start() error { } } + if wf.frrFactory != nil { + wf.frrFactory.Start(wf.stopChan) + for oType, synced := range waitForCacheSyncWithTimeout(wf.frrFactory, wf.stopChan) { + if !synced { + return fmt.Errorf("error in syncing cache for %v informer", oType) + } + } + } + return nil } @@ -609,6 +624,9 @@ func (wf *WatchFactory) Stop() { if wf.raFactory != nil { wf.raFactory.Shutdown() } + if wf.frrFactory != nil { + wf.frrFactory.Shutdown() + } } // NewNodeWatchFactory initializes a watch factory with significantly fewer @@ -810,6 +828,9 @@ func NewClusterManagerWatchFactory(ovnClientset *util.OVNClusterManagerClientset if err := routeadvertisementsapi.AddToScheme(routeadvertisementsscheme.Scheme); err != nil { return nil, err } + if err := frrapi.AddToScheme(frrscheme.Scheme); err != nil { + return nil, err + } // For Services and Endpoints, pre-populate the shared Informer with one that // has a label selector excluding headless services. @@ -929,6 +950,10 @@ func NewClusterManagerWatchFactory(ovnClientset *util.OVNClusterManagerClientset wf.raFactory = routeadvertisementsinformerfactory.NewSharedInformerFactory(ovnClientset.RouteAdvertisementsClient, resyncInterval) // make sure shared informer is created for a factory, so on wf.raFactory.Start() it is initialized and caches are synced. wf.raFactory.K8s().V1().RouteAdvertisements().Informer() + + wf.frrFactory = frrinformerfactory.NewSharedInformerFactory(ovnClientset.FRRClient, resyncInterval) + // make sure shared informer is created for a factory, so on wf.frrFactory.Start() it is initialized and caches are synced. + wf.frrFactory.Api().V1beta1().FRRConfigurations().Informer() } return wf, nil @@ -1612,6 +1637,10 @@ func (wf *WatchFactory) RouteAdvertisementsInformer() routeadvertisementsinforme return wf.raFactory.K8s().V1().RouteAdvertisements() } +func (wf *WatchFactory) FRRConfigurationsInformer() frrinformer.FRRConfigurationInformer { + return wf.frrFactory.Api().V1beta1().FRRConfigurations() +} + // withServiceNameAndNoHeadlessServiceSelector returns a LabelSelector (added to the // watcher for EndpointSlices) that will only choose EndpointSlices with a non-empty // "kubernetes.io/service-name" label and without "service.kubernetes.io/headless" diff --git a/go-controller/pkg/kube/kube.go b/go-controller/pkg/kube/kube.go index a287f058c7..d43b591112 100644 --- a/go-controller/pkg/kube/kube.go +++ b/go-controller/pkg/kube/kube.go @@ -5,6 +5,7 @@ import ( "encoding/json" ipamclaimsapi "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1" ipamclaimssclientset "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1/apis/clientset/versioned" + nadclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" ocpcloudnetworkapi "github.com/openshift/api/cloudnetwork/v1" ocpcloudnetworkclientset "github.com/openshift/client-go/cloudnetwork/clientset/versioned" adminpolicybasedrouteclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/clientset/versioned" @@ -84,6 +85,7 @@ type KubeOVN struct { APBRouteClient adminpolicybasedrouteclientset.Interface EgressQoSClient egressqosclientset.Interface IPAMClaimsClient ipamclaimssclientset.Interface + NADClient nadclientset.Interface } // SetAnnotationsOnPod takes the pod object and map of key/value string pairs to set as annotations @@ -463,3 +465,29 @@ func (k *KubeOVN) UpdateIPAMClaimIPs(updatedIPAMClaim *ipamclaimsapi.IPAMClaim) _, err := k.IPAMClaimsClient.K8sV1alpha1().IPAMClaims(updatedIPAMClaim.Namespace).UpdateStatus(context.TODO(), updatedIPAMClaim, metav1.UpdateOptions{}) return err } + +// SetAnnotationsOnNAD takes a NAD namespace and name and a map of key/value string pairs to set as annotations +func (k *KubeOVN) SetAnnotationsOnNAD(namespace, name string, annotations map[string]string, fieldManager string) error { + var err error + var patchData []byte + patch := struct { + Metadata map[string]interface{} `json:"metadata"` + }{ + Metadata: map[string]interface{}{ + "annotations": annotations, + }, + } + + patchData, err = json.Marshal(&patch) + if err != nil { + return err + } + + patchOptions := metav1.PatchOptions{} + if fieldManager != "" { + patchOptions.FieldManager = fieldManager + } + + _, err = k.NADClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespace).Patch(context.Background(), name, types.MergePatchType, patchData, patchOptions) + return err +} diff --git a/go-controller/pkg/libovsdb/ops/router.go b/go-controller/pkg/libovsdb/ops/router.go index 46eb6b5a82..3c95230a74 100644 --- a/go-controller/pkg/libovsdb/ops/router.go +++ b/go-controller/pkg/libovsdb/ops/router.go @@ -602,6 +602,30 @@ func FindLogicalRouterStaticRoutesWithPredicate(nbClient libovsdbclient.Client, return found, err } +// GetRouterLogicalRouterStaticRoutesWithPredicate looks up logical router +// static routes associated to the provided logical router from the cache based +// on a given predicate +func GetRouterLogicalRouterStaticRoutesWithPredicate(nbClient libovsdbclient.Client, router *nbdb.LogicalRouter, p logicalRouterStaticRoutePredicate) ([]*nbdb.LogicalRouterStaticRoute, error) { + r, err := GetLogicalRouter(nbClient, router) + if err != nil { + return nil, fmt.Errorf("failed to get router: %s, error: %w", router.Name, err) + } + + lrsrs := []*nbdb.LogicalRouterStaticRoute{} + for _, uuid := range r.StaticRoutes { + lrsr := &nbdb.LogicalRouterStaticRoute{UUID: uuid} + err := nbClient.Get(context.Background(), lrsr) + if err != nil { + return nil, err + } + if p(lrsr) { + lrsrs = append(lrsrs, lrsr) + } + } + + return lrsrs, nil +} + // CreateOrUpdateLogicalRouterStaticRoutesWithPredicateOps looks up a logical // router static route from the cache based on a given predicate. If it does not // exist, it creates the provided logical router static route. If it does, it @@ -664,25 +688,12 @@ func CreateOrReplaceLogicalRouterStaticRouteWithPredicate(nbClient libovsdbclien lrsr *nbdb.LogicalRouterStaticRoute, p logicalRouterStaticRoutePredicate, fields ...interface{}) error { lr := &nbdb.LogicalRouter{Name: routerName} - router, err := GetLogicalRouter(nbClient, lr) - if err != nil { - return fmt.Errorf("unable to get logical router %s: %w", routerName, err) - } - newPredicate := func(item *nbdb.LogicalRouterStaticRoute) bool { - for _, routeUUID := range router.StaticRoutes { - if routeUUID == item.UUID && p(item) { - return true - } - } - return false - } - routes, err := FindLogicalRouterStaticRoutesWithPredicate(nbClient, newPredicate) + routes, err := GetRouterLogicalRouterStaticRoutesWithPredicate(nbClient, lr, p) if err != nil { return fmt.Errorf("unable to get logical router static routes with predicate on router %s: %w", routerName, err) } var ops []libovsdb.Operation - m := newModelClient(nbClient) if len(routes) > 0 { lrsr.UUID = routes[0].UUID @@ -691,27 +702,7 @@ func CreateOrReplaceLogicalRouterStaticRouteWithPredicate(nbClient libovsdbclien if len(routes) > 1 { // should only be a single route remove all except the first routes = routes[1:] - opModels := make([]operationModel, 0, len(routes)+1) - router.StaticRoutes = []string{} - for _, route := range routes { - route := route - router.StaticRoutes = append(router.StaticRoutes, route.UUID) - opModel := operationModel{ - Model: route, - ErrNotFound: false, - BulkOp: false, - } - opModels = append(opModels, opModel) - } - opModel := operationModel{ - Model: router, - OnModelMutations: []interface{}{&router.StaticRoutes}, - ErrNotFound: true, - BulkOp: false, - } - opModels = append(opModels, opModel) - - ops, err = m.DeleteOps(nil, opModels...) + ops, err = DeleteLogicalRouterStaticRoutesOps(nbClient, ops, routerName, routes...) if err != nil { return err } @@ -768,9 +759,9 @@ func DeleteLogicalRouterStaticRoutesWithPredicateOps(nbClient libovsdbclient.Cli return m.DeleteOps(ops, opModels...) } -// DeleteLogicalRouterStaticRoutes deletes the logical router static routes and -// removes them from the provided logical router -func DeleteLogicalRouterStaticRoutes(nbClient libovsdbclient.Client, routerName string, lrsrs ...*nbdb.LogicalRouterStaticRoute) error { +// DeleteLogicalRouterStaticRoutesOps deletes the logical router static routes and +// returns the ops to remove them from the provided logical router +func DeleteLogicalRouterStaticRoutesOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, routerName string, lrsrs ...*nbdb.LogicalRouterStaticRoute) ([]libovsdb.Operation, error) { router := &nbdb.LogicalRouter{ Name: routerName, StaticRoutes: make([]string, 0, len(lrsrs)), @@ -780,24 +771,31 @@ func DeleteLogicalRouterStaticRoutes(nbClient libovsdbclient.Client, routerName for _, lrsr := range lrsrs { lrsr := lrsr router.StaticRoutes = append(router.StaticRoutes, lrsr.UUID) - opModel := operationModel{ - Model: lrsr, - ErrNotFound: false, - BulkOp: false, - } - opModels = append(opModels, opModel) } opModel := operationModel{ Model: router, OnModelMutations: []interface{}{&router.StaticRoutes}, - ErrNotFound: true, + ErrNotFound: false, BulkOp: false, } opModels = append(opModels, opModel) m := newModelClient(nbClient) - return m.Delete(opModels...) + return m.DeleteOps(ops, opModels...) +} + +// DeleteLogicalRouterStaticRoutes deletes the logical router static routes and +// removes them from the provided logical router +func DeleteLogicalRouterStaticRoutes(nbClient libovsdbclient.Client, routerName string, lrsrs ...*nbdb.LogicalRouterStaticRoute) error { + var ops []libovsdb.Operation + var err error + ops, err = DeleteLogicalRouterStaticRoutesOps(nbClient, ops, routerName, lrsrs...) + if err != nil { + return err + } + _, err = TransactAndCheck(nbClient, ops) + return err } // BFD ops diff --git a/go-controller/pkg/networkmanager/api.go b/go-controller/pkg/networkmanager/api.go index 58853d1ed3..a34340deeb 100644 --- a/go-controller/pkg/networkmanager/api.go +++ b/go-controller/pkg/networkmanager/api.go @@ -4,17 +4,29 @@ import ( "context" "errors" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "k8s.io/client-go/tools/record" ) var ErrNetworkControllerTopologyNotManaged = errors.New("no cluster network controller to manage topology") +const ( + // DefaultNetworkID is the default network. + DefaultNetworkID = 0 + + // MaxNetworks is the maximum number of networks allowed. + MaxNetworks = 4096 +) + // Interface is the main package entrypoint and provides network related // information to the rest of the project. type Interface interface { GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) - GetNetwork(networkName string) util.NetInfo + + // GetNetwork returns the network of the given name or nil if unknown + GetNetwork(name string) util.NetInfo + // DoWithLock takes care of locking and unlocking while iterating over all role primary user defined networks. DoWithLock(f func(network util.NetInfo) error) error GetActiveNetworkNamespaces(networkName string) ([]string, error) @@ -38,6 +50,7 @@ func Default() Controller { func NewForCluster( cm ControllerManager, wf watchFactory, + ovnClient *util.OVNClusterManagerClientset, recorder record.EventRecorder, ) (Controller, error) { return new( @@ -46,6 +59,7 @@ func NewForCluster( "", cm, wf, + ovnClient, recorder, ) } @@ -63,6 +77,7 @@ func NewForZone( cm, wf, nil, + nil, ) } @@ -79,6 +94,7 @@ func NewForNode( cm, wf, nil, + nil, ) } @@ -91,9 +107,10 @@ func new( node string, cm ControllerManager, wf watchFactory, + ovnClient *util.OVNClusterManagerClientset, recorder record.EventRecorder, ) (Controller, error) { - return newController(name, zone, node, cm, wf, recorder) + return newController(name, zone, node, cm, wf, ovnClient, recorder) } // ControllerManager manages controllers. Needs to be provided in order to build @@ -103,6 +120,10 @@ type ControllerManager interface { NewNetworkController(netInfo util.NetInfo) (NetworkController, error) GetDefaultNetworkController() ReconcilableNetworkController CleanupStaleNetworks(validNetworks ...util.NetInfo) error + + // Reconcile informs the manager of network changes that other managed + // network aware controllers might be interested in. + Reconcile(name string, old, new util.NetInfo) error } // ReconcilableNetworkController is a network controller that can reconcile @@ -152,7 +173,10 @@ func (nm defaultNetworkManager) GetActiveNetworkForNamespace(string) (util.NetIn return &util.DefaultNetInfo{}, nil } -func (nm defaultNetworkManager) GetNetwork(networkName string) util.NetInfo { +func (nm defaultNetworkManager) GetNetwork(name string) util.NetInfo { + if name != types.DefaultNetworkName { + return nil + } return &util.DefaultNetInfo{} } diff --git a/go-controller/pkg/networkmanager/nad_controller.go b/go-controller/pkg/networkmanager/nad_controller.go index 63d7ec6e1c..f4b0c3dc68 100644 --- a/go-controller/pkg/networkmanager/nad_controller.go +++ b/go-controller/pkg/networkmanager/nad_controller.go @@ -3,6 +3,7 @@ package networkmanager import ( "fmt" "reflect" + "strconv" "sync" "time" @@ -12,19 +13,22 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" coreinformers "k8s.io/client-go/informers/core/v1" - corev1listers "k8s.io/client-go/listers/core/v1" + corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + nadclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" nadinformers "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1" nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" rainformers "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/informers/externalversions/routeadvertisements/v1" userdefinednetworkinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/userdefinednetwork/v1" userdefinednetworklister "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/listers/userdefinednetwork/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" @@ -53,9 +57,11 @@ type nadController struct { nadLister nadlisters.NetworkAttachmentDefinitionLister udnLister userdefinednetworklister.UserDefinedNetworkLister cudnLister userdefinednetworklister.ClusterUserDefinedNetworkLister - namespaceLister corev1listers.NamespaceLister - controller controller.Controller - recorder record.EventRecorder + namespaceLister corelisters.NamespaceLister + nodeLister corelisters.NodeLister + + controller controller.Controller + recorder record.EventRecorder // networkController reconciles network specific controllers networkController *networkController @@ -65,6 +71,9 @@ type nadController struct { // primaryNADs holds a mapping of namespace to primary NAD names primaryNADs map[string]string + + networkIDAllocator id.Allocator + nadClient nadclientset.Interface } func newController( @@ -73,23 +82,39 @@ func newController( node string, cm ControllerManager, wf watchFactory, + ovnClient *util.OVNClusterManagerClientset, recorder record.EventRecorder, ) (*nadController, error) { c := &nadController{ name: fmt.Sprintf("[%s NAD controller]", name), recorder: recorder, nadLister: wf.NADInformer().Lister(), + nodeLister: wf.NodeCoreInformer().Lister(), networkController: newNetworkController(name, zone, node, cm, wf), nads: map[string]string{}, primaryNADs: map[string]string{}, } + if ovnClient != nil { + c.nadClient = ovnClient.NetworkAttchDefClient + } + + // this is cluster network manager, so we allocate network IDs + if zone == "" && node == "" { + c.networkIDAllocator = id.NewIDAllocator("NetworkIDs", MaxNetworks) + // Reserve the ID of the default network + err := c.networkIDAllocator.ReserveID(types.DefaultNetworkName, DefaultNetworkID) + if err != nil { + return nil, fmt.Errorf("failed to allocate default network ID: %w", err) + } + } + config := &controller.ControllerConfig[nettypes.NetworkAttachmentDefinition]{ RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), Informer: wf.NADInformer().Informer(), Lister: c.nadLister.List, Reconcile: c.sync, - ObjNeedsUpdate: nadNeedsUpdate, + ObjNeedsUpdate: c.nadNeedsUpdate, Threadiness: 1, } @@ -149,19 +174,64 @@ func (c *nadController) syncAll() (err error) { return fmt.Errorf("%s: failed to list NADs: %w", c.name, err) } + syncNAD := func(nad *nettypes.NetworkAttachmentDefinition) error { + key, err := cache.MetaNamespaceKeyFunc(nad) + if err != nil { + klog.Errorf("%s: failed to sync %v: %v", c.name, nad, err) + } + err = c.syncNAD(key, nad) + if err != nil { + return fmt.Errorf("%s: failed to sync %s: %v", c.name, key, err) + } + return nil + } + // create all networks with their updated list of NADs and only then start // the corresponding controllers so as to avoid to the extent possible the // errors and retries that would result if the controller attempted to // process pods attached with NADs we wouldn't otherwise know about yet + nadsWithoutID := []*nettypes.NetworkAttachmentDefinition{} for _, nad := range existingNADs { - key, err := cache.MetaNamespaceKeyFunc(nad) - if err != nil { - klog.Errorf("%s: failed to sync %v: %v", c.name, nad, err) + // skip NADs that are not annotated with an ID + if c.networkIDAllocator != nil && nad.Annotations[types.OvnNetworkIDAnnotation] == "" { + nadsWithoutID = append(nadsWithoutID, nad) continue } - err = c.syncNAD(key, nad) + err := syncNAD(nad) if err != nil { - return fmt.Errorf("%s: failed to sync %s: %v", c.name, key, err) + return err + } + } + + if len(nadsWithoutID) == 0 { + return nil + } + + // If we are missing IDs, get them from the nodes which is where we + // originally had them + klog.V(5).Infof("%s: %d NADs are missing the network ID annotation, fetching from nodes", c.name, len(nadsWithoutID)) + nodes, err := c.nodeLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("error listing nodes: %w", err) + } + for _, n := range nodes { + networkIdsMap, err := util.GetNodeNetworkIDsAnnotationNetworkIDs(n) + if err == nil { + for networkName, id := range networkIdsMap { + // Reserve the id for the network name. We can safely + // ignore any errors if there are duplicate ids or if + // two networks have the same id. We will assign network + // IDs anyway on sync. + _ = c.networkIDAllocator.ReserveID(networkName, id) + } + } + } + + // finally process the pending NADs + for _, nad := range existingNADs { + err := syncNAD(nad) + if err != nil { + return err } } @@ -272,6 +342,10 @@ func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefin } } + if err := c.handleNetworkID(oldNetwork, ensureNetwork, nad); err != nil { + return err + } + // this was a nad delete if ensureNetwork == nil { delete(c.nads, key) @@ -281,6 +355,14 @@ func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefin return err } + // if network ID has not been set and this is not the well known default + // network, need to wait until cluster nad controller allocates an ID for + // the network + if ensureNetwork.GetNetworkID() == util.InvalidID { + klog.V(4).Infof("%s: will wait for cluster manager to allocate an ID before ensuring network %s", c.name, nadNetworkName) + return nil + } + // ensure the network is associated with the NAD ensureNetwork.AddNADs(key) c.nads[key] = ensureNetwork.GetNetworkName() @@ -299,7 +381,13 @@ func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefin return nil } -func nadNeedsUpdate(oldNAD, newNAD *nettypes.NetworkAttachmentDefinition) bool { +// isOwnUpdate checks if an object was updated by us last, as indicated by its +// managed fields. Used to avoid reconciling an update that we made ourselves. +func isOwnUpdate(manager string, managedFields []metav1.ManagedFieldsEntry) bool { + return util.IsLastUpdatedByManager(manager, managedFields) +} + +func (c *nadController) nadNeedsUpdate(oldNAD, newNAD *nettypes.NetworkAttachmentDefinition) bool { if oldNAD == nil || newNAD == nil { return true } @@ -310,9 +398,15 @@ func nadNeedsUpdate(oldNAD, newNAD *nettypes.NetworkAttachmentDefinition) bool { return false } + if isOwnUpdate(c.name, newNAD.ManagedFields) { + return false + } + // also reconcile the network in case its route advertisements changed return !reflect.DeepEqual(oldNAD.Spec, newNAD.Spec) || - oldNAD.Annotations[types.OvnRouteAdvertisementsKey] != newNAD.Annotations[types.OvnRouteAdvertisementsKey] + oldNAD.Annotations[types.OvnRouteAdvertisementsKey] != newNAD.Annotations[types.OvnRouteAdvertisementsKey] || + oldNAD.Annotations[types.OvnNetworkIDAnnotation] != newNAD.Annotations[types.OvnNetworkIDAnnotation] || + oldNAD.Annotations[types.OvnNetworkNameAnnotation] != newNAD.Annotations[types.OvnNetworkNameAnnotation] } func (c *nadController) GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { @@ -382,15 +476,15 @@ func (c *nadController) GetNetwork(name string) util.NetInfo { return network } -func (nadController *nadController) GetActiveNetworkNamespaces(networkName string) ([]string, error) { +func (c *nadController) GetActiveNetworkNamespaces(networkName string) ([]string, error) { if !util.IsNetworkSegmentationSupportEnabled() { return []string{"default"}, nil } namespaces := make([]string, 0) - nadController.RLock() - defer nadController.RUnlock() - for namespaceName, primaryNAD := range nadController.primaryNADs { - nadNetworkName := nadController.nads[primaryNAD] + c.RLock() + defer c.RUnlock() + for namespaceName, primaryNAD := range c.primaryNADs { + nadNetworkName := c.nads[primaryNAD] if nadNetworkName != networkName { continue } @@ -401,26 +495,26 @@ func (nadController *nadController) GetActiveNetworkNamespaces(networkName strin // DoWithLock iterates over all role primary user defined networks and executes the given fn with each network as input. // An error will not block execution and instead all errors will be aggregated and returned when all networks are processed. -func (nadController *nadController) DoWithLock(f func(network util.NetInfo) error) error { +func (c *nadController) DoWithLock(f func(network util.NetInfo) error) error { if !util.IsNetworkSegmentationSupportEnabled() { defaultNetwork := &util.DefaultNetInfo{} return f(defaultNetwork) } - nadController.RLock() - defer nadController.RUnlock() + c.RLock() + defer c.RUnlock() var errs []error - for _, primaryNAD := range nadController.primaryNADs { + for _, primaryNAD := range c.primaryNADs { if primaryNAD == "" { continue } - netName := nadController.nads[primaryNAD] + netName := c.nads[primaryNAD] if netName == "" { // this should never happen where we have a nad keyed in the primaryNADs // map, but it doesn't exist in the nads map panic("NAD Controller broken consistency between primary NADs and cached NADs") } - network := nadController.networkController.getNetwork(netName) + network := c.networkController.getNetwork(netName) n := util.NewMutableNetInfo(network) // update the returned netInfo copy to only have the primary NAD for this namespace n.SetNADs(primaryNAD) @@ -430,3 +524,107 @@ func (nadController *nadController) DoWithLock(f func(network util.NetInfo) erro } return errors.Join(errs...) } + +// handleNetworkID finds out what the network ID should be for a new network and +// sets it on 'new'. The network ID is primarily found annotated in the NAD. If +// not annotated, it means it is still to be allocated. If this is not the NAD +// controller running in cluster manager, then we don't do anything as we are +// expected to wait until it happens. If this is the NAD controller running in +// cluster manager then a new ID is allocated and annotated on the NAD. The NAD +// controller running in cluster manager also releases here the network ID of a +// network that is being deleted. +func (c *nadController) handleNetworkID(old util.NetInfo, new util.MutableNetInfo, nad *nettypes.NetworkAttachmentDefinition) error { + if new != nil && new.IsDefault() { + return nil + } + + var err error + id := util.InvalidID + + // check what ID is currently annotated + if nad != nil && nad.Annotations[types.OvnNetworkIDAnnotation] != "" { + annotated := nad.Annotations[types.OvnNetworkIDAnnotation] + id, err = strconv.Atoi(annotated) + if err != nil { + return fmt.Errorf("failed to parse annotated network ID: %w", err) + } + } + + // this is not the cluster manager nad controller and we are not allocating + // so just return what we got from the annotation + if c.networkIDAllocator == nil { + if new != nil { + new.SetNetworkID(id) + } + return nil + } + + // release old ID if the network is being deleted + if old != nil && !old.IsDefault() && len(old.GetNADs()) == 0 { + c.networkIDAllocator.ReleaseID(old.GetNetworkName()) + } + + // nothing to allocate + if new == nil { + return nil + } + name := new.GetNetworkName() + + // an ID was annotated, check if it is free to use or stale + if id != util.InvalidID { + err = c.networkIDAllocator.ReserveID(name, id) + if err != nil { + // already reserved for a different network, allocate a new id + id = util.InvalidID + } + } + + // we don't have an ID, allocate a new one + if id == util.InvalidID { + id, err = c.networkIDAllocator.AllocateID(name) + if err != nil { + return fmt.Errorf("failed to allocate network ID: %w", err) + } + + // check if there is still a network running with that ID in the process + // of being stopped + other := c.networkController.getRunningNetwork(id) + if other != "" && c.networkController.getNetwork(other) == nil { + c.networkIDAllocator.ReleaseID(name) + return fmt.Errorf("found other network %s being stopped with allocated ID %d, will retry", other, id) + } + } + + // set and annotate the network ID + annotations := map[string]string{ + types.OvnNetworkNameAnnotation: name, + types.OvnNetworkIDAnnotation: strconv.Itoa(id), + } + if nad.Annotations[types.OvnNetworkNameAnnotation] == annotations[types.OvnNetworkNameAnnotation] { + delete(annotations, types.OvnNetworkNameAnnotation) + } + if nad.Annotations[types.OvnNetworkIDAnnotation] == annotations[types.OvnNetworkIDAnnotation] { + delete(annotations, types.OvnNetworkIDAnnotation) + } + if len(annotations) == 0 { + return nil + } + + k := kube.KubeOVN{ + NADClient: c.nadClient, + } + + err = k.SetAnnotationsOnNAD( + nad.Namespace, + nad.Name, + annotations, + c.name, + ) + if err != nil { + c.networkIDAllocator.ReleaseID(name) + return fmt.Errorf("failed to annotate network ID on NAD: %w", err) + } + new.SetNetworkID(id) + + return nil +} diff --git a/go-controller/pkg/networkmanager/nad_controller_test.go b/go-controller/pkg/networkmanager/nad_controller_test.go index de404ddb61..306c827669 100644 --- a/go-controller/pkg/networkmanager/nad_controller_test.go +++ b/go-controller/pkg/networkmanager/nad_controller_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "sync" "testing" @@ -12,10 +13,12 @@ import ( cnitypes "github.com/containernetworking/cni/pkg/types" nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" @@ -63,6 +66,10 @@ func testNetworkKey(nInfo util.NetInfo) string { return nInfo.GetNetworkName() + " " + nInfo.TopologyType() } +func networkFromTestNetworkKey(key string) string { + return key[:strings.LastIndex(key, " ")] +} + type testControllerManager struct { sync.Mutex @@ -101,6 +108,10 @@ func (tcm *testControllerManager) GetDefaultNetworkController() ReconcilableNetw return tcm.defaultNetwork } +func (tcm *testControllerManager) Reconcile(name string, old, new util.NetInfo) error { + return nil +} + func TestNADController(t *testing.T) { networkAPrimary := &ovncnitypes.NetConf{ Topology: types.Layer2Topology, @@ -452,11 +463,16 @@ func TestNADController(t *testing.T) { ReconcilableNetInfo: &util.DefaultNetInfo{}, }, } + fakeClient := util.GetOVNClientset().GetClusterManagerClientset() nadController := &nadController{ - nads: map[string]string{}, - primaryNADs: map[string]string{}, - networkController: newNetworkController("", "", "", tcm, nil), + nads: map[string]string{}, + primaryNADs: map[string]string{}, + networkController: newNetworkController("", "", "", tcm, nil), + networkIDAllocator: id.NewIDAllocator("NetworkIDs", MaxNetworks), + nadClient: fakeClient.NetworkAttchDefClient, } + err = nadController.networkIDAllocator.ReserveID(types.DefaultNetworkName, DefaultNetworkID) + g.Expect(err).ToNot(gomega.HaveOccurred()) netController := nadController.networkController g.Expect(nadController.networkController.Start()).To(gomega.Succeed()) @@ -471,6 +487,8 @@ func TestNADController(t *testing.T) { args.network.NADName = args.nad nad, err = buildNAD(name, namespace, args.network) g.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = fakeClient.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespace).Create(context.Background(), nad, v1.CreateOptions{}) + g.Expect(err).To(gomega.Or(gomega.Not(gomega.HaveOccurred()), gomega.MatchError(errors.IsAlreadyExists, "AlreadyExists"))) } err = nadController.syncNAD(args.nad, nad) @@ -506,6 +524,9 @@ func TestNADController(t *testing.T) { fmt.Sprintf("matching network config for network %s", name)) g.Expect(netController.networks[name].GetNADs()).To(gomega.ConsistOf(expected.nads), fmt.Sprintf("matching NADs for network %s", name)) + id, err := nadController.networkIDAllocator.AllocateID(name) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(netController.networks[name].GetNetworkID()).To(gomega.Equal(id)) // test that the actual controllers have the expected config and NADs if !netInfo.IsDefault() { @@ -514,6 +535,7 @@ func TestNADController(t *testing.T) { fmt.Sprintf("matching network config for network %s", name)) g.Expect(tcm.controllers[testNetworkKey].GetNADs()).To(gomega.ConsistOf(expected.nads), fmt.Sprintf("matching NADs for network %s", name)) + g.Expect(tcm.controllers[testNetworkKey].GetNetworkID()).To(gomega.Equal(id)) expectRunning = append(expectRunning, testNetworkKey) } }() @@ -534,6 +556,20 @@ func TestNADController(t *testing.T) { g.Expect(tcm.started).To(gomega.ContainElements(expectRunning), "started network controllers") g.Expect(tcm.stopped).To(gomega.ConsistOf(expectStopped), "stopped network controllers") g.Expect(tcm.cleaned).To(gomega.ConsistOf(expectStopped), "cleaned up network controllers") + + // if we reallocate all stopped networks, they should get a higher id than base if the previous id was released + base, err := nadController.networkIDAllocator.AllocateID("test") + g.Expect(err).ToNot(gomega.HaveOccurred()) + for _, stopped := range expectStopped { + network := networkFromTestNetworkKey(stopped) + if nadController.GetNetwork(network) != nil { + // this network is still running under different config + continue + } + id, err := nadController.networkIDAllocator.AllocateID(network) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(id).To(gomega.BeNumerically(">", base), "unexpected network ID for network %s", network) + } } g.Eventually(meetsExpectations).Should(gomega.Succeed()) @@ -597,8 +633,8 @@ func TestSyncAll(t *testing.T) { g.Expect(err).ToNot(gomega.HaveOccurred()) config.OVNKubernetesFeature.EnableNetworkSegmentation = true config.OVNKubernetesFeature.EnableMultiNetwork = true - fakeClient := util.GetOVNClientset().GetOVNKubeControllerClientset() - wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClient) + fakeClient := util.GetOVNClientset().GetClusterManagerClientset() + wf, err := factory.NewClusterManagerWatchFactory(fakeClient) g.Expect(err).ToNot(gomega.HaveOccurred()) tcm := &testControllerManager{ @@ -611,6 +647,7 @@ func TestSyncAll(t *testing.T) { controller, err := NewForCluster( tcm, wf, + fakeClient, nil, ) g.Expect(err).ToNot(gomega.HaveOccurred()) diff --git a/go-controller/pkg/networkmanager/network_controller.go b/go-controller/pkg/networkmanager/network_controller.go index e697a0ea36..85f04cf0a5 100644 --- a/go-controller/pkg/networkmanager/network_controller.go +++ b/go-controller/pkg/networkmanager/network_controller.go @@ -301,6 +301,13 @@ func (c *networkController) syncNetwork(network string) error { return nil } + // inform controller manager of upcoming changes so other controllers are + // aware + err = c.cm.Reconcile(network, have, want) + if err != nil { + return fmt.Errorf("failed to reconcile controller manager for network %s: %w", network, err) + } + // ensure the network controller err = c.ensureNetwork(want) if err != nil { @@ -322,7 +329,7 @@ func (c *networkController) ensureNetwork(network util.MutableNetInfo) error { if reconcilable != nil { err := reconcilable.Reconcile(network) if err != nil { - return fmt.Errorf("failed to reconcile network %s: %w", networkName, err) + return fmt.Errorf("failed to reconcile controller for network %s: %w", networkName, err) } return nil } @@ -521,3 +528,15 @@ func nodeNeedsUpdate(oldNode, newNode *corev1.Node) bool { return !reflect.DeepEqual(oldNode.Labels, newNode.Labels) || oldNode.Annotations[util.OvnNodeZoneName] != newNode.Annotations[util.OvnNodeZoneName] } + +func (c *networkController) getRunningNetwork(id int) string { + if id == DefaultNetworkID { + return types.DefaultNetworkName + } + for _, state := range c.getAllNetworkStates() { + if state.controller.GetNetworkID() == id { + return state.controller.GetNetworkName() + } + } + return "" +} diff --git a/go-controller/pkg/node/controllers/egressservice/egressservice_node.go b/go-controller/pkg/node/controllers/egressservice/egressservice_node.go index f3c5e16bf7..0faee4d575 100644 --- a/go-controller/pkg/node/controllers/egressservice/egressservice_node.go +++ b/go-controller/pkg/node/controllers/egressservice/egressservice_node.go @@ -1,21 +1,19 @@ package egressservice import ( + "context" "encoding/json" "fmt" - "strconv" "strings" "sync" "time" - "github.com/coreos/go-iptables/iptables" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" egressserviceapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1" egressserviceinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/egressservice/v1" egressservicelisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/listers/egressservice/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - nodeipt "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables" + nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -34,11 +32,20 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" + "sigs.k8s.io/knftables" ) const ( - Chain = "OVN-KUBE-EGRESS-SVC" // called from nat-POSTROUTING - IPRulePriority = 5000 // the priority of the ip rules created by the controller. Egress IP priority is 6000. + IPRulePriority = 5000 // the priority of the ip rules created by the controller. Egress IP priority is 6000. + + // chain for egress service NAT rules + NFTablesChainName = "egress-services" + + // map for IPv4 SNAT mappings + NFTablesMapV4 = "egress-service-snat-v4" + + // map for IPv6 SNAT mappings + NFTablesMapV6 = "egress-service-snat-v6" ) type Controller struct { @@ -217,7 +224,7 @@ func (c *Controller) Run(wg *sync.WaitGroup, threadiness int) error { return nil } -// Removes stale iptables/ip rules, updates the controller cache with the correct existing ones. +// Removes stale nftables/ip rules, updates the controller cache with the correct existing ones. func (c *Controller) repair() error { c.Lock() defer c.Unlock() @@ -327,7 +334,7 @@ func (c *Controller) repair() error { errorList = append(errorList, err) } - err = c.repairIPTables(v4EndpointsToSvcKey, v6EndpointsToSvcKey) + err = c.repairNFTables(v4EndpointsToSvcKey, v6EndpointsToSvcKey) if err != nil { errorList = append(errorList, err) } @@ -479,191 +486,123 @@ func (c *Controller) repairIPRules(v4EpsToServices, v6EpsToServices, cipsToServi return utilerrors.Join(errorList...) } -// Remove stale iptables rules, update caches with valid existing ones. -// In addition, verify that the first rule in the Chain matches the "returnMark": -// packets coming with that mark should not be evaluated for SNATing = we make sure -// the first rule in the Chain is a RETURN one for this mark. -// Valid iptables rules in this context are those that belong to an existing EgressService, their -// source points to an existing ep the service and the SNAT matches the service's LB. -func (c *Controller) repairIPTables(v4EpsToServices, v6EpsToServices map[string]string) error { - type esvcIPTRule struct { - svcKey string - ep string - lb string - mark int32 - } - - parseIPTRule := func(rule string) (*esvcIPTRule, error) { - split := strings.Fields(rule) - if len(split)%2 != 0 { - return nil, fmt.Errorf("expected rule %s to have a key-value format", rule) - } - - args := map[string]string{} - for i := 0; i < len(split); i += 2 { - args[split[i]] = split[i+1] - } - - svcKey := args["--comment"] - ep := args["-s"] - lb := args["--to-source"] - - strMark := args["--mark"] - mark := int32(0) - if strMark != "" { - parsedFWMark, err := strconv.ParseInt(strMark, 0, 32) - if err != nil { - return nil, fmt.Errorf("could not parse fwmark for rule %s, err: %w", rule, err) - } - mark = int32(parsedFWMark) - } - - return &esvcIPTRule{svcKey: svcKey, ep: ep, lb: lb, mark: mark}, nil - } - - snatRepair := func(proto iptables.Protocol, epsToSvcs map[string]string) error { - defaultFirstRule := c.defaultReturnRule(proto) // fetch the rule that should be first in the chain - - ipt, err := util.GetIPTablesHelper(proto) - if err != nil { - return err - } - - snatRules, err := ipt.List("nat", Chain) - if err != nil { - return err - } - - // we verify that the first rule is the "do not snat" one - if len(snatRules) == 0 { - return nodeipt.AddRules([]nodeipt.Rule{defaultFirstRule}, true) - } - - parsedFWMark, err := strconv.ParseInt(c.returnMark, 0, 0) - if err != nil { - return fmt.Errorf("could not parse %s as default return mark for egress services, err: %w", c.returnMark, err) - } - mark := int32(parsedFWMark) - - firstRule := snatRules[0] - parsed, err := parseIPTRule(firstRule) - doNotSNATAdded := false - if err != nil || parsed.mark != mark { - // The first rule is either malformed (err != nil) or it does not match - // the correct mark. In either case, we should add the correct rule anyways - // to be sure it is the first one. - err := nodeipt.AddRules([]nodeipt.Rule{defaultFirstRule}, false) - if err != nil { - return err - } - doNotSNATAdded = true - } - if !doNotSNATAdded { - // the do not snat rule was already present at the first position - snatRules = snatRules[1:] - } - - // now we run over the existing rules to determine which should be deleted - // and update the cache accordingly - rulesToDel := []string{} - for _, rule := range snatRules { - if rule == fmt.Sprintf("-N %s", Chain) { - // Ignore chain creation rule +// Remove stale nftables map elements, update caches with valid existing ones. Ensure that +// the chain exists and checks the correct returnMark before SNATting. Valid nftables +// elements in this context are those that belong to an existing EgressService; their +// source points to an existing endpoint of the service and the SNAT matches the service's +// LB. +func (c *Controller) repairNFTables(v4EpsToServices, v6EpsToServices map[string]string) error { + snatRepair := func(tx *knftables.Transaction, family utilnet.IPFamily, existingElements []*knftables.Element, epsToSvcs map[string]string) { + for _, elem := range existingElements { + if len(elem.Key) != 1 || len(elem.Value) != 1 || elem.Comment == nil { + // the element is malformed + tx.Delete(elem) continue } - parsed, err := parseIPTRule(rule) - if err != nil { - // the rule is malformed - rulesToDel = append(rulesToDel, rule) - continue - } + ep := elem.Key[0] + lb := elem.Value[0] + svcKey := *elem.Comment - svcKey := epsToSvcs[parsed.ep] - if svcKey != parsed.svcKey { - // the rule matches the wrong service - rulesToDel = append(rulesToDel, rule) + if svcKey != epsToSvcs[ep] { + // the element matches the wrong service + tx.Delete(elem) continue } svcState, found := c.services[svcKey] if !found { - // the rule matches a service that is no longer valid - rulesToDel = append(rulesToDel, rule) + // the element matches a service that is no longer valid + tx.Delete(elem) continue } lbToCompare := svcState.v4LB epsToAdd := svcState.v4Eps - if proto == iptables.ProtocolIPv6 { + if family == utilnet.IPv6 { lbToCompare = svcState.v6LB epsToAdd = svcState.v6Eps } - if lbToCompare != parsed.lb { - // the rule SNATs to the wrong IP - rulesToDel = append(rulesToDel, rule) + if lbToCompare != lb { + // the element SNATs to the wrong IP + tx.Delete(elem) continue } - // the rule is valid, we update the service's cache to not reconfigure it later. - epsToAdd.Insert(parsed.ep) - } - - errorList := []error{} - for _, rule := range rulesToDel { - args := strings.Fields(rule) - if len(args) < 2 { - continue - } - // strip "-A OVN-KUBE-EGRESS-SVC" - args = args[2:] - err := ipt.Delete("nat", Chain, args...) - if err != nil { - errorList = append(errorList, err) - } + // the element is valid, we update the service's cache to not reconfigure it later. + epsToAdd.Insert(ep) } + } - return utilerrors.Join(errorList...) + nft, err := nodenft.GetNFTablesHelper() + if err != nil { + return err } - errorList := []error{} - if config.IPv4Mode { - ipt, err := util.GetIPTablesHelper(iptables.ProtocolIPv4) - if err != nil { - errorList = append(errorList, err) - } + tx := nft.NewTransaction() + tx.Add(&knftables.Chain{ + Name: NFTablesChainName, + + Type: knftables.PtrTo(knftables.NATType), + Hook: knftables.PtrTo(knftables.PostroutingHook), + Priority: knftables.PtrTo(knftables.SNATPriority), + }) + tx.Flush(&knftables.Chain{ + Name: NFTablesChainName, + }) + tx.Add(&knftables.Rule{ + Chain: NFTablesChainName, + Rule: knftables.Concat( + "mark", "==", c.returnMark, + "return", + ), + Comment: knftables.PtrTo("DoNotSNAT"), + }) - err = ipt.NewChain("nat", Chain) - if err != nil { - klog.V(5).Infof("Could not create egress service nat chain: %v", err) - } + if config.IPv4Mode { + tx.Add(&knftables.Map{ + Name: NFTablesMapV4, + Type: "ipv4_addr : ipv4_addr", + }) + tx.Add(&knftables.Rule{ + Chain: NFTablesChainName, + Rule: knftables.Concat( + "snat to", "ip saddr map", "@", NFTablesMapV4, + ), + }) - err = snatRepair(iptables.ProtocolIPv4, v4EpsToServices) - if err != nil { - errorList = append(errorList, err) + existing, err := nft.ListElements(context.TODO(), "map", NFTablesMapV4) + if err != nil && !knftables.IsNotFound(err) { + return fmt.Errorf("could not list existing egress service map elements: %w", err) } - + snatRepair(tx, utilnet.IPv4, existing, v4EpsToServices) } if config.IPv6Mode { - ipt, err := util.GetIPTablesHelper(iptables.ProtocolIPv4) - if err != nil { - errorList = append(errorList, err) - } + tx.Add(&knftables.Map{ + Name: NFTablesMapV6, + Type: "ipv6_addr : ipv6_addr", + }) + tx.Add(&knftables.Rule{ + Chain: NFTablesChainName, + Rule: knftables.Concat( + "snat to", "ip6 saddr map", "@", NFTablesMapV6, + ), + }) - err = ipt.NewChain("nat", Chain) - if err != nil { - klog.V(5).Infof("Could not create egress service nat chain: %v", err) - } - - err = snatRepair(iptables.ProtocolIPv6, v6EpsToServices) - if err != nil { - errorList = append(errorList, err) + existing, err := nft.ListElements(context.TODO(), "map", NFTablesMapV6) + if err != nil && !knftables.IsNotFound(err) { + return fmt.Errorf("could not list existing egress service map elements: %w", err) } + snatRepair(tx, utilnet.IPv6, existing, v6EpsToServices) } - return utilerrors.Join(errorList...) + err = nft.Run(context.TODO(), tx) + if err != nil { + return err + } + return nil } func (c *Controller) runEgressServiceWorker(wg *sync.WaitGroup) { @@ -807,45 +746,57 @@ func (c *Controller) syncEgressService(key string) error { v4ToDelete := cachedState.v4Eps.Difference(v4Eps) v6ToDelete := cachedState.v6Eps.Difference(v6Eps) + nft, err := nodenft.GetNFTablesHelper() + if err != nil { + return err + } + tx := nft.NewTransaction() + if cachedState.v4LB != "" { for ep := range v4ToAdd { - err := nodeipt.AddRules([]nodeipt.Rule{snatIPTRuleFor(key, cachedState.v4LB, ep)}, true) - if err != nil { - return err - } + tx.Add(&knftables.Element{ + Map: NFTablesMapV4, + Key: []string{ep}, + Value: []string{cachedState.v4LB}, + Comment: knftables.PtrTo(key), + }) cachedState.v4Eps.Insert(ep) } for ep := range v4ToDelete { - err := nodeipt.DelRules([]nodeipt.Rule{snatIPTRuleFor(key, cachedState.v4LB, ep)}) - if err != nil { - return err - } - + tx.Delete(&knftables.Element{ + Map: NFTablesMapV4, + Key: []string{ep}, + }) cachedState.v4Eps.Delete(ep) } } if cachedState.v6LB != "" { for ep := range v6ToAdd { - err := nodeipt.AddRules([]nodeipt.Rule{snatIPTRuleFor(key, cachedState.v6LB, ep)}, true) - if err != nil { - return err - } - + tx.Add(&knftables.Element{ + Map: NFTablesMapV6, + Key: []string{ep}, + Value: []string{cachedState.v6LB}, + Comment: knftables.PtrTo(key), + }) cachedState.v6Eps.Insert(ep) } for ep := range v6ToDelete { - err := nodeipt.DelRules([]nodeipt.Rule{snatIPTRuleFor(key, cachedState.v6LB, ep)}) - if err != nil { - return err - } - + tx.Delete(&knftables.Element{ + Map: NFTablesMapV6, + Key: []string{ep}, + }) cachedState.v6Eps.Delete(ep) } } + err = nft.Run(context.TODO(), tx) + if err != nil { + return err + } + // At this point we finished handling the SNAT rules // Now we create the relevant ip rules according to the object's "Network" @@ -1010,26 +961,34 @@ func (c *Controller) allEndpointsFor(svc *corev1.Service, localOnly bool) (sets. // Clears all of the SNAT rules of the service. func (c *Controller) clearServiceSNATRules(key string, state *svcState) error { - for ip := range state.v4Eps { - err := nodeipt.DelRules([]nodeipt.Rule{snatIPTRuleFor(key, state.v4LB, ip)}) - if err != nil { - return err - } + nft, err := nodenft.GetNFTablesHelper() + if err != nil { + return err + } + tx := nft.NewTransaction() + for ip := range state.v4Eps { + tx.Delete(&knftables.Element{ + Map: NFTablesMapV4, + Key: []string{ip}, + }) state.v4Eps.Delete(ip) } state.v4LB = "" for ip := range state.v6Eps { - err := nodeipt.DelRules([]nodeipt.Rule{snatIPTRuleFor(key, state.v6LB, ip)}) - if err != nil { - return err - } - + tx.Delete(&knftables.Element{ + Map: NFTablesMapV6, + Key: []string{ip}, + }) state.v6Eps.Delete(ip) } state.v6LB = "" + err = nft.Run(context.TODO(), tx) + if err != nil { + return err + } return nil } @@ -1071,7 +1030,7 @@ func (c *Controller) clearServiceIPRules(state *svcState) error { return utilerrors.Join(errorList...) } -// Clears all of the iptables rules that relate to the service and removes it from the cache. +// Clears all of the nftables rules that relate to the service and removes it from the cache. func (c *Controller) clearServiceRulesAndRequeue(key string, state *svcState) error { state.stale = true @@ -1143,42 +1102,3 @@ func deleteNodePortIPRule(family string, priority int32, src string, srcPort int return nil } - -// Returns the SNAT rule that should be created for the given lb/endpoint -func snatIPTRuleFor(comment string, lb, ip string) nodeipt.Rule { - return nodeipt.Rule{ - Table: "nat", - Chain: Chain, - Args: []string{ - "-s", ip, - "-m", "comment", "--comment", comment, - "-j", "SNAT", - "--to-source", lb, - }, - Protocol: getIPTablesProtocol(ip), - } -} - -// getIPTablesProtocol returns the IPTables protocol matching the protocol (v4/v6) of provided IP string -func getIPTablesProtocol(ip string) iptables.Protocol { - if utilnet.IsIPv6String(ip) { - return iptables.ProtocolIPv6 - } - return iptables.ProtocolIPv4 -} - -// Returns the rule that should be first in the Chain. -// Packets coming with the controller's "returnMark" should not be evaluated for SNATing. -// The rule here is a "RETURN" for these packets. -func (c *Controller) defaultReturnRule(proto iptables.Protocol) nodeipt.Rule { - return nodeipt.Rule{ - Table: "nat", - Chain: Chain, - Args: []string{ - "-m", "mark", "--mark", string(c.returnMark), - "-m", "comment", "--comment", "DoNotSNAT", - "-j", "RETURN", - }, - Protocol: proto, - } -} diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index 18d1e67b3c..610bd7db68 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -342,24 +342,68 @@ func setOVSFlowTargets(node *kapi.Node) error { return nil } +// validateEncapIP returns false if there is an error or if the given IP is not known local IP address. +func validateEncapIP(encapIP string) (bool, error) { + links, err := netlink.LinkList() + if err != nil { + return false, fmt.Errorf("failed to get all the links on the node: %v", err) + } + for _, link := range links { + addrs, err := util.GetFilteredInterfaceAddrs(link, config.IPv4Mode, config.IPv6Mode) + if err != nil { + return false, err + } + for _, addr := range addrs { + if addr.IP.String() == encapIP { + return true, nil + } + } + } + return false, nil +} + func setupOVNNode(node *kapi.Node) error { var err error + nodePrimaryIP, err := util.GetNodePrimaryIP(node) + if err != nil { + return fmt.Errorf("failed to obtain local primary IP from node %q: %v", node.Name, err) + } + encapIP := config.Default.EncapIP if encapIP == "" { - encapIP, err = util.GetNodePrimaryIP(node) - if err != nil { - return fmt.Errorf("failed to obtain local IP from node %q: %v", node.Name, err) - } - config.Default.EncapIP = encapIP + config.Default.EffectiveEncapIP = nodePrimaryIP } else { // OVN allows `external_ids:ovn-encap-ip` to be a list of IPs separated by comma. + config.Default.EffectiveEncapIP = encapIP ovnEncapIps := strings.Split(encapIP, ",") for _, ovnEncapIp := range ovnEncapIps { if ip := net.ParseIP(strings.TrimSpace(ovnEncapIp)); ip == nil { return fmt.Errorf("invalid IP address %q in provided encap-ip setting %q", ovnEncapIp, encapIP) } } + // if there are more than one encap IPs, it must be configured explicitly. otherwise: + if len(ovnEncapIps) == 1 { + encapIP = ovnEncapIps[0] + if encapIP == nodePrimaryIP { + // the current encap IP is node primary IP, unset config.Default.EncapIP to indicate it is + // implicitly configured through the old external_ids:ovn-encap-ip value and needs to be updated + // if node primary IP changes. + config.Default.EncapIP = "" + } else { + // the encap IP may be incorrectly set or; + // previous implicitly set with the old primary node IP through the old external_ids:ovn-encap-ip value, + // that has changed when ovnkube-node is down. + // validate it to see if it is still a valid local IP address. + valid, err := validateEncapIP(encapIP) + if err != nil { + return fmt.Errorf("invalid encap IP %s: %v", encapIP, err) + } + if !valid { + return fmt.Errorf("invalid encap IP %s: does not exist", encapIP) + } + } + } } setExternalIdsCmd := []string{ @@ -367,7 +411,7 @@ func setupOVNNode(node *kapi.Node) error { "Open_vSwitch", ".", fmt.Sprintf("external_ids:ovn-encap-type=%s", config.Default.EncapType), - fmt.Sprintf("external_ids:ovn-encap-ip=%s", encapIP), + fmt.Sprintf("external_ids:ovn-encap-ip=%s", config.Default.EffectiveEncapIP), fmt.Sprintf("external_ids:ovn-remote-probe-interval=%d", config.Default.InactivityProbe), fmt.Sprintf("external_ids:ovn-openflow-probe-interval=%d", @@ -900,6 +944,7 @@ func (nc *DefaultNodeNetworkController) PreStart(ctx context.Context) error { } gatewaySetup.mgmtPorts = mgmtPorts gatewaySetup.mgmtPortConfig = mgmtPortConfig + gatewaySetup.nodeAddress = nodeAddr if err := util.SetNodeZone(nodeAnnotator, sbZone); err != nil { return fmt.Errorf("failed to set node zone annotation for node %s: %w", nc.name, err) @@ -909,9 +954,11 @@ func (nc *DefaultNodeNetworkController) PreStart(ctx context.Context) error { } // Connect ovn-controller to SBDB - for _, auth := range []config.OvnAuthConfig{config.OvnNorth, config.OvnSouth} { - if err := auth.SetDBAuth(); err != nil { - return fmt.Errorf("unable to set the authentication towards OVN local dbs") + if config.OvnKubeNode.Mode != types.NodeModeDPUHost { + for _, auth := range []config.OvnAuthConfig{config.OvnNorth, config.OvnSouth} { + if err := auth.SetDBAuth(); err != nil { + return fmt.Errorf("unable to set the authentication towards OVN local dbs") + } } } @@ -1435,11 +1482,11 @@ func (nc *DefaultNodeNetworkController) WatchNamespaces() error { // enough, it will return an error func (nc *DefaultNodeNetworkController) validateVTEPInterfaceMTU() error { // OVN allows `external_ids:ovn-encap-ip` to be a list of IPs separated by comma - ovnEncapIps := strings.Split(config.Default.EncapIP, ",") + ovnEncapIps := strings.Split(config.Default.EffectiveEncapIP, ",") for _, ip := range ovnEncapIps { ovnEncapIP := net.ParseIP(strings.TrimSpace(ip)) if ovnEncapIP == nil { - return fmt.Errorf("invalid IP address %q in provided encap-ip setting %q", ovnEncapIP, config.Default.EncapIP) + return fmt.Errorf("invalid IP address %q in provided encap-ip setting %q", ovnEncapIP, config.Default.EffectiveEncapIP) } interfaceName, mtu, err := util.GetIFNameAndMTUForAddress(ovnEncapIP) if err != nil { diff --git a/go-controller/pkg/node/default_node_network_controller_test.go b/go-controller/pkg/node/default_node_network_controller_test.go index a4a8306795..39246fbac6 100644 --- a/go-controller/pkg/node/default_node_network_controller_test.go +++ b/go-controller/pkg/node/default_node_network_controller_test.go @@ -74,7 +74,7 @@ var _ = Describe("Node", func() { } config.Default.MTU = configDefaultMTU - config.Default.EncapIP = "10.1.0.40" + config.Default.EffectiveEncapIP = "10.1.0.40" }) @@ -219,7 +219,7 @@ var _ = Describe("Node", func() { BeforeEach(func() { config.IPv4Mode = true config.IPv6Mode = false - config.Default.EncapIP = "10.1.0.40,10.2.0.50" + config.Default.EffectiveEncapIP = "10.1.0.40,10.2.0.50" netlinkOpsMock.On("LinkByIndex", 5).Return(netlinkLinkMock, nil) }) diff --git a/go-controller/pkg/node/egress_service_test.go b/go-controller/pkg/node/egress_service_test.go index 5c1b15ac52..cb90aca2b1 100644 --- a/go-controller/pkg/node/egress_service_test.go +++ b/go-controller/pkg/node/egress_service_test.go @@ -11,7 +11,7 @@ import ( egressserviceapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressservice" - nodeipt "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables" + nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" util "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/mocks" @@ -19,14 +19,23 @@ import ( v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/knftables" ) +const nftablesRulesEgressServicesBase = ` +add table inet ovn-kubernetes +add chain inet ovn-kubernetes egress-services { type nat hook postrouting priority 100 ; } +add map inet ovn-kubernetes egress-service-snat-v4 { type ipv4_addr : ipv4_addr ; } +add rule inet ovn-kubernetes egress-services mark == 0x3f0 return comment "DoNotSNAT" +add rule inet ovn-kubernetes egress-services snat to ip saddr map @egress-service-snat-v4 +` + var _ = Describe("Egress Service Operations", func() { var ( app *cli.App fakeOvnNode *FakeOVNNode fExec *ovntest.FakeExec - iptV4 util.IPTablesHelper + nft *knftables.Fake netlinkMock *mocks.NetLinkOps ) @@ -51,7 +60,7 @@ var _ = Describe("Egress Service Operations", func() { _, cidr4, _ := net.ParseCIDR("10.128.0.0/16") config.Default.ClusterSubnets = []config.CIDRNetworkEntry{{CIDR: cidr4, HostSubnetLength: 24}} - iptV4, _ = util.SetFakeIPTablesHelpers() + nft = nodenft.SetFakeNFTablesHelper() }) AfterEach(func() { @@ -61,13 +70,14 @@ var _ = Describe("Egress Service Operations", func() { }) Context("on egress service resource changes", func() { - It("repairs iptables and ip rules when stale entries are present", func() { + It("repairs nftables and ip rules when stale entries are present", func() { app.Action = func(ctx *cli.Context) error { fakeOvnNode.fakeExec.AddFakeCmd(&ovntest.ExpectedCmd{ Cmd: "ip -4 --json rule show", Output: "[{\"priority\":5000,\"src\":\"10.128.0.3\",\"table\":\"wrongTable\"},{\"priority\":5000,\"src\":\"goneEp\",\"table\":\"mynetwork\"}," + "{\"priority\":5000,\"src\":\"10.128.0.3\",\"table\":\"mynetwork\"},{\"priority\":5000,\"src\":\"10.129.0.2\",\"table\":\"mynetwork\"}," + "{\"priority\":5000,\"src\":\"10.128.0.33\",\"table\":\"mynetwork2\"},{\"priority\":5000,\"src\":\"10.129.0.3\",\"table\":\"mynetwork2\"}," + + "{\"priority\":5000,\"src\":\"10.128.0.4\",\"table\":\"mynetwork\"},{\"priority\":5000,\"src\":\"10.129.0.4\",\"table\":\"mynetwork\"}," + fmt.Sprintf("{\"priority\":5000,\"src\":\"%s\",\"sport\":31111,\"table\":\"mynetwork\"},", config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP) + fmt.Sprintf("{\"priority\":5000,\"src\":\"%s\",\"sport\":30300,\"table\":\"mynetwork2\"}]", config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP), Err: nil, @@ -85,73 +95,43 @@ var _ = Describe("Egress Service Operations", func() { Err: nil, }) - fakeRules := []nodeipt.Rule{ - { - Table: "nat", - Chain: "OVN-KUBE-EGRESS-SVC", - Args: []string{ - "-m", "mark", "--mark", "0x3f0", - "-m", "comment", "--comment", "DoNotSNAT", - "-j", "RETURN", - }, - }, - { - Table: "nat", - Chain: "OVN-KUBE-EGRESS-SVC", - Args: []string{ - "-s", "10.128.0.3", - "-m", "comment", "--comment", "namespace1/service1", - "-j", "SNAT", - "--to-source", "5.5.5.5", - }, - Protocol: getIPTablesProtocol("5.5.5.5"), - }, - { - Table: "nat", - Chain: "OVN-KUBE-EGRESS-SVC", - Args: []string{ - "-s", "10.128.0.88", // gone ep - "-m", "comment", "--comment", "namespace1/service1", - "-j", "SNAT", - "--to-source", "5.5.5.5", - }, - Protocol: getIPTablesProtocol("5.5.5.5"), - }, - { - Table: "nat", - Chain: "OVN-KUBE-EGRESS-SVC", - Args: []string{ - "-s", "10.128.0.3", - "-m", "comment", "--comment", "namespace1/service1", - "-j", "SNAT", - "--to-source", "5.200.5.12", // wrong lb - }, - Protocol: getIPTablesProtocol("5.5.5.5"), - }, - { - Table: "nat", - Chain: "OVN-KUBE-EGRESS-SVC", - Args: []string{ - "-s", "10.128.0.3", - "-m", "comment", "--comment", "namespace13service6", // gone service - "-j", "SNAT", - "--to-source", "1.2.3.4", - }, - Protocol: getIPTablesProtocol("5.5.5.5"), - }, - { - Table: "nat", - Chain: "OVN-KUBE-EGRESS-SVC", - Args: []string{ - "-s", "10.128.0.33", - "-m", "comment", "--comment", "namespace1/service2", // service with host=ALL - "-j", "SNAT", - "--to-source", "1.2.3.4", - }, - Protocol: getIPTablesProtocol("5.5.5.5"), - }, - } - Expect(appendIptRules(fakeRules)).To(Succeed()) + tx := nft.NewTransaction() + tx.Add(&knftables.Map{ + Name: egressservice.NFTablesMapV4, + Type: "ipv4_addr : ipv4_addr", + }) + tx.Add(&knftables.Element{ + Map: egressservice.NFTablesMapV4, + Key: []string{"10.128.0.3"}, + Value: []string{"5.5.5.5"}, + Comment: knftables.PtrTo("namespace1/service1"), + }) + tx.Add(&knftables.Element{ + Map: egressservice.NFTablesMapV4, + Key: []string{"10.128.0.88"}, // gone ep + Value: []string{"5.5.5.5"}, + Comment: knftables.PtrTo("namespace1/service1"), + }) + tx.Add(&knftables.Element{ + Map: egressservice.NFTablesMapV4, + Key: []string{"10.128.0.4"}, + Value: []string{"5.200.5.12"}, // wrong lb + Comment: knftables.PtrTo("namespace1/service3"), + }) + tx.Add(&knftables.Element{ + Map: egressservice.NFTablesMapV4, + Key: []string{"10.128.0.5"}, + Value: []string{"1.2.3.4"}, + Comment: knftables.PtrTo("namespace13service6"), // gone service + }) + tx.Add(&knftables.Element{ + Map: egressservice.NFTablesMapV4, + Key: []string{"10.128.0.33"}, + Value: []string{"1.2.3.4"}, + Comment: knftables.PtrTo("namespace1/service2"), // service with host=ALL + }) + + Expect(nft.Run(context.Background(), tx)).To(Succeed()) epPortName := "https" epPortValue := int32(443) @@ -250,23 +230,67 @@ var _ = Describe("Egress Service Operations", func() { []discovery.Endpoint{ep3}, []discovery.EndpointPort{epPort}) + egressService3 := egressserviceapi.EgressService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service3", + Namespace: "namespace1", + }, + Spec: egressserviceapi.EgressServiceSpec{ + Network: "mynetwork", + }, + Status: egressserviceapi.EgressServiceStatus{ + Host: fakeNodeName, + }, + } + service3 := *newService("service3", "namespace1", "10.129.0.4", + []v1.ServicePort{ + { + NodePort: int32(32222), + Protocol: v1.ProtocolTCP, + Port: int32(8080), + }, + }, + v1.ServiceTypeLoadBalancer, + []string{}, + v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{{ + IP: "6.6.6.6", + }}, + }, + }, + false, false, + ) + + ep3_1 := discovery.Endpoint{ + Addresses: []string{"10.128.0.4"}, + } + endpointSlice3 := *newEndpointSlice( + "service3", + "namespace1", + []discovery.Endpoint{ep3_1}, + []discovery.EndpointPort{epPort}) + fakeOvnNode.start(ctx, &v1.ServiceList{ Items: []v1.Service{ service, service2, + service3, }, }, &discovery.EndpointSliceList{ Items: []discovery.EndpointSlice{ endpointSlice, endpointSlice2, + endpointSlice3, }, }, &egressserviceapi.EgressServiceList{ Items: []egressserviceapi.EgressService{ egressService, egressService2, + egressService3, }, }, ) @@ -278,20 +302,12 @@ var _ = Describe("Egress Service Operations", func() { err = c.Run(fakeOvnNode.wg, 1) Expect(err).ToNot(HaveOccurred()) - expectedTables := map[string]util.FakeTable{ - "nat": { - "OVN-KUBE-EGRESS-SVC": []string{ - "-m mark --mark 0x3f0 -m comment --comment DoNotSNAT -j RETURN", - "-s 10.128.0.3 -m comment --comment namespace1/service1 -j SNAT --to-source 5.5.5.5", - }, - }, - "filter": {}, - "mangle": {}, - } - - f4 := iptV4.(*util.FakeIPTables) + expectedNFT := nftablesRulesEgressServicesBase + ` +add element inet ovn-kubernetes egress-service-snat-v4 { 10.128.0.3 comment "namespace1/service1" : 5.5.5.5 } +add element inet ovn-kubernetes egress-service-snat-v4 { 10.128.0.4 comment "namespace1/service3" : 6.6.6.6 } +` Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Expect(fakeOvnNode.fakeExec.CalledMatchesExpected()).To(BeTrue(), fakeOvnNode.fakeExec.ErrorDesc) @@ -387,35 +403,19 @@ var _ = Describe("Egress Service Operations", func() { err = c.Run(fakeOvnNode.wg, 1) Expect(err).ToNot(HaveOccurred()) - expectedTables := map[string]util.FakeTable{ - "nat": { - "OVN-KUBE-EGRESS-SVC": []string{ - "-m mark --mark 0x3f0 -m comment --comment DoNotSNAT -j RETURN", - "-s 10.128.0.3 -m comment --comment namespace1/service1 -j SNAT --to-source 5.5.5.5", - }, - }, - "filter": {}, - "mangle": {}, - } - - f4 := iptV4.(*util.FakeIPTables) + expectedNFT := nftablesRulesEgressServicesBase + ` +add element inet ovn-kubernetes egress-service-snat-v4 { 10.128.0.3 comment "namespace1/service1" : 5.5.5.5 } +` Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) - expectedTables = map[string]util.FakeTable{ - "nat": { - "OVN-KUBE-EGRESS-SVC": []string{"-m mark --mark 0x3f0 -m comment --comment DoNotSNAT -j RETURN"}, - }, - "filter": {}, - "mangle": {}, - } - err = fakeOvnNode.fakeClient.EgressServiceClient.K8sV1().EgressServices("namespace1").Delete(context.TODO(), "service1", metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred()) + expectedNFT = nftablesRulesEgressServicesBase Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Expect(fakeOvnNode.fakeExec.CalledMatchesExpected()).To(BeTrue(), fakeOvnNode.fakeExec.ErrorDesc) @@ -603,55 +603,23 @@ var _ = Describe("Egress Service Operations", func() { err = c.Run(fakeOvnNode.wg, 1) Expect(err).ToNot(HaveOccurred()) - expectedTables := map[string]util.FakeTable{ - "nat": { - "OVN-KUBE-EGRESS-SVC": []string{ - "-m mark --mark 0x3f0 -m comment --comment DoNotSNAT -j RETURN", - "-s 10.128.0.3 -m comment --comment namespace1/service1 -j SNAT --to-source 5.5.5.5", - "-s 10.128.0.11 -m comment --comment namespace1/service2 -j SNAT --to-source 5.5.5.6", - }, - }, - "filter": {}, - "mangle": {}, - } - // The order of the rules is determined by the order the services are processed. - // We check if one of the two tables match to not flake on ordering. - expectedTables2 := map[string]util.FakeTable{ - "nat": { - "OVN-KUBE-EGRESS-SVC": []string{ - "-m mark --mark 0x3f0 -m comment --comment DoNotSNAT -j RETURN", - "-s 10.128.0.11 -m comment --comment namespace1/service2 -j SNAT --to-source 5.5.5.6", - "-s 10.128.0.3 -m comment --comment namespace1/service1 -j SNAT --to-source 5.5.5.5", - }, - }, - "filter": {}, - "mangle": {}, - } - f4 := iptV4.(*util.FakeIPTables) + expectedNFT := nftablesRulesEgressServicesBase + ` +add element inet ovn-kubernetes egress-service-snat-v4 { 10.128.0.3 comment "namespace1/service1" : 5.5.5.5 } +add element inet ovn-kubernetes egress-service-snat-v4 { 10.128.0.11 comment "namespace1/service2" : 5.5.5.6 } +` Eventually(func() error { - err := f4.MatchState(expectedTables, nil) - if err == nil { - return nil - } - return f4.MatchState(expectedTables2, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) - expectedTables = map[string]util.FakeTable{ - "nat": { - "OVN-KUBE-EGRESS-SVC": []string{"-m mark --mark 0x3f0 -m comment --comment DoNotSNAT -j RETURN"}, - }, - "filter": {}, - "mangle": {}, - } - err = fakeOvnNode.fakeClient.EgressServiceClient.K8sV1().EgressServices("namespace1").Delete(context.TODO(), "service1", metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred()) err = fakeOvnNode.fakeClient.EgressServiceClient.K8sV1().EgressServices("namespace1").Delete(context.TODO(), "service2", metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred()) + expectedNFT = nftablesRulesEgressServicesBase Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Expect(fakeOvnNode.fakeExec.CalledMatchesExpected()).To(BeTrue(), fakeOvnNode.fakeExec.ErrorDesc) @@ -825,18 +793,9 @@ var _ = Describe("Egress Service Operations", func() { err = c.Run(fakeOvnNode.wg, 1) Expect(err).ToNot(HaveOccurred()) - expectedTables := map[string]util.FakeTable{ - "nat": { - "OVN-KUBE-EGRESS-SVC": []string{ - "-m mark --mark 0x3f0 -m comment --comment DoNotSNAT -j RETURN", - }, - }, - "filter": {}, - "mangle": {}, - } - f4 := iptV4.(*util.FakeIPTables) + expectedNFT := nftablesRulesEgressServicesBase Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Eventually(func() bool { @@ -870,8 +829,9 @@ var _ = Describe("Egress Service Operations", func() { err = fakeOvnNode.fakeClient.EgressServiceClient.K8sV1().EgressServices("namespace1").Delete(context.TODO(), "service2", metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred()) + expectedNFT = nftablesRulesEgressServicesBase Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Eventually(func() bool { @@ -988,18 +948,9 @@ var _ = Describe("Egress Service Operations", func() { err = c.Run(fakeOvnNode.wg, 1) Expect(err).ToNot(HaveOccurred()) - expectedTables := map[string]util.FakeTable{ - "nat": { - "OVN-KUBE-EGRESS-SVC": []string{ - "-m mark --mark 0x3f0 -m comment --comment DoNotSNAT -j RETURN", - }, - }, - "filter": {}, - "mangle": {}, - } - f4 := iptV4.(*util.FakeIPTables) + expectedNFT := nftablesRulesEgressServicesBase Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Eventually(func() bool { @@ -1016,8 +967,9 @@ var _ = Describe("Egress Service Operations", func() { _, err = fakeOvnNode.fakeClient.KubeClient.CoreV1().Services("namespace1").Update(context.TODO(), &service, metav1.UpdateOptions{}) Expect(err).ToNot(HaveOccurred()) + expectedNFT = nftablesRulesEgressServicesBase Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Eventually(func() bool { @@ -1034,8 +986,9 @@ var _ = Describe("Egress Service Operations", func() { _, err = fakeOvnNode.fakeClient.KubeClient.CoreV1().Services("namespace1").Update(context.TODO(), &service, metav1.UpdateOptions{}) Expect(err).ToNot(HaveOccurred()) + expectedNFT = nftablesRulesEgressServicesBase Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Eventually(func() bool { @@ -1055,8 +1008,9 @@ var _ = Describe("Egress Service Operations", func() { err = fakeOvnNode.fakeClient.EgressServiceClient.K8sV1().EgressServices("namespace1").Delete(context.TODO(), "service1", metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred()) + expectedNFT = nftablesRulesEgressServicesBase Eventually(func() error { - return f4.MatchState(expectedTables, nil) + return nodenft.MatchNFTRules(expectedNFT, nft.Dump()) }).ShouldNot(HaveOccurred()) Eventually(func() bool { diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 9a5efb64d0..f4c3142096 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -453,14 +453,10 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -1308,13 +1304,11 @@ OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0` fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", "-s 169.254.169.1 -j MASQUERADE", "-s 10.1.1.0/24 -j MASQUERADE", }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": { "FORWARD": []string{ diff --git a/go-controller/pkg/node/gateway_iptables.go b/go-controller/pkg/node/gateway_iptables.go index 0f9728e971..51d033de1e 100644 --- a/go-controller/pkg/node/gateway_iptables.go +++ b/go-controller/pkg/node/gateway_iptables.go @@ -14,7 +14,6 @@ import ( "github.com/coreos/go-iptables/iptables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressservice" nodeipt "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" utilerrors "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" @@ -94,16 +93,6 @@ func ensureChain(table, chain string) error { func getGatewayInitRules(chain string, proto iptables.Protocol) []nodeipt.Rule { iptRules := []nodeipt.Rule{} - if chain == egressservice.Chain { - return []nodeipt.Rule{ - { - Table: "nat", - Chain: "POSTROUTING", - Args: []string{"-j", chain}, - Protocol: proto, - }, - } - } if chain == iptableITPChain { iptRules = append(iptRules, nodeipt.Rule{ @@ -564,7 +553,7 @@ func addChaintoTable(ipt util.IPTablesHelper, tableName, chain string) { func handleGatewayIPTables(iptCallback func(rules []nodeipt.Rule) error, genGatewayChainRules func(chain string, proto iptables.Protocol) []nodeipt.Rule) error { rules := make([]nodeipt.Rule, 0) // (NOTE: Order is important, add jump to iptableETPChain before jump to NP/EIP chains) - for _, chain := range []string{iptableITPChain, egressservice.Chain, iptableNodePortChain, iptableExternalIPChain, iptableETPChain} { + for _, chain := range []string{iptableITPChain, iptableNodePortChain, iptableExternalIPChain, iptableETPChain} { for _, proto := range clusterIPTablesProtocols() { ipt, err := util.GetIPTablesHelper(proto) if err != nil { diff --git a/go-controller/pkg/node/gateway_localnet_linux_test.go b/go-controller/pkg/node/gateway_localnet_linux_test.go index b410dd6051..41a9e1c3a7 100644 --- a/go-controller/pkg/node/gateway_localnet_linux_test.go +++ b/go-controller/pkg/node/gateway_localnet_linux_test.go @@ -414,16 +414,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-EXTERNALIP": []string{ fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -494,16 +490,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-EXTERNALIP": []string{ fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -576,16 +568,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -668,9 +656,6 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, @@ -678,8 +663,7 @@ var _ = Describe("Node Operations", func() { "OVN-KUBE-ETP": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP.String(), service.Spec.Ports[0].NodePort), }, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -768,9 +752,6 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, @@ -778,9 +759,8 @@ var _ = Describe("Node Operations", func() { fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Status.LoadBalancer.Ingress[0].IP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -869,9 +849,6 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, @@ -884,8 +861,7 @@ var _ = Describe("Node Operations", func() { fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP.String(), service.Spec.Ports[0].NodePort), fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP.String(), service.Spec.Ports[0].NodePort), }, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -1011,9 +987,6 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-EXTERNALIP": []string{ fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Status.LoadBalancer.Ingress[0].IP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), @@ -1025,8 +998,7 @@ var _ = Describe("Node Operations", func() { fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%d -m statistic --mode random --probability 0.5000000000", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, ep1.Addresses[0], int32(service.Spec.Ports[0].TargetPort.IntValue())), fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%d -m statistic --mode random --probability 1.0000000000", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, ep2.Addresses[0], int32(service.Spec.Ports[0].TargetPort.IntValue())), }, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -1124,9 +1096,6 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, @@ -1134,9 +1103,8 @@ var _ = Describe("Node Operations", func() { fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Status.LoadBalancer.Ingress[0].IP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -1239,9 +1207,6 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, @@ -1253,8 +1218,7 @@ var _ = Describe("Node Operations", func() { fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Status.LoadBalancer.Ingress[0].IP, service.Spec.Ports[0].Port, config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP.String(), service.Spec.Ports[0].NodePort), fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP.String(), service.Spec.Ports[0].NodePort), }, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -1359,16 +1323,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIPs[0], service.Spec.Ports[0].Port), }, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -1458,16 +1418,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-EXTERNALIP": []string{ fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIPv4, service.Spec.Ports[0].Port, clusterIPv4, service.Spec.Ports[0].Port), }, - "OVN-KUBE-NODEPORT": []string{}, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-NODEPORT": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -1556,14 +1512,10 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -1641,14 +1593,10 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -1730,18 +1678,14 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-EXTERNALIP": []string{ fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, - "OVN-KUBE-NODEPORT": []string{}, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-NODEPORT": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -1777,12 +1721,8 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -1964,14 +1904,10 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -2066,16 +2002,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, nodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -2111,12 +2043,8 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -2199,9 +2127,6 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, @@ -2209,8 +2134,7 @@ var _ = Describe("Node Operations", func() { "OVN-KUBE-ETP": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP.String(), service.Spec.Ports[0].NodePort), }, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -2250,12 +2174,8 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -2342,16 +2262,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -2398,12 +2314,8 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, + "OVN-KUBE-ITP": []string{}, }, "filter": {}, "mangle": { @@ -2496,16 +2408,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -2550,12 +2458,8 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ITP": []string{}, + "OVN-KUBE-ETP": []string{}, }, "filter": {}, "mangle": { @@ -2641,16 +2545,12 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, "OVN-KUBE-EXTERNALIP": []string{}, "OVN-KUBE-ITP": []string{}, "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": {}, "mangle": { @@ -2700,11 +2600,7 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, }, "filter": {}, "mangle": { @@ -2794,9 +2690,6 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-NODEPORT": []string{ fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, service.Spec.ClusterIP, service.Spec.Ports[0].Port), }, @@ -2804,8 +2697,7 @@ var _ = Describe("Node Operations", func() { "OVN-KUBE-ITP": []string{ fmt.Sprintf("-p %s -d %s --dport %d -j REDIRECT --to-port %d", service.Spec.Ports[0].Protocol, service.Spec.ClusterIP, service.Spec.Ports[0].Port, int32(service.Spec.Ports[0].TargetPort.IntValue())), }, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ETP": []string{}, }, "filter": {}, "mangle": { @@ -2850,12 +2742,8 @@ var _ = Describe("Node Operations", func() { "-j OVN-KUBE-NODEPORT", "-j OVN-KUBE-ITP", }, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-ETP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, + "OVN-KUBE-ITP": []string{}, + "OVN-KUBE-ETP": []string{}, }, "filter": {}, "mangle": { @@ -2904,12 +2792,8 @@ var _ = Describe("Node Operations", func() { }, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-EXTERNALIP": []string{}, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": { "FORWARD": []string{ @@ -2963,12 +2847,8 @@ var _ = Describe("Node Operations", func() { }, "OVN-KUBE-NODEPORT": []string{}, "OVN-KUBE-EXTERNALIP": []string{}, - "POSTROUTING": []string{ - "-j OVN-KUBE-EGRESS-SVC", - }, "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-EGRESS-SVC": []string{}, }, "filter": { "FORWARD": []string{}, diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index 4d8517f3af..a5f9a7fc7c 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -17,7 +17,6 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressservice" nodeipt "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/linkmanager" nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" @@ -1044,7 +1043,7 @@ func (npw *nodePortWatcher) SyncServices(services []interface{}) error { // sync netfilter rules once only for Full mode if !npw.dpuMode { // (NOTE: Order is important, add jump to iptableETPChain before jump to NP/EIP chains) - for _, chain := range []string{iptableITPChain, egressservice.Chain, iptableNodePortChain, iptableExternalIPChain, iptableETPChain} { + for _, chain := range []string{iptableITPChain, iptableNodePortChain, iptableExternalIPChain, iptableETPChain} { if err = recreateIPTRules("nat", chain, keepIPTRules); err != nil { errors = append(errors, err) } @@ -1185,7 +1184,7 @@ func (npw *nodePortWatcher) DeleteEndpointSlice(epSlice *discovery.EndpointSlice if svcConfig, exists := npw.updateServiceInfo(*namespacedName, nil, &hasLocalHostNetworkEp, localEndpoints); exists { netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(namespacedName.Namespace) if err != nil { - return fmt.Errorf("error getting active network for service %s in namespace %s: %w", svc.Name, svc.Namespace, err) + return fmt.Errorf("error getting active network for service %s/%s: %w", namespacedName.Namespace, namespacedName.Name, err) } // Lock the cache mutex here so we don't miss a service delete during an endpoint delete @@ -2304,8 +2303,10 @@ func newGateway( // very unlikely - somehow node has lost its IP address klog.Errorf("Failed to re-generate gateway flows after address change: %v", err) } - npw, _ := gw.nodePortWatcher.(*nodePortWatcher) - npw.updateGatewayIPs(gw.nodeIPManager) + if gw.nodePortWatcher != nil { + npw, _ := gw.nodePortWatcher.(*nodePortWatcher) + npw.updateGatewayIPs(gw.nodeIPManager) + } // Services create OpenFlow flows as well, need to update them all if gw.servicesRetryFramework != nil { if errs := gw.addAllServices(); len(errs) > 0 { diff --git a/go-controller/pkg/node/gateway_udn.go b/go-controller/pkg/node/gateway_udn.go index 6e7fb72a7b..e0f6035919 100644 --- a/go-controller/pkg/node/gateway_udn.go +++ b/go-controller/pkg/node/gateway_udn.go @@ -293,7 +293,7 @@ func (udng *UserDefinedNetworkGateway) AddNetwork() error { if err != nil { return fmt.Errorf("could not create management port netdevice for network %s: %w", udng.GetNetworkName(), err) } - vrfDeviceName := util.GetVRFDeviceNameForUDN(udng.networkID) + vrfDeviceName := util.GetNetworkVRFName(udng.NetInfo) vrfTableId := util.CalculateRouteTableID(mplink.Attrs().Index) routes, err := udng.computeRoutesForUDN(vrfTableId, mplink) if err != nil { @@ -367,7 +367,7 @@ func (udng *UserDefinedNetworkGateway) GetNetworkRuleMetadata() string { // DelNetwork will be responsible to remove all plumbings // used by this UDN on the gateway side func (udng *UserDefinedNetworkGateway) DelNetwork() error { - vrfDeviceName := util.GetVRFDeviceNameForUDN(udng.networkID) + vrfDeviceName := util.GetNetworkVRFName(udng.NetInfo) // delete the iprules for this network if err := udng.ruleManager.DeleteWithMetadata(udng.GetNetworkRuleMetadata()); err != nil { return fmt.Errorf("unable to delete iprules for network %s, err: %v", udng.GetNetworkName(), err) diff --git a/go-controller/pkg/node/node_ip_handler_linux.go b/go-controller/pkg/node/node_ip_handler_linux.go index 0afd0dbed9..eb5c368eff 100644 --- a/go-controller/pkg/node/node_ip_handler_linux.go +++ b/go-controller/pkg/node/node_ip_handler_linux.go @@ -50,12 +50,12 @@ func newAddressManager(nodeName string, k kube.Interface, config *managementPort // newAddressManagerInternal creates a new address manager; this function is // only expose for testcases to disable netlink subscription to ensure // reproducibility of unit tests. -func newAddressManagerInternal(nodeName string, k kube.Interface, config *managementPortConfig, watchFactory factory.NodeWatchFactory, gwBridge *bridgeConfiguration, useNetlink bool) *addressManager { +func newAddressManagerInternal(nodeName string, k kube.Interface, mgmtConfig *managementPortConfig, watchFactory factory.NodeWatchFactory, gwBridge *bridgeConfiguration, useNetlink bool) *addressManager { mgr := &addressManager{ nodeName: nodeName, watchFactory: watchFactory, cidrs: sets.New[string](), - mgmtPortConfig: config, + mgmtPortConfig: mgmtConfig, gatewayBridge: gwBridge, OnChanged: func() {}, useNetlink: useNetlink, @@ -228,7 +228,7 @@ func (c *addressManager) handleNodePrimaryAddrChange() { klog.Errorf("Address Manager failed to check node primary address change: %v", err) return } - if nodePrimaryAddrChanged { + if nodePrimaryAddrChanged && config.Default.EncapIP == "" { klog.Infof("Node primary address changed to %v. Updating OVN encap IP.", c.nodePrimaryAddr) updateOVNEncapIPAndReconnect(c.nodePrimaryAddr) } @@ -536,6 +536,7 @@ func updateOVNEncapIPAndReconnect(newIP net.IP) { } } + config.Default.EffectiveEncapIP = newIP.String() confCmd := []string{ "set", "Open_vSwitch", diff --git a/go-controller/pkg/node/secondary_node_network_controller_test.go b/go-controller/pkg/node/secondary_node_network_controller_test.go index 9151875fd1..bfb345bc27 100644 --- a/go-controller/pkg/node/secondary_node_network_controller_test.go +++ b/go-controller/pkg/node/secondary_node_network_controller_test.go @@ -337,7 +337,7 @@ var _ = Describe("SecondaryNodeNetworkController: UserDefinedPrimaryNetwork Gate Expect(err).NotTo(HaveOccurred()) By("check management interface and VRF device is created for the network") - vrfDeviceName := util.GetVRFDeviceNameForUDN(netID) + vrfDeviceName := util.GetNetworkVRFName(NetInfo) vrfLink, err := util.GetNetLinkOps().LinkByName(vrfDeviceName) Expect(err).NotTo(HaveOccurred()) Expect(vrfLink.Type()).To(Equal("vrf")) diff --git a/go-controller/pkg/node/udn_isolation.go b/go-controller/pkg/node/udn_isolation.go index 92e879426b..83ee56fbca 100644 --- a/go-controller/pkg/node/udn_isolation.go +++ b/go-controller/pkg/node/udn_isolation.go @@ -87,6 +87,7 @@ func NewUDNHostIsolationManager(ipv4, ipv6 bool, podInformer coreinformers.PodIn // Start must be called on node setup. func (m *UDNHostIsolationManager) Start(ctx context.Context) error { + klog.Infof("Starting UDN host isolation manager") // find kubelet cgroup path. // kind cluster uses "kubelet.slice/kubelet.service", while OCP cluster uses "system.slice/kubelet.service". // as long as ovn-k node is running as a privileged container, we can access the host cgroup directory. @@ -363,7 +364,10 @@ func (m *UDNHostIsolationManager) podInitialSync() error { // ignore openPorts parse error in initial sync pi, _, err := m.getPodInfo(podKey, pod) if err != nil { - return err + // don't fail because of one pod error on initial sync as it may cause crashloop. + // expect pod event to come later with correct/updated annotations. + klog.Warningf("UDNHostIsolationManager failed to get pod info for pod %s/%s on initial sync: %v", pod.Name, pod.Namespace, err) + continue } if pi == nil { // this pod doesn't need to be updated @@ -452,6 +456,10 @@ func (m *UDNHostIsolationManager) getPodInfo(podKey string, pod *v1.Pod) (*podIn if pod == nil { return pi, nil, nil } + if util.PodWantsHostNetwork(pod) { + // host network pods can't be isolated by IP + return nil, nil, nil + } // only add pods with primary UDN primaryUDN, err := m.isPodPrimaryUDN(pod) if err != nil { diff --git a/go-controller/pkg/node/udn_isolation_test.go b/go-controller/pkg/node/udn_isolation_test.go index abc1d33a88..ffb6f5fd4b 100644 --- a/go-controller/pkg/node/udn_isolation_test.go +++ b/go-controller/pkg/node/udn_isolation_test.go @@ -314,6 +314,36 @@ add rule inet ovn-kubernetes udn-isolation ip6 daddr @udn-pod-default-ips-v6 dro } }) + It("correctly handles host-network and not ready pods on initial sync", func() { + hostNetPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostnet", + UID: ktypes.UID("hostnet"), + Namespace: defaultNamespace, + }, + } + hostNetPod.Spec.HostNetwork = true + notReadyPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "notready", + UID: ktypes.UID("notready"), + Namespace: defaultNamespace, + }, + } + + fakeClient = util.GetOVNClientset(hostNetPod, notReadyPod).GetNodeClientset() + var err error + wf, err = factory.NewNodeWatchFactory(fakeClient, "node1") + Expect(err).NotTo(HaveOccurred()) + manager = NewUDNHostIsolationManager(true, true, wf.PodCoreInformer()) + nft = nodenft.SetFakeNFTablesHelper() + manager.nft = nft + + Expect(wf.Start()).To(Succeed()) + Expect(manager.setupUDNIsolationFromHost()).To(Succeed()) + Expect(manager.podInitialSync()).To(Succeed()) + }) + It("correctly generates initial rules", func() { start() Expect(nft.Dump()).To(Equal(getExpectedDump(nil, nil))) diff --git a/go-controller/pkg/node/vrfmanager/vrf_manager.go b/go-controller/pkg/node/vrfmanager/vrf_manager.go index 8ea9793bf1..75acb0cb4e 100644 --- a/go-controller/pkg/node/vrfmanager/vrf_manager.go +++ b/go-controller/pkg/node/vrfmanager/vrf_manager.go @@ -3,14 +3,12 @@ package vrfmanager import ( "fmt" - "strings" "sync" "time" "github.com/containernetworking/plugins/pkg/ns" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" utilerrors "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" @@ -178,11 +176,14 @@ func (vrfm *Controller) sync(vrf vrf) error { vrfLink, err := util.GetNetLinkOps().LinkByName(vrf.name) var mustRecreate bool if err == nil { - if vrfLink.Type() != "vrf" { + vrfDev, ok := vrfLink.(*netlink.Vrf) + if !ok { return fmt.Errorf("node has another non VRF device with same name %s", vrf.name) } - vrfDev, ok := vrfLink.(*netlink.Vrf) - if ok && vrfDev.Table != vrf.table { + if vrfDev.Table < util.RoutingTableIDStart { + return fmt.Errorf("node has another VRF device with same name %s that is not managed by ovn-kubernetes", vrf.name) + } + if vrfDev.Table != vrf.table { klog.Warningf("Found a conflict with existing VRF device table id for VRF device %s, recreating it", vrf.name) err = vrfm.deleteVRF(vrfLink) if err != nil { @@ -245,6 +246,9 @@ func (vrfm *Controller) AddVRF(name string, slaveInterface string, table uint32, if len(name) > 15 { return fmt.Errorf("VRF Manager: VRF name %s must be within 15 characters", name) } + if table < util.RoutingTableIDStart { + return fmt.Errorf("VRF Manager: cannot manage a VRF %s with table %d lower than %d", name, table, util.RoutingTableIDStart) + } var ( vrfDev vrf ok bool @@ -296,9 +300,6 @@ func (vrfm *Controller) AddVRFRoutes(name string, routes []netlink.Route) error // Repair deletes stale VRF device(s) on the host. This helps remove // device(s) for which DeleteVRF is never invoked. -// Assumptions: 1) The validVRFs list must contain device for which AddVRF -// is already invoked. 2) The device name(s) in validVRFs are suffixed -// with "-vrf" and prefixed with "mp". func (vrfm *Controller) Repair(validVRFs sets.Set[string]) error { vrfm.mu.Lock() defer vrfm.mu.Unlock() @@ -313,20 +314,25 @@ func (vrfm *Controller) repair(validVRFs sets.Set[string]) error { } for _, link := range links { - name := link.Attrs().Name - // Skip if the link is not a vrf type or name is not suffixed with -vrf or is not prefixed with mp. - if link.Type() != "vrf" || !strings.HasSuffix(name, types.UDNVRFDeviceSuffix) || - !strings.HasPrefix(name, types.UDNVRFDevicePrefix) { + vrf, isVRF := link.(*netlink.Vrf) + if !isVRF { + // not a vrf device + continue + } + if vrf.Table < util.RoutingTableIDStart { + // vrf device not managed by us continue } + name := vrf.Name if validVRFs.Has(name) { + // vrf not stale continue } err = util.GetNetLinkOps().LinkDelete(link) if err != nil { klog.Errorf("VRF Manager: error deleting stale VRF device %s, err: %v", name, err) } - delete(vrfm.vrfs, link.Attrs().Index) + delete(vrfm.vrfs, vrf.Index) } return nil } diff --git a/go-controller/pkg/node/vrfmanager/vrf_manager_test.go b/go-controller/pkg/node/vrfmanager/vrf_manager_test.go index 179c188deb..f85cd1441b 100644 --- a/go-controller/pkg/node/vrfmanager/vrf_manager_test.go +++ b/go-controller/pkg/node/vrfmanager/vrf_manager_test.go @@ -21,8 +21,9 @@ import ( var ( c *Controller - vrfLinkName1 = "mp100-udn-vrf" + vrfLinkName1 = "blue" vrfLinkName2 = "mp200-udn-vrf" + vrfLinkName3 = "red-not-udn" ) var _ = ginkgo.Describe("VRF manager", func() { @@ -31,10 +32,8 @@ var _ = ginkgo.Describe("VRF manager", func() { enslaveLinkName1 = "dev100" enslaveLinkName2 = "dev101" nlMock *mocks.NetLinkOps - vrfLinkMock1 *netlink_mocks.Link enslaveLinkMock1 *netlink_mocks.Link enslaveLinkMock2 *netlink_mocks.Link - vrfLinkMock2 *netlink_mocks.Link ) linkIndexes := map[string]int{ @@ -42,6 +41,7 @@ var _ = ginkgo.Describe("VRF manager", func() { enslaveLinkName1: 2, enslaveLinkName2: 3, vrfLinkName2: 4, + vrfLinkName3: 5, } masterIndexes := map[string]int{ @@ -49,6 +49,13 @@ var _ = ginkgo.Describe("VRF manager", func() { enslaveLinkName1: 1, enslaveLinkName2: 1, vrfLinkName2: 0, + vrfLinkName3: 0, + } + + vrfTables := map[string]uint32{ + vrfLinkName1: 1000, + vrfLinkName2: 2000, + vrfLinkName3: 999, } getLinkIndex := func(linkName string) int { @@ -67,29 +74,41 @@ var _ = ginkgo.Describe("VRF manager", func() { return masterIndex } + getVRFTable := func(linkName string) uint32 { + table, ok := vrfTables[linkName] + if !ok { + panic(fmt.Sprintf("failed to find table for vrf %q", linkName)) + } + return table + } + + buildVRF := func(name string) *netlink.Vrf { + return &netlink.Vrf{ + LinkAttrs: netlink.LinkAttrs{Name: name, MasterIndex: getLinkMasterIndex(name), Index: getLinkIndex(name)}, + Table: getVRFTable(name), + } + } + ginkgo.BeforeEach(func() { c = NewController(routemanager.NewController()) nlMock = &mocks.NetLinkOps{} - vrfLinkMock1 = new(netlink_mocks.Link) + enslaveLinkMock1 = new(netlink_mocks.Link) enslaveLinkMock2 = new(netlink_mocks.Link) - vrfLinkMock2 = new(netlink_mocks.Link) util.SetNetLinkOpMockInst(nlMock) - nlMock.On("LinkByName", vrfLinkName1).Return(vrfLinkMock1, nil) + nlMock.On("LinkByName", vrfLinkName1).Return(buildVRF(vrfLinkName1), nil) nlMock.On("LinkByName", enslaveLinkName1).Return(enslaveLinkMock1, nil) nlMock.On("LinkByName", enslaveLinkName2).Return(enslaveLinkMock2, nil) nlMock.On("IsLinkNotFoundError", mock.Anything).Return(true) nlMock.On("LinkAdd", mock.Anything).Return(nil) nlMock.On("LinkSetUp", mock.Anything).Return(nil) - vrfLinkMock1.On("Type").Return("vrf") - vrfLinkMock1.On("Attrs").Return(&netlink.LinkAttrs{Name: vrfLinkName1, MasterIndex: getLinkMasterIndex(vrfLinkName1), Index: getLinkIndex(vrfLinkName1)}, nil) - nlMock.On("LinkByName", vrfLinkName2).Return(vrfLinkMock2, nil) - vrfLinkMock2.On("Type").Return("vrf") - vrfLinkMock2.On("Attrs").Return(&netlink.LinkAttrs{Name: vrfLinkName2, MasterIndex: getLinkMasterIndex(vrfLinkName2), Index: getLinkIndex(vrfLinkName2), OperState: netlink.OperUp}, nil) - nlMock.On("LinkDelete", vrfLinkMock2).Return(nil) + nlMock.On("LinkByName", vrfLinkName2).Return(buildVRF(vrfLinkName2), nil) + nlMock.On("LinkDelete", buildVRF(vrfLinkName2)).Return(nil) + + nlMock.On("LinkByName", vrfLinkName3).Return(buildVRF(vrfLinkName3), nil) }) ginkgo.AfterEach(func() { @@ -98,28 +117,44 @@ var _ = ginkgo.Describe("VRF manager", func() { ginkgo.Context("VRFs", func() { ginkgo.It("add VRF with a slave interface", func() { - nlMock.On("LinkList").Return([]netlink.Link{vrfLinkMock1, enslaveLinkMock1}, nil) + nlMock.On("LinkList").Return([]netlink.Link{buildVRF(vrfLinkName1), enslaveLinkMock1}, nil) enslaveLinkMock1.On("Attrs").Return(&netlink.LinkAttrs{Name: enslaveLinkName1, MasterIndex: 0, Index: getLinkIndex(enslaveLinkName1)}, nil) - nlMock.On("LinkSetMaster", enslaveLinkMock1, vrfLinkMock1).Return(nil) - err := c.AddVRF(vrfLinkName1, enslaveLinkName1, 10, nil) + nlMock.On("LinkSetMaster", enslaveLinkMock1, buildVRF(vrfLinkName1)).Return(nil) + err := c.AddVRF(vrfLinkName1, enslaveLinkName1, getVRFTable(vrfLinkName1), nil) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) + ginkgo.It("fails if we add a VRF with a long name", func() { + err := c.AddVRF("this.name.is.too.long", "other", 0, nil) + gomega.Expect(err).Should(gomega.HaveOccurred()) + }) + + ginkgo.It("fails if we add a VRF with a non managed routing table", func() { + err := c.AddVRF("this.name.is.ok", "other", 999, nil) + gomega.Expect(err).Should(gomega.HaveOccurred()) + }) + + ginkgo.It("fails if we add VRF with same name as existing non-managed VRF", func() { + nlMock.On("LinkList").Return([]netlink.Link{buildVRF(vrfLinkName3)}, nil) + err := c.AddVRF(vrfLinkName3, "other", 3000, nil) + gomega.Expect(err).Should(gomega.HaveOccurred()) + }) + ginkgo.It("delete VRF", func() { - err := c.AddVRF(vrfLinkName2, "", 20, nil) + err := c.AddVRF(vrfLinkName2, "", getVRFTable(vrfLinkName2), nil) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = c.DeleteVRF(vrfLinkName2) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) ginkgo.It("reconcile VRFs", func() { - nlMock.On("LinkList").Return([]netlink.Link{vrfLinkMock1, vrfLinkMock2, enslaveLinkMock1}, nil) + nlMock.On("LinkList").Return([]netlink.Link{buildVRF(vrfLinkName1), buildVRF(vrfLinkName2), enslaveLinkMock1}, nil) enslaveLinkMock1.On("Attrs").Return(&netlink.LinkAttrs{Name: enslaveLinkName1, MasterIndex: getLinkMasterIndex(enslaveLinkName1), Index: getLinkIndex(enslaveLinkName1)}, nil) - err := c.AddVRF(vrfLinkName1, enslaveLinkName1, 10, nil) + err := c.AddVRF(vrfLinkName1, enslaveLinkName1, getVRFTable(vrfLinkName1), nil) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - err = c.AddVRF(vrfLinkName2, "", 20, nil) + err = c.AddVRF(vrfLinkName2, "", getVRFTable(vrfLinkName2), nil) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - err = util.GetNetLinkOps().LinkDelete(vrfLinkMock2) + err = util.GetNetLinkOps().LinkDelete(buildVRF(vrfLinkName2)) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) enslaveLinkMock1.On("Type").Return("dummy") err = c.reconcile() @@ -130,7 +165,7 @@ var _ = ginkgo.Describe("VRF manager", func() { }) ginkgo.It("repair VRFs", func() { - nlMock.On("LinkList").Return([]netlink.Link{vrfLinkMock1, vrfLinkMock2, enslaveLinkMock1}, nil) + nlMock.On("LinkList").Return([]netlink.Link{buildVRF(vrfLinkName1), buildVRF(vrfLinkName2), buildVRF(vrfLinkName3), enslaveLinkMock1}, nil) enslaveLinkMock1.On("Attrs").Return(&netlink.LinkAttrs{Name: enslaveLinkName1, MasterIndex: getLinkMasterIndex(enslaveLinkName1), Index: getLinkIndex(enslaveLinkName1)}, nil) enslaveLinkMock1.On("Type").Return("dummy") validVRFs := make(sets.Set[string]) @@ -202,9 +237,9 @@ var _ = ginkgo.Describe("VRF manager tests with a network namespace", func() { ovntest.OnSupportedPlatformsIt("ensure VRF manager is reconciling configured VRF devices correctly", func() { err := testNS.Do(func(ns.NetNS) error { defer ginkgo.GinkgoRecover() - err := c.AddVRF(vrfLinkName1, "", 10, nil) + err := c.AddVRF(vrfLinkName1, "", 1000, nil) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - err = c.AddVRF(vrfLinkName2, "", 20, nil) + err = c.AddVRF(vrfLinkName2, "", 2000, nil) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) wg3 := &sync.WaitGroup{} diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index b15a00e64b..38f7d1b834 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -24,6 +24,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/observability" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" lsm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/routeimport" zoneic "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/zone_interconnect" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/persistentips" ovnretry "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" @@ -177,6 +178,8 @@ type BaseNetworkController struct { ovnClusterLRPToJoinIfAddrs []*net.IPNet observManager *observability.Manager + + routeImportManager routeimport.Manager } // BaseSecondaryNetworkController structure holds per-network fields and network specific diff --git a/go-controller/pkg/ovn/base_network_controller_namespace.go b/go-controller/pkg/ovn/base_network_controller_namespace.go index 7523ae3643..76ad058c2c 100644 --- a/go-controller/pkg/ovn/base_network_controller_namespace.go +++ b/go-controller/pkg/ovn/base_network_controller_namespace.go @@ -68,18 +68,21 @@ func getNamespaceAddrSetDbIDs(namespaceName, controller string) *libovsdbops.DbO }) } +func (bnc *BaseNetworkController) shouldWatchNamespaces() bool { + // Watch namespaces only if one of the following conditions is met: + // - The network is the default network. + // - The network is primary, and network segmentation is enabled. + // - The network is secondary, and multi NetworkPolicies are enabled. + return bnc.IsDefault() || + bnc.IsPrimaryNetwork() && util.IsNetworkSegmentationSupportEnabled() || + bnc.IsSecondary() && util.IsMultiNetworkPoliciesSupportEnabled() +} + // WatchNamespaces starts the watching of namespace resource and calls // back the appropriate handler logic func (bnc *BaseNetworkController) WatchNamespaces() error { - if bnc.IsPrimaryNetwork() && !util.IsNetworkSegmentationSupportEnabled() { - // For primary user defined networks, we don't have to watch namespace events if - // network segmentation support is not enabled. - return nil - } - - if bnc.IsSecondary() && !util.IsMultiNetworkPoliciesSupportEnabled() { - // For secondary networks, we don't have to watch namespace events if - // multi-network policy support is not enabled. + if !bnc.shouldWatchNamespaces() { + klog.Infof("Ignoring namespaces events for network: %s", bnc.GetNetworkName()) return nil } diff --git a/go-controller/pkg/ovn/base_network_controller_namespace_test.go b/go-controller/pkg/ovn/base_network_controller_namespace_test.go new file mode 100644 index 0000000000..dbb20dfc67 --- /dev/null +++ b/go-controller/pkg/ovn/base_network_controller_namespace_test.go @@ -0,0 +1,84 @@ +package ovn + +import ( + "testing" + + cnitypes "github.com/containernetworking/cni/pkg/types" + "github.com/stretchr/testify/assert" + + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +func TestBaseNetworkController_shouldWatchNamespaces(t *testing.T) { + tests := []struct { + name string + netCfg *ovntypes.NetConf + enableNetSeg, enableMultiNetPolicies, expectedReturn bool + }{ + { + name: "should watch namespaces for default network", + netCfg: &ovntypes.NetConf{ + NetConf: cnitypes.NetConf{Name: types.DefaultNetworkName}, + }, + expectedReturn: true, + }, + { + name: "should watch namespaces for primary network when network segmentation is enabled", + netCfg: &ovntypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "primary"}, + Topology: types.Layer3Topology, + Role: types.NetworkRolePrimary, + }, + enableNetSeg: true, + expectedReturn: true, + }, + { + name: "should watch namespaces for secondary network when multi NetworkPolicies are enabled", + netCfg: &ovntypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "secondary"}, + Topology: types.Layer3Topology, + Role: types.NetworkRoleSecondary, + }, + enableMultiNetPolicies: true, + expectedReturn: true, + }, + { + name: "should not watch namespaces for primary network when network segmentation is disabled", + netCfg: &ovntypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "primary"}, + Topology: types.Layer3Topology, + Role: types.NetworkRolePrimary, + }, + expectedReturn: false, + }, + { + name: "should not watch namespaces for secondary network when multi NetworkPolicies is disabled", + netCfg: &ovntypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "secondary"}, + Topology: types.Layer3Topology, + Role: types.NetworkRoleSecondary, + }, + expectedReturn: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + util.PrepareTestConfig() + config.OVNKubernetesFeature.EnableMultiNetwork = tt.enableNetSeg || tt.enableMultiNetPolicies + config.OVNKubernetesFeature.EnableNetworkSegmentation = tt.enableNetSeg + config.OVNKubernetesFeature.EnableMultiNetworkPolicy = tt.enableMultiNetPolicies + netInfo, err := util.NewNetInfo(tt.netCfg) + assert.Nil(t, err, "failed to create network info") + bnc := &BaseNetworkController{ + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), + } + if tt.expectedReturn != bnc.shouldWatchNamespaces() { + t.Fail() + } + assert.Equal(t, tt.expectedReturn, bnc.shouldWatchNamespaces()) + }) + } +} diff --git a/go-controller/pkg/ovn/controller/egressservice/egressservice_zone.go b/go-controller/pkg/ovn/controller/egressservice/egressservice_zone.go index 316e9df283..bb0eba6970 100644 --- a/go-controller/pkg/ovn/controller/egressservice/egressservice_zone.go +++ b/go-controller/pkg/ovn/controller/egressservice/egressservice_zone.go @@ -44,7 +44,7 @@ const ( ) type InitClusterEgressPoliciesFunc func(client libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, - ni util.NetInfo, clusterSubnets []*net.IPNet, controllerName string) error + ni util.NetInfo, clusterSubnets []*net.IPNet, controllerName, routerName string) error type EnsureNoRerouteNodePoliciesFunc func(client libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, networkName, controllerName, clusterRouter string, nodeLister corelisters.NodeLister, v4, v6 bool) error type DeleteLegacyDefaultNoRerouteNodePoliciesFunc func(nbClient libovsdbclient.Client, clusterRouter, nodeName string) error @@ -60,10 +60,9 @@ type Controller struct { stopCh <-chan struct{} sync.Mutex - initClusterEgressPolicies InitClusterEgressPoliciesFunc - ensureNoRerouteNodePolicies EnsureNoRerouteNodePoliciesFunc - deleteLegacyDefaultNoRerouteNodePolicies DeleteLegacyDefaultNoRerouteNodePoliciesFunc - createDefaultRouteToExternalForIC CreateDefaultRouteToExternalFunc + initClusterEgressPolicies InitClusterEgressPoliciesFunc + ensureNoRerouteNodePolicies EnsureNoRerouteNodePoliciesFunc + createDefaultRouteToExternalForIC CreateDefaultRouteToExternalFunc services map[string]*svcState // svc key -> state, for services that have sourceIPBy LBIP nodes map[string]*nodeState // node name -> state, contains nodes that host an egress service @@ -121,7 +120,6 @@ func NewController( addressSetFactory addressset.AddressSetFactory, initClusterEgressPolicies InitClusterEgressPoliciesFunc, ensureNoRerouteNodePolicies EnsureNoRerouteNodePoliciesFunc, - deleteLegacyDefaultNoRerouteNodePolicies DeleteLegacyDefaultNoRerouteNodePoliciesFunc, createDefaultRouteToExternalForIC CreateDefaultRouteToExternalFunc, stopCh <-chan struct{}, esInformer egressserviceinformer.EgressServiceInformer, @@ -132,20 +130,19 @@ func NewController( klog.Info("Setting up event handlers for Egress Services") c := &Controller{ - NetInfo: netInfo, - controllerName: controllerName, - client: client, - nbClient: nbClient, - addressSetFactory: addressSetFactory, - initClusterEgressPolicies: initClusterEgressPolicies, - ensureNoRerouteNodePolicies: ensureNoRerouteNodePolicies, - deleteLegacyDefaultNoRerouteNodePolicies: deleteLegacyDefaultNoRerouteNodePolicies, - createDefaultRouteToExternalForIC: createDefaultRouteToExternalForIC, - stopCh: stopCh, - services: map[string]*svcState{}, - nodes: map[string]*nodeState{}, - nodesZoneState: map[string]bool{}, - zone: zone, + NetInfo: netInfo, + controllerName: controllerName, + client: client, + nbClient: nbClient, + addressSetFactory: addressSetFactory, + initClusterEgressPolicies: initClusterEgressPolicies, + ensureNoRerouteNodePolicies: ensureNoRerouteNodePolicies, + createDefaultRouteToExternalForIC: createDefaultRouteToExternalForIC, + stopCh: stopCh, + services: map[string]*svcState{}, + nodes: map[string]*nodeState{}, + nodesZoneState: map[string]bool{}, + zone: zone, } c.egressServiceLister = esInformer.Lister() @@ -232,7 +229,7 @@ func (c *Controller) Run(wg *sync.WaitGroup, threadiness int) error { klog.Errorf("Failed to repair Egress Services entries: %v", err) } subnets := util.GetAllClusterSubnetsFromEntries(c.Subnets()) - err = c.initClusterEgressPolicies(c.nbClient, c.addressSetFactory, &util.DefaultNetInfo{}, subnets, c.controllerName) + err = c.initClusterEgressPolicies(c.nbClient, c.addressSetFactory, c, subnets, c.controllerName, c.GetNetworkScopedClusterRouterName()) if err != nil { klog.Errorf("Failed to init Egress Services cluster policies: %v", err) } diff --git a/go-controller/pkg/ovn/controller/egressservice/egressservice_zone_node.go b/go-controller/pkg/ovn/controller/egressservice/egressservice_zone_node.go index 48cd530c72..0dd52e309f 100644 --- a/go-controller/pkg/ovn/controller/egressservice/egressservice_zone_node.go +++ b/go-controller/pkg/ovn/controller/egressservice/egressservice_zone_node.go @@ -122,11 +122,6 @@ func (c *Controller) syncNode(key string) error { } else { delete(c.nodesZoneState, nodeName) } - - if err := c.deleteLegacyDefaultNoRerouteNodePolicies(c.nbClient, c.GetNetworkScopedClusterRouterName(), nodeName); err != nil { - return err - } - // We ensure node no re-route policies contemplating possible node IP // address changes regardless of allocated services. network := util.DefaultNetInfo{} diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index 9322ae9fbf..64958d7fe2 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -172,14 +172,21 @@ type Controller struct { // Run will not return until stopCh is closed. workers determines how many // endpoints will be handled in parallel. -func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair, useLBGroups, useTemplates bool) error { - defer utilruntime.HandleCrash() - defer c.queue.ShutDown() +func (c *Controller) Run(workers int, stopCh <-chan struct{}, wg *sync.WaitGroup, runRepair, useLBGroups, useTemplates bool) error { + wg.Add(1) + go func() { + defer utilruntime.HandleCrash() + defer wg.Done() + // wait until we're told to stop + <-stopCh + + klog.Infof("Shutting down controller %s for network=%s", controllerName, c.netInfo.GetNetworkName()) + c.queue.ShutDown() + }() c.useLBGroups = useLBGroups c.useTemplates = useTemplates klog.Infof("Starting controller %s for network=%s", controllerName, c.netInfo.GetNetworkName()) - defer klog.Infof("Shutting down controller %s for network=%s", controllerName, c.netInfo.GetNetworkName()) nodeHandler, err := c.nodeTracker.Start(c.nodeInformer) if err != nil { @@ -245,7 +252,6 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair, useLBGr go wait.Until(c.worker, c.workerLoopPeriod, stopCh) } - <-stopCh return nil } diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index 16c78d7364..67efe46f1f 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -29,6 +29,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/external_ids_syncer/nat" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/external_ids_syncer/port_group" lsm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/routeimport" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/topology" zoneic "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/zone_interconnect" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" @@ -148,12 +149,13 @@ func NewDefaultNetworkController( cnci *CommonNetworkControllerInfo, observManager *observability.Manager, networkManager networkmanager.Interface, + routeImportManager routeimport.Manager, eIPController *EgressIPController, portCache *PortCache, ) (*DefaultNetworkController, error) { stopChan := make(chan struct{}) wg := &sync.WaitGroup{} - return newDefaultNetworkControllerCommon(cnci, stopChan, wg, nil, networkManager, observManager, eIPController, portCache) + return newDefaultNetworkControllerCommon(cnci, stopChan, wg, nil, networkManager, routeImportManager, observManager, eIPController, portCache) } func newDefaultNetworkControllerCommon( @@ -162,6 +164,7 @@ func newDefaultNetworkControllerCommon( defaultWg *sync.WaitGroup, addressSetFactory addressset.AddressSetFactory, networkManager networkmanager.Interface, + routeImportManager routeimport.Manager, observManager *observability.Manager, eIPController *EgressIPController, portCache *PortCache, @@ -225,6 +228,7 @@ func newDefaultNetworkControllerCommon( cancelableCtx: util.NewCancelableContext(), observManager: observManager, networkManager: networkManager, + routeImportManager: routeImportManager, }, externalGatewayRouteInfo: apbExternalRouteController.ExternalGWRouteInfoCache, eIPC: eIPController, @@ -363,6 +367,9 @@ func (oc *DefaultNetworkController) Stop() { if oc.efNodeController != nil { controller.Stop(oc.efNodeController) } + if oc.routeImportManager != nil { + oc.routeImportManager.ForgetNetwork(oc.GetNetworkName()) + } close(oc.stopChan) oc.cancelableCtx.Cancel() @@ -422,6 +429,14 @@ func (oc *DefaultNetworkController) Init(ctx context.Context) error { return err } + // Add ourselves to the route import manager + if oc.routeImportManager != nil { + err := oc.routeImportManager.AddNetwork(oc.GetNetInfo()) + if err != nil { + return fmt.Errorf("failed to add default network to the route import manager: %v", err) + } + } + return nil } @@ -619,12 +634,21 @@ func (oc *DefaultNetworkController) Reconcile(netInfo util.NetInfo) error { return fmt.Errorf("failed to reconcile network %s: %w", oc.GetNetworkName(), err) } + reconcileRoutes := oc.routeImportManager != nil && oc.routeImportManager.NeedsReconciliation(netInfo) + // update network information, point of no return err = util.ReconcileNetInfo(oc.ReconcilableNetInfo, netInfo) if err != nil { klog.Errorf("Failed to reconcile network %s: %v", oc.GetNetworkName(), err) } + if reconcileRoutes { + err = oc.routeImportManager.ReconcileNetwork(oc.GetNetworkName()) + if err != nil { + return err + } + } + for _, node := range retryNodes { oc.gatewaysFailed.Store(node.Name, true) err = oc.retryNodes.AddRetryObjWithAddNoBackoff(node) diff --git a/go-controller/pkg/ovn/egressip.go b/go-controller/pkg/ovn/egressip.go index 4ae5191cd8..53f20e2edd 100644 --- a/go-controller/pkg/ovn/egressip.go +++ b/go-controller/pkg/ovn/egressip.go @@ -1181,6 +1181,18 @@ func (e *EgressIPController) SyncLocalNodeZonesCache() error { return nil } +// getALocalZoneNodeName fetches the first local OVN zone Node. Support for multiple Nodes per OVN zone is not supported +// and neither is changing a Nodes OVN zone. This function supports said assumptions. +func (e *EgressIPController) getALocalZoneNodeName() (string, error) { + nodeNames := e.nodeZoneState.GetKeys() + for _, nodeName := range nodeNames { + if isLocal, ok := e.nodeZoneState.Load(nodeName); ok && isLocal { + return nodeName, nil + } + } + return "", fmt.Errorf("failed to find a local OVN zone Node") +} + func (e *EgressIPController) syncStaleAddressSetIPs(egressIPCache egressIPCache) error { for _, networkPodCache := range egressIPCache.egressIPNameToPods { for networkName, podCache := range networkPodCache { @@ -1209,12 +1221,12 @@ func (e *EgressIPController) syncStaleAddressSetIPs(egressIPCache egressIPCache) } // syncStaleGWMarkRules removes stale or invalid LRP that packet mark. They are attached to egress nodes gateway router. -// It adds expected LRPs that packet mark. +// It adds expected LRPs that packet mark. This is valid only for L3 networks. func (e *EgressIPController) syncStaleGWMarkRules(egressIPCache egressIPCache) error { // Delete all stale LRPs then add missing LRPs // This func assumes one node per zone. It determines if an LRP is a valid local LRP. It doesn't determine if the // LRP is attached to the correct GW router - if !util.IsNetworkSegmentationSupportEnabled() || !config.OVNKubernetesFeature.EnableInterconnect { + if !isEgressIPForUDNSupported() { return nil } for _, networkPodCache := range egressIPCache.egressIPNameToPods { @@ -1426,9 +1438,18 @@ func (e *EgressIPController) syncPodAssignmentCache(egressIPCache egressIPCache) if ni == nil { return fmt.Errorf("failed to get active network for network name %q", networkName) } - reRoutePolicies, err := libovsdbops.FindALogicalRouterPoliciesWithPredicate(e.nbClient, ni.GetNetworkScopedClusterRouterName(), p1) + routerName := ni.GetNetworkScopedClusterRouterName() + if ni.TopologyType() == types.Layer2Topology { + // no support for multiple Nodes per OVN zone, therefore pick the first local zone node + localNodeName, err := e.getALocalZoneNodeName() + if err != nil { + return err + } + routerName = ni.GetNetworkScopedGWRouterName(localNodeName) + } + reRoutePolicies, err := libovsdbops.FindALogicalRouterPoliciesWithPredicate(e.nbClient, routerName, p1) if err != nil { - return err + return fmt.Errorf("failed to retrieve a logical router polices attached to router %s: %w", routerName, err) } // not scoped by network since all NATs selected by the follow predicate select only CDN NATs p2 := func(item *nbdb.NAT) bool { @@ -1730,7 +1751,17 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { continue } redirectCache[ni.GetNetworkName()] = map[string]redirectIPs{} - cache.networkToRouter[ni.GetNetworkName()] = ni.GetNetworkScopedClusterRouterName() + var localNodeName string + if localZoneNodes.Len() > 0 { + localNodeName = localZoneNodes.UnsortedList()[0] + } + routerName, err := getTopologyScopedRouterName(ni, localNodeName) + if err != nil { + klog.Errorf("Failed to get network topology scoped router name for network %s attached to namespace %s, stale objects may remain: %v", + ni.GetNetworkName(), namespace.Name, err) + continue + } + cache.networkToRouter[ni.GetNetworkName()] = routerName for _, node := range nodes { r := redirectIPs{} mgmtPort := &nbdb.LogicalSwitchPort{Name: ni.GetNetworkScopedK8sMgmtIntfName(node.Name)} @@ -1763,14 +1794,14 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { if localZoneNodes.Has(node.Name) { if e.v4 { - if gatewayRouterIP, err := e.getGatewayRouterJoinIP(ni, node.Name, false); err != nil { + if gatewayRouterIP, err := e.getGatewayNextHop(ni, node.Name, false); err != nil { klog.V(5).Infof("Unable to retrieve gateway IP for node: %s, protocol is IPv4: err: %v", node.Name, err) } else { r.v4Gateway = gatewayRouterIP.String() } } if e.v6 { - if gatewayRouterIP, err := e.getGatewayRouterJoinIP(ni, node.Name, true); err != nil { + if gatewayRouterIP, err := e.getGatewayNextHop(ni, node.Name, true); err != nil { klog.V(5).Infof("Unable to retrieve gateway IP for node: %s, protocol is IPv6: err: %v", node.Name, err) } else { r.v6Gateway = gatewayRouterIP.String() @@ -1950,12 +1981,11 @@ func (e *EgressIPController) addEgressNode(node *corev1.Node) error { // on IC if the node is gone, then the ovn_cluster_router is also gone along with all // the routes on it. processNetworkFn := func(ni util.NetInfo) error { - clusterSubnetsNetEntry := ni.Subnets() - if len(clusterSubnetsNetEntry) == 0 { + if ni.TopologyType() == types.Layer2Topology || len(ni.Subnets()) == 0 { return nil } if err := libovsdbutil.CreateDefaultRouteToExternal(e.nbClient, ni.GetNetworkScopedClusterRouterName(), - ni.GetNetworkScopedGWRouterName(node.Name), clusterSubnetsNetEntry); err != nil { + ni.GetNetworkScopedGWRouterName(node.Name), ni.Subnets()); err != nil { return fmt.Errorf("failed to create route to external for network %s: %v", ni.GetNetworkName(), err) } return nil @@ -1967,7 +1997,7 @@ func (e *EgressIPController) addEgressNode(node *corev1.Node) error { if err := processNetworkFn(ni); err != nil { return fmt.Errorf("failed to process default network: %v", err) } - if !util.IsNetworkSegmentationSupportEnabled() { + if !isEgressIPForUDNSupported() { return nil } if err := e.networkManager.DoWithLock(processNetworkFn); err != nil { @@ -1989,29 +2019,29 @@ func (e *EgressIPController) addEgressNode(node *corev1.Node) error { func (e *EgressIPController) initClusterEgressPolicies(nodes []interface{}) error { // Init default network defaultNetInfo := e.networkManager.GetNetwork(types.DefaultNetworkName) + localNodeName, err := e.getALocalZoneNodeName() + if err != nil { + klog.Warningf(err.Error()) + } subnets := util.GetAllClusterSubnetsFromEntries(defaultNetInfo.Subnets()) - if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, defaultNetInfo, subnets, e.controllerName); err != nil { + if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, defaultNetInfo, subnets, e.controllerName, defaultNetInfo.GetNetworkScopedClusterRouterName()); err != nil { return fmt.Errorf("failed to initialize networks cluster logical router egress policies for the default network: %v", err) } - for _, node := range nodes { - node := node.(*kapi.Node) - if err := DeleteLegacyDefaultNoRerouteNodePolicies(e.nbClient, defaultNetInfo.GetNetworkScopedClusterRouterName(), node.Name); err != nil { - return fmt.Errorf("failed to delete legacy default no reroute to nodes for node %s: %v", node.Name, err) - } - } + return e.networkManager.DoWithLock(func(network util.NetInfo) error { if network.GetNetworkName() == types.DefaultNetworkName { return nil } subnets = util.GetAllClusterSubnetsFromEntries(network.Subnets()) - if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, network, subnets, e.controllerName); err != nil { - return fmt.Errorf("failed to initialize networks cluster logical router egress policies for network %s: %v", network.GetNetworkName(), err) + if len(subnets) == 0 { + return nil } - for _, node := range nodes { - node := node.(*kapi.Node) - if err := DeleteLegacyDefaultNoRerouteNodePolicies(e.nbClient, network.GetNetworkScopedClusterRouterName(), node.Name); err != nil { - return fmt.Errorf("failed to delete legacy default no reroute node policies for node %s and network %s: %v", node.Name, network.GetNetworkName(), err) - } + routerName, err := getTopologyScopedRouterName(network, localNodeName) + if err != nil { + return err + } + if err = InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, network, subnets, e.controllerName, routerName); err != nil { + return fmt.Errorf("failed to initialize networks cluster logical router egress policies for network %s: %v", network.GetNetworkName(), err) } return nil }) @@ -2020,7 +2050,10 @@ func (e *EgressIPController) initClusterEgressPolicies(nodes []interface{}) erro // InitClusterEgressPolicies creates the global no reroute policies and address-sets // required by the egressIP and egressServices features. func InitClusterEgressPolicies(nbClient libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, ni util.NetInfo, - clusterSubnets []*net.IPNet, controllerName string) error { + clusterSubnets []*net.IPNet, controllerName, routerName string) error { + if len(clusterSubnets) == 0 { + return nil + } var v4ClusterSubnet, v6ClusterSubnet []*net.IPNet for _, subnet := range clusterSubnets { if utilnet.IsIPv6CIDR(subnet) { @@ -2049,15 +2082,14 @@ func InitClusterEgressPolicies(nbClient libovsdbclient.Client, addressSetFactory return fmt.Errorf("network %s: failed to parse IPv6 join subnet: %v", ni.GetNetworkName(), err) } } - router := ni.GetNetworkScopedClusterRouterName() - if err = createDefaultNoReroutePodPolicies(nbClient, ni.GetNetworkName(), controllerName, router, v4ClusterSubnet, v6ClusterSubnet); err != nil { + if err = createDefaultNoReroutePodPolicies(nbClient, ni.GetNetworkName(), controllerName, routerName, v4ClusterSubnet, v6ClusterSubnet); err != nil { return fmt.Errorf("failed to create no reroute policies for pods on network %s: %v", ni.GetNetworkName(), err) } - if err = createDefaultNoRerouteServicePolicies(nbClient, ni.GetNetworkName(), controllerName, router, v4ClusterSubnet, v6ClusterSubnet, + if err = createDefaultNoRerouteServicePolicies(nbClient, ni.GetNetworkName(), controllerName, routerName, v4ClusterSubnet, v6ClusterSubnet, v4JoinSubnet, v6JoinSubnet); err != nil { return fmt.Errorf("failed to create no reroute policies for services on network %s: %v", ni.GetNetworkName(), err) } - if err = createDefaultNoRerouteReplyTrafficPolicy(nbClient, ni.GetNetworkName(), controllerName, router); err != nil { + if err = createDefaultNoRerouteReplyTrafficPolicy(nbClient, ni.GetNetworkName(), controllerName, routerName); err != nil { return fmt.Errorf("failed to create no reroute reply traffic policy for network %s: %v", ni.GetNetworkName(), err) } @@ -2082,9 +2114,9 @@ func InitClusterEgressPolicies(nbClient libovsdbclient.Client, addressSetFactory return fmt.Errorf("cannot ensure that addressSet for egressService pods %s exists %v", egresssvc.EgressServiceServedPodsAddrSetName, err) } - if !ni.IsDefault() && util.IsNetworkSegmentationSupportEnabled() { + if !ni.IsDefault() && isEgressIPForUDNSupported() { v4, v6 := len(v4ClusterSubnet) > 0, len(v6ClusterSubnet) > 0 - if err = ensureDefaultNoRerouteUDNEnabledSvcPolicies(nbClient, addressSetFactory, ni, controllerName, v4, v6); err != nil { + if err = ensureDefaultNoRerouteUDNEnabledSvcPolicies(nbClient, addressSetFactory, ni, controllerName, routerName, v4, v6); err != nil { return fmt.Errorf("failed to ensure no reroute for UDN enabled services for network %s: %v", ni.GetNetworkName(), err) } } @@ -2242,44 +2274,72 @@ func (e *EgressIPController) addPodEgressIPAssignment(ni util.NetInfo, egressIPN } isOVNNetwork := util.IsOVNNetwork(parsedNodeEIPConfig, eIPIP) nextHopIP, err := e.getNextHop(ni, status.Node, status.EgressIP, egressIPName, isLocalZoneEgressNode, isOVNNetwork) - if err != nil || nextHopIP == "" { + if err != nil { return fmt.Errorf("failed to determine next hop for pod %s/%s when configuring egress IP %s"+ " IP %s: %v", pod.Namespace, pod.Name, egressIPName, status.EgressIP, err) } + if nextHopIP == "" { + return fmt.Errorf("could not calculate the next hop for pod %s/%s when configuring egress IP %s"+ + " IP %s", pod.Namespace, pod.Name, egressIPName, status.EgressIP) + } var ops []ovsdb.Operation if loadedEgressNode && isLocalZoneEgressNode { + // create NATs for CDNs only + // create LRPs with allow action (aka GW policy marks) only for L3 UDNs. + // L2 UDNs require LRPs with reroute action with a pkt_mark option attached to GW router. if isOVNNetwork { if ni.IsDefault() { ops, err = e.createNATRuleOps(ni, nil, podIPs, status, egressIPName, pod.Namespace, pod.Name) if err != nil { return fmt.Errorf("unable to create NAT rule ops for status: %v, err: %v", status, err) } - } else if ni.IsSecondary() { + + } else if ni.IsSecondary() && ni.TopologyType() == types.Layer3Topology { + // not required for L2 because we always have LRPs using reroute action to pkt mark ops, err = e.createGWMarkPolicyOps(ni, ops, podIPs, status, mark, pod.Namespace, pod.Name, egressIPName) if err != nil { return fmt.Errorf("unable to create GW router LRP ops to packet mark pod %s/%s: %v", pod.Namespace, pod.Name, err) } } } - if config.OVNKubernetesFeature.EnableInterconnect && !isOVNNetwork && (loadedPodNode && !isLocalZonePod) { - // configure reroute for non-local-zone pods on egress nodes - ops, err = e.createReroutePolicyOps(ni, ops, podIPs, status, mark, egressIPName, nextHopIP, pod.Namespace, pod.Name) + if config.OVNKubernetesFeature.EnableInterconnect && ni.IsDefault() && !isOVNNetwork && (loadedPodNode && !isLocalZonePod) { + // For CDNs, configure LRP with reroute action for non-local-zone pods on egress nodes to support redirect to local management port + // when the egress IP is assigned to a host secondary interface + routerName, err := getTopologyScopedRouterName(ni, pod.Spec.NodeName) + if err != nil { + return err + } + ops, err = e.createReroutePolicyOps(ni, ops, podIPs, status, mark, egressIPName, nextHopIP, routerName, pod.Namespace, pod.Name) if err != nil { return fmt.Errorf("unable to create logical router policy ops %v, err: %v", status, err) } } } - // exec when node is local OR when pods are local + // For L2, we always attach an LRP with reroute action to the Nodes gateway router. If the pod is remote, use the local zone Node name to generate the GW router name. + nodeName := pod.Spec.NodeName + if loadedEgressNode && loadedPodNode && !isLocalZonePod && isLocalZoneEgressNode && ni.IsSecondary() && ni.TopologyType() == types.Layer2Topology { + nodeName = status.Node + } + routerName, err := getTopologyScopedRouterName(ni, nodeName) + if err != nil { + return err + } + + // exec when node is local OR when pods are local or L2 UDN // don't add a reroute policy if the egress node towards which we are adding this doesn't exist - if loadedEgressNode && loadedPodNode && isLocalZonePod { - ops, err = e.createReroutePolicyOps(ni, ops, podIPs, status, mark, egressIPName, nextHopIP, pod.Namespace, pod.Name) - if err != nil { - return fmt.Errorf("unable to create logical router policy ops, err: %v", err) + if loadedEgressNode && loadedPodNode { + if isLocalZonePod || (isLocalZoneEgressNode && ni.IsSecondary() && ni.TopologyType() == types.Layer2Topology) { + ops, err = e.createReroutePolicyOps(ni, ops, podIPs, status, mark, egressIPName, nextHopIP, routerName, pod.Namespace, pod.Name) + if err != nil { + return fmt.Errorf("unable to create logical router policy ops, err: %v", err) + } } - ops, err = e.deleteExternalGWPodSNATOps(ni, ops, pod, podIPs, status, isOVNNetwork) - if err != nil { - return err + if isLocalZonePod { + ops, err = e.deleteExternalGWPodSNATOps(ni, ops, pod, podIPs, status, isOVNNetwork) + if err != nil { + return err + } } } _, err = libovsdbops.TransactAndCheck(e.nbClient, ops) @@ -2324,13 +2384,22 @@ func (e *EgressIPController) deletePodEgressIPAssignment(ni util.NetInfo, egress } } } + // For L2, we always attach an LRP with reroute action to the Nodes gateway router. If the pod is remote, use the local zone Node name to generate the GW router name. + nodeName := pod.Spec.NodeName + if !isLocalZonePod && isLocalZoneEgressNode && ni.IsSecondary() && ni.TopologyType() == types.Layer2Topology { + nodeName = status.Node + } + routerName, err := getTopologyScopedRouterName(ni, nodeName) + if err != nil { + return err + } var ops []ovsdb.Operation if !loadedPodNode || isLocalZonePod { // node is deleted (we can't determine zone so we always try and nuke OR pod is local to zone) ops, err = e.addExternalGWPodSNATOps(ni, nil, pod.Namespace, pod.Name, status) if err != nil { return err } - ops, err = e.deleteReroutePolicyOps(ni, ops, status, egressIPName, nextHopIP, pod.Namespace, pod.Name) + ops, err = e.deleteReroutePolicyOps(ni, ops, status, egressIPName, nextHopIP, routerName, pod.Namespace, pod.Name) if errors.Is(err, libovsdbclient.ErrNotFound) { // if the gateway router join IP setup is already gone, then don't count it as error. klog.Warningf("Unable to delete logical router policy, err: %v", err) @@ -2340,9 +2409,9 @@ func (e *EgressIPController) deletePodEgressIPAssignment(ni util.NetInfo, egress } if loadedEgressNode && isLocalZoneEgressNode { - if config.OVNKubernetesFeature.EnableInterconnect && !isOVNNetwork && (!loadedPodNode || !isLocalZonePod) { // node is deleted (we can't determine zone so we always try and nuke OR pod is remote to zone) - // delete reroute for non-local-zone pods on egress nodes - ops, err = e.deleteReroutePolicyOps(ni, ops, status, egressIPName, nextHopIP, pod.Namespace, pod.Name) + if config.OVNKubernetesFeature.EnableInterconnect && ni.IsDefault() && !isOVNNetwork && (!loadedPodNode || !isLocalZonePod) { // node is deleted (we can't determine zone so we always try and nuke OR pod is remote to zone) + // For CDNs, delete reroute for non-local-zone pods on egress nodes when the egress IP is assigned to a secondary host interface + ops, err = e.deleteReroutePolicyOps(ni, ops, status, egressIPName, nextHopIP, routerName, pod.Namespace, pod.Name) if err != nil { return fmt.Errorf("unable to delete logical router static route ops %v, err: %v", status, err) } @@ -2352,7 +2421,7 @@ func (e *EgressIPController) deletePodEgressIPAssignment(ni util.NetInfo, egress if err != nil { return fmt.Errorf("unable to delete NAT rule for status: %v, err: %v", status, err) } - } else if ni.IsSecondary() { + } else if ni.IsSecondary() && ni.TopologyType() == types.Layer3Topology { ops, err = e.deleteGWMarkPolicyOps(ni, ops, status, pod.Namespace, pod.Name, egressIPName) if err != nil { return fmt.Errorf("unable to create GW router packet mark LRPs delete ops for pod %s/%s: %v", pod.Namespace, pod.Name, err) @@ -2456,13 +2525,100 @@ func (e *EgressIPController) deleteExternalGWPodSNATOps(ni util.NetInfo, ops []o return ops, nil } -func (e *EgressIPController) getGatewayRouterJoinIP(ni util.NetInfo, node string, wantsIPv6 bool) (net.IP, error) { - gatewayIPs, err := libovsdbutil.GetLRPAddrs(e.nbClient, types.GWRouterToJoinSwitchPrefix+ni.GetNetworkScopedGWRouterName(node)) +// getGatewayNextHop determines the next hop for a given Node considering the network topology type +// For layer 3, next hop is gateway routers 'router to join' port IP +// For layer 2, it's the callers responsibility to ensure that the egress node is remote because a LRP should not be created +func (e *EgressIPController) getGatewayNextHop(ni util.NetInfo, nodeName string, isIPv6 bool) (net.IP, error) { + // fetch gateway router 'router to join' port IP + if ni.TopologyType() == types.Layer3Topology { + return e.getRouterPortIP(types.GWRouterToJoinSwitchPrefix+ni.GetNetworkScopedGWRouterName(nodeName), isIPv6) + } + + // If egress node is local, retrieve the external default gateway next hops from the Node L3 gateway annotation. + // We must pick one of the next hops to add to the LRP reroute next hops to not break ECMP. + // If an egress node is remote, retrieve the remote Nodes gateway router 'router to switch' port IP + // from the Node annotation. + // FIXME: remove gathering the required information from a Node annotations as this approach does not scale + // FIXME: we do not respect multiple default gateway next hops and instead pick the first IP that matches the IP family of the EIP + if ni.TopologyType() == types.Layer2Topology { + node, err := e.watchFactory.GetNode(nodeName) + if err != nil { + return nil, fmt.Errorf("failed to retrive node %s: %w", nodeName, err) + } + localNode, err := e.getALocalZoneNodeName() + if err != nil { + return nil, err + } + // Node is local + if localNode == nodeName { + nextHopIPs, err := util.ParseNodeL3GatewayAnnotation(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + // remote node may not have the annotation yet, suppress it + return nil, types.NewSuppressedError(err) + } + return nil, fmt.Errorf("failed to get the node %s L3 gateway annotation: %w", node.Name, err) + } + if len(nextHopIPs.NextHops) == 0 { + return nil, fmt.Errorf("l3 gateway annotation on the node %s has empty next hop IPs. Next hop is required for Layer 2 networks", node.Name) + } + ip, err := util.MatchFirstIPFamily(isIPv6, nextHopIPs.NextHops) + if err != nil { + return nil, fmt.Errorf("unable to find a next hop IP from the node %s L3 gateway annotation that is equal "+ + "to the EgressIP IP family (is IPv6: %v): %v", node.Name, isIPv6, err) + } + return ip, nil + } + // Node is remote + // fetch Node gateway routers 'router to switch' port IP + if isIPv6 { + return util.ParseNodeGatewayRouterJoinIPv6(node, ni.GetNetworkName()) + } + return util.ParseNodeGatewayRouterJoinIPv4(node, ni.GetNetworkName()) + } + return nil, fmt.Errorf("unsupported network topology %s", ni.TopologyType()) +} + +// getLocalMgmtPortNextHop attempts the given networks local management port. If the information is not available because +// of management port deletion, no error will be returned. +func (e *EgressIPController) getLocalMgmtPortNextHop(ni util.NetInfo, egressNodeName, egressIPName, egressIPIP string, isIPv6 bool) (string, error) { + mgmtPort := &nbdb.LogicalSwitchPort{Name: ni.GetNetworkScopedK8sMgmtIntfName(egressNodeName)} + mgmtPort, err := libovsdbops.GetLogicalSwitchPort(e.nbClient, mgmtPort) + if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { + return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s "+ + "because unable to get management port: %v", egressIPName, egressNodeName, err) + } else if err != nil { + klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get management switch port: %v", + egressIPName, egressIPIP, err) + return "", nil + } + allMgmtPortAddresses := mgmtPort.GetAddresses() + if len(allMgmtPortAddresses) == 0 { + return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s"+ + "because failed to retrieve a MAC and IP address entry from management switch port %s", egressIPName, egressNodeName, mgmtPort.Name) + } + // select first MAC & IP(s) entry + mgmtPortAddresses := strings.Fields(allMgmtPortAddresses[0]) + if len(mgmtPortAddresses) < 2 { + return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s"+ + "because management switch port %s does not contain expected MAC address and one or more IP addresses", egressIPName, egressNodeName, mgmtPort.Name) + } + // filter out the MAC address which is always the first entry within the slice + mgmtPortAddresses = mgmtPortAddresses[1:] + nextHopIP, err := util.MatchIPStringFamily(isIPv6, mgmtPortAddresses) if err != nil { - return nil, fmt.Errorf("attempt at finding node gateway router network information failed, err: %w", err) + return "", fmt.Errorf("failed to find a management port %s IP matching the IP family of the EgressIP: %v", mgmtPort.Name, err) + } + return nextHopIP, nil +} + +func (e *EgressIPController) getRouterPortIP(portName string, wantsIPv6 bool) (net.IP, error) { + gatewayIPs, err := libovsdbutil.GetLRPAddrs(e.nbClient, portName) + if err != nil { + return nil, fmt.Errorf("failed to retrieve port %s IP(s): %w", portName, err) } if gatewayIP, err := util.MatchFirstIPNetFamily(wantsIPv6, gatewayIPs); err != nil { - return nil, fmt.Errorf("could not find gateway IP for node %s with family %v: %v", node, wantsIPv6, err) + return nil, fmt.Errorf("failed to find IP for port %s for IP family %v: %v", portName, wantsIPv6, err) } else { return gatewayIP.IP, nil } @@ -2493,68 +2649,48 @@ func (e *EgressIPController) getTransitIP(nodeName string, wantsIPv6 bool) (stri return nodeTransitIP.IP.String(), nil } -// getNextHop attempts to determine whether an egress IP should be routed through the primary OVN network or through -// a secondary host network. If we failed to look up the information required to determine this, an error will be returned -// however if we are able to lookup the information, but it doesnt exist, called must be able to tolerate a blank next hop -// and no error returned. This means we searched successfully but could not find the information required to generate the next hop. +// getNextHop attempts to determine whether an egress IP should be routed through the Nodes primary network interface (isOVNetwork = true) +// or through a secondary host network (isOVNNetwork = false). If we failed to look up the information required to determine this, an error will be returned +// however if the information to determine the next hop IP doesn't exist, caller must be able to tolerate a empty next hop +// and no error returned. This means we searched successfully but could not find the information required to generate the next hop IP. func (e *EgressIPController) getNextHop(ni util.NetInfo, egressNodeName, egressIP, egressIPName string, isLocalZoneEgressNode, isOVNNetwork bool) (string, error) { - var nextHopIP string - var err error isEgressIPv6 := utilnet.IsIPv6String(egressIP) - // NOTE: No need to check if status.node exists or not in the cache, we are calling this function only if it - // is present in the nodeZoneState cache. Since we call it with lock on cache, we are safe here. - if isLocalZoneEgressNode { + if isLocalZoneEgressNode || ni.TopologyType() == types.Layer2Topology { + // isOVNNetwork is true when an EgressIP is "assigned" to the Nodes primary interface (breth0). Ext traffic will egress breth0. + // is OVNNetwork is false when the EgressIP is assigned to a host secondary interface (not breth0). Ext traffic will egress this interface. if isOVNNetwork { - gatewayRouterIP, err := e.getGatewayRouterJoinIP(ni, egressNodeName, isEgressIPv6) + gatewayRouterIP, err := e.getGatewayNextHop(ni, egressNodeName, isEgressIPv6) + // return error only when we failed to retrieve the gateway IP. Do not return error when we can never get this IP (gw deleted) if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { return "", fmt.Errorf("unable to retrieve gateway IP for node: %s, protocol is IPv6: %v, err: %w", egressNodeName, isEgressIPv6, err) } else if err != nil { - klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get gateway "+ - "router join IP: %v", egressIPName, egressIP, err) + klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get Node %s gateway "+ + "router IP: %v", egressIPName, egressIP, egressNodeName, err) return "", nil } - nextHopIP = gatewayRouterIP.String() + return gatewayRouterIP.String(), nil } else { - mgmtPort := &nbdb.LogicalSwitchPort{Name: ni.GetNetworkScopedK8sMgmtIntfName(egressNodeName)} - mgmtPort, err := libovsdbops.GetLogicalSwitchPort(e.nbClient, mgmtPort) - if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { - return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s "+ - "because unable to get management port: %v", egressIPName, egressNodeName, err) - } else if err != nil { - klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get management switch port: %v", - egressIPName, egressIP, err) - return "", nil - } - allMgmtPortAddresses := mgmtPort.GetAddresses() - if len(allMgmtPortAddresses) == 0 { - return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s"+ - "because management switch port %s does not have a MAC and IP address entry", egressIPName, egressNodeName, mgmtPort.Name) - } - // select first MAC & IP(s) entry - mgmtPortAddresses := strings.Fields(allMgmtPortAddresses[0]) - if len(mgmtPortAddresses) < 2 { - return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s"+ - "because management switch port %s does not contain expected MAC address and one or more IP addresses", egressIPName, egressNodeName, mgmtPort.Name) - } - // filter out the MAC address which is always the first entry within the slice - mgmtPortAddresses = mgmtPortAddresses[1:] - nextHopIP, err = util.MatchIPStringFamily(isEgressIPv6, mgmtPortAddresses) - if err != nil { - return "", fmt.Errorf("failed to find a management port %s IP matching the IP family of the EgressIP: %v", mgmtPort.Name, err) + // for an egress IP assigned to a host secondary interface, next hop IP is the networks management port IP + if ni.IsSecondary() { + return "", fmt.Errorf("egress IP assigned to a host secondary interface for a user defined network (network name %s) is unsupported", ni.GetNetworkName()) } + return e.getLocalMgmtPortNextHop(ni, egressNodeName, egressIPName, egressIP, isEgressIPv6) } - } else if config.OVNKubernetesFeature.EnableInterconnect { - // fetch node annotation of the egress node - nextHopIP, err = e.getTransitIP(egressNodeName, isEgressIPv6) + } + + if config.OVNKubernetesFeature.EnableInterconnect { + nextHopIP, err := e.getTransitIP(egressNodeName, isEgressIPv6) if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { return "", fmt.Errorf("unable to fetch transit switch IP for node %s: %v", egressNodeName, err) } else if err != nil { klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get transit switch IP: %v", egressIPName, egressIP, err) + return "", nil } + return nextHopIP, nil } - return nextHopIP, nil + return "", nil } // createReroutePolicyOps creates an operation that does idempotent updates of the @@ -2570,7 +2706,7 @@ func (e *EgressIPController) getNextHop(ni util.NetInfo, egressNodeName, egressI // enabled, the appropriate transit switch port. // This function should be called with lock on nodeZoneState cache key status.Node func (e *EgressIPController) createReroutePolicyOps(ni util.NetInfo, ops []ovsdb.Operation, podIPNets []*net.IPNet, status egressipv1.EgressIPStatusItem, - mark util.EgressIPMark, egressIPName, nextHopIP, podNamespace, podName string) ([]ovsdb.Operation, error) { + mark util.EgressIPMark, egressIPName, nextHopIP, routerName, podNamespace, podName string) ([]ovsdb.Operation, error) { isEgressIPv6 := utilnet.IsIPv6String(status.EgressIP) ipFamily := getEIPIPFamily(isEgressIPv6) options := make(map[string]string) @@ -2580,10 +2716,10 @@ func (e *EgressIPController) createReroutePolicyOps(ni util.NetInfo, ops []ovsdb } addPktMarkToLRPOptions(options, mark.String()) } - var err error dbIDs := getEgressIPLRPReRouteDbIDs(egressIPName, podNamespace, podName, ipFamily, ni.GetNetworkName(), e.controllerName) p := libovsdbops.GetPredicate[*nbdb.LogicalRouterPolicy](dbIDs, nil) // Handle all pod IPs that match the egress IP address family + var err error for _, podIPNet := range util.MatchAllIPNetFamily(isEgressIPv6, podIPNets) { lrp := nbdb.LogicalRouterPolicy{ @@ -2594,9 +2730,9 @@ func (e *EgressIPController) createReroutePolicyOps(ni util.NetInfo, ops []ovsdb ExternalIDs: dbIDs.GetExternalIDs(), Options: options, } - ops, err = libovsdbops.CreateOrAddNextHopsToLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), &lrp, p) + ops, err = libovsdbops.CreateOrAddNextHopsToLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, routerName, &lrp, p) if err != nil { - return nil, fmt.Errorf("error creating logical router policy %+v on router %s: %v", lrp, ni.GetNetworkScopedClusterRouterName(), err) + return nil, fmt.Errorf("error creating logical router policy %+v on router %s: %v", lrp, routerName, err) } } return ops, nil @@ -2613,7 +2749,7 @@ func (e *EgressIPController) createReroutePolicyOps(ni util.NetInfo, ops []ovsdb // which will break HA momentarily // This function should be called with lock on nodeZoneState cache key status.Node func (e *EgressIPController) deleteReroutePolicyOps(ni util.NetInfo, ops []ovsdb.Operation, status egressipv1.EgressIPStatusItem, - egressIPName, nextHopIP, podNamespace, podName string) ([]ovsdb.Operation, error) { + egressIPName, nextHopIP, routerName, podNamespace, podName string) ([]ovsdb.Operation, error) { isEgressIPv6 := utilnet.IsIPv6String(status.EgressIP) ipFamily := getEIPIPFamily(isEgressIPv6) var err error @@ -2621,19 +2757,19 @@ func (e *EgressIPController) deleteReroutePolicyOps(ni util.NetInfo, ops []ovsdb dbIDs := getEgressIPLRPReRouteDbIDs(egressIPName, podNamespace, podName, ipFamily, ni.GetNetworkName(), e.controllerName) p := libovsdbops.GetPredicate[*nbdb.LogicalRouterPolicy](dbIDs, nil) if nextHopIP != "" { - ops, err = libovsdbops.DeleteNextHopFromLogicalRouterPoliciesWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), p, nextHopIP) + ops, err = libovsdbops.DeleteNextHopFromLogicalRouterPoliciesWithPredicateOps(e.nbClient, ops, routerName, p, nextHopIP) if err != nil { return nil, fmt.Errorf("error removing nexthop IP %s from egress ip %s policies on router %s: %v", - nextHopIP, egressIPName, ni.GetNetworkScopedClusterRouterName(), err) + nextHopIP, egressIPName, routerName, err) } } else { klog.Errorf("Caller failed to pass next hop for EgressIP %s and IP %s. Deleting all LRPs. This will break HA momentarily", egressIPName, status.EgressIP) // since next hop was not found, delete everything to ensure no stale entries however this will break load // balancing between hops, but we offer no guarantees except one of the EIPs will work - ops, err = libovsdbops.DeleteLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), p) + ops, err = libovsdbops.DeleteLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, routerName, p) if err != nil { - return nil, fmt.Errorf("failed to create logical router policy delete operations on %s: %v", ni.GetNetworkScopedClusterRouterName(), err) + return nil, fmt.Errorf("failed to create logical router policy delete operations on %s: %v", routerName, err) } } return ops, nil @@ -2742,12 +2878,20 @@ func (e *EgressIPController) deleteEgressIPStatusSetup(ni util.NetInfo, name str } if nextHopIP != "" { - ops, err = libovsdbops.DeleteNextHopFromLogicalRouterPoliciesWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), policyPredNextHop, nextHopIP) + router := ni.GetNetworkScopedClusterRouterName() + if ni.TopologyType() == types.Layer2Topology { + nodeName, err := e.getALocalZoneNodeName() + if err != nil { + return err + } + router = ni.GetNetworkScopedGWRouterName(nodeName) + } + ops, err = libovsdbops.DeleteNextHopFromLogicalRouterPoliciesWithPredicateOps(e.nbClient, ops, router, policyPredNextHop, nextHopIP) if err != nil { return fmt.Errorf("error removing nexthop IP %s from egress ip %s policies on router %s: %v", - nextHopIP, name, ni.GetNetworkScopedClusterRouterName(), err) + nextHopIP, name, router, err) } - } else if ops, err = e.ensureOnlyValidNextHops(ni, name, ops); err != nil { + } else if ops, err = e.ensureOnlyValidNextHops(ni, name, status.Node, ops); err != nil { return err } @@ -2779,7 +2923,7 @@ func (e *EgressIPController) deleteEgressIPStatusSetup(ni util.NetInfo, name str return nil } -func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name string, ops []libovsdb.Operation) ([]libovsdb.Operation, error) { +func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name, nodeName string, ops []libovsdb.Operation) ([]libovsdb.Operation, error) { // When no nextHopIP is found, This may happen when node object is already deleted. // So compare validNextHopIPs associated with current eIP.Status and Nexthops present // in the LogicalRouterPolicy, then delete nexthop(s) from LogicalRouterPolicy if @@ -2789,15 +2933,19 @@ func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name strin strings.HasPrefix(item.ExternalIDs[libovsdbops.ObjectNameKey.String()], name+dbIDEIPNamePodDivider) && item.ExternalIDs[libovsdbops.NetworkKey.String()] == ni.GetNetworkName() } + routerName, err := getTopologyScopedRouterName(ni, nodeName) + if err != nil { + return ops, err + } eIP, err := e.watchFactory.GetEgressIP(name) if err != nil && !apierrors.IsNotFound(err) { return ops, fmt.Errorf("error retrieving EgressIP %s object for updating logical router policy nexthops, err: %w", name, err) } else if err != nil && apierrors.IsNotFound(err) { // EgressIP object is not found, so delete LRP associated with it. - ops, err = libovsdbops.DeleteLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), policyPred) + ops, err = libovsdbops.DeleteLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, routerName, policyPred) if err != nil { return ops, fmt.Errorf("error creating ops to remove logical router policy for EgressIP %s from router %s: %v", - name, ni.GetNetworkScopedClusterRouterName(), err) + name, routerName, err) } } else { validNextHopIPs := make(sets.Set[string]) @@ -2815,10 +2963,10 @@ func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name strin return ops, fmt.Errorf("error finding logical router policy for EgressIP %s: %v", name, err) } if len(validNextHopIPs) == 0 { - ops, err = libovsdbops.DeleteLogicalRouterPoliciesOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), reRoutePolicies...) + ops, err = libovsdbops.DeleteLogicalRouterPoliciesOps(e.nbClient, ops, routerName, reRoutePolicies...) if err != nil { return ops, fmt.Errorf("error creating ops to remove logical router policy for EgressIP %s from router %s: %v", - name, ni.GetNetworkScopedClusterRouterName(), err) + name, routerName, err) } return ops, nil } @@ -2827,10 +2975,10 @@ func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name strin if validNextHopIPs.Has(nextHop) { continue } - ops, err = libovsdbops.DeleteNextHopsFromLogicalRouterPolicyOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), []*nbdb.LogicalRouterPolicy{policy}, nextHop) + ops, err = libovsdbops.DeleteNextHopsFromLogicalRouterPolicyOps(e.nbClient, ops, routerName, []*nbdb.LogicalRouterPolicy{policy}, nextHop) if err != nil { return ops, fmt.Errorf("error creating ops to remove stale next hop IP %s from logical router policy for EgressIP %s from router %s: %v", - nextHop, name, ni.GetNetworkScopedClusterRouterName(), err) + nextHop, name, routerName, err) } } } @@ -2920,34 +3068,40 @@ func createDefaultNoRerouteServicePolicies(nbClient libovsdbclient.Client, netwo return nil } -func (e *EgressIPController) ensureL3ClusterRouterPoliciesForNetwork(ni util.NetInfo) error { +func (e *EgressIPController) ensureRouterPoliciesForNetwork(ni util.NetInfo) error { e.nodeUpdateMutex.Lock() defer e.nodeUpdateMutex.Unlock() - subnets := util.GetAllClusterSubnetsFromEntries(ni.Subnets()) - if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, ni, subnets, e.controllerName); err != nil { + subnetEntries := ni.Subnets() + subnets := util.GetAllClusterSubnetsFromEntries(subnetEntries) + if len(subnets) == 0 { + return nil + } + localNode, err := e.getALocalZoneNodeName() + if err != nil { + return err + } + routerName, err := getTopologyScopedRouterName(ni, localNode) + if err != nil { + return err + } + if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, ni, subnets, e.controllerName, routerName); err != nil { return fmt.Errorf("failed to initialize networks cluster logical router egress policies for the default network: %v", err) } - err := ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, ni.GetNetworkName(), ni.GetNetworkScopedClusterRouterName(), + err = ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, ni.GetNetworkName(), routerName, e.controllerName, listers.NewNodeLister(e.watchFactory.NodeInformer().GetIndexer()), e.v4, e.v6) if err != nil { return fmt.Errorf("failed to ensure no reroute node policies for network %s: %v", ni.GetNetworkName(), err) } - if !config.OVNKubernetesFeature.EnableInterconnect { - return nil - } - nodes := e.nodeZoneState.GetKeys() - for _, node := range nodes { - if isLocal, ok := e.nodeZoneState.Load(node); ok && isLocal { - if err := libovsdbutil.CreateDefaultRouteToExternal(e.nbClient, ni.GetNetworkScopedClusterRouterName(), - ni.GetNetworkScopedGWRouterName(node), ni.Subnets()); err != nil { - return fmt.Errorf("failed to create route to external for network %s: %v", ni.GetNetworkName(), err) - } + if config.OVNKubernetesFeature.EnableInterconnect && ni.TopologyType() == types.Layer3Topology { + if err := libovsdbutil.CreateDefaultRouteToExternal(e.nbClient, routerName, + ni.GetNetworkScopedGWRouterName(localNode), subnetEntries); err != nil { + return fmt.Errorf("failed to create route to external for network %s: %v", ni.GetNetworkName(), err) } } return nil } -func (e *EgressIPController) ensureL3SwitchPoliciesForNode(ni util.NetInfo, nodeName string) error { +func (e *EgressIPController) ensureSwitchPoliciesForNode(ni util.NetInfo, nodeName string) error { e.nodeUpdateMutex.Lock() defer e.nodeUpdateMutex.Unlock() ops, err := e.ensureDefaultNoReRouteQosRulesForNode(ni, nodeName, nil) @@ -2963,10 +3117,10 @@ func (e *EgressIPController) ensureL3SwitchPoliciesForNode(ni util.NetInfo, node // createDefaultNoRerouteReplyTrafficPolicies ensures any traffic which is a response/reply from the egressIP pods // will not be re-routed to egress-nodes. This ensures EIP can work well with ETP=local // this policy is ipFamily neutral -func createDefaultNoRerouteReplyTrafficPolicy(nbClient libovsdbclient.Client, network, controller, clusterRouter string) error { +func createDefaultNoRerouteReplyTrafficPolicy(nbClient libovsdbclient.Client, network, controller, routerName string) error { match := fmt.Sprintf("pkt.mark == %d", types.EgressIPReplyTrafficConnectionMark) dbIDs := getEgressIPLRPNoReRouteDbIDs(types.DefaultNoRereoutePriority, ReplyTrafficNoReroute, IPFamilyValue, network, controller) - if err := createLogicalRouterPolicy(nbClient, clusterRouter, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create no-reroute reply traffic policies, err: %v", err) } return nil @@ -2974,18 +3128,18 @@ func createDefaultNoRerouteReplyTrafficPolicy(nbClient libovsdbclient.Client, ne // createDefaultNoReroutePodPolicies ensures egress pods east<->west traffic with regular pods, // i.e: ensuring that an egress pod can still communicate with a regular pod / service backed by regular pods -func createDefaultNoReroutePodPolicies(nbClient libovsdbclient.Client, network, controller, clusterRouter string, v4ClusterSubnet, v6ClusterSubnet []*net.IPNet) error { +func createDefaultNoReroutePodPolicies(nbClient libovsdbclient.Client, network, controller, routerName string, v4ClusterSubnet, v6ClusterSubnet []*net.IPNet) error { for _, v4Subnet := range v4ClusterSubnet { match := fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Subnet.String(), v4Subnet.String()) dbIDs := getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, network, controller) - if err := createLogicalRouterPolicy(nbClient, clusterRouter, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create IPv4 no-reroute pod policies, err: %v", err) } } for _, v6Subnet := range v6ClusterSubnet { match := fmt.Sprintf("ip6.src == %s && ip6.dst == %s", v6Subnet.String(), v6Subnet.String()) dbIDs := getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV6, network, controller) - if err := createLogicalRouterPolicy(nbClient, clusterRouter, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create IPv6 no-reroute pod policies, err: %v", err) } } @@ -3153,14 +3307,23 @@ func (e *EgressIPController) ensureDefaultNoRerouteNodePolicies() error { if err != nil { return fmt.Errorf("failed to ensure default no reroute policies for nodes for default network: %v", err) } - if !util.IsNetworkSegmentationSupportEnabled() { + if !isEgressIPForUDNSupported() { return nil } if err = e.networkManager.DoWithLock(func(network util.NetInfo) error { if network.GetNetworkName() == types.DefaultNetworkName { return nil } - err = ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, network.GetNetworkName(), network.GetNetworkScopedClusterRouterName(), + routerName := network.GetNetworkScopedClusterRouterName() + if network.TopologyType() == types.Layer2Topology { + // assume one node per zone only. Multi nodes per zone not supported. + nodeName, err := e.getALocalZoneNodeName() + if err != nil { + return err + } + routerName = network.GetNetworkScopedGWRouterName(nodeName) + } + err = ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, network.GetNetworkName(), routerName, e.controllerName, nodeLister, e.v4, e.v6) if err != nil { return fmt.Errorf("failed to ensure default no reroute policies for nodes for network %s: %v", network.GetNetworkName(), err) @@ -3322,23 +3485,6 @@ func createLogicalRouterPolicy(nbClient libovsdbclient.Client, clusterRouter, ma return nil } -// DeleteLegacyDefaultNoRerouteNodePolicies deletes the older EIP node reroute policies -// called from syncFunction and is a one time operation -// sample: 101 ip4.src == 10.244.0.0/16 && ip4.dst == 172.18.0.2/32 allow -func DeleteLegacyDefaultNoRerouteNodePolicies(nbClient libovsdbclient.Client, clusterRouter, node string) error { - p := func(item *nbdb.LogicalRouterPolicy) bool { - if item.Priority != types.DefaultNoRereoutePriority { - return false - } - nodeName, ok := item.ExternalIDs["node"] - if !ok { - return false - } - return nodeName == node - } - return libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(nbClient, clusterRouter, p) -} - func (e *EgressIPController) buildSNATFromEgressIPStatus(ni util.NetInfo, podIP net.IP, status egressipv1.EgressIPStatusItem, egressIPName, podNamespace, podName string) (*nbdb.NAT, error) { logicalIP := &net.IPNet{ IP: podIP, @@ -3417,7 +3563,7 @@ func (e *EgressIPController) getNetworkFromPodAssignment(podKey string) util.Net } func ensureDefaultNoRerouteUDNEnabledSvcPolicies(nbClient libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, - ni util.NetInfo, controllerName string, v4, v6 bool) error { + ni util.NetInfo, controllerName, routerName string, v4, v6 bool) error { var err error var as addressset.AddressSet // fetch the egressIP pods address-set @@ -3471,14 +3617,14 @@ func ensureDefaultNoRerouteUDNEnabledSvcPolicies(nbClient libovsdbclient.Client, // Create global allow policy for UDN enabled service traffic if v4 && matchV4 != "" { dbIDs = getEgressIPLRPNoReRouteDbIDs(types.DefaultNoRereoutePriority, NoReRouteUDNPodToCDNSvc, IPFamilyValueV4, ni.GetNetworkName(), controllerName) - if err := createLogicalRouterPolicy(nbClient, ni.GetNetworkScopedClusterRouterName(), matchV4, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, matchV4, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create IPv4 no-rerouteUDN pod to CDN svc, err: %v", err) } } if v6 && matchV6 != "" { dbIDs = getEgressIPLRPNoReRouteDbIDs(types.DefaultNoRereoutePriority, NoReRouteUDNPodToCDNSvc, IPFamilyValueV6, ni.GetNetworkName(), controllerName) - if err := createLogicalRouterPolicy(nbClient, ni.GetNetworkScopedClusterRouterName(), matchV6, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, matchV6, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create IPv6 no-reroute UDN pod to CDN svc policies, err: %v", err) } } @@ -3497,7 +3643,7 @@ func getPodNamespaceAndNameFromKey(podKey string) (string, string) { func getEgressIPPktMark(eipName string, annotations map[string]string) util.EgressIPMark { var err error var mark util.EgressIPMark - if util.IsNetworkSegmentationSupportEnabled() && util.IsEgressIPMarkSet(annotations) { + if isEgressIPForUDNSupported() && util.IsEgressIPMarkSet(annotations) { mark, err = util.ParseEgressIPMark(annotations) if err != nil { klog.Errorf("Failed to get EgressIP %s packet mark from annotations: %v", eipName, err) @@ -3528,3 +3674,20 @@ func getEIPIPFamily(isIPv6 bool) egressIPFamilyValue { func addPktMarkToLRPOptions(options map[string]string, mark string) { options["pkt_mark"] = mark } + +// getTopologyScopedRouterName returns the router name that we attach polices to support EgressIP depending on network topology +// For Layer 3, we return the network scoped OVN "cluster router" name. For layer 2, we return a Nodes network scoped OVN gateway router name. +func getTopologyScopedRouterName(ni util.NetInfo, nodeName string) (string, error) { + if ni.TopologyType() == types.Layer2Topology { + if nodeName == "" { + return "", fmt.Errorf("node name is required to determine the Nodes gateway router name") + } + return ni.GetNetworkScopedGWRouterName(nodeName), nil + } + return ni.GetNetworkScopedClusterRouterName(), nil +} + +func isEgressIPForUDNSupported() bool { + return config.OVNKubernetesFeature.EnableInterconnect && + config.OVNKubernetesFeature.EnableNetworkSegmentation +} diff --git a/go-controller/pkg/ovn/egressip_test.go b/go-controller/pkg/ovn/egressip_test.go index 02296a38cc..daeedd83e1 100644 --- a/go-controller/pkg/ovn/egressip_test.go +++ b/go-controller/pkg/ovn/egressip_test.go @@ -7303,8 +7303,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" if !isNode1Local && isNode2Local { return nil } - recordedEvent := <-fakeOvn.fakeRecorder.Events - gomega.Expect(recordedEvent).To(gomega.ContainSubstring("EgressIP object egressip-2 will not be configured for pod egressip-namespace_egress-pod since another egressIP object egressip is serving it, this is undefined")) + gomega.Eventually(fakeOvn.fakeRecorder.Events).Should(gomega.Receive( + gomega.ContainSubstring("EgressIP object egressip-2 will not be configured for pod egressip-namespace_egress-pod since another egressIP object egressip is serving it, this is undefined"), + )) assignedEIP := egressIPs1[0] var pas *podAssignmentState @@ -7581,9 +7582,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[1]]).To(gomega.Equal("")) g.Expect(pas.standbyEgressIPNames.Has(egressIP2Name)).To(gomega.BeTrue()) }).Should(gomega.Succeed()) - gomega.Eventually(func() string { - return <-fakeOvn.fakeRecorder.Events - }).Should(gomega.ContainSubstring("EgressIP object egressip-2 will not be configured for pod egressip-namespace_egress-pod since another egressIP object egressip is serving it, this is undefined")) + gomega.Eventually(fakeOvn.fakeRecorder.Events).Should(gomega.Receive( + gomega.ContainSubstring("EgressIP object egressip-2 will not be configured for pod egressip-namespace_egress-pod since another egressIP object egressip is serving it, this is undefined"), + )) gomega.Eventually(getEgressIPStatusLen(egressIP2Name)).Should(gomega.Equal(1)) egressIPs2, nodes2 = getEgressIPStatus(egressIP2Name) diff --git a/go-controller/pkg/ovn/egressip_udn_l2_test.go b/go-controller/pkg/ovn/egressip_udn_l2_test.go new file mode 100644 index 0000000000..0e8d5fa402 --- /dev/null +++ b/go-controller/pkg/ovn/egressip_udn_l2_test.go @@ -0,0 +1,2555 @@ +package ovn + +import ( + "context" + "fmt" + "net" + + cnitypes "github.com/containernetworking/cni/pkg/types" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + egressipv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/udnenabledsvc" + libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/urfave/cli/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = ginkgo.Describe("EgressIP Operations for user defined network with topology L2", func() { + var ( + app *cli.App + fakeOvn *FakeOVN + ) + + const ( + nadName1 = "nad1" + networkName1 = "network1" + networkName1_ = networkName1 + "_" + node1Name = "node1" + v4Net1 = "20.128.0.0/14" + v4Node1Net1 = "20.128.0.0/16" + v4Pod1IPNode1Net1 = "20.128.0.5" + node1DefaultRtoJIP = "100.64.0.1" + node1DefaultRtoJIPCIDR = node1DefaultRtoJIP + "/16" + node1Network1RtoSIP = "100.65.0.2" + node1Network1RtoSIPCIDR = node1Network1RtoSIP + "/16" + podName3 = "egress-pod3" + v4Pod2IPNode1Net1 = "20.128.0.6" + v4Node1Tsp = "100.88.0.2" + node2Name = "node2" + v4Node2Net1 = "20.129.0.0/16" + v4Node2Tsp = "100.88.0.3" + podName4 = "egress-pod4" + v4Pod1IPNode2Net1 = "20.129.0.2" + v4Pod2IPNode2Net1 = "20.129.0.3" + node2DefaultRtoJIP = "100.66.0.1" + node2DefaultRtoJIPCIDR = node2DefaultRtoJIP + "/16" + node2Network1RtoSIP = "100.67.0.2" + node2Network1RtoSIPCIDR = node2Network1RtoSIP + "/16" + eIP1Mark = 50000 + eIP2Mark = 50001 + layer2SwitchName = "ovn_layer2_switch" + gwIP = "192.168.126.1" + gwIP2 = "192.168.127.1" + secondaryNetworkID = "2" + ) + + getEgressIPStatusLen := func(egressIPName string) func() int { + return func() int { + tmp, err := fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Get(context.TODO(), egressIPName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return len(tmp.Status.Items) + } + } + + getIPNetWithIP := func(cidr string) *net.IPNet { + ip, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + panic(err.Error()) + } + ipNet.IP = ip + return ipNet + } + + setPrimaryNetworkAnnot := func(pod *corev1.Pod, nadName, cidr string) { + var err error + hwAddr, _ := net.ParseMAC("00:00:5e:00:53:01") + pod.Annotations, err = util.MarshalPodAnnotation(pod.Annotations, + &util.PodAnnotation{ + IPs: []*net.IPNet{getIPNetWithIP(cidr)}, + MAC: hwAddr, + Role: "primary", + }, + nadName) + if err != nil { + panic(err.Error()) + } + } + + ginkgo.BeforeEach(func() { + // Restore global default values before each testcase + gomega.Expect(config.PrepareTestConfig()).Should(gomega.Succeed()) + config.OVNKubernetesFeature.EnableEgressIP = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableInterconnect = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.Gateway.Mode = config.GatewayModeShared + config.OVNKubernetesFeature.EgressIPNodeHealthCheckPort = 1234 + + app = cli.NewApp() + app.Name = "test" + app.Flags = config.Flags + + fakeOvn = NewFakeOVN(false) + }) + + ginkgo.AfterEach(func() { + fakeOvn.shutdown() + // Restore global default values + gomega.Expect(config.PrepareTestConfig()).Should(gomega.Succeed()) + }) + + ginkgo.Context("sync", func() { + ginkgo.It("should remove stale LRPs and configures missing LRPs", func() { + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + nadName := util.GetNADName(eipNamespace2, nadName1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) + egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNRemote, nadName, fmt.Sprintf("%s%s", v4Pod2IPNode2Net1, util.GetIPFullMaskString(v4Pod2IPNode2Net1))) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop":"192.168.126.1", "next-hops": ["192.168.126.1"]}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + }, + }, + } + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP2, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), // stale gateway + getReRoutePolicyForController(egressIPName, eipNamespace2, podName, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP2, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), // stale pod + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName, IPFamilyValueV4, netInfo.GetNetworkName())}, // stale policies + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDNLocal, egressPodUDNLocal, egressPodCDNRemote, egressPodUDNRemote}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + fakeOvn.controller.logicalPortCache.add(&egressPodUDNLocal, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + fakeOvn.controller.eIPC.zone = node1.Name + fakeOvn.controller.zone = node1.Name + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(fakeOvn.networkManager.Start()).Should(gomega.Succeed()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID", "egressip-nat2-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP2, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNRemote.Namespace, egressPodCDNRemote.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.NAT{ + UUID: "egressip-nat2-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getReRoutePolicyForController(egressIPName, eipNamespace2, podName4, v4Pod2IPNode2Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName()), + "udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), "udn-enabled-svc-no-reroute-UUID", + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("EgressIP update", func() { + ginkgo.It("should update UDN and CDN config", func() { + // Test steps: + // update an EIP selecting a pod on an UDN and another pod on a CDN + // EIP egresses locally and remote + // EIP egresses remote + // EIP egresses locally and remote + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + nadName := util.GetNADName(eipNamespace2, nadName1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) + egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNRemote, nadName, fmt.Sprintf("%s%s", v4Pod2IPNode2Net1, util.GetIPFullMaskString(v4Pod2IPNode2Net1))) + + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that selects pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDNLocal, egressPodUDNLocal, egressPodCDNRemote, egressPodUDNRemote}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.zone = node1.Name + fakeOvn.eIPController.zone = node1.Name + fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + secConInfo, ok := fakeOvn.secondaryControllers[networkName1] + gomega.Expect(ok).To(gomega.BeTrue()) + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer fakeOvn.networkManager.Stop() + // simulate Start() of secondary network controller + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(secConInfo.bnc.GetNetInfo()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(secConInfo.bnc.GetNetInfo(), node1Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + secConInfo.bnc.logicalPortCache.add(&egressPodUDNLocal, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + _, err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Create(context.TODO(), &eIP, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID", "egressip-nat2-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP2, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNRemote.Namespace, egressPodCDNRemote.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.NAT{ + UUID: "egressip-nat2-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getReRoutePolicyForController(egressIPName, eipNamespace2, podName4, v4Pod2IPNode2Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{ + "udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName())}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + ginkgo.By("patch EgressIP status to ensure remote node is egressable only") + oneNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + err = patchEgressIP(fakeOvn.controller.kube.PatchEgressIP, eIP.Name, generateEgressIPPatches(eIP1Mark, oneNodeStatus)...) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(getEgressIPStatusLen(eIP.Name)).Should(gomega.Equal(1)) + expectedDatabaseStateOneEgressNode := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName())}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateOneEgressNode)) + + ginkgo.By("restore both nodes as egressable") + err = patchEgressIP(fakeOvn.controller.kube.PatchEgressIP, eIP.Name, generateEgressIPPatches(eIP1Mark, twoNodeStatus)...) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(getEgressIPStatusLen(eIP.Name)).Should(gomega.Equal(2)) + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("EgressIP delete", func() { + ginkgo.It("should del UDN and CDN config", func() { + // Test steps: + // One EIP selecting a pod on an UDN and another pod on a CDN + // EIP egresses locally and remote + // Delete EIP + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + nadName := util.GetNADName(eipNamespace2, nadName1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) + egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNRemote, nadName, fmt.Sprintf("%s%s", v4Pod2IPNode2Net1, util.GetIPFullMaskString(v4Pod2IPNode2Net1))) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that selects pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDNLocal, egressPodUDNLocal, egressPodCDNRemote, egressPodUDNRemote}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + fakeOvn.controller.zone = node1.Name + fakeOvn.eIPController.zone = node1.Name + err = fakeOvn.eIPController.SyncLocalNodeZonesCache() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer fakeOvn.networkManager.Stop() + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + secConInfo, ok := fakeOvn.secondaryControllers[networkName1] + gomega.Expect(ok).To(gomega.BeTrue()) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + secConInfo.bnc.logicalPortCache.add(&egressPodUDNLocal, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + _, err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Create(context.TODO(), &eIP, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID", "egressip-nat2-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP2, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNRemote.Namespace, egressPodCDNRemote.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.NAT{ + UUID: "egressip-nat2-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getReRoutePolicyForController(egressIPName, eipNamespace2, podName4, v4Pod2IPNode2Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{ + "udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName()), + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + ginkgo.By("delete EgressIP") + err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Delete(context.TODO(), eIP.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressIPServedPodsASCDNv4.Addresses = nil + egressIPServedPodsASUDNv4.Addresses = nil + expectedDatabaseState := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName())}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: secConInfo.bnc.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("Namespace update", func() { + ginkgo.It("should update UDN and CDN config", func() { + // Test steps: + // create an EIP not selecting a pod on an UDN and another pod on a CDN because namespace labels aren't selected + // EIP egresses locally and remote + // Update namespace to match EIP selectors + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, nil) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, nil) + egressPodCDN := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDN := *newPodWithLabels(eipNamespace2, podName2, node1Name, podV4IP2, egressPodLabel) + + nadNsName := util.GetNADName(eipNamespace2, nadName1) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadNsName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that doesnt select pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDN, egressPodUDN}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDN, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + fakeOvn.controller.zone = node1Name + fakeOvn.controller.eIPC.zone = node1Name + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + err = fakeOvn.eIPController.SyncLocalNodeZonesCache() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer fakeOvn.networkManager.Stop() + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + secConInfo, ok := fakeOvn.secondaryControllers[networkName1] + gomega.Expect(ok).To(gomega.BeTrue()) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + secConInfo.bnc.logicalPortCache.add(&egressPodUDN, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + ginkgo.By("update namespaces with label so its now selected by EgressIP") + egressCDNNamespace.Labels = egressPodLabel + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.Background(), egressCDNNamespace, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressUDNNamespace.Labels = egressPodLabel + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.Background(), egressUDNNamespace, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDN.Namespace, egressPodCDN.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDN.Namespace, egressPodCDN.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", + "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName())}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("Namespace delete", func() { + ginkgo.It("should delete UDN and CDN config", func() { + // Test steps: + // create an EIP selecting a pod on an UDN and another pod on a CDN + // EIP egresses locally and remote + // Delete namespace + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDN := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDN := *newPodWithLabels(eipNamespace2, podName2, node1Name, podV4IP2, egressPodLabel) + + nadNsName := util.GetNADName(eipNamespace2, nadName1) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadNsName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that selects pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDN, egressPodUDN}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDN, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + fakeOvn.controller.zone = node1Name + fakeOvn.eIPController.zone = node1Name + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + fakeOvn.controller.logicalPortCache.add(&egressPodUDN, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer fakeOvn.networkManager.Stop() + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDN.Namespace, egressPodCDN.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDN.Namespace, egressPodCDN.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", + "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName())}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("Pod update", func() { + ginkgo.It("should update UDN and CDN config", func() { + // Test steps: + // create an EIP no pods + // Create multiple pods, some selected by EIP selectors and some not + // EIP egresses locally and remote + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + nadName := util.GetNADName(eipNamespace2, nadName1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, nil) + egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, nil) + egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) + egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNRemote, nadName, fmt.Sprintf("%s%s", v4Pod2IPNode2Net1, util.GetIPFullMaskString(v4Pod2IPNode2Net1))) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that doesnt select pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedSwitchName(node1.Name), + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDNLocal, egressPodUDNLocal, egressPodCDNRemote, egressPodUDNRemote}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + fakeOvn.controller.zone = node1Name + fakeOvn.eIPController.zone = node1Name + err = fakeOvn.eIPController.SyncLocalNodeZonesCache() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer fakeOvn.networkManager.Stop() + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + secConInfo, ok := fakeOvn.secondaryControllers[networkName1] + gomega.Expect(ok).To(gomega.BeTrue()) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + secConInfo.bnc.logicalPortCache.add(&egressPodUDNLocal, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + ginkgo.By("update pod with label so its now selected by EgressIP") + egressPodCDNLocal.Labels = egressPodLabel + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(eipNamespace).Update(context.Background(), &egressPodCDNLocal, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressPodUDNLocal.Labels = egressPodLabel + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(eipNamespace2).Update(context.Background(), &egressPodUDNLocal, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID", "egressip-nat2-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP2, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNRemote.Namespace, egressPodCDNRemote.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.NAT{ + UUID: "egressip-nat2-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getReRoutePolicyForController(egressIPName, eipNamespace2, podName4, v4Pod2IPNode2Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", + "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName()), + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) +}) diff --git a/go-controller/pkg/ovn/egressip_udn_l3_test.go b/go-controller/pkg/ovn/egressip_udn_l3_test.go index 78461f6d24..34dec4d824 100644 --- a/go-controller/pkg/ovn/egressip_udn_l3_test.go +++ b/go-controller/pkg/ovn/egressip_udn_l3_test.go @@ -35,24 +35,25 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol ) const ( - nadName1 = "nad1" - networkName1 = "network1" - networkName1_ = networkName1 + "_" - node1Name = "node1" - v4Net1 = "20.128.0.0/14" - v4Node1Net1 = "20.128.0.0/16" - v4Pod1IPNode1Net1 = "20.128.0.5" - podName3 = "egress-pod3" - v4Pod2IPNode1Net1 = "20.128.0.6" - v4Node1Tsp = "100.88.0.2" - node2Name = "node2" - v4Node2Net1 = "20.129.0.0/16" - v4Node2Tsp = "100.88.0.3" - podName4 = "egress-pod4" - v4Pod1IPNode2Net1 = "20.129.0.2" - v4Pod2IPNode2Net1 = "20.129.0.3" - eIP1Mark = 50000 - eIP2Mark = 50001 + nadName1 = "nad1" + networkName1 = "network1" + networkName1_ = networkName1 + "_" + node1Name = "node1" + v4Net1 = "20.128.0.0/14" + v4Node1Net1 = "20.128.0.0/16" + v4Pod1IPNode1Net1 = "20.128.0.5" + podName3 = "egress-pod3" + v4Pod2IPNode1Net1 = "20.128.0.6" + v4Node1Tsp = "100.88.0.2" + node2Name = "node2" + v4Node2Net1 = "20.129.0.0/16" + v4Node2Tsp = "100.88.0.3" + podName4 = "egress-pod4" + v4Pod1IPNode2Net1 = "20.129.0.2" + v4Pod2IPNode2Net1 = "20.129.0.3" + eIP1Mark = 50000 + eIP2Mark = 50001 + secondaryNetworkID = "2" ) getEgressIPStatusLen := func(egressIPName string) func() int { @@ -147,6 +148,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol netconf, ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} netInfo, err := util.NewNetInfo(&netconf) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -220,9 +222,9 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol Ports: []string{"k8s-" + node1Name + "-UUID"}, }, // UDN start - //getGWPktMarkLRPForController(eIP1Mark, egressIPName, eipNamespace2, podName3, v4Pod2IPNode1Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), - //getGWPktMarkLRPForController(eIP2Mark, egressIPName, eipNamespace2, podName4, v4Pod1IPNode2Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), //stale EIP mark - //getGWPktMarkLRPForController(eIP2Mark, egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), //stale EIP mark + getGWPktMarkLRPForController(eIP1Mark, egressIPName, eipNamespace2, podName3, v4Pod2IPNode1Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), + getGWPktMarkLRPForController(eIP2Mark, egressIPName, eipNamespace2, podName4, v4Pod1IPNode2Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), //stale EIP mark + getGWPktMarkLRPForController(eIP2Mark, egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), //stale EIP mark &nbdb.LogicalRouterPort{ UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + networkName1_ + node1.Name + "-UUID", Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + networkName1_ + node1.Name, @@ -277,6 +279,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol // watch EgressIP depends on UDN enabled svcs address set being available c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) go func() { + defer ginkgo.GinkgoRecover() gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) }() // Add pod IPs to CDN cache @@ -291,11 +294,12 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) fakeOvn.controller.eIPC.zone = node1.Name fakeOvn.controller.zone = node1.Name - err = fakeOvn.eIPController.ensureL3ClusterRouterPoliciesForNetwork(netInfo) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3SwitchPoliciesForNode(netInfo, node1Name) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(fakeOvn.networkManager.Start()).Should(gomega.Succeed()) + defer fakeOvn.networkManager.Stop() err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() @@ -519,6 +523,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol netconf, ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} netInfo, err := util.NewNetInfo(&netconf) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -645,6 +650,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol // watch EgressIP depends on UDN enabled svcs address set being available c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) go func() { + defer ginkgo.GinkgoRecover() gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) }() // Add pod IPs to CDN cache @@ -655,12 +661,15 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) secConInfo, ok := fakeOvn.secondaryControllers[networkName1] gomega.Expect(ok).To(gomega.BeTrue()) + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer fakeOvn.networkManager.Stop() // simulate Start() of secondary network controller - err = fakeOvn.eIPController.ensureL3ClusterRouterPoliciesForNetwork(secConInfo.bnc.GetNetInfo()) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(secConInfo.bnc.GetNetInfo()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3SwitchPoliciesForNode(secConInfo.bnc.GetNetInfo(), node1Name) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(secConInfo.bnc.GetNetInfo(), node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1033,6 +1042,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol netconf, ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} netInfo, err := util.NewNetInfo(&netconf) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1159,6 +1169,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol // watch EgressIP depends on UDN enabled svcs address set being available c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) go func() { + defer ginkgo.GinkgoRecover() gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) }() // Add pod IPs to CDN cache @@ -1169,6 +1180,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.eIPController.zone = node1.Name err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer fakeOvn.networkManager.Stop() err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() @@ -1523,6 +1535,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol netconf, ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} netInfo, err := util.NewNetInfo(&netconf) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1652,6 +1665,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol // watch EgressIP depends on UDN enabled svcs address set being available c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) go func() { + defer ginkgo.GinkgoRecover() gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) }() // Add pod IPs to CDN cache @@ -1660,11 +1674,14 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.logicalPortCache.add(&egressPodCDN, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) fakeOvn.controller.zone = node1Name fakeOvn.controller.eIPC.zone = node1Name + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3ClusterRouterPoliciesForNetwork(netInfo) + defer fakeOvn.networkManager.Stop() + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3SwitchPoliciesForNode(netInfo, node1Name) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1882,6 +1899,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol netconf, ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} netInfo, err := util.NewNetInfo(&netconf) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2011,6 +2029,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol // watch EgressIP depends on UDN enabled svcs address set being available c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) go func() { + defer ginkgo.GinkgoRecover() gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) }() // Add pod IPs to CDN cache @@ -2027,9 +2046,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.logicalPortCache.add(&egressPodUDN, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3ClusterRouterPoliciesForNetwork(netInfo) + defer fakeOvn.networkManager.Stop() + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3SwitchPoliciesForNode(netInfo, node1Name) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2238,6 +2258,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol netconf, ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} netInfo, err := util.NewNetInfo(&netconf) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2367,6 +2388,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol // watch EgressIP depends on UDN enabled svcs address set being available c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) go func() { + defer ginkgo.GinkgoRecover() gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) }() // Add pod IPs to CDN cache @@ -2375,6 +2397,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer fakeOvn.networkManager.Stop() fakeOvn.controller.zone = node1Name fakeOvn.eIPController.zone = node1Name err = fakeOvn.controller.WatchEgressIPNamespaces() diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 9d5f81d8c0..798635e87d 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -306,10 +306,10 @@ func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { return fmt.Errorf("failed to get node logical switches which have other-config set: %v", err) } - staleNodes := sets.NewString() + staleSwitches := sets.NewString() for _, nodeSwitch := range nodeSwitches { if nodeSwitch.Name != types.TransitSwitch && !foundNodes.Has(nodeSwitch.Name) { - staleNodes.Insert(nodeSwitch.Name) + staleSwitches.Insert(nodeSwitch.Name) } } @@ -321,7 +321,7 @@ func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { } nodeName := strings.TrimPrefix(item.Name, types.ExternalSwitchPrefix) if nodeName != item.Name && len(nodeName) > 0 && !foundNodes.Has(nodeName) { - staleNodes.Insert(nodeName) + staleSwitches.Insert(nodeName) return true } return false @@ -339,7 +339,7 @@ func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { } nodeName := strings.TrimPrefix(item.Name, types.GWRouterPrefix) if nodeName != item.Name && len(nodeName) > 0 && !foundNodes.Has(nodeName) { - staleNodes.Insert(nodeName) + staleSwitches.Insert(nodeName) return true } return false @@ -350,9 +350,9 @@ func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { } // Cleanup stale nodes (including gateway routers and external logical switches) - for _, staleNode := range staleNodes.UnsortedList() { - if err := oc.cleanupNodeResources(staleNode); err != nil { - return fmt.Errorf("failed to cleanup node resources:%s, err:%w", staleNode, err) + for _, staleSwitch := range staleSwitches.UnsortedList() { + if err := oc.cleanupNodeResources(staleSwitch); err != nil { + return fmt.Errorf("failed to cleanup node resources:%s, err:%w", staleSwitch, err) } } diff --git a/go-controller/pkg/ovn/multicast_test.go b/go-controller/pkg/ovn/multicast_test.go index 16b667ef78..7735146ba5 100644 --- a/go-controller/pkg/ovn/multicast_test.go +++ b/go-controller/pkg/ovn/multicast_test.go @@ -259,11 +259,11 @@ func getNodeData(netInfo util.NetInfo, nodeName string) []libovsdb.TestData { } } -func newNodeWithNad(nad *nadapi.NetworkAttachmentDefinition, networkName string) *v1.Node { +func newNodeWithNad(nad *nadapi.NetworkAttachmentDefinition, networkName, networkID string) *v1.Node { n := newNode(nodeName, "192.168.126.202/24") if nad != nil { n.Annotations["k8s.ovn.org/node-subnets"] = fmt.Sprintf("{\"default\":\"192.168.126.202/24\", \"%s\":\"192.168.127.202/24\"}", networkName) - n.Annotations["k8s.ovn.org/network-ids"] = fmt.Sprintf("{\"default\":\"0\",\"%s\":\"50\"}", networkName) + n.Annotations["k8s.ovn.org/network-ids"] = fmt.Sprintf("{\"default\":\"0\",\"%s\":\"%s\"}", networkName, networkID) n.Annotations["k8s.ovn.org/node-mgmt-port-mac-addresses"] = fmt.Sprintf("{\"default\":\"96:8f:e8:25:a2:e5\",\"%s\":\"d6:bc:85:32:30:fb\"}", networkName) n.Annotations["k8s.ovn.org/node-chassis-id"] = "abdcef" n.Annotations["k8s.ovn.org/l3-gateway-config"] = "{\"default\":{\"mac-address\":\"52:54:00:e2:ed:d0\",\"ip-addresses\":[\"10.1.1.10/24\"],\"ip-address\":\"10.1.1.10/24\",\"next-hops\":[\"10.1.1.1\"],\"next-hop\":\"10.1.1.1\"}}" @@ -341,18 +341,22 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { gomegaFormatMaxLength int networkName = "bluenet" nadName = "rednad" + networkID = "50" nadFromIPMode = func(useIPv4, useIPv6 bool) *nadapi.NetworkAttachmentDefinition { + var nad *nadapi.NetworkAttachmentDefinition if useIPv4 && useIPv6 { - return ovntest.GenerateNAD(networkName, nadName, namespaceName1, + nad = ovntest.GenerateNAD(networkName, nadName, namespaceName1, types.Layer3Topology, "100.128.0.0/16,ae70::66/60", types.NetworkRolePrimary) } else if useIPv4 { - return ovntest.GenerateNAD(networkName, nadName, namespaceName1, + nad = ovntest.GenerateNAD(networkName, nadName, namespaceName1, types.Layer3Topology, "100.128.0.0/16", types.NetworkRolePrimary) } else { - return ovntest.GenerateNAD(networkName, nadName, namespaceName1, + nad = ovntest.GenerateNAD(networkName, nadName, namespaceName1, types.Layer3Topology, "ae70::66/60", types.NetworkRolePrimary) } + nad.Annotations = map[string]string{types.OvnNetworkIDAnnotation: networkID} + return nad } ) @@ -666,7 +670,7 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { config.IPv6Mode = useIPv6 netInfo := getNetInfoFromNAD(nad) - node := newNodeWithNad(nad, networkName) + node := newNodeWithNad(nad, networkName, networkID) namespace1 := *newNamespace(namespaceName1) pods, tPods, tPodIPs := createTestPods(nodeName, namespaceName1, useIPv4, useIPv6) @@ -741,7 +745,7 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { namespace1 := *newNamespace(longNameSpace1Name) longNameSpace2Name := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijl" // create with 63 characters namespace2 := *newNamespace(longNameSpace2Name) - node := newNodeWithNad(nad, networkName) + node := newNodeWithNad(nad, networkName, networkID) objs := []runtime.Object{ &v1.NamespaceList{ @@ -816,7 +820,7 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { netInfo := getNetInfoFromNAD(nad) namespace1 := *newNamespace(namespaceName1) - node := newNodeWithNad(nad, networkName) + node := newNodeWithNad(nad, networkName, networkID) _, tPods, tPodIPs := createTestPods(nodeName, namespaceName1, useIPv4, useIPv6) ports := []string{} diff --git a/go-controller/pkg/ovn/multihoming_test.go b/go-controller/pkg/ovn/multihoming_test.go index b2adfe441c..f04052c651 100644 --- a/go-controller/pkg/ovn/multihoming_test.go +++ b/go-controller/pkg/ovn/multihoming_test.go @@ -281,6 +281,11 @@ func (em *secondaryNetworkExpectationMachine) expectedLogicalSwitchesAndPortsWit if _, alreadyAdded := alreadyAddedManagementElements[pod.nodeName]; !alreadyAdded && em.isInterconnectCluster && ocInfo.bnc.TopologyType() == ovntypes.Layer3Topology { transitSwitchName := ocInfo.bnc.GetNetworkName() + "_transit_switch" + extIDs := map[string]string{ + ovntypes.NetworkExternalID: ocInfo.bnc.GetNetworkName(), + ovntypes.NetworkRoleExternalID: util.GetUserDefinedNetworkRole(ocInfo.bnc.IsPrimaryNetwork()), + ovntypes.TopologyExternalID: ocInfo.bnc.TopologyType(), + } data = append(data, &nbdb.LogicalSwitch{ UUID: transitSwitchName + "-UUID", Name: transitSwitchName, @@ -291,6 +296,7 @@ func (em *secondaryNetworkExpectationMachine) expectedLogicalSwitchesAndPortsWit "requested-tnl-key": "16711685", "mcast_snoop": "true", }, + ExternalIDs: extIDs, }) } if _, alreadyAdded := alreadyAddedManagementElements[pod.nodeName]; !alreadyAdded && diff --git a/go-controller/pkg/ovn/ovn.go b/go-controller/pkg/ovn/ovn.go index 85847710ff..ce1b4cf92f 100644 --- a/go-controller/pkg/ovn/ovn.go +++ b/go-controller/pkg/ovn/ovn.go @@ -469,17 +469,14 @@ func shouldUpdateNode(node, oldNode *kapi.Node) (bool, error) { } func (oc *DefaultNetworkController) StartServiceController(wg *sync.WaitGroup, runRepair bool) error { - wg.Add(1) - go func() { - defer wg.Done() - useLBGroups := oc.clusterLoadBalancerGroupUUID != "" - // use 5 workers like most of the kubernetes controllers in the - // kubernetes controller-manager - err := oc.svcController.Run(5, oc.stopChan, runRepair, useLBGroups, oc.svcTemplateSupport) - if err != nil { - klog.Errorf("Error running OVN Kubernetes Services controller: %v", err) - } - }() + useLBGroups := oc.clusterLoadBalancerGroupUUID != "" + // use 5 workers like most of the kubernetes controllers in the + // kubernetes controller-manager + err := oc.svcController.Run(5, oc.stopChan, wg, runRepair, useLBGroups, oc.svcTemplateSupport) + if err != nil { + + return fmt.Errorf("error running OVN Kubernetes Services controller: %v", err) + } return nil } @@ -487,14 +484,13 @@ func (oc *DefaultNetworkController) InitEgressServiceZoneController() (*egresssv // If the EgressIP controller is enabled it will take care of creating the // "no reroute" policies - we can pass "noop" functions to the egress service controller. initClusterEgressPolicies := func(nbClient libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, ni util.NetInfo, - clusterSubnets []*net.IPNet, controllerName string) error { + clusterSubnets []*net.IPNet, controllerName, routerName string) error { return nil } ensureNodeNoReroutePolicies := func(nbClient libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, network, router, controller string, nodeLister listers.NodeLister, v4, v6 bool) error { return nil } - deleteLegacyDefaultNoRerouteNodePolicies := func(libovsdbclient.Client, string, string) error { return nil } // used only when IC=true createDefaultNodeRouteToExternal := func(nbClient libovsdbclient.Client, clusterRouter, gwRouterName string, clusterSubnets []config.CIDRNetworkEntry) error { return nil @@ -503,12 +499,11 @@ func (oc *DefaultNetworkController) InitEgressServiceZoneController() (*egresssv if !config.OVNKubernetesFeature.EnableEgressIP { initClusterEgressPolicies = InitClusterEgressPolicies ensureNodeNoReroutePolicies = ensureDefaultNoRerouteNodePolicies - deleteLegacyDefaultNoRerouteNodePolicies = DeleteLegacyDefaultNoRerouteNodePolicies createDefaultNodeRouteToExternal = libovsdbutil.CreateDefaultRouteToExternal } return egresssvc_zone.NewController(oc.GetNetInfo(), DefaultNetworkControllerName, oc.client, oc.nbClient, oc.addressSetFactory, - initClusterEgressPolicies, ensureNodeNoReroutePolicies, deleteLegacyDefaultNoRerouteNodePolicies, + initClusterEgressPolicies, ensureNodeNoReroutePolicies, createDefaultNodeRouteToExternal, oc.stopChan, oc.watchFactory.EgressServiceInformer(), oc.watchFactory.ServiceCoreInformer(), oc.watchFactory.EndpointSliceCoreInformer(), diff --git a/go-controller/pkg/ovn/ovn_test.go b/go-controller/pkg/ovn/ovn_test.go index 8aae325f54..276ebe1e15 100644 --- a/go-controller/pkg/ovn/ovn_test.go +++ b/go-controller/pkg/ovn/ovn_test.go @@ -378,7 +378,7 @@ func NewOvnController( return nil, err } - dnc, err := newDefaultNetworkControllerCommon(cnci, stopChan, wg, addressSetFactory, networkManager, nil, eIPController, portCache) + dnc, err := newDefaultNetworkControllerCommon(cnci, stopChan, wg, addressSetFactory, networkManager, nil, nil, eIPController, portCache) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if nbZoneFailed { @@ -504,7 +504,7 @@ func (o *FakeOVN) NewSecondaryNetworkController(netattachdef *nettypes.NetworkAt } secondaryController = &l3Controller.BaseSecondaryNetworkController case types.Layer2Topology: - l2Controller, err := NewSecondaryLayer2NetworkController(cnci, nInfo, o.networkManager.Interface()) + l2Controller, err := NewSecondaryLayer2NetworkController(cnci, nInfo, o.networkManager.Interface(), o.eIPController, o.portCache) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if o.asf != nil { // use fake asf only when enabled l2Controller.addressSetFactory = asf diff --git a/go-controller/pkg/ovn/routeimport/route_import.go b/go-controller/pkg/ovn/routeimport/route_import.go new file mode 100644 index 0000000000..c3dc6ede00 --- /dev/null +++ b/go-controller/pkg/ovn/routeimport/route_import.go @@ -0,0 +1,544 @@ +package routeimport + +import ( + "fmt" + "net" + "strconv" + "strings" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/ovn-org/libovsdb/client" + "github.com/ovn-org/libovsdb/ovsdb" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + controllerutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + nbdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" +) + +const ( + subscribePeriod = 1 * time.Second + subscribeBuffer = 100 + reconcileDelay = 500 * time.Millisecond + noTable = -1 + controllerExternalIDKey = string(nbdbops.OwnerControllerKey) + controllerName = "RouteImport" +) + +type Manager interface { + // AddNetwork instructs the manager to continously reconcile BGP routes from + // the network host vrf to the network gateway router. A network can only be + // added once otherwise an error will be returned. + AddNetwork(network util.NetInfo) error + + // NeedsReconciliation checks the provided network information against the + // stored one and returns whether there is any change requires + // reconciliation. If the network is not known to the manager, it returns + // false. + NeedsReconciliation(network util.NetInfo) bool + + // ReconcileNetwork triggers a manual reconciliation. + ReconcileNetwork(name string) error + + // ForgetNetwork instructs the manager to stop reconciling BGP routes from + // the network host vrf to the network gateway router. + ForgetNetwork(name string) +} + +type Controller interface { + Manager + Start() error + Stop() +} + +func New(node string, nbClient client.Client) Controller { + c := &controller{ + ctx: util.NewCancelableContext(), + node: node, + nbClient: nbClient, + networkIDs: map[int]string{}, + networks: map[string]*netInfo{}, + tables: map[int]int{}, + log: klog.LoggerWithName(klog.Background(), controllerName), + netlink: util.GetNetLinkOps(), + } + + c.reconciler = controllerutil.NewReconciler( + controllerName, + &controllerutil.ReconcilerConfig{ + Threadiness: 1, + Reconcile: c.syncNetwork, + RateLimiter: workqueue.NewTypedItemFastSlowRateLimiter[string](time.Second, 5*time.Second, 5), + }, + ) + + return c +} + +type netInfo struct { + util.NetInfo + id int + table int +} + +type controller struct { + ctx util.CancelableContext + nbClient client.Client + node string + log logr.Logger + reconciler controllerutil.Reconciler + netlink util.NetLinkOps + + sync.RWMutex + networkIDs map[int]string + networks map[string]*netInfo + tables map[int]int +} + +func (c *controller) AddNetwork(network util.NetInfo) error { + c.Lock() + defer c.Unlock() + + networkID := network.GetNetworkID() + if c.networkIDs[networkID] != "" { + return fmt.Errorf("already tracking network %q with ID %d", + c.networkIDs[networkID], + networkID, + ) + } + + name := network.GetNetworkName() + if c.networks[name] != nil { + // this shouldn't happen as the network ID is correlated uniquely with + // the network name, but do the check anyway in case this is not being + // handled correctly + return fmt.Errorf("already tracking network name %q", name) + } + + info := &netInfo{NetInfo: network, id: networkID, table: noTable} + if network.IsDefault() { + c.tables[unix.RT_TABLE_MAIN] = networkID + info.table = unix.RT_TABLE_MAIN + } + c.networkIDs[networkID] = name + c.networks[name] = info + + c.log.V(5).Info("Started tracking network", "name", name, "id", networkID) + c.reconcile(name) + + return nil +} + +func (c *controller) ForgetNetwork(name string) { + c.Lock() + defer c.Unlock() + + network := c.networks[name] + if network == nil { + return + } + + delete(c.networkIDs, network.id) + delete(c.networks, name) + + c.log.V(5).Info("Stopped tracking network", "name", name) +} + +func (c *controller) NeedsReconciliation(network util.NetInfo) bool { + c.RLock() + defer c.RUnlock() + + if c.networks[network.GetNetworkName()] == nil { + return false + } + + // TODO check if overlay mode changed + return false +} + +func (c *controller) ReconcileNetwork(name string) error { + c.RLock() + defer c.RUnlock() + if c.networks[name] == nil { + return fmt.Errorf("unknown network with name %q", name) + } + c.log.V(5).Info("Reconciling network", "name", name) + c.reconcile(name) + return nil +} + +func (c *controller) Start() error { + defer c.log.Info("Controller started") + c.subscribe(c.ctx.Done()) + return controllerutil.Start(c.reconciler) +} + +func (c *controller) Stop() { + controllerutil.Stop(c.reconciler) + c.ctx.Cancel() + c.log.Info("Controller stopped") +} + +func (c *controller) subscribe(stop <-chan struct{}) { + go func() { + onError := func(err error) { + c.log.Error(err, "Error on netlink route event subscription") + } + routeEventCh := subscribeNetlinkRouteEvents(c.netlink, stop, onError) + subscribeTicker := time.NewTicker(subscribePeriod) + defer subscribeTicker.Stop() + for { + select { + case <-stop: + return + case <-subscribeTicker.C: + if routeEventCh != nil { + continue + } + routeEventCh = subscribeNetlinkRouteEvents(c.netlink, stop, onError) + case r, open := <-routeEventCh: + if !open { + routeEventCh = subscribeNetlinkRouteEvents(c.netlink, stop, onError) + continue + } + c.log.V(5).Info("Received route event", "event", r) + c.syncRouteUpdate(&r) + } + } + }() + + go func() { + onError := func(err error) { + c.log.Error(err, "Error on netlink link event subscription") + } + linkEventCh := subscribeNetlinkLinkEvents(c.netlink, stop, onError) + subscribeTicker := time.NewTicker(subscribePeriod) + defer subscribeTicker.Stop() + for { + select { + case <-stop: + return + case <-subscribeTicker.C: + if linkEventCh != nil { + continue + } + linkEventCh = subscribeNetlinkLinkEvents(c.netlink, stop, onError) + case l, open := <-linkEventCh: + if !open { + c.tables = map[int]int{} + linkEventCh = subscribeNetlinkLinkEvents(c.netlink, stop, onError) + continue + } + c.log.V(5).Info("Received link event", "event", l) + c.syncLinkUpdate(&l) + } + } + }() +} + +func (c *controller) syncRouteUpdate(update *netlink.RouteUpdate) { + if update.Protocol != unix.RTPROT_BGP { + return + } + + table := update.Table + network := c.getNetworkForTable(table) + if network != nil { + c.reconcile(network.GetNetworkName()) + } +} + +func (c *controller) syncLinkUpdate(update *netlink.LinkUpdate) { + vrf, isVrf := update.Link.(*netlink.Vrf) + if !isVrf { + return + } + + name := vrf.Name + if !strings.HasPrefix(name, types.UDNVRFDevicePrefix) { + return + } + if !strings.HasSuffix(name, types.UDNVRFDeviceSuffix) { + return + } + + id, err := strconv.Atoi(name[len(types.UDNVRFDevicePrefix) : len(name)-len(types.UDNVRFDeviceSuffix)]) + if err != nil { + c.log.Error(err, "Failed to parse network ID from device name", "name", name) + return + } + + c.Lock() + defer c.Unlock() + + table := int(vrf.Table) + network := c.networkIDs[id] + info := c.networks[network] + current := c.tables[table] == id + var old int + if info != nil { + old = info.table + } + + switch update.IfInfomsg.Type { + case unix.RTM_DELLINK: + if !current { + c.log.Info("Ignoring VRF delete for old network", "network", id) + return + } + delete(c.tables, table) + table = noTable + case unix.RTM_NEWLINK: + delete(c.tables, old) + c.tables[table] = id + default: + c.log.Info("Unexpected VRF update event type", "type", update.IfInfomsg.Type) + return + } + if info != nil { + info.table = table + } + + needsReconcile := info != nil && table != noTable && !current + + c.log.V(5).Info("Associated table with network", "table", table, "network", id, "needsReconcile", needsReconcile) + if needsReconcile { + c.reconcile(network) + } +} + +func (c *controller) reconcile(network string) { + c.reconciler.ReconcileAfter(network, reconcileDelay) +} + +type route struct { + dst string + gw string +} + +type stringer struct { + v any +} + +func (s stringer) String() string { + return fmt.Sprintf("%v", s.v) +} + +func (c *controller) syncNetwork(network string) error { + start := time.Now() + c.log.V(5).Info("Reconciling network", "network", network) + + info := c.getNetwork(network) + if info == nil { + return nil + } + router := info.GetNetworkScopedGWRouterName(c.node) + + // skip routes in the pod network + // TODO do not skip these routes in no overlay mode + ignoreSubnets := make([]*net.IPNet, len(info.Subnets())) + for i, subnet := range info.Subnets() { + ignoreSubnets[i] = subnet.CIDR + } + + table := c.getTableForNetwork(info.id) + if table == noTable { + return nil + } + + expected, err := c.getBGPRoutes(table, ignoreSubnets) + if err != nil { + return err + } + + actual, uuids, err := c.getOVNRoutes(router) + if err != nil { + return fmt.Errorf("failed to get routes from OVN: %w", err) + } + + deletes := actual.Difference(expected) + adds := expected.Difference(actual) + if len(deletes)+len(adds) == 0 { + c.log.V(5).Info("Found no updates for router", "router", router) + return nil + } + c.log.V(5).Info("Found updates for router", "router", router, "adds", stringer{adds}, "deletes", stringer{deletes}) + + var errs []error + var ops []ovsdb.Operation + + p := func(new, db *nbdb.LogicalRouterStaticRoute) bool { + return db.ExternalIDs[controllerExternalIDKey] == controllerName && db.IPPrefix == new.IPPrefix && db.Nexthop == new.Nexthop + } + for add := range adds { + lrsr := &nbdb.LogicalRouterStaticRoute{ + UUID: uuids[add], + IPPrefix: add.dst, + Nexthop: add.gw, + ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}, + } + p := func(db *nbdb.LogicalRouterStaticRoute) bool { return p(lrsr, db) } + ops, err = nbdbops.CreateOrUpdateLogicalRouterStaticRoutesWithPredicateOps(c.nbClient, ops, router, lrsr, p) + if err != nil { + err := fmt.Errorf("failed to add routes on router %s: %w", router, err) + errs = append(errs, err) + continue + } + } + + lrsrs := make([]*nbdb.LogicalRouterStaticRoute, 0, len(deletes)) + for delete := range deletes { + lrsrs = append(lrsrs, &nbdb.LogicalRouterStaticRoute{UUID: uuids[delete]}) + } + if len(lrsrs) > 0 { + ops, err = nbdbops.DeleteLogicalRouterStaticRoutesOps(c.nbClient, ops, router, lrsrs...) + if err != nil { + err := fmt.Errorf("failed to delete routes on router %s: %w", router, err) + errs = append(errs, err) + } + } + + _, err = nbdbops.TransactAndCheck(c.nbClient, ops) + if err != nil { + err := fmt.Errorf("failed to transact ops %v: %w", ops, err) + errs = append(errs, err) + } + + err = errors.Join(errs...) + c.log.V(5).Info("Reconciled network", "network", network, "took", time.Since(start), "ops", ops, "errors", err) + return err +} + +func (c *controller) getBGPRoutes(table int, ignoreSubnets []*net.IPNet) (sets.Set[route], error) { + start := time.Now() + filter := &netlink.Route{ + Protocol: unix.RTPROT_BGP, + Table: table, + } + nlroutes, err := c.netlink.RouteListFiltered(netlink.FAMILY_ALL, filter, netlink.RT_FILTER_PROTOCOL|netlink.RT_FILTER_TABLE) + if err != nil { + return nil, fmt.Errorf("failed to list BGP routes: %w", err) + } + + routes := sets.New[route]() + for _, nlroute := range nlroutes { + if util.IsContainedInAnyCIDR(nlroute.Dst, ignoreSubnets...) { + continue + } + routes.Insert(routesFromNetlinkRoute(&nlroute)...) + } + + c.log.V(5).Info("Listed BGP routes", "table", table, "routes", stringer{routes}, "took", time.Since(start)) + return routes, nil +} + +func (c *controller) getOVNRoutes(router string) (sets.Set[route], map[route]string, error) { + start := time.Now() + lr := &nbdb.LogicalRouter{ + Name: router, + } + p := func(lrsr *nbdb.LogicalRouterStaticRoute) bool { + return lrsr.ExternalIDs[controllerExternalIDKey] == controllerName + } + lrsrs, err := nbdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(c.nbClient, lr, p) + if err != nil { + return nil, nil, fmt.Errorf("failed to get routes from router %s: %w", router, err) + } + uuids := make(map[route]string, len(lrsrs)) + routes := make(sets.Set[route], len(lrsrs)) + for _, lrsr := range lrsrs { + r := route{dst: lrsr.IPPrefix, gw: lrsr.Nexthop} + routes.Insert(r) + uuids[r] = lrsr.UUID + } + c.log.V(5).Info("Listed OVN routes", "router", router, "routes", stringer{routes}, "took", time.Since(start)) + return routes, uuids, nil +} + +func (c *controller) getNetwork(network string) *netInfo { + c.RLock() + defer c.RUnlock() + return c.networks[network] +} + +func (c *controller) getTableForNetwork(network int) int { + c.RLock() + defer c.RUnlock() + if info := c.networks[c.networkIDs[network]]; info != nil { + return info.table + } + return noTable +} + +func (c *controller) getNetworkForTable(table int) *netInfo { + c.RLock() + defer c.RUnlock() + if network, known := c.tables[table]; known { + return c.networks[c.networkIDs[network]] + } + return nil +} + +func routesFromNetlinkRoute(r *netlink.Route) []route { + validIP := func(ip string) bool { + if ip == "" || ip == "" { + return false + } + return true + } + if r.Dst == nil { + return nil + } + dst := r.Dst.String() + if !validIP(dst) { + return nil + } + var routes []route + gw := r.Gw.String() + if validIP(gw) { + routes = append(routes, route{dst: dst, gw: gw}) + } + for _, nh := range r.MultiPath { + gw = nh.Gw.String() + if validIP(gw) { + routes = append(routes, route{dst: dst, gw: gw}) + } + } + return routes +} + +func subscribeNetlinkRouteEvents(nlops util.NetLinkOps, stopCh <-chan struct{}, onError func(error)) chan netlink.RouteUpdate { + routeEventCh := make(chan netlink.RouteUpdate, subscribeBuffer) + options := netlink.RouteSubscribeOptions{ + ErrorCallback: onError, + ListExisting: true, + } + err := nlops.RouteSubscribeWithOptions(routeEventCh, stopCh, options) + if err != nil { + onError(err) + return nil + } + return routeEventCh +} + +func subscribeNetlinkLinkEvents(nlops util.NetLinkOps, stopCh <-chan struct{}, onError func(error)) chan netlink.LinkUpdate { + linkEventCh := make(chan netlink.LinkUpdate, subscribeBuffer) + options := netlink.LinkSubscribeOptions{ + ErrorCallback: onError, + ListExisting: true, + } + if err := nlops.LinkSubscribeWithOptions(linkEventCh, stopCh, options); err != nil { + onError(err) + return nil + } + return linkEventCh +} diff --git a/go-controller/pkg/ovn/routeimport/route_import_test.go b/go-controller/pkg/ovn/routeimport/route_import_test.go new file mode 100644 index 0000000000..bbad567afc --- /dev/null +++ b/go-controller/pkg/ovn/routeimport/route_import_test.go @@ -0,0 +1,469 @@ +package routeimport + +import ( + "errors" + "sync" + "testing" + + "github.com/go-logr/logr/testr" + "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" + "k8s.io/client-go/util/workqueue" + + controllerutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + ovntesting "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/mocks" +) + +func Test_controller_syncNetwork(t *testing.T) { + node := "testnode" + defaultNetwork := &util.DefaultNetInfo{} + type fields struct { + networkIDs map[int]string + networks map[string]*netInfo + tables map[int]int + } + type args struct { + network string + } + tests := []struct { + name string + fields fields + args args + initial []libovsdb.TestData + expected []libovsdb.TestData + routes []netlink.Route + routesErr bool + wantErr bool + }{ + { + name: "reconciled ignored if network not known", + args: args{"default"}, + }, + { + name: "reconciled ignored if network table not known", + args: args{"default"}, + fields: fields{ + networkIDs: map[int]string{0: "default"}, + networks: map[string]*netInfo{"default": {NetInfo: defaultNetwork, id: noTable}}, + }, + }, + { + name: "fails if kernel routes cannot be fetched", + args: args{"default"}, + fields: fields{ + networkIDs: map[int]string{0: "default"}, + networks: map[string]*netInfo{"default": {NetInfo: defaultNetwork, id: 0, table: unix.RT_TABLE_MAIN}}, + }, + routesErr: true, + wantErr: true, + }, + { + name: "fails if OVN routes cannot be fetched (i.e. router does not exist)", + args: args{"default"}, + fields: fields{ + networkIDs: map[int]string{0: "default"}, + networks: map[string]*netInfo{"default": {NetInfo: defaultNetwork, id: 0, table: unix.RT_TABLE_MAIN}}, + }, + wantErr: true, + }, + { + name: "adds and removes routes as necessary", + args: args{"default"}, + fields: fields{ + networkIDs: map[int]string{0: "default"}, + networks: map[string]*netInfo{"default": {NetInfo: defaultNetwork, id: 0, table: unix.RT_TABLE_MAIN}}, + }, + initial: []libovsdb.TestData{ + &nbdb.LogicalRouter{Name: defaultNetwork.GetNetworkScopedGWRouterName(node), StaticRoutes: []string{"keep-1", "keep-2", "remove"}}, + &nbdb.LogicalRouterStaticRoute{UUID: "keep-1", IPPrefix: "1.1.1.0/24", Nexthop: "1.1.1.1", ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + &nbdb.LogicalRouterStaticRoute{UUID: "keep-2", IPPrefix: "5.5.5.0/24", Nexthop: "5.5.5.1"}, + &nbdb.LogicalRouterStaticRoute{UUID: "remove", IPPrefix: "6.6.6.0/24", Nexthop: "6.6.6.1", ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + }, + routes: []netlink.Route{ + {Dst: ovntesting.MustParseIPNet("1.1.1.0/24"), Gw: ovntesting.MustParseIP("1.1.1.1")}, + {Dst: ovntesting.MustParseIPNet("2.2.2.0/24"), Gw: ovntesting.MustParseIP("2.2.2.1")}, + {Dst: ovntesting.MustParseIPNet("3.3.3.0/24"), MultiPath: []*netlink.NexthopInfo{{Gw: ovntesting.MustParseIP("3.3.3.1")}, {Gw: ovntesting.MustParseIP("3.3.3.2")}}}, + }, + expected: []libovsdb.TestData{ + &nbdb.LogicalRouter{UUID: "router", Name: defaultNetwork.GetNetworkScopedGWRouterName(node), StaticRoutes: []string{"keep-1", "keep-2", "add-1", "add-2", "add-3"}}, + &nbdb.LogicalRouterStaticRoute{UUID: "keep-1", IPPrefix: "1.1.1.0/24", Nexthop: "1.1.1.1", ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + &nbdb.LogicalRouterStaticRoute{UUID: "keep-2", IPPrefix: "5.5.5.0/24", Nexthop: "5.5.5.1"}, + &nbdb.LogicalRouterStaticRoute{UUID: "add-1", IPPrefix: "2.2.2.0/24", Nexthop: "2.2.2.1", ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + &nbdb.LogicalRouterStaticRoute{UUID: "add-2", IPPrefix: "3.3.3.0/24", Nexthop: "3.3.3.1", ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + &nbdb.LogicalRouterStaticRoute{UUID: "add-3", IPPrefix: "3.3.3.0/24", Nexthop: "3.3.3.2", ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + nlmock := &mocks.NetLinkOps{} + if info := tt.fields.networks[tt.args.network]; info != nil && info.table != 0 { + matchFilter := func(r *netlink.Route) bool { + return r != nil && r.Equal(netlink.Route{Protocol: unix.RTPROT_BGP, Table: info.table}) + } + nlcall := nlmock.On("RouteListFiltered", netlink.FAMILY_ALL, mock.MatchedBy(matchFilter), netlink.RT_FILTER_PROTOCOL|netlink.RT_FILTER_TABLE) + if tt.routesErr { + nlcall.Return(nil, errors.New("test error")) + } else { + nlcall.Return(tt.routes, nil) + } + } + + client, ctx, err := libovsdb.NewNBTestHarness(libovsdb.TestSetup{NBData: tt.initial}, nil) + g.Expect(err).ToNot(gomega.HaveOccurred()) + t.Cleanup(ctx.Cleanup) + + c := &controller{ + nbClient: client, + node: node, + log: testr.New(t), + networkIDs: tt.fields.networkIDs, + networks: tt.fields.networks, + tables: tt.fields.tables, + netlink: nlmock, + } + + err = c.syncNetwork(tt.args.network) + if tt.wantErr { + g.Expect(err).To(gomega.HaveOccurred()) + return + } + + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(client).To(libovsdb.HaveData(tt.expected...)) + }) + } +} + +func Test_controller_syncRouteUpdate(t *testing.T) { + defaultNetwork := &util.DefaultNetInfo{} + type fields struct { + networkIDs map[int]string + networks map[string]*netInfo + tables map[int]int + } + type args struct { + update *netlink.RouteUpdate + } + tests := []struct { + name string + fields fields + args args + expected []string + }{ + { + name: "ignores route updates with protocol != BGP", + args: args{&netlink.RouteUpdate{Route: netlink.Route{Protocol: unix.RTPROT_STATIC}}}, + }, + { + name: "ignores route updates for unknown tables", + args: args{&netlink.RouteUpdate{Route: netlink.Route{Protocol: unix.RTPROT_BGP, Table: unix.RT_TABLE_UNSPEC}}}, + }, + { + name: "ignores route updates for unknown networks", + fields: fields{tables: map[int]int{unix.RT_TABLE_MAIN: 0}}, + args: args{&netlink.RouteUpdate{Route: netlink.Route{Protocol: unix.RTPROT_BGP, Table: unix.RT_TABLE_MAIN}}}, + }, + { + name: "processes route updates", + fields: fields{ + networkIDs: map[int]string{0: "default"}, + networks: map[string]*netInfo{"default": {NetInfo: defaultNetwork, id: 0, table: unix.RT_TABLE_MAIN}}, + tables: map[int]int{unix.RT_TABLE_MAIN: 0}, + }, + args: args{&netlink.RouteUpdate{Route: netlink.Route{Protocol: unix.RTPROT_BGP, Table: unix.RT_TABLE_MAIN}}}, + expected: []string{"default"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + var reconciled []string + var m sync.Mutex + reconcile := func(key string) error { + m.Lock() + defer m.Unlock() + reconciled = append(reconciled, key) + return nil + } + matchReconcile := func(g gomega.Gomega, expected []string) { + m.Lock() + defer m.Unlock() + g.Expect(reconciled).To(gomega.Equal(expected)) + } + r := controllerutil.NewReconciler( + "test", + &controllerutil.ReconcilerConfig{Reconcile: reconcile, Threadiness: 1, RateLimiter: workqueue.NewTypedItemFastSlowRateLimiter[string](0, 0, 0)}) + c := &controller{ + log: testr.New(t), + networkIDs: tt.fields.networkIDs, + networks: tt.fields.networks, + tables: tt.fields.tables, + reconciler: r, + } + err := controllerutil.Start(r) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + c.syncRouteUpdate(tt.args.update) + + g.Eventually(matchReconcile).WithArguments(tt.expected).Should(gomega.Succeed()) + g.Consistently(matchReconcile).WithArguments(tt.expected).Should(gomega.Succeed()) + }) + } +} + +func Test_controller_syncLinkUpdate(t *testing.T) { + someNetwork := &util.DefaultNetInfo{} + type fields struct { + networkIDs map[int]string + networks map[string]*netInfo + tables map[int]int + } + type args struct { + update *netlink.LinkUpdate + } + tests := []struct { + name string + fields fields + args args + expectTables map[int]int + expectReconciles []string + }{ + { + name: "ignores link updates with type != VRF", + args: args{&netlink.LinkUpdate{Link: &netlink.Dummy{}}}, + }, + { + name: "ignores link updates with incorrect prefix", + args: args{&netlink.LinkUpdate{Link: &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "something10" + types.UDNVRFDeviceSuffix}}}}, + }, + { + name: "ignores link updates with incorrect suffix", + args: args{&netlink.LinkUpdate{Link: &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: types.UDNVRFDevicePrefix + "10-something"}}}}, + }, + { + name: "ignores link updates with incorrect format", + args: args{&netlink.LinkUpdate{Link: &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: types.UDNVRFDevicePrefix + "something" + types.UDNVRFDeviceSuffix}}}}, + }, + { + name: "ignores unknown event types", + fields: fields{ + networkIDs: map[int]string{1: "net1"}, + networks: map[string]*netInfo{"net1": {NetInfo: someNetwork, id: 1, table: 2}}, + tables: map[int]int{2: 1}, + }, + args: args{&netlink.LinkUpdate{ + IfInfomsg: nl.IfInfomsg{IfInfomsg: unix.IfInfomsg{Type: unix.RTM_BASE}}, + Link: &netlink.Vrf{LinkAttrs: netlink.LinkAttrs{Name: types.UDNVRFDevicePrefix + "1" + types.UDNVRFDeviceSuffix}, Table: 2}}, + }, + expectTables: map[int]int{2: 1}, + }, + { + name: "ignores removal of old VRFs", + fields: fields{ + networkIDs: map[int]string{1: "net1", 2: "net2"}, + networks: map[string]*netInfo{"net1": {NetInfo: someNetwork, id: 1, table: 2}, "net2": {NetInfo: someNetwork, id: 2, table: 2}}, + tables: map[int]int{2: 2}, + }, + args: args{&netlink.LinkUpdate{ + IfInfomsg: nl.IfInfomsg{IfInfomsg: unix.IfInfomsg{Type: unix.RTM_DELLINK}}, + Link: &netlink.Vrf{LinkAttrs: netlink.LinkAttrs{Name: types.UDNVRFDevicePrefix + "1" + types.UDNVRFDeviceSuffix}, Table: 2}}, + }, + expectTables: map[int]int{2: 2}, + }, + { + name: "processes link removals", + fields: fields{ + networkIDs: map[int]string{1: "net1"}, + networks: map[string]*netInfo{"net1": {NetInfo: someNetwork, id: 1, table: 2}}, + tables: map[int]int{2: 1}, + }, + args: args{&netlink.LinkUpdate{ + IfInfomsg: nl.IfInfomsg{IfInfomsg: unix.IfInfomsg{Type: unix.RTM_DELLINK}}, + Link: &netlink.Vrf{LinkAttrs: netlink.LinkAttrs{Name: types.UDNVRFDevicePrefix + "1" + types.UDNVRFDeviceSuffix}, Table: 2}}, + }, + expectTables: map[int]int{}, + }, + { + name: "does not reconcile on link updates with no actual changes", + fields: fields{ + networkIDs: map[int]string{1: "net1"}, + networks: map[string]*netInfo{"net1": {NetInfo: someNetwork, id: 1, table: 2}}, + tables: map[int]int{2: 1}, + }, + args: args{&netlink.LinkUpdate{ + IfInfomsg: nl.IfInfomsg{IfInfomsg: unix.IfInfomsg{Type: unix.RTM_NEWLINK}}, + Link: &netlink.Vrf{LinkAttrs: netlink.LinkAttrs{Name: types.UDNVRFDevicePrefix + "1" + types.UDNVRFDeviceSuffix}, Table: 2}}, + }, + expectTables: map[int]int{2: 1}, + }, + { + name: "does reconcile on link updates with actual changes", + fields: fields{ + networkIDs: map[int]string{1: "net1"}, + networks: map[string]*netInfo{"net1": {NetInfo: someNetwork, id: 1, table: 2}}, + tables: map[int]int{2: 1}, + }, + args: args{&netlink.LinkUpdate{ + IfInfomsg: nl.IfInfomsg{IfInfomsg: unix.IfInfomsg{Type: unix.RTM_NEWLINK}}, + Link: &netlink.Vrf{LinkAttrs: netlink.LinkAttrs{Name: types.UDNVRFDevicePrefix + "1" + types.UDNVRFDeviceSuffix}, Table: 3}}, + }, + expectTables: map[int]int{3: 1}, + expectReconciles: []string{"net1"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + var reconciled []string + var m sync.Mutex + reconcile := func(key string) error { + m.Lock() + defer m.Unlock() + reconciled = append(reconciled, key) + return nil + } + matchReconcile := func(g gomega.Gomega, expected []string) { + m.Lock() + defer m.Unlock() + g.Expect(reconciled).To(gomega.Equal(expected)) + } + r := controllerutil.NewReconciler( + "test", + &controllerutil.ReconcilerConfig{Reconcile: reconcile, Threadiness: 1, RateLimiter: workqueue.NewTypedItemFastSlowRateLimiter[string](0, 0, 0)}) + c := &controller{ + log: testr.New(t), + networkIDs: tt.fields.networkIDs, + networks: tt.fields.networks, + tables: tt.fields.tables, + reconciler: r, + } + err := controllerutil.Start(r) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + c.syncLinkUpdate(tt.args.update) + + g.Expect(c.tables).To(gomega.Equal(tt.expectTables)) + g.Eventually(matchReconcile).WithArguments(tt.expectReconciles).Should(gomega.Succeed()) + g.Consistently(matchReconcile).WithArguments(tt.expectReconciles).Should(gomega.Succeed()) + }) + } +} + +func Test_controller_subscribe(t *testing.T) { + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + + var m sync.Mutex + var routeEventCh chan<- netlink.RouteUpdate + var linkEventCh chan<- netlink.LinkUpdate + setRouteEventCh := func(ch chan<- netlink.RouteUpdate) { + m.Lock() + defer m.Unlock() + routeEventCh = ch + } + setLinkEventCh := func(ch chan<- netlink.LinkUpdate) { + m.Lock() + defer m.Unlock() + linkEventCh = ch + } + isRouteEventChSet := func(g gomega.Gomega) { + m.Lock() + defer m.Unlock() + g.Expect(routeEventCh).ToNot(gomega.BeNil()) + } + isLinkEventChSet := func(g gomega.Gomega) { + m.Lock() + defer m.Unlock() + g.Expect(linkEventCh).ToNot(gomega.BeNil()) + } + + matchOptions := func(options any) bool { + switch o := options.(type) { + case netlink.RouteSubscribeOptions: + return o.ListExisting == true + case netlink.LinkSubscribeOptions: + return o.ListExisting == true + } + return false + } + + var stopArg <-chan struct{} = stop + nlmock := &mocks.NetLinkOps{} + nlmock.On("RouteSubscribeWithOptions", mock.AnythingOfType("chan<- netlink.RouteUpdate"), stopArg, mock.MatchedBy(matchOptions)). + Run(func(args mock.Arguments) { setRouteEventCh(args.Get(0).(chan<- netlink.RouteUpdate)) }). + Return(nil).Twice() + nlmock.On("RouteSubscribeWithOptions", mock.AnythingOfType("chan<- netlink.RouteUpdate"), stopArg, mock.MatchedBy(matchOptions)). + Return(errors.New("test error")).Twice() + nlmock.On("RouteSubscribeWithOptions", mock.AnythingOfType("chan<- netlink.RouteUpdate"), stopArg, mock.MatchedBy(matchOptions)). + Run(func(args mock.Arguments) { setRouteEventCh(args.Get(0).(chan<- netlink.RouteUpdate)) }). + Return(nil).Once() + + nlmock.On("LinkSubscribeWithOptions", mock.AnythingOfType("chan<- netlink.LinkUpdate"), stopArg, mock.MatchedBy(matchOptions)). + Run(func(args mock.Arguments) { setLinkEventCh(args.Get(0).(chan<- netlink.LinkUpdate)) }). + Return(nil).Twice() + nlmock.On("LinkSubscribeWithOptions", mock.AnythingOfType("chan<- netlink.LinkUpdate"), stopArg, mock.MatchedBy(matchOptions)). + Return(errors.New("test error")).Twice() + nlmock.On("LinkSubscribeWithOptions", mock.AnythingOfType("chan<- netlink.LinkUpdate"), stopArg, mock.MatchedBy(matchOptions)). + Run(func(args mock.Arguments) { setLinkEventCh(args.Get(0).(chan<- netlink.LinkUpdate)) }). + Return(nil).Once() + nlmock.On("LinkSubscribeWithOptions", mock.AnythingOfType("chan<- netlink.LinkUpdate"), stopArg, mock.MatchedBy(matchOptions)). + Run(func(args mock.Arguments) { setLinkEventCh(args.Get(0).(chan<- netlink.LinkUpdate)) }). + Return(errors.New("test error")) + + c := &controller{ + log: testr.New(t), + netlink: nlmock, + tables: map[int]int{1: 1}, + } + + c.subscribe(stop) + + g := gomega.NewWithT(t) + + g.Eventually(isRouteEventChSet).Should(gomega.Succeed()) + g.Eventually(isLinkEventChSet).Should(gomega.Succeed()) + + rch := routeEventCh + routeEventCh = nil + close(rch) + + g.Eventually(isRouteEventChSet).WithTimeout(subscribePeriod * 3).Should(gomega.Succeed()) + + rch = routeEventCh + routeEventCh = nil + close(rch) + + g.Eventually(isRouteEventChSet).WithTimeout(subscribePeriod * 3).Should(gomega.Succeed()) + + lch := linkEventCh + linkEventCh = nil + close(lch) + + g.Eventually(isLinkEventChSet).WithTimeout(subscribePeriod * 3).Should(gomega.Succeed()) + + lch = linkEventCh + linkEventCh = nil + close(lch) + + g.Eventually(isLinkEventChSet).WithTimeout(subscribePeriod * 3).Should(gomega.Succeed()) + + lch = linkEventCh + linkEventCh = nil + close(lch) + + g.Eventually(isLinkEventChSet).WithTimeout(subscribePeriod * 3).Should(gomega.Succeed()) + g.Expect(c.tables).To(gomega.BeEmpty()) +} diff --git a/go-controller/pkg/ovn/secondary_layer2_network_controller.go b/go-controller/pkg/ovn/secondary_layer2_network_controller.go index 0de1e9c4d6..179c86beef 100644 --- a/go-controller/pkg/ovn/secondary_layer2_network_controller.go +++ b/go-controller/pkg/ovn/secondary_layer2_network_controller.go @@ -256,6 +256,9 @@ type SecondaryLayer2NetworkController struct { // Controller in charge of services svcController *svccontroller.Controller + + // EgressIP controller utilized only to initialize a network with OVN polices to support EgressIP functionality. + eIPController *EgressIPController } // NewSecondaryLayer2NetworkController create a new OVN controller for the given secondary layer2 nad @@ -263,6 +266,8 @@ func NewSecondaryLayer2NetworkController( cnci *CommonNetworkControllerInfo, netInfo util.NetInfo, networkManager networkmanager.Interface, + eIPController *EgressIPController, + portCache *PortCache, ) (*SecondaryLayer2NetworkController, error) { stopChan := make(chan struct{}) @@ -275,23 +280,6 @@ func NewSecondaryLayer2NetworkController( lsManagerFactoryFn = lsm.NewL2SwitchManagerForUserDefinedPrimaryNetwork } - var svcController *svccontroller.Controller - if util.IsNetworkSegmentationSupportEnabled() { - var err error - svcController, err = svccontroller.NewController( - cnci.client, cnci.nbClient, - cnci.watchFactory.ServiceCoreInformer(), - cnci.watchFactory.EndpointSliceCoreInformer(), - cnci.watchFactory.NodeCoreInformer(), - networkManager, - cnci.recorder, - netInfo, - ) - if err != nil { - return nil, fmt.Errorf("unable to create new service controller while creating new layer2 network controller: %w", err) - } - } - oc := &SecondaryLayer2NetworkController{ BaseSecondaryLayer2NetworkController: BaseSecondaryLayer2NetworkController{ @@ -301,7 +289,7 @@ func NewSecondaryLayer2NetworkController( controllerName: getNetworkControllerName(netInfo.GetNetworkName()), ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), lsManager: lsManagerFactoryFn(), - logicalPortCache: NewPortCache(stopChan), + logicalPortCache: portCache, namespaces: make(map[string]*namespaceInfo), namespacesMutex: sync.Mutex{}, addressSetFactory: addressSetFactory, @@ -319,13 +307,29 @@ func NewSecondaryLayer2NetworkController( mgmtPortFailed: sync.Map{}, syncZoneICFailed: sync.Map{}, gatewayManagers: sync.Map{}, - svcController: svcController, + eIPController: eIPController, } if config.OVNKubernetesFeature.EnableInterconnect { oc.zoneICHandler = zoneinterconnect.NewZoneInterconnectHandler(oc.GetNetInfo(), oc.nbClient, oc.sbClient, oc.watchFactory) } + if util.IsNetworkSegmentationSupportEnabled() { + var err error + oc.svcController, err = svccontroller.NewController( + cnci.client, cnci.nbClient, + cnci.watchFactory.ServiceCoreInformer(), + cnci.watchFactory.EndpointSliceCoreInformer(), + cnci.watchFactory.NodeCoreInformer(), + networkManager, + cnci.recorder, + oc.GetNetInfo(), + ) + if err != nil { + return nil, fmt.Errorf("unable to create new service controller while creating new layer2 network controller: %w", err) + } + } + if oc.allocatesPodAnnotation() { var claimsReconciler persistentips.PersistentAllocations if oc.allowPersistentIPs() { @@ -338,7 +342,7 @@ func NewSecondaryLayer2NetworkController( claimsReconciler = ipamClaimsReconciler } oc.podAnnotationAllocator = pod.NewPodAnnotationAllocator( - netInfo, + oc.GetNetInfo(), cnci.watchFactory.PodCoreInformer().Lister(), cnci.kube, claimsReconciler) @@ -542,22 +546,31 @@ func (oc *SecondaryLayer2NetworkController) addUpdateLocalNodeEvent(node *corev1 } } } - if util.IsNetworkSegmentationSupportEnabled() && oc.IsPrimaryNetwork() { - if nSyncs.syncMgmtPort { - // Layer 2 networks have a single, large subnet, that's the one - // associated to the controller. Take the management port IP from - // there. - subnets := oc.Subnets() - hostSubnets := make([]*net.IPNet, 0, len(subnets)) - for _, subnet := range oc.Subnets() { - hostSubnets = append(hostSubnets, subnet.CIDR) - } - if _, err := oc.syncNodeManagementPort(node, oc.GetNetworkScopedSwitchName(types.OVNLayer2Switch), oc.GetNetworkScopedGWRouterName(node.Name), hostSubnets); err != nil { - errs = append(errs, err) - oc.mgmtPortFailed.Store(node.Name, true) - } else { - oc.mgmtPortFailed.Delete(node.Name) - } + + if nSyncs.syncMgmtPort { + // Layer 2 networks have a single, large subnet, that's the one + // associated to the controller. Take the management port IP from + // there. + subnets := oc.Subnets() + hostSubnets := make([]*net.IPNet, 0, len(subnets)) + for _, subnet := range oc.Subnets() { + hostSubnets = append(hostSubnets, subnet.CIDR) + } + if _, err := oc.syncNodeManagementPort(node, oc.GetNetworkScopedSwitchName(types.OVNLayer2Switch), + oc.GetNetworkScopedGWRouterName(node.Name), hostSubnets); err != nil { + errs = append(errs, err) + oc.mgmtPortFailed.Store(node.Name, true) + } else { + oc.mgmtPortFailed.Delete(node.Name) + } + } + + if config.OVNKubernetesFeature.EnableEgressIP { + if err := oc.eIPController.ensureRouterPoliciesForNetwork(oc.GetNetInfo()); err != nil { + errs = append(errs, fmt.Errorf("failed to ensure EgressIP router policies for network %s: %v", oc.GetNetworkName(), err)) + } + if err := oc.eIPController.ensureSwitchPoliciesForNode(oc.GetNetInfo(), node.Name); err != nil { + errs = append(errs, fmt.Errorf("failed to ensure EgressIP switch policies for network %s: %v", oc.GetNetworkName(), err)) } } } @@ -788,16 +801,12 @@ func (oc *SecondaryLayer2NetworkController) gatewayOptions() []GatewayOption { } func (oc *SecondaryLayer2NetworkController) StartServiceController(wg *sync.WaitGroup, runRepair bool) error { - wg.Add(1) - go func() { - defer wg.Done() - useLBGroups := oc.clusterLoadBalancerGroupUUID != "" - // use 5 workers like most of the kubernetes controllers in the kubernetes controller-manager - // do not use LB templates for UDNs - OVN bug https://issues.redhat.com/browse/FDP-988 - err := oc.svcController.Run(5, oc.stopChan, runRepair, useLBGroups, false) - if err != nil { - klog.Errorf("Error running OVN Kubernetes Services controller for network %s: %v", oc.GetNetworkName(), err) - } - }() + useLBGroups := oc.clusterLoadBalancerGroupUUID != "" + // use 5 workers like most of the kubernetes controllers in the kubernetes controller-manager + // do not use LB templates for UDNs - OVN bug https://issues.redhat.com/browse/FDP-988 + err := oc.svcController.Run(5, oc.stopChan, wg, runRepair, useLBGroups, false) + if err != nil { + return fmt.Errorf("error running OVN Kubernetes Services controller for network %s: %v", oc.GetNetworkName(), err) + } return nil } diff --git a/go-controller/pkg/ovn/secondary_layer2_network_controller_test.go b/go-controller/pkg/ovn/secondary_layer2_network_controller_test.go index 28ff111643..427bef1faa 100644 --- a/go-controller/pkg/ovn/secondary_layer2_network_controller_test.go +++ b/go-controller/pkg/ovn/secondary_layer2_network_controller_test.go @@ -93,6 +93,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer2 network", func() { Expect(err).NotTo(HaveOccurred()) Expect(setupFakeOvnForLayer2Topology(fakeOvn, initialDB, netInfo, testNode, podInfo, pod)).To(Succeed()) + defer fakeOvn.networkManager.Stop() // for layer2 on interconnect, it is the cluster manager that // allocates the OVN annotation; on unit tests, this just @@ -199,6 +200,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer2 network", func() { Expect(err).NotTo(HaveOccurred()) Expect(setupFakeOvnForLayer2Topology(fakeOvn, initialDB, netInfo, testNode, sourcePodInfo, sourcePod)).To(Succeed()) + defer fakeOvn.networkManager.Stop() // for layer2 on interconnect, it is the cluster manager that // allocates the OVN annotation; on unit tests, this just @@ -350,6 +352,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer2 network", func() { *netConf, ) Expect(err).NotTo(HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} const nodeIPv4CIDR = "192.168.126.202/24" testNode, err := newNodeWithSecondaryNets(nodeName, nodeIPv4CIDR, netInfo) @@ -424,6 +427,8 @@ var _ = Describe("OVN Multi-Homed pod operations for layer2 network", func() { networkConfig, nodeName, fakeNetworkManager, + nil, + NewPortCache(ctx.Done()), ).Cleanup()).To(Succeed()) Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData([]libovsdbtest.TestData{nbZone})) @@ -673,8 +678,10 @@ func newSecondaryLayer2NetworkController( netInfo util.NetInfo, nodeName string, networkManager networkmanager.Interface, + eIPController *EgressIPController, + portCache *PortCache, ) *SecondaryLayer2NetworkController { - layer2NetworkController, _ := NewSecondaryLayer2NetworkController(cnci, netInfo, networkManager) + layer2NetworkController, _ := NewSecondaryLayer2NetworkController(cnci, netInfo, networkManager, eIPController, portCache) layer2NetworkController.gatewayManagers.Store( nodeName, newDummyGatewayManager(cnci.kube, cnci.nbClient, netInfo, cnci.watchFactory, nodeName), @@ -705,6 +712,7 @@ func setupFakeOvnForLayer2Topology(fakeOvn *FakeOVN, initialDB libovsdbtest.Test *netInfo.netconf(), ) Expect(err).NotTo(HaveOccurred()) + nad.Annotations = map[string]string{ovntypes.OvnNetworkIDAnnotation: secondaryNetworkID} By("setting up the OVN DB without any entities in it") Expect(netInfo.setupOVNDependencies(&initialDB)).To(Succeed()) diff --git a/go-controller/pkg/ovn/secondary_layer3_network_controller.go b/go-controller/pkg/ovn/secondary_layer3_network_controller.go index 0a862562d9..441011511f 100644 --- a/go-controller/pkg/ovn/secondary_layer3_network_controller.go +++ b/go-controller/pkg/ovn/secondary_layer3_network_controller.go @@ -320,30 +320,9 @@ func NewSecondaryLayer3NetworkController( stopChan := make(chan struct{}) ipv4Mode, ipv6Mode := netInfo.IPMode() - var zoneICHandler *zoneic.ZoneInterconnectHandler - if config.OVNKubernetesFeature.EnableInterconnect { - zoneICHandler = zoneic.NewZoneInterconnectHandler(netInfo, cnci.nbClient, cnci.sbClient, cnci.watchFactory) - } addressSetFactory := addressset.NewOvnAddressSetFactory(cnci.nbClient, ipv4Mode, ipv6Mode) - var svcController *svccontroller.Controller - if util.IsNetworkSegmentationSupportEnabled() && netInfo.IsPrimaryNetwork() { - var err error - svcController, err = svccontroller.NewController( - cnci.client, cnci.nbClient, - cnci.watchFactory.ServiceCoreInformer(), - cnci.watchFactory.EndpointSliceCoreInformer(), - cnci.watchFactory.NodeCoreInformer(), - networkManager, - cnci.recorder, - netInfo, - ) - if err != nil { - return nil, fmt.Errorf("unable to create new service controller for network=%s: %w", netInfo.GetNetworkName(), err) - } - } - oc := &SecondaryLayer3NetworkController{ BaseSecondaryNetworkController: BaseSecondaryNetworkController{ BaseNetworkController: BaseNetworkController{ @@ -361,7 +340,6 @@ func NewSecondaryLayer3NetworkController( stopChan: stopChan, wg: &sync.WaitGroup{}, localZoneNodes: &sync.Map{}, - zoneICHandler: zoneICHandler, cancelableCtx: util.NewCancelableContext(), networkManager: networkManager, }, @@ -373,13 +351,32 @@ func NewSecondaryLayer3NetworkController( gatewaysFailed: sync.Map{}, gatewayTopologyFactory: topology.NewGatewayTopologyFactory(cnci.nbClient), gatewayManagers: sync.Map{}, - svcController: svcController, eIPController: eIPController, } + if config.OVNKubernetesFeature.EnableInterconnect { + oc.zoneICHandler = zoneic.NewZoneInterconnectHandler(oc.GetNetInfo(), cnci.nbClient, cnci.sbClient, cnci.watchFactory) + } + + if util.IsNetworkSegmentationSupportEnabled() && netInfo.IsPrimaryNetwork() { + var err error + oc.svcController, err = svccontroller.NewController( + cnci.client, cnci.nbClient, + cnci.watchFactory.ServiceCoreInformer(), + cnci.watchFactory.EndpointSliceCoreInformer(), + cnci.watchFactory.NodeCoreInformer(), + networkManager, + cnci.recorder, + oc.GetNetInfo(), + ) + if err != nil { + return nil, fmt.Errorf("unable to create new service controller for network=%s: %w", netInfo.GetNetworkName(), err) + } + } + if oc.allocatesPodAnnotation() { podAnnotationAllocator := pod.NewPodAnnotationAllocator( - netInfo, + oc.GetNetInfo(), cnci.watchFactory.PodCoreInformer().Lister(), cnci.kube, nil) @@ -764,11 +761,11 @@ func (oc *SecondaryLayer3NetworkController) addUpdateLocalNodeEvent(node *kapi.N } if config.OVNKubernetesFeature.EnableEgressIP && util.IsNetworkSegmentationSupportEnabled() && oc.IsPrimaryNetwork() { - if err = oc.eIPController.ensureL3ClusterRouterPoliciesForNetwork(oc.GetNetInfo()); err != nil { - errs = append(errs, fmt.Errorf("failed to add network %s to EgressIP controller: %v", oc.GetNetworkName(), err)) + if err = oc.eIPController.ensureRouterPoliciesForNetwork(oc.GetNetInfo()); err != nil { + errs = append(errs, fmt.Errorf("failed to ensure EgressIP router polices for network %s: %v", oc.GetNetworkName(), err)) } - if err = oc.eIPController.ensureL3SwitchPoliciesForNode(oc.GetNetInfo(), node.Name); err != nil { - errs = append(errs, fmt.Errorf("failed to ensure EgressIP switch policies: %v", err)) + if err = oc.eIPController.ensureSwitchPoliciesForNode(oc.GetNetInfo(), node.Name); err != nil { + errs = append(errs, fmt.Errorf("failed to ensure EgressIP switch policies for network %s: %v", oc.GetNetworkName(), err)) } } @@ -1075,16 +1072,12 @@ func (oc *SecondaryLayer3NetworkController) gatewayManagerForNode(nodeName strin } func (oc *SecondaryLayer3NetworkController) StartServiceController(wg *sync.WaitGroup, runRepair bool) error { - wg.Add(1) - go func() { - defer wg.Done() - useLBGroups := oc.clusterLoadBalancerGroupUUID != "" - // use 5 workers like most of the kubernetes controllers in the kubernetes controller-manager - // do not use LB templates for UDNs - OVN bug https://issues.redhat.com/browse/FDP-988 - err := oc.svcController.Run(5, oc.stopChan, runRepair, useLBGroups, false) - if err != nil { - klog.Errorf("Error running OVN Kubernetes Services controller for network %s: %v", oc.GetNetworkName(), err) - } - }() + useLBGroups := oc.clusterLoadBalancerGroupUUID != "" + // use 5 workers like most of the kubernetes controllers in the kubernetes controller-manager + // do not use LB templates for UDNs - OVN bug https://issues.redhat.com/browse/FDP-988 + err := oc.svcController.Run(5, oc.stopChan, wg, runRepair, useLBGroups, false) + if err != nil { + return fmt.Errorf("error running OVN Kubernetes Services controller for network %s: %v", oc.GetNetworkName(), err) + } return nil } diff --git a/go-controller/pkg/ovn/secondary_layer3_network_controller_test.go b/go-controller/pkg/ovn/secondary_layer3_network_controller_test.go index 9a5adeb2b0..c587ab9105 100644 --- a/go-controller/pkg/ovn/secondary_layer3_network_controller_test.go +++ b/go-controller/pkg/ovn/secondary_layer3_network_controller_test.go @@ -31,7 +31,6 @@ import ( libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" testnm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" - ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" networkingv1 "k8s.io/api/networking/v1" ) @@ -49,6 +48,7 @@ const ( nadName = "blue-net" ns = "namespace1" secondaryNetworkName = "isolatednet" + secondaryNetworkID = "2" denyPolicyName = "deny-all-policy" denyPG = "deny-port-group" ) @@ -113,6 +113,7 @@ var _ = Describe("OVN Multi-Homed pod operations", func() { *netInfo.netconf(), ) Expect(err).NotTo(HaveOccurred()) + nad.Annotations = map[string]string{types.OvnNetworkIDAnnotation: secondaryNetworkID} Expect(netInfo.setupOVNDependencies(&initialDB)).To(Succeed()) if netInfo.isPrimary { networkConfig, err := util.NewNetInfo(netInfo.netconf()) @@ -311,6 +312,7 @@ var _ = Describe("OVN Multi-Homed pod operations", func() { *netConf, ) Expect(err).NotTo(HaveOccurred()) + nad.Annotations = map[string]string{types.OvnNetworkIDAnnotation: secondaryNetworkID} mutableNetworkConfig := util.NewMutableNetInfo(networkConfig) mutableNetworkConfig.SetNADs(util.GetNADName(nad.Namespace, nad.Name)) @@ -325,7 +327,7 @@ var _ = Describe("OVN Multi-Homed pod operations", func() { testNode, err := newNodeWithSecondaryNets(nodeName, nodeIPv4CIDR, netInfo) Expect(err).NotTo(HaveOccurred()) - nbZone := &nbdb.NBGlobal{Name: ovntypes.OvnDefaultZone, UUID: ovntypes.OvnDefaultZone} + nbZone := &nbdb.NBGlobal{Name: types.OvnDefaultZone, UUID: types.OvnDefaultZone} defaultNetExpectations := emptyDefaultClusterNetworkNodeSwitch(podInfo.nodeName) defaultNetExpectations = append(defaultNetExpectations, nbZone) gwConfig, err := util.ParseNodeL3GatewayAnnotation(testNode) @@ -636,7 +638,7 @@ func newNodeWithSecondaryNets(nodeName string, nodeIPv4CIDR string, netInfos ... "k8s.ovn.org/zone-name": "global", "k8s.ovn.org/l3-gateway-config": fmt.Sprintf("{\"default\":{\"mode\":\"shared\",\"bridge-id\":\"breth0\",\"interface-id\":\"breth0_ovn-worker\",\"mac-address\":%q,\"ip-addresses\":[%[2]q],\"ip-address\":%[2]q,\"next-hops\":[%[3]q],\"next-hop\":%[3]q,\"node-port-enable\":\"true\",\"vlan-id\":\"0\"}}", util.IPAddrToHWAddr(nodeIP), nodeCIDR, nextHopIP), util.OvnNodeChassisID: "abdcef", - "k8s.ovn.org/network-ids": "{\"default\":\"0\",\"isolatednet\":\"2\"}", + "k8s.ovn.org/network-ids": fmt.Sprintf("{\"default\":\"0\",\"isolatednet\":\"%s\"}", secondaryNetworkID), util.OVNNodeGRLRPAddrs: fmt.Sprintf("{\"isolatednet\":{\"ipv4\":%q}}", gwRouterJoinIPAddress()), "k8s.ovn.org/udn-layer2-node-gateway-router-lrp-tunnel-ids": "{\"isolatednet\":\"25\"}", }, @@ -745,7 +747,7 @@ func expectedGWRouterPlusNATAndStaticRoutes( } func expectedStaticMACBindings(gwRouterName string, ips []net.IP) []libovsdbtest.TestData { - lrpName := fmt.Sprintf("%s%s", ovntypes.GWRouterToExtSwitchPrefix, gwRouterName) + lrpName := fmt.Sprintf("%s%s", types.GWRouterToExtSwitchPrefix, gwRouterName) var bindings []libovsdbtest.TestData for _, ip := range ips { bindings = append(bindings, &nbdb.StaticMACBinding{ diff --git a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go index 5136378814..1e41e32d9b 100644 --- a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go +++ b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go @@ -159,12 +159,15 @@ func getTransitSwitchName(nInfo util.NetInfo) string { } func (zic *ZoneInterconnectHandler) createOrUpdateTransitSwitch(networkID int) error { + externalIDs := make(map[string]string) + if zic.IsSecondary() { + externalIDs = getSecondaryNetTransitSwitchExtIDs(zic.GetNetworkName(), zic.TopologyType(), zic.IsPrimaryNetwork()) + } ts := &nbdb.LogicalSwitch{ - Name: zic.networkTransitSwitchName, + Name: zic.networkTransitSwitchName, + ExternalIDs: externalIDs, } - zic.addTransitSwitchConfig(ts, networkID) - // Create transit switch if it doesn't exist if err := libovsdbops.CreateOrUpdateLogicalSwitch(zic.nbClient, ts); err != nil { return fmt.Errorf("failed to create/update transit switch %s: %w", zic.networkTransitSwitchName, err) @@ -777,3 +780,11 @@ func (zic *ZoneInterconnectHandler) getNetworkIdFromNodes(nodes []*corev1.Node) return util.InvalidID, fmt.Errorf("could not find network ID: %w", err) } + +func getSecondaryNetTransitSwitchExtIDs(networkName, topology string, isPrimaryUDN bool) map[string]string { + return map[string]string{ + types.NetworkExternalID: networkName, + types.NetworkRoleExternalID: util.GetUserDefinedNetworkRole(isPrimaryUDN), + types.TopologyExternalID: topology, + } +} diff --git a/go-controller/pkg/testing/libovsdb/matchers.go b/go-controller/pkg/testing/libovsdb/matchers.go index 038e3df4d8..f9e85e6014 100644 --- a/go-controller/pkg/testing/libovsdb/matchers.go +++ b/go-controller/pkg/testing/libovsdb/matchers.go @@ -224,9 +224,11 @@ func HaveEmptyData() gomegatypes.GomegaMatcher { } func haveData(ignoreUUIDs, nameUUIDs bool, expected []TestData) gomegatypes.GomegaMatcher { - if e, ok := expected[0].([]TestData); len(expected) == 1 && ok { - // flatten - expected = e + if len(expected) == 1 { + if e, ok := expected[0].([]TestData); ok { + // flatten + expected = e + } } matchers := []*testDataMatcher{} for _, e := range expected { diff --git a/go-controller/pkg/testing/net.go b/go-controller/pkg/testing/net.go index c95265f242..0ecc3fb125 100644 --- a/go-controller/pkg/testing/net.go +++ b/go-controller/pkg/testing/net.go @@ -1,6 +1,7 @@ package testing import ( + "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/vishvananda/netlink" ) @@ -8,6 +9,7 @@ import ( // AddLink sets up a dummy link for testing networking implementations // on the node side func AddLink(name string) netlink.Link { + ginkgo.GinkgoHelper() err := netlink.LinkAdd(&netlink.Dummy{ LinkAttrs: netlink.LinkAttrs{ Name: name, @@ -22,6 +24,7 @@ func AddLink(name string) netlink.Link { } func DelLink(name string) { + ginkgo.GinkgoHelper() origLink, err := netlink.LinkByName(name) Expect(err).NotTo(HaveOccurred()) err = netlink.LinkDel(origLink) @@ -29,6 +32,7 @@ func DelLink(name string) { } func AddVRFLink(name string, tableId uint32) netlink.Link { + ginkgo.GinkgoHelper() vrfLink := &netlink.Vrf{ LinkAttrs: netlink.LinkAttrs{Name: name}, Table: tableId, diff --git a/go-controller/pkg/testing/networkmanager/fake.go b/go-controller/pkg/testing/networkmanager/fake.go index bd9774a6b9..d52bbb39e6 100644 --- a/go-controller/pkg/testing/networkmanager/fake.go +++ b/go-controller/pkg/testing/networkmanager/fake.go @@ -40,6 +40,10 @@ func (fcm *FakeControllerManager) GetDefaultNetworkController() networkmanager.R return nil } +func (fcm *FakeControllerManager) Reconcile(name string, old, new util.NetInfo) error { + return nil +} + type FakeNetworkManager struct { // namespace -> netInfo PrimaryNetworks map[string]util.NetInfo @@ -84,8 +88,5 @@ func (nc *FakeNetworkManager) DoWithLock(f func(network util.NetInfo) error) err errs = append(errs, err) } } - if len(errs) > 0 { - return errors.Join(errs...) - } - return nil + return errors.Join(errs...) } diff --git a/go-controller/pkg/testing/parse.go b/go-controller/pkg/testing/parse.go index 103777eda5..56889dbc70 100644 --- a/go-controller/pkg/testing/parse.go +++ b/go-controller/pkg/testing/parse.go @@ -3,6 +3,7 @@ package testing import ( "fmt" "net" + "strconv" ) // MustParseIP is like net.ParseIP but it panics on error; use this for converting @@ -69,3 +70,11 @@ func MustParseMAC(macStr string) net.HardwareAddr { } return mac } + +func MustAtoi(s string) int { + i, err := strconv.Atoi(s) + if err != nil { + panic(fmt.Sprintf("Could not parse %q as a int: %v", s, err)) + } + return i +} diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index 22d92cbf24..9c6afae5ba 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -159,6 +159,12 @@ const ( // DefaultNetworkLabelSelector is the label that needs to be matched on a // selector to select the default network DefaultNetworkLabelSelector = OvnK8sPrefix + "/default-network" + // OvnNetworkNameAnnotation is the name of the network annotated on the NAD + // by cluster manager nad controller + OvnNetworkNameAnnotation = OvnK8sPrefix + "/network-name" + // OvnNetworkIDAnnotation is a unique network identifier annotated on the + // NAD by cluster manager nad controller + OvnNetworkIDAnnotation = OvnK8sPrefix + "/network-id" // Deprecated: we used to set topology version as an annotation on the node. We don't do this anymore. OvnK8sTopoAnno = OvnK8sPrefix + "/" + "topology-version" diff --git a/go-controller/pkg/util/fake_client.go b/go-controller/pkg/util/fake_client.go index e11ec5bb39..db4ffa32a9 100644 --- a/go-controller/pkg/util/fake_client.go +++ b/go-controller/pkg/util/fake_client.go @@ -5,6 +5,8 @@ import ( mnpfake "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/client/clientset/versioned/fake" nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" nadfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" + frrapi "github.com/metallb/frr-k8s/api/v1beta1" + frrfake "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake" ocpcloudnetworkapi "github.com/openshift/api/cloudnetwork/v1" ocpnetworkapiv1alpha1 "github.com/openshift/api/network/v1alpha1" cloudservicefake "github.com/openshift/client-go/cloudnetwork/clientset/versioned/fake" @@ -48,6 +50,7 @@ func GetOVNClientset(objects ...runtime.Object) *OVNClientset { dnsNameResolverObjects := []runtime.Object{} udnObjects := []runtime.Object{} raObjects := []runtime.Object{} + frrObjects := []runtime.Object{} for _, object := range objects { switch object.(type) { case *egressip.EgressIP: @@ -74,6 +77,8 @@ func GetOVNClientset(objects ...runtime.Object) *OVNClientset { udnObjects = append(udnObjects, object) case *routeadvertisements.RouteAdvertisements: raObjects = append(raObjects, object) + case *frrapi.FRRConfiguration: + frrObjects = append(frrObjects, object) default: v1Objects = append(v1Objects, object) } @@ -100,6 +105,7 @@ func GetOVNClientset(objects ...runtime.Object) *OVNClientset { OCPNetworkClient: ocpnetworkclientfake.NewSimpleClientset(dnsNameResolverObjects...), UserDefinedNetworkClient: udnfake.NewSimpleClientset(udnObjects...), RouteAdvertisementsClient: routeadvertisementsfake.NewSimpleClientset(raObjects...), + FRRClient: frrfake.NewSimpleClientset(frrObjects...), } } diff --git a/go-controller/pkg/util/kube.go b/go-controller/pkg/util/kube.go index 6822a68d83..0042093e58 100644 --- a/go-controller/pkg/util/kube.go +++ b/go-controller/pkg/util/kube.go @@ -18,6 +18,7 @@ import ( certificatesv1 "k8s.io/api/certificates/v1" kapi "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -38,6 +39,7 @@ import ( ipamclaimssclientset "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1/apis/clientset/versioned" multinetworkpolicyclientset "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/client/clientset/versioned" networkattchmentdefclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" + frrclientset "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" ocpcloudnetworkclientset "github.com/openshift/client-go/cloudnetwork/clientset/versioned" ocpnetworkclientset "github.com/openshift/client-go/network/clientset/versioned" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" @@ -67,6 +69,7 @@ type OVNClientset struct { IPAMClaimsClient ipamclaimssclientset.Interface UserDefinedNetworkClient userdefinednetworkclientset.Interface RouteAdvertisementsClient routeadvertisementsclientset.Interface + FRRClient frrclientset.Interface } // OVNMasterClientset @@ -85,6 +88,7 @@ type OVNMasterClientset struct { NetworkAttchDefClient networkattchmentdefclientset.Interface UserDefinedNetworkClient userdefinednetworkclientset.Interface RouteAdvertisementsClient routeadvertisementsclientset.Interface + FRRClient frrclientset.Interface } // OVNKubeControllerClientset @@ -128,6 +132,7 @@ type OVNClusterManagerClientset struct { OCPNetworkClient ocpnetworkclientset.Interface UserDefinedNetworkClient userdefinednetworkclientset.Interface RouteAdvertisementsClient routeadvertisementsclientset.Interface + FRRClient frrclientset.Interface } const ( @@ -156,6 +161,7 @@ func (cs *OVNClientset) GetMasterClientset() *OVNMasterClientset { NetworkAttchDefClient: cs.NetworkAttchDefClient, UserDefinedNetworkClient: cs.UserDefinedNetworkClient, RouteAdvertisementsClient: cs.RouteAdvertisementsClient, + FRRClient: cs.FRRClient, } } @@ -210,6 +216,7 @@ func (cs *OVNClientset) GetClusterManagerClientset() *OVNClusterManagerClientset OCPNetworkClient: cs.OCPNetworkClient, UserDefinedNetworkClient: cs.UserDefinedNetworkClient, RouteAdvertisementsClient: cs.RouteAdvertisementsClient, + FRRClient: cs.FRRClient, } } @@ -509,6 +516,11 @@ func NewOVNClientset(conf *config.KubernetesConfig) (*OVNClientset, error) { return nil, err } + frrClientset, err := frrclientset.NewForConfig(kconfig) + if err != nil { + return nil, err + } + return &OVNClientset{ KubeClient: kclientset, ANPClient: anpClientset, @@ -524,6 +536,7 @@ func NewOVNClientset(conf *config.KubernetesConfig) (*OVNClientset, error) { IPAMClaimsClient: ipamClaimsClientset, UserDefinedNetworkClient: userDefinedNetworkClientSet, RouteAdvertisementsClient: routeAdvertisementsClientset, + FRRClient: frrClientset, }, nil } @@ -911,3 +924,22 @@ func getEndpointsFromEndpointSlices(endpointSlices []*discovery.EndpointSlice) [ func GetConntrackZone() int { return config.Default.ConntrackZone } + +// IsLastUpdatedByManager checks if an object was updated by the manager last, +// as indicated by a set of managed fields. +func IsLastUpdatedByManager(manager string, managedFields []metav1.ManagedFieldsEntry) bool { + var lastUpdateTheirs, lastUpdateOurs time.Time + for _, managedFieldEntry := range managedFields { + switch managedFieldEntry.Manager { + case manager: + if managedFieldEntry.Time.Time.After(lastUpdateOurs) { + lastUpdateOurs = managedFieldEntry.Time.Time + } + default: + if managedFieldEntry.Time.Time.After(lastUpdateTheirs) { + lastUpdateTheirs = managedFieldEntry.Time.Time + } + } + } + return lastUpdateOurs.After(lastUpdateTheirs) +} diff --git a/go-controller/pkg/util/mocks/NetLinkOps.go b/go-controller/pkg/util/mocks/NetLinkOps.go index ea7374c40e..7c7c7ef121 100644 --- a/go-controller/pkg/util/mocks/NetLinkOps.go +++ b/go-controller/pkg/util/mocks/NetLinkOps.go @@ -433,6 +433,24 @@ func (_m *NetLinkOps) LinkSetVfHardwareAddr(pfLink netlink.Link, vfIndex int, hw return r0 } +// LinkSubscribeWithOptions provides a mock function with given fields: ch, done, options +func (_m *NetLinkOps) LinkSubscribeWithOptions(ch chan<- netlink.LinkUpdate, done <-chan struct{}, options netlink.LinkSubscribeOptions) error { + ret := _m.Called(ch, done, options) + + if len(ret) == 0 { + panic("no return value specified for LinkSubscribeWithOptions") + } + + var r0 error + if rf, ok := ret.Get(0).(func(chan<- netlink.LinkUpdate, <-chan struct{}, netlink.LinkSubscribeOptions) error); ok { + r0 = rf(ch, done, options) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // NeighAdd provides a mock function with given fields: neigh func (_m *NetLinkOps) NeighAdd(neigh *netlink.Neigh) error { ret := _m.Called(neigh) @@ -613,6 +631,24 @@ func (_m *NetLinkOps) RouteReplace(route *netlink.Route) error { return r0 } +// RouteSubscribeWithOptions provides a mock function with given fields: ch, done, options +func (_m *NetLinkOps) RouteSubscribeWithOptions(ch chan<- netlink.RouteUpdate, done <-chan struct{}, options netlink.RouteSubscribeOptions) error { + ret := _m.Called(ch, done, options) + + if len(ret) == 0 { + panic("no return value specified for RouteSubscribeWithOptions") + } + + var r0 error + if rf, ok := ret.Get(0).(func(chan<- netlink.RouteUpdate, <-chan struct{}, netlink.RouteSubscribeOptions) error); ok { + r0 = rf(ch, done, options) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // RuleListFiltered provides a mock function with given fields: family, filter, filterMask func (_m *NetLinkOps) RuleListFiltered(family int, filter *netlink.Rule, filterMask uint64) ([]netlink.Rule, error) { ret := _m.Called(family, filter, filterMask) diff --git a/go-controller/pkg/util/multi_network.go b/go-controller/pkg/util/multi_network.go index 2d2a4bb7dc..a2ab9dc42e 100644 --- a/go-controller/pkg/util/multi_network.go +++ b/go-controller/pkg/util/multi_network.go @@ -32,6 +32,7 @@ var ( type NetInfo interface { // static information, not expected to change. GetNetworkName() string + GetNetworkID() int IsDefault() bool IsPrimaryNetwork() bool IsSecondary() bool @@ -98,6 +99,10 @@ type DefaultNetInfo struct { type MutableNetInfo interface { NetInfo + // SetNetworkID sets the network ID before any controller handles the + // network + SetNetworkID(id int) + // NADs referencing a network SetNADs(nadName ...string) AddNADs(nadName ...string) @@ -209,11 +214,11 @@ func reconcilable(netInfo NetInfo) ReconcilableNetInfo { type mutableNetInfo struct { sync.RWMutex - nads sets.Set[string] + // id of the network. It's mutable because is set on day-1 but it can't be + // changed or reconciled on day-2 + id int - // Each network can be selected by multiple routeadvertisements each with a - // different target VRF and different selected node. Represent this through - // a map of node names to VRFs. + nads sets.Set[string] podNetworkAdvertisements map[string][]string eipAdvertisements map[string][]string @@ -243,18 +248,29 @@ func (l *mutableNetInfo) reconcile(r NetInfo) { } func (l *mutableNetInfo) equals(r *mutableNetInfo) bool { + if (l == nil) != (r == nil) { + return false + } + if l == r { + return true + } l.RLock() defer l.RUnlock() r.RLock() defer r.RUnlock() - return reflect.DeepEqual(l.nads, r.nads) && + return reflect.DeepEqual(l.id, r.id) && + reflect.DeepEqual(l.nads, r.nads) && reflect.DeepEqual(l.podNetworkAdvertisements, r.podNetworkAdvertisements) && reflect.DeepEqual(l.eipAdvertisements, r.eipAdvertisements) } func (l *mutableNetInfo) copyFrom(r *mutableNetInfo) { + if l == r { + return + } aux := mutableNetInfo{} r.RLock() + aux.id = r.id aux.nads = r.nads.Clone() aux.setPodNetworkAdvertisedOnVRFs(r.podNetworkAdvertisements) aux.setEgressIPAdvertisedAtNodes(r.eipAdvertisements) @@ -262,12 +278,23 @@ func (l *mutableNetInfo) copyFrom(r *mutableNetInfo) { r.RUnlock() l.Lock() defer l.Unlock() + l.id = aux.id l.nads = aux.nads l.podNetworkAdvertisements = aux.podNetworkAdvertisements l.eipAdvertisements = aux.eipAdvertisements l.namespaces = aux.namespaces } +func (nInfo *mutableNetInfo) GetNetworkID() int { + return nInfo.id +} + +func (nInfo *mutableNetInfo) SetNetworkID(id int) { + nInfo.Lock() + defer nInfo.Unlock() + nInfo.id = id +} + func (nInfo *mutableNetInfo) SetPodNetworkAdvertisedVRFs(podAdvertisements map[string][]string) { nInfo.Lock() defer nInfo.Unlock() @@ -851,6 +878,7 @@ func newLayer3NetConfInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) joinSubnets: joinSubnets, mtu: netconf.MTU, mutableNetInfo: mutableNetInfo{ + id: InvalidID, nads: sets.Set[string]{}, }, } @@ -877,6 +905,7 @@ func newLayer2NetConfInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) mtu: netconf.MTU, allowPersistentIPs: netconf.AllowPersistentIPs, mutableNetInfo: mutableNetInfo{ + id: InvalidID, nads: sets.Set[string]{}, }, } @@ -900,6 +929,7 @@ func newLocalnetNetConfInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error allowPersistentIPs: netconf.AllowPersistentIPs, physicalNetworkName: netconf.PhysicalNetworkName, mutableNetInfo: mutableNetInfo{ + id: InvalidID, nads: sets.Set[string]{}, }, } @@ -1055,6 +1085,18 @@ func newNetInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) { return ni, nil } +// GetAnnotatedNetworkName gets the network name annotated by cluster manager +// nad controller +func GetAnnotatedNetworkName(netattachdef *nettypes.NetworkAttachmentDefinition) string { + if netattachdef == nil { + return "" + } + if netattachdef.Name == types.DefaultNetworkName && netattachdef.Namespace == config.Kubernetes.OVNConfigNamespace { + return types.DefaultNetworkName + } + return netattachdef.Annotations[types.OvnNetworkNameAnnotation] +} + // ParseNADInfo parses config in NAD spec and return a NetAttachDefInfo object for secondary networks func ParseNADInfo(netattachdef *nettypes.NetworkAttachmentDefinition) (NetInfo, error) { netconf, err := ParseNetConf(netattachdef) @@ -1319,3 +1361,23 @@ func AllowsPersistentIPs(netInfo NetInfo) bool { func IsPodNetworkAdvertisedAtNode(netInfo NetInfo, node string) bool { return len(netInfo.GetPodNetworkAdvertisedOnNodeVRFs(node)) > 0 } + +func GetNetworkVRFName(netInfo NetInfo) string { + if netInfo.GetNetworkName() == types.DefaultNetworkName { + return types.DefaultNetworkName + } + // use the CUDN network name as the VRF name if possible + vrfDeviceName := strings.TrimPrefix(netInfo.GetNetworkName(), "cluster.udn.") + switch { + case len(vrfDeviceName) > 15: + // not possible if longer than the maximum device name length + fallthrough + case vrfDeviceName == netInfo.GetNetworkName(): + // this is not a CUDN + fallthrough + case vrfDeviceName == types.DefaultNetworkName: + // can't be the default network name + return fmt.Sprintf("%s%d%s", types.UDNVRFDevicePrefix, netInfo.GetNetworkID(), types.UDNVRFDeviceSuffix) + } + return vrfDeviceName +} diff --git a/go-controller/pkg/util/net.go b/go-controller/pkg/util/net.go index b13e3da96a..5061bba8b8 100644 --- a/go-controller/pkg/util/net.go +++ b/go-controller/pkg/util/net.go @@ -14,7 +14,7 @@ import ( ) const ( - routingTableIDStart = 1000 + RoutingTableIDStart = 1000 ) var ErrorNoIP = errors.New("no IP available") @@ -254,6 +254,18 @@ func MatchAllIPStringFamily(isIPv6 bool, ipStrings []string) ([]string, error) { return nil, ErrorNoIP } +// MatchAllCIDRStringFamily loops through the array of string and returns a slice +// of addresses in the same IP Family, based on input flag isIPv6. +func MatchAllIPNetsStringFamily(isIPv6 bool, ipnets []string) []string { + var out []string + for _, ipnet := range ipnets { + if utilnet.IsIPv6CIDRString(ipnet) == isIPv6 { + out = append(out, ipnet) + } + } + return out +} + // IsContainedInAnyCIDR returns true if ipnet is contained in any of ipnets func IsContainedInAnyCIDR(ipnet *net.IPNet, ipnets ...*net.IPNet) bool { for _, container := range ipnets { @@ -332,5 +344,5 @@ func IPNetsIPToStringSlice(ips []*net.IPNet) []string { // CalculateRouteTableID will calculate route table ID based on the network // interface index func CalculateRouteTableID(ifIndex int) int { - return ifIndex + routingTableIDStart + return ifIndex + RoutingTableIDStart } diff --git a/go-controller/pkg/util/net_linux.go b/go-controller/pkg/util/net_linux.go index cbe71abb8c..254bd157cc 100644 --- a/go-controller/pkg/util/net_linux.go +++ b/go-controller/pkg/util/net_linux.go @@ -53,6 +53,8 @@ type NetLinkOps interface { NeighList(linkIndex, family int) ([]netlink.Neigh, error) ConntrackDeleteFilter(table netlink.ConntrackTableType, family netlink.InetFamily, filter netlink.CustomConntrackFilter) (uint, error) LinkSetVfHardwareAddr(pfLink netlink.Link, vfIndex int, hwaddr net.HardwareAddr) error + RouteSubscribeWithOptions(ch chan<- netlink.RouteUpdate, done <-chan struct{}, options netlink.RouteSubscribeOptions) error + LinkSubscribeWithOptions(ch chan<- netlink.LinkUpdate, done <-chan struct{}, options netlink.LinkSubscribeOptions) error } type defaultNetLinkOps struct { @@ -187,6 +189,14 @@ func (defaultNetLinkOps) ConntrackDeleteFilter(table netlink.ConntrackTableType, return netlink.ConntrackDeleteFilter(table, family, filter) } +func (defaultNetLinkOps) RouteSubscribeWithOptions(ch chan<- netlink.RouteUpdate, done <-chan struct{}, options netlink.RouteSubscribeOptions) error { + return netlink.RouteSubscribeWithOptions(ch, done, options) +} + +func (defaultNetLinkOps) LinkSubscribeWithOptions(ch chan<- netlink.LinkUpdate, done <-chan struct{}, options netlink.LinkSubscribeOptions) error { + return netlink.LinkSubscribeWithOptions(ch, done, options) +} + func getFamily(ip net.IP) int { if utilnet.IsIPv6(ip) { return netlink.FAMILY_V6 diff --git a/go-controller/pkg/util/node_annotations.go b/go-controller/pkg/util/node_annotations.go index 6ca7e18f7a..33a5d36d78 100644 --- a/go-controller/pkg/util/node_annotations.go +++ b/go-controller/pkg/util/node_annotations.go @@ -907,6 +907,25 @@ func ParseNodeGatewayRouterJoinIPv4(node *kapi.Node, netName string) (net.IP, er return ip, nil } +// ParseNodeGatewayRouterJoinIPv6 returns the IPv6 address for the node's gateway router port +// stored in the 'OVNNodeGRLRPAddrs' annotation +func ParseNodeGatewayRouterJoinIPv6(node *kapi.Node, netName string) (net.IP, error) { + primaryIfAddr, err := ParseNodeGatewayRouterJoinNetwork(node, netName) + if err != nil { + return nil, err + } + if primaryIfAddr.IPv6 == "" { + return nil, fmt.Errorf("failed to find an IPv6 address for gateway route interface in node: %s, net: %s, "+ + "annotation values: %+v", node, netName, primaryIfAddr) + } + + ip, _, err := net.ParseCIDR(primaryIfAddr.IPv6) + if err != nil { + return nil, fmt.Errorf("failed to parse gateway router IPv6 address %s, err: %w", primaryIfAddr.IPv6, err) + } + return ip, nil +} + // ParseNodeGatewayRouterJoinAddrs returns the IPv4 and/or IPv6 addresses for the node's gateway router port // stored in the 'OVNNodeGRLRPAddrs' annotation func ParseNodeGatewayRouterJoinAddrs(node *kapi.Node, netName string) ([]*net.IPNet, error) { diff --git a/go-controller/pkg/util/subnet_annotations.go b/go-controller/pkg/util/subnet_annotations.go index 11964dfbb8..72ec00bf4f 100644 --- a/go-controller/pkg/util/subnet_annotations.go +++ b/go-controller/pkg/util/subnet_annotations.go @@ -209,3 +209,9 @@ func ParseNodesHostSubnetAnnotation(nodes []*kapi.Node, netName string) ([]*net. } return allSubnets, nil } + +// ParseNodeHostSubnetsAnnotation parses parses the "k8s.ovn.org/node-subnets" annotation +// for all the networks +func ParseNodeHostSubnetsAnnotation(node *kapi.Node) (map[string][]*net.IPNet, error) { + return parseSubnetAnnotation(node.Annotations, ovnNodeSubnets) +} diff --git a/go-controller/pkg/util/util.go b/go-controller/pkg/util/util.go index f3f7ab33fb..c7eff45fed 100644 --- a/go-controller/pkg/util/util.go +++ b/go-controller/pkg/util/util.go @@ -137,10 +137,6 @@ func GetNetworkScopedK8sMgmtHostIntfName(networkID uint) string { return intfName } -func GetVRFDeviceNameForUDN(networkID int) string { - return fmt.Sprintf("%s%d%s", types.UDNVRFDevicePrefix, networkID, types.UDNVRFDeviceSuffix) -} - // GetWorkerFromGatewayRouter determines a node's corresponding worker switch name from a gateway router name func GetWorkerFromGatewayRouter(gr string) string { return strings.TrimPrefix(gr, types.GWRouterPrefix) diff --git a/go-controller/vendor/github.com/emicklei/go-restful/v3/CHANGES.md b/go-controller/vendor/github.com/emicklei/go-restful/v3/CHANGES.md index 5edd5a7ca9..92b78048e2 100644 --- a/go-controller/vendor/github.com/emicklei/go-restful/v3/CHANGES.md +++ b/go-controller/vendor/github.com/emicklei/go-restful/v3/CHANGES.md @@ -1,6 +1,24 @@ # Change history of go-restful -## [v3.11.0] - 2023-08-19 + +## [v3.12.1] - 2024-05-28 + +- fix misroute when dealing multiple webservice with regex (#549) (thanks Haitao Chen) + +## [v3.12.0] - 2024-03-11 + +- add Flush method #529 (#538) +- fix: Improper handling of empty POST requests (#543) + +## [v3.11.3] - 2024-01-09 + +- better not have 2 tags on one commit + +## [v3.11.1, v3.11.2] - 2024-01-09 + +- fix by restoring custom JSON handler functions (Mike Beaumont #540) + +## [v3.12.0] - 2023-08-19 - restored behavior as <= v3.9.0 with option to change path strategy using TrimRightSlashEnabled. diff --git a/go-controller/vendor/github.com/emicklei/go-restful/v3/README.md b/go-controller/vendor/github.com/emicklei/go-restful/v3/README.md index e3e30080ec..7234604e47 100644 --- a/go-controller/vendor/github.com/emicklei/go-restful/v3/README.md +++ b/go-controller/vendor/github.com/emicklei/go-restful/v3/README.md @@ -2,7 +2,6 @@ go-restful ========== package for building REST-style Web Services using Google Go -[![Build Status](https://travis-ci.org/emicklei/go-restful.png)](https://travis-ci.org/emicklei/go-restful) [![Go Report Card](https://goreportcard.com/badge/github.com/emicklei/go-restful)](https://goreportcard.com/report/github.com/emicklei/go-restful) [![GoDoc](https://godoc.org/github.com/emicklei/go-restful?status.svg)](https://pkg.go.dev/github.com/emicklei/go-restful) [![codecov](https://codecov.io/gh/emicklei/go-restful/branch/master/graph/badge.svg)](https://codecov.io/gh/emicklei/go-restful) @@ -95,8 +94,7 @@ There are several hooks to customize the behavior of the go-restful package. - Trace logging - Compression - Encoders for other serializers -- Use [jsoniter](https://github.com/json-iterator/go) by building this package using a build tag, e.g. `go build -tags=jsoniter .` -- Use the package variable `TrimRightSlashEnabled` (default true) to control the behavior of matching routes that end with a slash `/` +- Use the package variable `TrimRightSlashEnabled` (default true) to control the behavior of matching routes that end with a slash `/` ## Resources diff --git a/go-controller/vendor/github.com/emicklei/go-restful/v3/compress.go b/go-controller/vendor/github.com/emicklei/go-restful/v3/compress.go index 1ff239f99f..80adf55fdf 100644 --- a/go-controller/vendor/github.com/emicklei/go-restful/v3/compress.go +++ b/go-controller/vendor/github.com/emicklei/go-restful/v3/compress.go @@ -49,6 +49,16 @@ func (c *CompressingResponseWriter) CloseNotify() <-chan bool { return c.writer.(http.CloseNotifier).CloseNotify() } +// Flush is part of http.Flusher interface. Noop if the underlying writer doesn't support it. +func (c *CompressingResponseWriter) Flush() { + flusher, ok := c.writer.(http.Flusher) + if !ok { + // writer doesn't support http.Flusher interface + return + } + flusher.Flush() +} + // Close the underlying compressor func (c *CompressingResponseWriter) Close() error { if c.isCompressorClosed() { diff --git a/go-controller/vendor/github.com/emicklei/go-restful/v3/curly.go b/go-controller/vendor/github.com/emicklei/go-restful/v3/curly.go index ba1fc5d5f1..6fd2bcd5a1 100644 --- a/go-controller/vendor/github.com/emicklei/go-restful/v3/curly.go +++ b/go-controller/vendor/github.com/emicklei/go-restful/v3/curly.go @@ -46,10 +46,10 @@ func (c CurlyRouter) SelectRoute( // selectRoutes return a collection of Route from a WebService that matches the path tokens from the request. func (c CurlyRouter) selectRoutes(ws *WebService, requestTokens []string) sortableCurlyRoutes { candidates := make(sortableCurlyRoutes, 0, 8) - for _, each := range ws.routes { - matches, paramCount, staticCount := c.matchesRouteByPathTokens(each.pathParts, requestTokens, each.hasCustomVerb) + for _, eachRoute := range ws.routes { + matches, paramCount, staticCount := c.matchesRouteByPathTokens(eachRoute.pathParts, requestTokens, eachRoute.hasCustomVerb) if matches { - candidates.add(curlyRoute{each, paramCount, staticCount}) // TODO make sure Routes() return pointers? + candidates.add(curlyRoute{eachRoute, paramCount, staticCount}) // TODO make sure Routes() return pointers? } } sort.Sort(candidates) @@ -72,7 +72,7 @@ func (c CurlyRouter) matchesRouteByPathTokens(routeTokens, requestTokens []strin return false, 0, 0 } requestToken := requestTokens[i] - if routeHasCustomVerb && hasCustomVerb(routeToken){ + if routeHasCustomVerb && hasCustomVerb(routeToken) { if !isMatchCustomVerb(routeToken, requestToken) { return false, 0, 0 } @@ -129,44 +129,52 @@ func (c CurlyRouter) detectRoute(candidateRoutes sortableCurlyRoutes, httpReques // detectWebService returns the best matching webService given the list of path tokens. // see also computeWebserviceScore func (c CurlyRouter) detectWebService(requestTokens []string, webServices []*WebService) *WebService { - var best *WebService + var bestWs *WebService score := -1 - for _, each := range webServices { - matches, eachScore := c.computeWebserviceScore(requestTokens, each.pathExpr.tokens) + for _, eachWS := range webServices { + matches, eachScore := c.computeWebserviceScore(requestTokens, eachWS.pathExpr.tokens) if matches && (eachScore > score) { - best = each + bestWs = eachWS score = eachScore } } - return best + return bestWs } // computeWebserviceScore returns whether tokens match and // the weighted score of the longest matching consecutive tokens from the beginning. -func (c CurlyRouter) computeWebserviceScore(requestTokens []string, tokens []string) (bool, int) { - if len(tokens) > len(requestTokens) { +func (c CurlyRouter) computeWebserviceScore(requestTokens []string, routeTokens []string) (bool, int) { + if len(routeTokens) > len(requestTokens) { return false, 0 } score := 0 - for i := 0; i < len(tokens); i++ { - each := requestTokens[i] - other := tokens[i] - if len(each) == 0 && len(other) == 0 { + for i := 0; i < len(routeTokens); i++ { + eachRequestToken := requestTokens[i] + eachRouteToken := routeTokens[i] + if len(eachRequestToken) == 0 && len(eachRouteToken) == 0 { score++ continue } - if len(other) > 0 && strings.HasPrefix(other, "{") { + if len(eachRouteToken) > 0 && strings.HasPrefix(eachRouteToken, "{") { // no empty match - if len(each) == 0 { + if len(eachRequestToken) == 0 { return false, score } - score += 1 + score++ + + if colon := strings.Index(eachRouteToken, ":"); colon != -1 { + // match by regex + matchesToken, _ := c.regularMatchesPathToken(eachRouteToken, colon, eachRequestToken) + if matchesToken { + score++ // extra score for regex match + } + } } else { // not a parameter - if each != other { + if eachRequestToken != eachRouteToken { return false, score } - score += (len(tokens) - i) * 10 //fuzzy + score += (len(routeTokens) - i) * 10 //fuzzy } } return true, score diff --git a/go-controller/vendor/github.com/emicklei/go-restful/v3/entity_accessors.go b/go-controller/vendor/github.com/emicklei/go-restful/v3/entity_accessors.go index 66dfc824f5..9808752acd 100644 --- a/go-controller/vendor/github.com/emicklei/go-restful/v3/entity_accessors.go +++ b/go-controller/vendor/github.com/emicklei/go-restful/v3/entity_accessors.go @@ -5,11 +5,18 @@ package restful // that can be found in the LICENSE file. import ( + "encoding/json" "encoding/xml" "strings" "sync" ) +var ( + MarshalIndent = json.MarshalIndent + NewDecoder = json.NewDecoder + NewEncoder = json.NewEncoder +) + // EntityReaderWriter can read and write values using an encoding such as JSON,XML. type EntityReaderWriter interface { // Read a serialized version of the value from the request. diff --git a/go-controller/vendor/github.com/emicklei/go-restful/v3/json.go b/go-controller/vendor/github.com/emicklei/go-restful/v3/json.go deleted file mode 100644 index 871165166a..0000000000 --- a/go-controller/vendor/github.com/emicklei/go-restful/v3/json.go +++ /dev/null @@ -1,11 +0,0 @@ -// +build !jsoniter - -package restful - -import "encoding/json" - -var ( - MarshalIndent = json.MarshalIndent - NewDecoder = json.NewDecoder - NewEncoder = json.NewEncoder -) diff --git a/go-controller/vendor/github.com/emicklei/go-restful/v3/jsoniter.go b/go-controller/vendor/github.com/emicklei/go-restful/v3/jsoniter.go deleted file mode 100644 index 11b8f8ae7f..0000000000 --- a/go-controller/vendor/github.com/emicklei/go-restful/v3/jsoniter.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build jsoniter - -package restful - -import "github.com/json-iterator/go" - -var ( - json = jsoniter.ConfigCompatibleWithStandardLibrary - MarshalIndent = json.MarshalIndent - NewDecoder = json.NewDecoder - NewEncoder = json.NewEncoder -) diff --git a/go-controller/vendor/github.com/emicklei/go-restful/v3/jsr311.go b/go-controller/vendor/github.com/emicklei/go-restful/v3/jsr311.go index 07a0c91e94..a9b3faaa81 100644 --- a/go-controller/vendor/github.com/emicklei/go-restful/v3/jsr311.go +++ b/go-controller/vendor/github.com/emicklei/go-restful/v3/jsr311.go @@ -155,7 +155,7 @@ func (r RouterJSR311) detectRoute(routes []Route, httpRequest *http.Request) (*R method, length := httpRequest.Method, httpRequest.Header.Get("Content-Length") if (method == http.MethodPost || method == http.MethodPut || - method == http.MethodPatch) && length == "" { + method == http.MethodPatch) && (length == "" || length == "0") { return nil, NewError( http.StatusUnsupportedMediaType, fmt.Sprintf("415: Unsupported Media Type\n\nAvailable representations: %s", strings.Join(available, ", ")), diff --git a/go-controller/vendor/github.com/go-logr/logr/testr/testr.go b/go-controller/vendor/github.com/go-logr/logr/testr/testr.go new file mode 100644 index 0000000000..5eabe2b6fd --- /dev/null +++ b/go-controller/vendor/github.com/go-logr/logr/testr/testr.go @@ -0,0 +1,167 @@ +/* +Copyright 2019 The logr Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testr provides support for using logr in tests. +package testr + +import ( + "testing" + + "github.com/go-logr/logr" + "github.com/go-logr/logr/funcr" +) + +// New returns a logr.Logger that prints through a testing.T object. +// Info logs are only enabled at V(0). +func New(t *testing.T) logr.Logger { + return NewWithOptions(t, Options{}) +} + +// Options carries parameters which influence the way logs are generated. +type Options struct { + // LogTimestamp tells the logger to add a "ts" key to log + // lines. This has some overhead, so some users might not want + // it. + LogTimestamp bool + + // Verbosity tells the logger which V logs to be write. + // Higher values enable more logs. + Verbosity int +} + +// NewWithOptions returns a logr.Logger that prints through a testing.T object. +// In contrast to the simpler New, output formatting can be configured. +func NewWithOptions(t *testing.T, opts Options) logr.Logger { + l := &testlogger{ + testloggerInterface: newLoggerInterfaceWithOptions(t, opts), + } + return logr.New(l) +} + +// TestingT is an interface wrapper around testing.T, testing.B and testing.F. +type TestingT interface { + Helper() + Log(args ...any) +} + +// NewWithInterface returns a logr.Logger that prints through a +// TestingT object. +// In contrast to the simpler New, output formatting can be configured. +func NewWithInterface(t TestingT, opts Options) logr.Logger { + l := newLoggerInterfaceWithOptions(t, opts) + return logr.New(&l) +} + +func newLoggerInterfaceWithOptions(t TestingT, opts Options) testloggerInterface { + return testloggerInterface{ + t: t, + Formatter: funcr.NewFormatter(funcr.Options{ + LogTimestamp: opts.LogTimestamp, + Verbosity: opts.Verbosity, + }), + } +} + +// Underlier exposes access to the underlying testing.T instance. Since +// callers only have a logr.Logger, they have to know which +// implementation is in use, so this interface is less of an +// abstraction and more of a way to test type conversion. +type Underlier interface { + GetUnderlying() *testing.T +} + +// UnderlierInterface exposes access to the underlying TestingT instance. Since +// callers only have a logr.Logger, they have to know which +// implementation is in use, so this interface is less of an +// abstraction and more of a way to test type conversion. +type UnderlierInterface interface { + GetUnderlying() TestingT +} + +// Info logging implementation shared between testLogger and testLoggerInterface. +func logInfo(t TestingT, formatInfo func(int, string, []any) (string, string), level int, msg string, kvList ...any) { + prefix, args := formatInfo(level, msg, kvList) + t.Helper() + if prefix != "" { + args = prefix + ": " + args + } + t.Log(args) +} + +// Error logging implementation shared between testLogger and testLoggerInterface. +func logError(t TestingT, formatError func(error, string, []any) (string, string), err error, msg string, kvList ...any) { + prefix, args := formatError(err, msg, kvList) + t.Helper() + if prefix != "" { + args = prefix + ": " + args + } + t.Log(args) +} + +// This type exists to wrap and modify the method-set of testloggerInterface. +// In particular, it changes the GetUnderlying() method. +type testlogger struct { + testloggerInterface +} + +func (l testlogger) GetUnderlying() *testing.T { + // This method is defined on testlogger, so the only type this could + // possibly be is testing.T, even though that's not guaranteed by the type + // system itself. + return l.t.(*testing.T) //nolint:forcetypeassert +} + +type testloggerInterface struct { + funcr.Formatter + t TestingT +} + +func (l testloggerInterface) WithName(name string) logr.LogSink { + l.Formatter.AddName(name) + return &l +} + +func (l testloggerInterface) WithValues(kvList ...any) logr.LogSink { + l.Formatter.AddValues(kvList) + return &l +} + +func (l testloggerInterface) GetCallStackHelper() func() { + return l.t.Helper +} + +func (l testloggerInterface) Info(level int, msg string, kvList ...any) { + l.t.Helper() + logInfo(l.t, l.FormatInfo, level, msg, kvList...) +} + +func (l testloggerInterface) Error(err error, msg string, kvList ...any) { + l.t.Helper() + logError(l.t, l.FormatError, err, msg, kvList...) +} + +func (l testloggerInterface) GetUnderlying() TestingT { + return l.t +} + +// Assert conformance to the interfaces. +var _ logr.LogSink = &testlogger{} +var _ logr.CallStackHelperLogSink = &testlogger{} +var _ Underlier = &testlogger{} + +var _ logr.LogSink = &testloggerInterface{} +var _ logr.CallStackHelperLogSink = &testloggerInterface{} +var _ UnderlierInterface = &testloggerInterface{} diff --git a/go-controller/vendor/github.com/go-openapi/jsonpointer/.golangci.yml b/go-controller/vendor/github.com/go-openapi/jsonpointer/.golangci.yml new file mode 100644 index 0000000000..22f8d21cca --- /dev/null +++ b/go-controller/vendor/github.com/go-openapi/jsonpointer/.golangci.yml @@ -0,0 +1,61 @@ +linters-settings: + govet: + check-shadowing: true + golint: + min-confidence: 0 + gocyclo: + min-complexity: 45 + maligned: + suggest-new: true + dupl: + threshold: 200 + goconst: + min-len: 2 + min-occurrences: 3 + +linters: + enable-all: true + disable: + - maligned + - unparam + - lll + - gochecknoinits + - gochecknoglobals + - funlen + - godox + - gocognit + - whitespace + - wsl + - wrapcheck + - testpackage + - nlreturn + - gomnd + - exhaustivestruct + - goerr113 + - errorlint + - nestif + - godot + - gofumpt + - paralleltest + - tparallel + - thelper + - ifshort + - exhaustruct + - varnamelen + - gci + - depguard + - errchkjson + - inamedparam + - nonamedreturns + - musttag + - ireturn + - forcetypeassert + - cyclop + # deprecated linters + - deadcode + - interfacer + - scopelint + - varcheck + - structcheck + - golint + - nosnakecase diff --git a/go-controller/vendor/github.com/go-openapi/jsonpointer/README.md b/go-controller/vendor/github.com/go-openapi/jsonpointer/README.md index 813788aff1..0108f1d572 100644 --- a/go-controller/vendor/github.com/go-openapi/jsonpointer/README.md +++ b/go-controller/vendor/github.com/go-openapi/jsonpointer/README.md @@ -1,6 +1,10 @@ -# gojsonpointer [![Build Status](https://travis-ci.org/go-openapi/jsonpointer.svg?branch=master)](https://travis-ci.org/go-openapi/jsonpointer) [![codecov](https://codecov.io/gh/go-openapi/jsonpointer/branch/master/graph/badge.svg)](https://codecov.io/gh/go-openapi/jsonpointer) [![Slack Status](https://slackin.goswagger.io/badge.svg)](https://slackin.goswagger.io) +# gojsonpointer [![Build Status](https://github.com/go-openapi/jsonpointer/actions/workflows/go-test.yml/badge.svg)](https://github.com/go-openapi/jsonpointer/actions?query=workflow%3A"go+test") [![codecov](https://codecov.io/gh/go-openapi/jsonpointer/branch/master/graph/badge.svg)](https://codecov.io/gh/go-openapi/jsonpointer) + +[![Slack Status](https://slackin.goswagger.io/badge.svg)](https://slackin.goswagger.io) +[![license](http://img.shields.io/badge/license-Apache%20v2-orange.svg)](https://raw.githubusercontent.com/go-openapi/jsonpointer/master/LICENSE) +[![Go Reference](https://pkg.go.dev/badge/github.com/go-openapi/jsonpointer.svg)](https://pkg.go.dev/github.com/go-openapi/jsonpointer) +[![Go Report Card](https://goreportcard.com/badge/github.com/go-openapi/jsonpointer)](https://goreportcard.com/report/github.com/go-openapi/jsonpointer) -[![license](http://img.shields.io/badge/license-Apache%20v2-orange.svg)](https://raw.githubusercontent.com/go-openapi/jsonpointer/master/LICENSE) [![GoDoc](https://godoc.org/github.com/go-openapi/jsonpointer?status.svg)](http://godoc.org/github.com/go-openapi/jsonpointer) An implementation of JSON Pointer - Go language ## Status diff --git a/go-controller/vendor/github.com/go-openapi/jsonpointer/pointer.go b/go-controller/vendor/github.com/go-openapi/jsonpointer/pointer.go index 7df9853def..d970c7cf44 100644 --- a/go-controller/vendor/github.com/go-openapi/jsonpointer/pointer.go +++ b/go-controller/vendor/github.com/go-openapi/jsonpointer/pointer.go @@ -26,6 +26,7 @@ package jsonpointer import ( + "encoding/json" "errors" "fmt" "reflect" @@ -40,6 +41,7 @@ const ( pointerSeparator = `/` invalidStart = `JSON pointer must be empty or start with a "` + pointerSeparator + notFound = `Can't find the pointer in the document` ) var jsonPointableType = reflect.TypeOf(new(JSONPointable)).Elem() @@ -48,13 +50,13 @@ var jsonSetableType = reflect.TypeOf(new(JSONSetable)).Elem() // JSONPointable is an interface for structs to implement when they need to customize the // json pointer process type JSONPointable interface { - JSONLookup(string) (interface{}, error) + JSONLookup(string) (any, error) } // JSONSetable is an interface for structs to implement when they need to customize the // json pointer process type JSONSetable interface { - JSONSet(string, interface{}) error + JSONSet(string, any) error } // New creates a new json pointer for the given string @@ -81,9 +83,7 @@ func (p *Pointer) parse(jsonPointerString string) error { err = errors.New(invalidStart) } else { referenceTokens := strings.Split(jsonPointerString, pointerSeparator) - for _, referenceToken := range referenceTokens[1:] { - p.referenceTokens = append(p.referenceTokens, referenceToken) - } + p.referenceTokens = append(p.referenceTokens, referenceTokens[1:]...) } } @@ -91,38 +91,58 @@ func (p *Pointer) parse(jsonPointerString string) error { } // Get uses the pointer to retrieve a value from a JSON document -func (p *Pointer) Get(document interface{}) (interface{}, reflect.Kind, error) { +func (p *Pointer) Get(document any) (any, reflect.Kind, error) { return p.get(document, swag.DefaultJSONNameProvider) } // Set uses the pointer to set a value from a JSON document -func (p *Pointer) Set(document interface{}, value interface{}) (interface{}, error) { +func (p *Pointer) Set(document any, value any) (any, error) { return document, p.set(document, value, swag.DefaultJSONNameProvider) } // GetForToken gets a value for a json pointer token 1 level deep -func GetForToken(document interface{}, decodedToken string) (interface{}, reflect.Kind, error) { +func GetForToken(document any, decodedToken string) (any, reflect.Kind, error) { return getSingleImpl(document, decodedToken, swag.DefaultJSONNameProvider) } // SetForToken gets a value for a json pointer token 1 level deep -func SetForToken(document interface{}, decodedToken string, value interface{}) (interface{}, error) { +func SetForToken(document any, decodedToken string, value any) (any, error) { return document, setSingleImpl(document, value, decodedToken, swag.DefaultJSONNameProvider) } -func getSingleImpl(node interface{}, decodedToken string, nameProvider *swag.NameProvider) (interface{}, reflect.Kind, error) { +func isNil(input any) bool { + if input == nil { + return true + } + + kind := reflect.TypeOf(input).Kind() + switch kind { //nolint:exhaustive + case reflect.Ptr, reflect.Map, reflect.Slice, reflect.Chan: + return reflect.ValueOf(input).IsNil() + default: + return false + } +} + +func getSingleImpl(node any, decodedToken string, nameProvider *swag.NameProvider) (any, reflect.Kind, error) { rValue := reflect.Indirect(reflect.ValueOf(node)) kind := rValue.Kind() + if isNil(node) { + return nil, kind, fmt.Errorf("nil value has not field %q", decodedToken) + } - if rValue.Type().Implements(jsonPointableType) { - r, err := node.(JSONPointable).JSONLookup(decodedToken) + switch typed := node.(type) { + case JSONPointable: + r, err := typed.JSONLookup(decodedToken) if err != nil { return nil, kind, err } return r, kind, nil + case *any: // case of a pointer to interface, that is not resolved by reflect.Indirect + return getSingleImpl(*typed, decodedToken, nameProvider) } - switch kind { + switch kind { //nolint:exhaustive case reflect.Struct: nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken) if !ok { @@ -159,7 +179,7 @@ func getSingleImpl(node interface{}, decodedToken string, nameProvider *swag.Nam } -func setSingleImpl(node, data interface{}, decodedToken string, nameProvider *swag.NameProvider) error { +func setSingleImpl(node, data any, decodedToken string, nameProvider *swag.NameProvider) error { rValue := reflect.Indirect(reflect.ValueOf(node)) if ns, ok := node.(JSONSetable); ok { // pointer impl @@ -170,7 +190,7 @@ func setSingleImpl(node, data interface{}, decodedToken string, nameProvider *sw return node.(JSONSetable).JSONSet(decodedToken, data) } - switch rValue.Kind() { + switch rValue.Kind() { //nolint:exhaustive case reflect.Struct: nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken) if !ok { @@ -210,7 +230,7 @@ func setSingleImpl(node, data interface{}, decodedToken string, nameProvider *sw } -func (p *Pointer) get(node interface{}, nameProvider *swag.NameProvider) (interface{}, reflect.Kind, error) { +func (p *Pointer) get(node any, nameProvider *swag.NameProvider) (any, reflect.Kind, error) { if nameProvider == nil { nameProvider = swag.DefaultJSONNameProvider @@ -231,8 +251,7 @@ func (p *Pointer) get(node interface{}, nameProvider *swag.NameProvider) (interf if err != nil { return nil, knd, err } - node, kind = r, knd - + node = r } rValue := reflect.ValueOf(node) @@ -241,11 +260,11 @@ func (p *Pointer) get(node interface{}, nameProvider *swag.NameProvider) (interf return node, kind, nil } -func (p *Pointer) set(node, data interface{}, nameProvider *swag.NameProvider) error { +func (p *Pointer) set(node, data any, nameProvider *swag.NameProvider) error { knd := reflect.ValueOf(node).Kind() if knd != reflect.Ptr && knd != reflect.Struct && knd != reflect.Map && knd != reflect.Slice && knd != reflect.Array { - return fmt.Errorf("only structs, pointers, maps and slices are supported for setting values") + return errors.New("only structs, pointers, maps and slices are supported for setting values") } if nameProvider == nil { @@ -284,7 +303,7 @@ func (p *Pointer) set(node, data interface{}, nameProvider *swag.NameProvider) e continue } - switch kind { + switch kind { //nolint:exhaustive case reflect.Struct: nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken) if !ok { @@ -363,6 +382,128 @@ func (p *Pointer) String() string { return pointerString } +func (p *Pointer) Offset(document string) (int64, error) { + dec := json.NewDecoder(strings.NewReader(document)) + var offset int64 + for _, ttk := range p.DecodedTokens() { + tk, err := dec.Token() + if err != nil { + return 0, err + } + switch tk := tk.(type) { + case json.Delim: + switch tk { + case '{': + offset, err = offsetSingleObject(dec, ttk) + if err != nil { + return 0, err + } + case '[': + offset, err = offsetSingleArray(dec, ttk) + if err != nil { + return 0, err + } + default: + return 0, fmt.Errorf("invalid token %#v", tk) + } + default: + return 0, fmt.Errorf("invalid token %#v", tk) + } + } + return offset, nil +} + +func offsetSingleObject(dec *json.Decoder, decodedToken string) (int64, error) { + for dec.More() { + offset := dec.InputOffset() + tk, err := dec.Token() + if err != nil { + return 0, err + } + switch tk := tk.(type) { + case json.Delim: + switch tk { + case '{': + if err = drainSingle(dec); err != nil { + return 0, err + } + case '[': + if err = drainSingle(dec); err != nil { + return 0, err + } + } + case string: + if tk == decodedToken { + return offset, nil + } + default: + return 0, fmt.Errorf("invalid token %#v", tk) + } + } + return 0, fmt.Errorf("token reference %q not found", decodedToken) +} + +func offsetSingleArray(dec *json.Decoder, decodedToken string) (int64, error) { + idx, err := strconv.Atoi(decodedToken) + if err != nil { + return 0, fmt.Errorf("token reference %q is not a number: %v", decodedToken, err) + } + var i int + for i = 0; i < idx && dec.More(); i++ { + tk, err := dec.Token() + if err != nil { + return 0, err + } + + if delim, isDelim := tk.(json.Delim); isDelim { + switch delim { + case '{': + if err = drainSingle(dec); err != nil { + return 0, err + } + case '[': + if err = drainSingle(dec); err != nil { + return 0, err + } + } + } + } + + if !dec.More() { + return 0, fmt.Errorf("token reference %q not found", decodedToken) + } + return dec.InputOffset(), nil +} + +// drainSingle drains a single level of object or array. +// The decoder has to guarantee the beginning delim (i.e. '{' or '[') has been consumed. +func drainSingle(dec *json.Decoder) error { + for dec.More() { + tk, err := dec.Token() + if err != nil { + return err + } + if delim, isDelim := tk.(json.Delim); isDelim { + switch delim { + case '{': + if err = drainSingle(dec); err != nil { + return err + } + case '[': + if err = drainSingle(dec); err != nil { + return err + } + } + } + } + + // Consumes the ending delim + if _, err := dec.Token(); err != nil { + return err + } + return nil +} + // Specific JSON pointer encoding here // ~0 => ~ // ~1 => / @@ -377,14 +518,14 @@ const ( // Unescape unescapes a json pointer reference token string to the original representation func Unescape(token string) string { - step1 := strings.Replace(token, encRefTok1, decRefTok1, -1) - step2 := strings.Replace(step1, encRefTok0, decRefTok0, -1) + step1 := strings.ReplaceAll(token, encRefTok1, decRefTok1) + step2 := strings.ReplaceAll(step1, encRefTok0, decRefTok0) return step2 } // Escape escapes a pointer reference token string func Escape(token string) string { - step1 := strings.Replace(token, decRefTok0, encRefTok0, -1) - step2 := strings.Replace(step1, decRefTok1, encRefTok1, -1) + step1 := strings.ReplaceAll(token, decRefTok0, encRefTok0) + step2 := strings.ReplaceAll(step1, decRefTok1, encRefTok1) return step2 } diff --git a/go-controller/vendor/github.com/go-openapi/jsonreference/.golangci.yml b/go-controller/vendor/github.com/go-openapi/jsonreference/.golangci.yml index 013fc1943a..22f8d21cca 100644 --- a/go-controller/vendor/github.com/go-openapi/jsonreference/.golangci.yml +++ b/go-controller/vendor/github.com/go-openapi/jsonreference/.golangci.yml @@ -1,50 +1,61 @@ linters-settings: govet: check-shadowing: true + golint: + min-confidence: 0 gocyclo: - min-complexity: 30 + min-complexity: 45 maligned: suggest-new: true dupl: - threshold: 100 + threshold: 200 goconst: min-len: 2 - min-occurrences: 4 - paralleltest: - ignore-missing: true + min-occurrences: 3 + linters: enable-all: true disable: - maligned + - unparam - lll + - gochecknoinits - gochecknoglobals + - funlen - godox - gocognit - whitespace - wsl - - funlen - - gochecknoglobals - - gochecknoinits - - scopelint - wrapcheck - - exhaustivestruct - - exhaustive - - nlreturn - testpackage - - gci - - gofumpt - - goerr113 + - nlreturn - gomnd - - tparallel + - exhaustivestruct + - goerr113 + - errorlint - nestif - godot - - errorlint - - varcheck - - interfacer - - deadcode - - golint + - gofumpt + - paralleltest + - tparallel + - thelper - ifshort + - exhaustruct + - varnamelen + - gci + - depguard + - errchkjson + - inamedparam + - nonamedreturns + - musttag + - ireturn + - forcetypeassert + - cyclop + # deprecated linters + - deadcode + - interfacer + - scopelint + - varcheck - structcheck + - golint - nosnakecase - - varnamelen - - exhaustruct diff --git a/go-controller/vendor/github.com/go-openapi/jsonreference/README.md b/go-controller/vendor/github.com/go-openapi/jsonreference/README.md index b94753aa52..c7fc2049c1 100644 --- a/go-controller/vendor/github.com/go-openapi/jsonreference/README.md +++ b/go-controller/vendor/github.com/go-openapi/jsonreference/README.md @@ -1,15 +1,19 @@ -# gojsonreference [![Build Status](https://travis-ci.org/go-openapi/jsonreference.svg?branch=master)](https://travis-ci.org/go-openapi/jsonreference) [![codecov](https://codecov.io/gh/go-openapi/jsonreference/branch/master/graph/badge.svg)](https://codecov.io/gh/go-openapi/jsonreference) [![Slack Status](https://slackin.goswagger.io/badge.svg)](https://slackin.goswagger.io) +# gojsonreference [![Build Status](https://github.com/go-openapi/jsonreference/actions/workflows/go-test.yml/badge.svg)](https://github.com/go-openapi/jsonreference/actions?query=workflow%3A"go+test") [![codecov](https://codecov.io/gh/go-openapi/jsonreference/branch/master/graph/badge.svg)](https://codecov.io/gh/go-openapi/jsonreference) + +[![Slack Status](https://slackin.goswagger.io/badge.svg)](https://slackin.goswagger.io) +[![license](http://img.shields.io/badge/license-Apache%20v2-orange.svg)](https://raw.githubusercontent.com/go-openapi/jsonreference/master/LICENSE) +[![Go Reference](https://pkg.go.dev/badge/github.com/go-openapi/jsonreference.svg)](https://pkg.go.dev/github.com/go-openapi/jsonreference) +[![Go Report Card](https://goreportcard.com/badge/github.com/go-openapi/jsonreference)](https://goreportcard.com/report/github.com/go-openapi/jsonreference) -[![license](http://img.shields.io/badge/license-Apache%20v2-orange.svg)](https://raw.githubusercontent.com/go-openapi/jsonreference/master/LICENSE) [![GoDoc](https://godoc.org/github.com/go-openapi/jsonreference?status.svg)](http://godoc.org/github.com/go-openapi/jsonreference) An implementation of JSON Reference - Go language ## Status Feature complete. Stable API ## Dependencies -https://github.com/go-openapi/jsonpointer +* https://github.com/go-openapi/jsonpointer ## References -http://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-07 -http://tools.ietf.org/html/draft-pbryan-zyp-json-ref-03 +* http://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-07 +* http://tools.ietf.org/html/draft-pbryan-zyp-json-ref-03 diff --git a/go-controller/vendor/github.com/go-openapi/swag/.gitignore b/go-controller/vendor/github.com/go-openapi/swag/.gitignore index d69b53accc..c4b1b64f04 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/.gitignore +++ b/go-controller/vendor/github.com/go-openapi/swag/.gitignore @@ -2,3 +2,4 @@ secrets.yml vendor Godeps .idea +*.out diff --git a/go-controller/vendor/github.com/go-openapi/swag/.golangci.yml b/go-controller/vendor/github.com/go-openapi/swag/.golangci.yml index bf503e4000..80e2be0042 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/.golangci.yml +++ b/go-controller/vendor/github.com/go-openapi/swag/.golangci.yml @@ -4,14 +4,14 @@ linters-settings: golint: min-confidence: 0 gocyclo: - min-complexity: 25 + min-complexity: 45 maligned: suggest-new: true dupl: - threshold: 100 + threshold: 200 goconst: min-len: 3 - min-occurrences: 2 + min-occurrences: 3 linters: enable-all: true @@ -20,35 +20,41 @@ linters: - lll - gochecknoinits - gochecknoglobals - - nlreturn - - testpackage + - funlen + - godox + - gocognit + - whitespace + - wsl - wrapcheck + - testpackage + - nlreturn - gomnd - - exhaustive - exhaustivestruct - goerr113 - - wsl - - whitespace - - gofumpt - - godot + - errorlint - nestif - - godox - - funlen - - gci - - gocognit + - godot + - gofumpt - paralleltest + - tparallel - thelper - ifshort - - gomoddirectives - - cyclop - - forcetypeassert - - ireturn - - tagliatelle - - varnamelen - - goimports - - tenv - - golint - exhaustruct - - nilnil + - varnamelen + - gci + - depguard + - errchkjson + - inamedparam - nonamedreturns + - musttag + - ireturn + - forcetypeassert + - cyclop + # deprecated linters + - deadcode + - interfacer + - scopelint + - varcheck + - structcheck + - golint - nosnakecase diff --git a/go-controller/vendor/github.com/go-openapi/swag/BENCHMARK.md b/go-controller/vendor/github.com/go-openapi/swag/BENCHMARK.md new file mode 100644 index 0000000000..e7f28ed6b7 --- /dev/null +++ b/go-controller/vendor/github.com/go-openapi/swag/BENCHMARK.md @@ -0,0 +1,52 @@ +# Benchmarks + +## Name mangling utilities + +```bash +go test -bench XXX -run XXX -benchtime 30s +``` + +### Benchmarks at b3e7a5386f996177e4808f11acb2aa93a0f660df + +``` +goos: linux +goarch: amd64 +pkg: github.com/go-openapi/swag +cpu: Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz +BenchmarkToXXXName/ToGoName-4 862623 44101 ns/op 10450 B/op 732 allocs/op +BenchmarkToXXXName/ToVarName-4 853656 40728 ns/op 10468 B/op 734 allocs/op +BenchmarkToXXXName/ToFileName-4 1268312 27813 ns/op 9785 B/op 617 allocs/op +BenchmarkToXXXName/ToCommandName-4 1276322 27903 ns/op 9785 B/op 617 allocs/op +BenchmarkToXXXName/ToHumanNameLower-4 895334 40354 ns/op 10472 B/op 731 allocs/op +BenchmarkToXXXName/ToHumanNameTitle-4 882441 40678 ns/op 10566 B/op 749 allocs/op +``` + +### Benchmarks after PR #79 + +~ x10 performance improvement and ~ /100 memory allocations. + +``` +goos: linux +goarch: amd64 +pkg: github.com/go-openapi/swag +cpu: Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz +BenchmarkToXXXName/ToGoName-4 9595830 3991 ns/op 42 B/op 5 allocs/op +BenchmarkToXXXName/ToVarName-4 9194276 3984 ns/op 62 B/op 7 allocs/op +BenchmarkToXXXName/ToFileName-4 17002711 2123 ns/op 147 B/op 7 allocs/op +BenchmarkToXXXName/ToCommandName-4 16772926 2111 ns/op 147 B/op 7 allocs/op +BenchmarkToXXXName/ToHumanNameLower-4 9788331 3749 ns/op 92 B/op 6 allocs/op +BenchmarkToXXXName/ToHumanNameTitle-4 9188260 3941 ns/op 104 B/op 6 allocs/op +``` + +``` +goos: linux +goarch: amd64 +pkg: github.com/go-openapi/swag +cpu: AMD Ryzen 7 5800X 8-Core Processor +BenchmarkToXXXName/ToGoName-16 18527378 1972 ns/op 42 B/op 5 allocs/op +BenchmarkToXXXName/ToVarName-16 15552692 2093 ns/op 62 B/op 7 allocs/op +BenchmarkToXXXName/ToFileName-16 32161176 1117 ns/op 147 B/op 7 allocs/op +BenchmarkToXXXName/ToCommandName-16 32256634 1137 ns/op 147 B/op 7 allocs/op +BenchmarkToXXXName/ToHumanNameLower-16 18599661 1946 ns/op 92 B/op 6 allocs/op +BenchmarkToXXXName/ToHumanNameTitle-16 17581353 2054 ns/op 105 B/op 6 allocs/op +``` diff --git a/go-controller/vendor/github.com/go-openapi/swag/README.md b/go-controller/vendor/github.com/go-openapi/swag/README.md index 217f6fa505..a729222998 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/README.md +++ b/go-controller/vendor/github.com/go-openapi/swag/README.md @@ -1,7 +1,8 @@ -# Swag [![Build Status](https://travis-ci.org/go-openapi/swag.svg?branch=master)](https://travis-ci.org/go-openapi/swag) [![codecov](https://codecov.io/gh/go-openapi/swag/branch/master/graph/badge.svg)](https://codecov.io/gh/go-openapi/swag) [![Slack Status](https://slackin.goswagger.io/badge.svg)](https://slackin.goswagger.io) +# Swag [![Build Status](https://github.com/go-openapi/swag/actions/workflows/go-test.yml/badge.svg)](https://github.com/go-openapi/swag/actions?query=workflow%3A"go+test") [![codecov](https://codecov.io/gh/go-openapi/swag/branch/master/graph/badge.svg)](https://codecov.io/gh/go-openapi/swag) +[![Slack Status](https://slackin.goswagger.io/badge.svg)](https://slackin.goswagger.io) [![license](http://img.shields.io/badge/license-Apache%20v2-orange.svg)](https://raw.githubusercontent.com/go-openapi/swag/master/LICENSE) -[![GoDoc](https://godoc.org/github.com/go-openapi/swag?status.svg)](http://godoc.org/github.com/go-openapi/swag) +[![Go Reference](https://pkg.go.dev/badge/github.com/go-openapi/swag.svg)](https://pkg.go.dev/github.com/go-openapi/swag) [![Go Report Card](https://goreportcard.com/badge/github.com/go-openapi/swag)](https://goreportcard.com/report/github.com/go-openapi/swag) Contains a bunch of helper functions for go-openapi and go-swagger projects. @@ -18,4 +19,5 @@ You may also use it standalone for your projects. This repo has only few dependencies outside of the standard library: -* YAML utilities depend on gopkg.in/yaml.v2 +* YAML utilities depend on `gopkg.in/yaml.v3` +* `github.com/mailru/easyjson v0.7.7` diff --git a/go-controller/vendor/github.com/go-openapi/swag/initialism_index.go b/go-controller/vendor/github.com/go-openapi/swag/initialism_index.go new file mode 100644 index 0000000000..20a359bb60 --- /dev/null +++ b/go-controller/vendor/github.com/go-openapi/swag/initialism_index.go @@ -0,0 +1,202 @@ +// Copyright 2015 go-swagger maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package swag + +import ( + "sort" + "strings" + "sync" +) + +var ( + // commonInitialisms are common acronyms that are kept as whole uppercased words. + commonInitialisms *indexOfInitialisms + + // initialisms is a slice of sorted initialisms + initialisms []string + + // a copy of initialisms pre-baked as []rune + initialismsRunes [][]rune + initialismsUpperCased [][]rune + + isInitialism func(string) bool + + maxAllocMatches int +) + +func init() { + // Taken from https://github.com/golang/lint/blob/3390df4df2787994aea98de825b964ac7944b817/lint.go#L732-L769 + configuredInitialisms := map[string]bool{ + "ACL": true, + "API": true, + "ASCII": true, + "CPU": true, + "CSS": true, + "DNS": true, + "EOF": true, + "GUID": true, + "HTML": true, + "HTTPS": true, + "HTTP": true, + "ID": true, + "IP": true, + "IPv4": true, + "IPv6": true, + "JSON": true, + "LHS": true, + "OAI": true, + "QPS": true, + "RAM": true, + "RHS": true, + "RPC": true, + "SLA": true, + "SMTP": true, + "SQL": true, + "SSH": true, + "TCP": true, + "TLS": true, + "TTL": true, + "UDP": true, + "UI": true, + "UID": true, + "UUID": true, + "URI": true, + "URL": true, + "UTF8": true, + "VM": true, + "XML": true, + "XMPP": true, + "XSRF": true, + "XSS": true, + } + + // a thread-safe index of initialisms + commonInitialisms = newIndexOfInitialisms().load(configuredInitialisms) + initialisms = commonInitialisms.sorted() + initialismsRunes = asRunes(initialisms) + initialismsUpperCased = asUpperCased(initialisms) + maxAllocMatches = maxAllocHeuristic(initialismsRunes) + + // a test function + isInitialism = commonInitialisms.isInitialism +} + +func asRunes(in []string) [][]rune { + out := make([][]rune, len(in)) + for i, initialism := range in { + out[i] = []rune(initialism) + } + + return out +} + +func asUpperCased(in []string) [][]rune { + out := make([][]rune, len(in)) + + for i, initialism := range in { + out[i] = []rune(upper(trim(initialism))) + } + + return out +} + +func maxAllocHeuristic(in [][]rune) int { + heuristic := make(map[rune]int) + for _, initialism := range in { + heuristic[initialism[0]]++ + } + + var maxAlloc int + for _, val := range heuristic { + if val > maxAlloc { + maxAlloc = val + } + } + + return maxAlloc +} + +// AddInitialisms add additional initialisms +func AddInitialisms(words ...string) { + for _, word := range words { + // commonInitialisms[upper(word)] = true + commonInitialisms.add(upper(word)) + } + // sort again + initialisms = commonInitialisms.sorted() + initialismsRunes = asRunes(initialisms) + initialismsUpperCased = asUpperCased(initialisms) +} + +// indexOfInitialisms is a thread-safe implementation of the sorted index of initialisms. +// Since go1.9, this may be implemented with sync.Map. +type indexOfInitialisms struct { + sortMutex *sync.Mutex + index *sync.Map +} + +func newIndexOfInitialisms() *indexOfInitialisms { + return &indexOfInitialisms{ + sortMutex: new(sync.Mutex), + index: new(sync.Map), + } +} + +func (m *indexOfInitialisms) load(initial map[string]bool) *indexOfInitialisms { + m.sortMutex.Lock() + defer m.sortMutex.Unlock() + for k, v := range initial { + m.index.Store(k, v) + } + return m +} + +func (m *indexOfInitialisms) isInitialism(key string) bool { + _, ok := m.index.Load(key) + return ok +} + +func (m *indexOfInitialisms) add(key string) *indexOfInitialisms { + m.index.Store(key, true) + return m +} + +func (m *indexOfInitialisms) sorted() (result []string) { + m.sortMutex.Lock() + defer m.sortMutex.Unlock() + m.index.Range(func(key, _ interface{}) bool { + k := key.(string) + result = append(result, k) + return true + }) + sort.Sort(sort.Reverse(byInitialism(result))) + return +} + +type byInitialism []string + +func (s byInitialism) Len() int { + return len(s) +} +func (s byInitialism) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} +func (s byInitialism) Less(i, j int) bool { + if len(s[i]) != len(s[j]) { + return len(s[i]) < len(s[j]) + } + + return strings.Compare(s[i], s[j]) > 0 +} diff --git a/go-controller/vendor/github.com/go-openapi/swag/loading.go b/go-controller/vendor/github.com/go-openapi/swag/loading.go index 00038c3773..783442fddf 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/loading.go +++ b/go-controller/vendor/github.com/go-openapi/swag/loading.go @@ -21,6 +21,7 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "runtime" "strings" @@ -40,43 +41,97 @@ var LoadHTTPBasicAuthPassword = "" var LoadHTTPCustomHeaders = map[string]string{} // LoadFromFileOrHTTP loads the bytes from a file or a remote http server based on the path passed in -func LoadFromFileOrHTTP(path string) ([]byte, error) { - return LoadStrategy(path, os.ReadFile, loadHTTPBytes(LoadHTTPTimeout))(path) +func LoadFromFileOrHTTP(pth string) ([]byte, error) { + return LoadStrategy(pth, os.ReadFile, loadHTTPBytes(LoadHTTPTimeout))(pth) } // LoadFromFileOrHTTPWithTimeout loads the bytes from a file or a remote http server based on the path passed in // timeout arg allows for per request overriding of the request timeout -func LoadFromFileOrHTTPWithTimeout(path string, timeout time.Duration) ([]byte, error) { - return LoadStrategy(path, os.ReadFile, loadHTTPBytes(timeout))(path) +func LoadFromFileOrHTTPWithTimeout(pth string, timeout time.Duration) ([]byte, error) { + return LoadStrategy(pth, os.ReadFile, loadHTTPBytes(timeout))(pth) } -// LoadStrategy returns a loader function for a given path or uri -func LoadStrategy(path string, local, remote func(string) ([]byte, error)) func(string) ([]byte, error) { - if strings.HasPrefix(path, "http") { +// LoadStrategy returns a loader function for a given path or URI. +// +// The load strategy returns the remote load for any path starting with `http`. +// So this works for any URI with a scheme `http` or `https`. +// +// The fallback strategy is to call the local loader. +// +// The local loader takes a local file system path (absolute or relative) as argument, +// or alternatively a `file://...` URI, **without host** (see also below for windows). +// +// There are a few liberalities, initially intended to be tolerant regarding the URI syntax, +// especially on windows. +// +// Before the local loader is called, the given path is transformed: +// - percent-encoded characters are unescaped +// - simple paths (e.g. `./folder/file`) are passed as-is +// - on windows, occurrences of `/` are replaced by `\`, so providing a relative path such a `folder/file` works too. +// +// For paths provided as URIs with the "file" scheme, please note that: +// - `file://` is simply stripped. +// This means that the host part of the URI is not parsed at all. +// For example, `file:///folder/file" becomes "/folder/file`, +// but `file://localhost/folder/file` becomes `localhost/folder/file` on unix systems. +// Similarly, `file://./folder/file` yields `./folder/file`. +// - on windows, `file://...` can take a host so as to specify an UNC share location. +// +// Reminder about windows-specifics: +// - `file://host/folder/file` becomes an UNC path like `\\host\folder\file` (no port specification is supported) +// - `file:///c:/folder/file` becomes `C:\folder\file` +// - `file://c:/folder/file` is tolerated (without leading `/`) and becomes `c:\folder\file` +func LoadStrategy(pth string, local, remote func(string) ([]byte, error)) func(string) ([]byte, error) { + if strings.HasPrefix(pth, "http") { return remote } - return func(pth string) ([]byte, error) { - upth, err := pathUnescape(pth) + + return func(p string) ([]byte, error) { + upth, err := url.PathUnescape(p) if err != nil { return nil, err } - if strings.HasPrefix(pth, `file://`) { - if runtime.GOOS == "windows" { - // support for canonical file URIs on windows. - // Zero tolerance here for dodgy URIs. - u, _ := url.Parse(upth) - if u.Host != "" { - // assume UNC name (volume share) - // file://host/share/folder\... ==> \\host\share\path\folder - // NOTE: UNC port not yet supported - upth = strings.Join([]string{`\`, u.Host, u.Path}, `\`) - } else { - // file:///c:/folder/... ==> just remove the leading slash - upth = strings.TrimPrefix(upth, `file:///`) - } - } else { - upth = strings.TrimPrefix(upth, `file://`) + if !strings.HasPrefix(p, `file://`) { + // regular file path provided: just normalize slashes + return local(filepath.FromSlash(upth)) + } + + if runtime.GOOS != "windows" { + // crude processing: this leaves full URIs with a host with a (mostly) unexpected result + upth = strings.TrimPrefix(upth, `file://`) + + return local(filepath.FromSlash(upth)) + } + + // windows-only pre-processing of file://... URIs + + // support for canonical file URIs on windows. + u, err := url.Parse(filepath.ToSlash(upth)) + if err != nil { + return nil, err + } + + if u.Host != "" { + // assume UNC name (volume share) + // NOTE: UNC port not yet supported + + // when the "host" segment is a drive letter: + // file://C:/folder/... => C:\folder + upth = path.Clean(strings.Join([]string{u.Host, u.Path}, `/`)) + if !strings.HasSuffix(u.Host, ":") && u.Host[0] != '.' { + // tolerance: if we have a leading dot, this can't be a host + // file://host/share/folder\... ==> \\host\share\path\folder + upth = "//" + upth + } + } else { + // no host, let's figure out if this is a drive letter + upth = strings.TrimPrefix(upth, `file://`) + first, _, _ := strings.Cut(strings.TrimPrefix(u.Path, "/"), "/") + if strings.HasSuffix(first, ":") { + // drive letter in the first segment: + // file:///c:/folder/... ==> strip the leading slash + upth = strings.TrimPrefix(upth, `/`) } } diff --git a/go-controller/vendor/github.com/go-openapi/swag/name_lexem.go b/go-controller/vendor/github.com/go-openapi/swag/name_lexem.go index aa7f6a9bb8..8bb64ac32f 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/name_lexem.go +++ b/go-controller/vendor/github.com/go-openapi/swag/name_lexem.go @@ -14,74 +14,80 @@ package swag -import "unicode" +import ( + "unicode" + "unicode/utf8" +) type ( - nameLexem interface { - GetUnsafeGoName() string - GetOriginal() string - IsInitialism() bool - } + lexemKind uint8 - initialismNameLexem struct { + nameLexem struct { original string matchedInitialism string + kind lexemKind } +) - casualNameLexem struct { - original string - } +const ( + lexemKindCasualName lexemKind = iota + lexemKindInitialismName ) -func newInitialismNameLexem(original, matchedInitialism string) *initialismNameLexem { - return &initialismNameLexem{ +func newInitialismNameLexem(original, matchedInitialism string) nameLexem { + return nameLexem{ + kind: lexemKindInitialismName, original: original, matchedInitialism: matchedInitialism, } } -func newCasualNameLexem(original string) *casualNameLexem { - return &casualNameLexem{ +func newCasualNameLexem(original string) nameLexem { + return nameLexem{ + kind: lexemKindCasualName, original: original, } } -func (l *initialismNameLexem) GetUnsafeGoName() string { - return l.matchedInitialism -} +func (l nameLexem) GetUnsafeGoName() string { + if l.kind == lexemKindInitialismName { + return l.matchedInitialism + } + + var ( + first rune + rest string + ) -func (l *casualNameLexem) GetUnsafeGoName() string { - var first rune - var rest string for i, orig := range l.original { if i == 0 { first = orig continue } + if i > 0 { rest = l.original[i:] break } } + if len(l.original) > 1 { - return string(unicode.ToUpper(first)) + lower(rest) + b := poolOfBuffers.BorrowBuffer(utf8.UTFMax + len(rest)) + defer func() { + poolOfBuffers.RedeemBuffer(b) + }() + b.WriteRune(unicode.ToUpper(first)) + b.WriteString(lower(rest)) + return b.String() } return l.original } -func (l *initialismNameLexem) GetOriginal() string { +func (l nameLexem) GetOriginal() string { return l.original } -func (l *casualNameLexem) GetOriginal() string { - return l.original -} - -func (l *initialismNameLexem) IsInitialism() bool { - return true -} - -func (l *casualNameLexem) IsInitialism() bool { - return false +func (l nameLexem) IsInitialism() bool { + return l.kind == lexemKindInitialismName } diff --git a/go-controller/vendor/github.com/go-openapi/swag/post_go18.go b/go-controller/vendor/github.com/go-openapi/swag/post_go18.go deleted file mode 100644 index f5228b82c0..0000000000 --- a/go-controller/vendor/github.com/go-openapi/swag/post_go18.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2015 go-swagger maintainers -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build go1.8 -// +build go1.8 - -package swag - -import "net/url" - -func pathUnescape(path string) (string, error) { - return url.PathUnescape(path) -} diff --git a/go-controller/vendor/github.com/go-openapi/swag/post_go19.go b/go-controller/vendor/github.com/go-openapi/swag/post_go19.go deleted file mode 100644 index 7c7da9c088..0000000000 --- a/go-controller/vendor/github.com/go-openapi/swag/post_go19.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2015 go-swagger maintainers -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build go1.9 -// +build go1.9 - -package swag - -import ( - "sort" - "sync" -) - -// indexOfInitialisms is a thread-safe implementation of the sorted index of initialisms. -// Since go1.9, this may be implemented with sync.Map. -type indexOfInitialisms struct { - sortMutex *sync.Mutex - index *sync.Map -} - -func newIndexOfInitialisms() *indexOfInitialisms { - return &indexOfInitialisms{ - sortMutex: new(sync.Mutex), - index: new(sync.Map), - } -} - -func (m *indexOfInitialisms) load(initial map[string]bool) *indexOfInitialisms { - m.sortMutex.Lock() - defer m.sortMutex.Unlock() - for k, v := range initial { - m.index.Store(k, v) - } - return m -} - -func (m *indexOfInitialisms) isInitialism(key string) bool { - _, ok := m.index.Load(key) - return ok -} - -func (m *indexOfInitialisms) add(key string) *indexOfInitialisms { - m.index.Store(key, true) - return m -} - -func (m *indexOfInitialisms) sorted() (result []string) { - m.sortMutex.Lock() - defer m.sortMutex.Unlock() - m.index.Range(func(key, value interface{}) bool { - k := key.(string) - result = append(result, k) - return true - }) - sort.Sort(sort.Reverse(byInitialism(result))) - return -} diff --git a/go-controller/vendor/github.com/go-openapi/swag/pre_go18.go b/go-controller/vendor/github.com/go-openapi/swag/pre_go18.go deleted file mode 100644 index 2757d9b95f..0000000000 --- a/go-controller/vendor/github.com/go-openapi/swag/pre_go18.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2015 go-swagger maintainers -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !go1.8 -// +build !go1.8 - -package swag - -import "net/url" - -func pathUnescape(path string) (string, error) { - return url.QueryUnescape(path) -} diff --git a/go-controller/vendor/github.com/go-openapi/swag/pre_go19.go b/go-controller/vendor/github.com/go-openapi/swag/pre_go19.go deleted file mode 100644 index 0565db377b..0000000000 --- a/go-controller/vendor/github.com/go-openapi/swag/pre_go19.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2015 go-swagger maintainers -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !go1.9 -// +build !go1.9 - -package swag - -import ( - "sort" - "sync" -) - -// indexOfInitialisms is a thread-safe implementation of the sorted index of initialisms. -// Before go1.9, this may be implemented with a mutex on the map. -type indexOfInitialisms struct { - getMutex *sync.Mutex - index map[string]bool -} - -func newIndexOfInitialisms() *indexOfInitialisms { - return &indexOfInitialisms{ - getMutex: new(sync.Mutex), - index: make(map[string]bool, 50), - } -} - -func (m *indexOfInitialisms) load(initial map[string]bool) *indexOfInitialisms { - m.getMutex.Lock() - defer m.getMutex.Unlock() - for k, v := range initial { - m.index[k] = v - } - return m -} - -func (m *indexOfInitialisms) isInitialism(key string) bool { - m.getMutex.Lock() - defer m.getMutex.Unlock() - _, ok := m.index[key] - return ok -} - -func (m *indexOfInitialisms) add(key string) *indexOfInitialisms { - m.getMutex.Lock() - defer m.getMutex.Unlock() - m.index[key] = true - return m -} - -func (m *indexOfInitialisms) sorted() (result []string) { - m.getMutex.Lock() - defer m.getMutex.Unlock() - for k := range m.index { - result = append(result, k) - } - sort.Sort(sort.Reverse(byInitialism(result))) - return -} diff --git a/go-controller/vendor/github.com/go-openapi/swag/split.go b/go-controller/vendor/github.com/go-openapi/swag/split.go index a1825fb7dc..274727a866 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/split.go +++ b/go-controller/vendor/github.com/go-openapi/swag/split.go @@ -15,124 +15,269 @@ package swag import ( + "bytes" + "sync" "unicode" + "unicode/utf8" ) -var nameReplaceTable = map[rune]string{ - '@': "At ", - '&': "And ", - '|': "Pipe ", - '$': "Dollar ", - '!': "Bang ", - '-': "", - '_': "", -} - type ( splitter struct { - postSplitInitialismCheck bool initialisms []string + initialismsRunes [][]rune + initialismsUpperCased [][]rune // initialisms cached in their trimmed, upper-cased version + postSplitInitialismCheck bool + } + + splitterOption func(*splitter) + + initialismMatch struct { + body []rune + start, end int + complete bool + } + initialismMatches []initialismMatch +) + +type ( + // memory pools of temporary objects. + // + // These are used to recycle temporarily allocated objects + // and relieve the GC from undue pressure. + + matchesPool struct { + *sync.Pool } - splitterOption func(*splitter) *splitter + buffersPool struct { + *sync.Pool + } + + lexemsPool struct { + *sync.Pool + } + + splittersPool struct { + *sync.Pool + } ) -// split calls the splitter; splitter provides more control and post options +var ( + // poolOfMatches holds temporary slices for recycling during the initialism match process + poolOfMatches = matchesPool{ + Pool: &sync.Pool{ + New: func() any { + s := make(initialismMatches, 0, maxAllocMatches) + + return &s + }, + }, + } + + poolOfBuffers = buffersPool{ + Pool: &sync.Pool{ + New: func() any { + return new(bytes.Buffer) + }, + }, + } + + poolOfLexems = lexemsPool{ + Pool: &sync.Pool{ + New: func() any { + s := make([]nameLexem, 0, maxAllocMatches) + + return &s + }, + }, + } + + poolOfSplitters = splittersPool{ + Pool: &sync.Pool{ + New: func() any { + s := newSplitter() + + return &s + }, + }, + } +) + +// nameReplaceTable finds a word representation for special characters. +func nameReplaceTable(r rune) (string, bool) { + switch r { + case '@': + return "At ", true + case '&': + return "And ", true + case '|': + return "Pipe ", true + case '$': + return "Dollar ", true + case '!': + return "Bang ", true + case '-': + return "", true + case '_': + return "", true + default: + return "", false + } +} + +// split calls the splitter. +// +// Use newSplitter for more control and options func split(str string) []string { - lexems := newSplitter().split(str) - result := make([]string, 0, len(lexems)) + s := poolOfSplitters.BorrowSplitter() + lexems := s.split(str) + result := make([]string, 0, len(*lexems)) - for _, lexem := range lexems { + for _, lexem := range *lexems { result = append(result, lexem.GetOriginal()) } + poolOfLexems.RedeemLexems(lexems) + poolOfSplitters.RedeemSplitter(s) return result } -func (s *splitter) split(str string) []nameLexem { - return s.toNameLexems(str) -} - -func newSplitter(options ...splitterOption) *splitter { - splitter := &splitter{ +func newSplitter(options ...splitterOption) splitter { + s := splitter{ postSplitInitialismCheck: false, initialisms: initialisms, + initialismsRunes: initialismsRunes, + initialismsUpperCased: initialismsUpperCased, } for _, option := range options { - splitter = option(splitter) + option(&s) } - return splitter + return s } // withPostSplitInitialismCheck allows to catch initialisms after main split process -func withPostSplitInitialismCheck(s *splitter) *splitter { +func withPostSplitInitialismCheck(s *splitter) { s.postSplitInitialismCheck = true +} + +func (p matchesPool) BorrowMatches() *initialismMatches { + s := p.Get().(*initialismMatches) + *s = (*s)[:0] // reset slice, keep allocated capacity + return s } -type ( - initialismMatch struct { - start, end int - body []rune - complete bool +func (p buffersPool) BorrowBuffer(size int) *bytes.Buffer { + s := p.Get().(*bytes.Buffer) + s.Reset() + + if s.Cap() < size { + s.Grow(size) } - initialismMatches []*initialismMatch -) -func (s *splitter) toNameLexems(name string) []nameLexem { + return s +} + +func (p lexemsPool) BorrowLexems() *[]nameLexem { + s := p.Get().(*[]nameLexem) + *s = (*s)[:0] // reset slice, keep allocated capacity + + return s +} + +func (p splittersPool) BorrowSplitter(options ...splitterOption) *splitter { + s := p.Get().(*splitter) + s.postSplitInitialismCheck = false // reset options + for _, apply := range options { + apply(s) + } + + return s +} + +func (p matchesPool) RedeemMatches(s *initialismMatches) { + p.Put(s) +} + +func (p buffersPool) RedeemBuffer(s *bytes.Buffer) { + p.Put(s) +} + +func (p lexemsPool) RedeemLexems(s *[]nameLexem) { + p.Put(s) +} + +func (p splittersPool) RedeemSplitter(s *splitter) { + p.Put(s) +} + +func (m initialismMatch) isZero() bool { + return m.start == 0 && m.end == 0 +} + +func (s splitter) split(name string) *[]nameLexem { nameRunes := []rune(name) matches := s.gatherInitialismMatches(nameRunes) + if matches == nil { + return poolOfLexems.BorrowLexems() + } + return s.mapMatchesToNameLexems(nameRunes, matches) } -func (s *splitter) gatherInitialismMatches(nameRunes []rune) initialismMatches { - matches := make(initialismMatches, 0) +func (s splitter) gatherInitialismMatches(nameRunes []rune) *initialismMatches { + var matches *initialismMatches for currentRunePosition, currentRune := range nameRunes { - newMatches := make(initialismMatches, 0, len(matches)) + // recycle these allocations as we loop over runes + // with such recycling, only 2 slices should be allocated per call + // instead of o(n). + newMatches := poolOfMatches.BorrowMatches() // check current initialism matches - for _, match := range matches { - if keepCompleteMatch := match.complete; keepCompleteMatch { - newMatches = append(newMatches, match) - continue - } + if matches != nil { // skip first iteration + for _, match := range *matches { + if keepCompleteMatch := match.complete; keepCompleteMatch { + *newMatches = append(*newMatches, match) + continue + } - // drop failed match - currentMatchRune := match.body[currentRunePosition-match.start] - if !s.initialismRuneEqual(currentMatchRune, currentRune) { - continue - } + // drop failed match + currentMatchRune := match.body[currentRunePosition-match.start] + if currentMatchRune != currentRune { + continue + } - // try to complete ongoing match - if currentRunePosition-match.start == len(match.body)-1 { - // we are close; the next step is to check the symbol ahead - // if it is a small letter, then it is not the end of match - // but beginning of the next word - - if currentRunePosition < len(nameRunes)-1 { - nextRune := nameRunes[currentRunePosition+1] - if newWord := unicode.IsLower(nextRune); newWord { - // oh ok, it was the start of a new word - continue + // try to complete ongoing match + if currentRunePosition-match.start == len(match.body)-1 { + // we are close; the next step is to check the symbol ahead + // if it is a small letter, then it is not the end of match + // but beginning of the next word + + if currentRunePosition < len(nameRunes)-1 { + nextRune := nameRunes[currentRunePosition+1] + if newWord := unicode.IsLower(nextRune); newWord { + // oh ok, it was the start of a new word + continue + } } + + match.complete = true + match.end = currentRunePosition } - match.complete = true - match.end = currentRunePosition + *newMatches = append(*newMatches, match) } - - newMatches = append(newMatches, match) } // check for new initialism matches - for _, initialism := range s.initialisms { - initialismRunes := []rune(initialism) - if s.initialismRuneEqual(initialismRunes[0], currentRune) { - newMatches = append(newMatches, &initialismMatch{ + for i := range s.initialisms { + initialismRunes := s.initialismsRunes[i] + if initialismRunes[0] == currentRune { + *newMatches = append(*newMatches, initialismMatch{ start: currentRunePosition, body: initialismRunes, complete: false, @@ -140,24 +285,28 @@ func (s *splitter) gatherInitialismMatches(nameRunes []rune) initialismMatches { } } + if matches != nil { + poolOfMatches.RedeemMatches(matches) + } matches = newMatches } + // up to the caller to redeem this last slice return matches } -func (s *splitter) mapMatchesToNameLexems(nameRunes []rune, matches initialismMatches) []nameLexem { - nameLexems := make([]nameLexem, 0) +func (s splitter) mapMatchesToNameLexems(nameRunes []rune, matches *initialismMatches) *[]nameLexem { + nameLexems := poolOfLexems.BorrowLexems() - var lastAcceptedMatch *initialismMatch - for _, match := range matches { + var lastAcceptedMatch initialismMatch + for _, match := range *matches { if !match.complete { continue } - if firstMatch := lastAcceptedMatch == nil; firstMatch { - nameLexems = append(nameLexems, s.breakCasualString(nameRunes[:match.start])...) - nameLexems = append(nameLexems, s.breakInitialism(string(match.body))) + if firstMatch := lastAcceptedMatch.isZero(); firstMatch { + s.appendBrokenDownCasualString(nameLexems, nameRunes[:match.start]) + *nameLexems = append(*nameLexems, s.breakInitialism(string(match.body))) lastAcceptedMatch = match @@ -169,63 +318,66 @@ func (s *splitter) mapMatchesToNameLexems(nameRunes []rune, matches initialismMa } middle := nameRunes[lastAcceptedMatch.end+1 : match.start] - nameLexems = append(nameLexems, s.breakCasualString(middle)...) - nameLexems = append(nameLexems, s.breakInitialism(string(match.body))) + s.appendBrokenDownCasualString(nameLexems, middle) + *nameLexems = append(*nameLexems, s.breakInitialism(string(match.body))) lastAcceptedMatch = match } // we have not found any accepted matches - if lastAcceptedMatch == nil { - return s.breakCasualString(nameRunes) - } - - if lastAcceptedMatch.end+1 != len(nameRunes) { + if lastAcceptedMatch.isZero() { + *nameLexems = (*nameLexems)[:0] + s.appendBrokenDownCasualString(nameLexems, nameRunes) + } else if lastAcceptedMatch.end+1 != len(nameRunes) { rest := nameRunes[lastAcceptedMatch.end+1:] - nameLexems = append(nameLexems, s.breakCasualString(rest)...) + s.appendBrokenDownCasualString(nameLexems, rest) } - return nameLexems -} + poolOfMatches.RedeemMatches(matches) -func (s *splitter) initialismRuneEqual(a, b rune) bool { - return a == b + return nameLexems } -func (s *splitter) breakInitialism(original string) nameLexem { +func (s splitter) breakInitialism(original string) nameLexem { return newInitialismNameLexem(original, original) } -func (s *splitter) breakCasualString(str []rune) []nameLexem { - segments := make([]nameLexem, 0) - currentSegment := "" +func (s splitter) appendBrokenDownCasualString(segments *[]nameLexem, str []rune) { + currentSegment := poolOfBuffers.BorrowBuffer(len(str)) // unlike strings.Builder, bytes.Buffer initial storage can reused + defer func() { + poolOfBuffers.RedeemBuffer(currentSegment) + }() addCasualNameLexem := func(original string) { - segments = append(segments, newCasualNameLexem(original)) + *segments = append(*segments, newCasualNameLexem(original)) } addInitialismNameLexem := func(original, match string) { - segments = append(segments, newInitialismNameLexem(original, match)) + *segments = append(*segments, newInitialismNameLexem(original, match)) } - addNameLexem := func(original string) { - if s.postSplitInitialismCheck { - for _, initialism := range s.initialisms { - if upper(initialism) == upper(original) { - addInitialismNameLexem(original, initialism) + var addNameLexem func(string) + if s.postSplitInitialismCheck { + addNameLexem = func(original string) { + for i := range s.initialisms { + if isEqualFoldIgnoreSpace(s.initialismsUpperCased[i], original) { + addInitialismNameLexem(original, s.initialisms[i]) + return } } - } - addCasualNameLexem(original) + addCasualNameLexem(original) + } + } else { + addNameLexem = addCasualNameLexem } - for _, rn := range string(str) { - if replace, found := nameReplaceTable[rn]; found { - if currentSegment != "" { - addNameLexem(currentSegment) - currentSegment = "" + for _, rn := range str { + if replace, found := nameReplaceTable(rn); found { + if currentSegment.Len() > 0 { + addNameLexem(currentSegment.String()) + currentSegment.Reset() } if replace != "" { @@ -236,27 +388,121 @@ func (s *splitter) breakCasualString(str []rune) []nameLexem { } if !unicode.In(rn, unicode.L, unicode.M, unicode.N, unicode.Pc) { - if currentSegment != "" { - addNameLexem(currentSegment) - currentSegment = "" + if currentSegment.Len() > 0 { + addNameLexem(currentSegment.String()) + currentSegment.Reset() } continue } if unicode.IsUpper(rn) { - if currentSegment != "" { - addNameLexem(currentSegment) + if currentSegment.Len() > 0 { + addNameLexem(currentSegment.String()) } - currentSegment = "" + currentSegment.Reset() } - currentSegment += string(rn) + currentSegment.WriteRune(rn) + } + + if currentSegment.Len() > 0 { + addNameLexem(currentSegment.String()) } +} + +// isEqualFoldIgnoreSpace is the same as strings.EqualFold, but +// it ignores leading and trailing blank spaces in the compared +// string. +// +// base is assumed to be composed of upper-cased runes, and be already +// trimmed. +// +// This code is heavily inspired from strings.EqualFold. +func isEqualFoldIgnoreSpace(base []rune, str string) bool { + var i, baseIndex int + // equivalent to b := []byte(str), but without data copy + b := hackStringBytes(str) + + for i < len(b) { + if c := b[i]; c < utf8.RuneSelf { + // fast path for ASCII + if c != ' ' && c != '\t' { + break + } + i++ + + continue + } + + // unicode case + r, size := utf8.DecodeRune(b[i:]) + if !unicode.IsSpace(r) { + break + } + i += size + } + + if i >= len(b) { + return len(base) == 0 + } + + for _, baseRune := range base { + if i >= len(b) { + break + } + + if c := b[i]; c < utf8.RuneSelf { + // single byte rune case (ASCII) + if baseRune >= utf8.RuneSelf { + return false + } + + baseChar := byte(baseRune) + if c != baseChar && + !('a' <= c && c <= 'z' && c-'a'+'A' == baseChar) { + return false + } + + baseIndex++ + i++ + + continue + } + + // unicode case + r, size := utf8.DecodeRune(b[i:]) + if unicode.ToUpper(r) != baseRune { + return false + } + baseIndex++ + i += size + } + + if baseIndex != len(base) { + return false + } + + // all passed: now we should only have blanks + for i < len(b) { + if c := b[i]; c < utf8.RuneSelf { + // fast path for ASCII + if c != ' ' && c != '\t' { + return false + } + i++ + + continue + } + + // unicode case + r, size := utf8.DecodeRune(b[i:]) + if !unicode.IsSpace(r) { + return false + } - if currentSegment != "" { - addNameLexem(currentSegment) + i += size } - return segments + return true } diff --git a/go-controller/vendor/github.com/go-openapi/swag/string_bytes.go b/go-controller/vendor/github.com/go-openapi/swag/string_bytes.go new file mode 100644 index 0000000000..90745d5ca9 --- /dev/null +++ b/go-controller/vendor/github.com/go-openapi/swag/string_bytes.go @@ -0,0 +1,8 @@ +package swag + +import "unsafe" + +// hackStringBytes returns the (unsafe) underlying bytes slice of a string. +func hackStringBytes(str string) []byte { + return unsafe.Slice(unsafe.StringData(str), len(str)) +} diff --git a/go-controller/vendor/github.com/go-openapi/swag/util.go b/go-controller/vendor/github.com/go-openapi/swag/util.go index d971fbe34b..5051401c49 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/util.go +++ b/go-controller/vendor/github.com/go-openapi/swag/util.go @@ -18,76 +18,25 @@ import ( "reflect" "strings" "unicode" + "unicode/utf8" ) -// commonInitialisms are common acronyms that are kept as whole uppercased words. -var commonInitialisms *indexOfInitialisms - -// initialisms is a slice of sorted initialisms -var initialisms []string - -var isInitialism func(string) bool - // GoNamePrefixFunc sets an optional rule to prefix go names // which do not start with a letter. // +// The prefix function is assumed to return a string that starts with an upper case letter. +// // e.g. to help convert "123" into "{prefix}123" // // The default is to prefix with "X" var GoNamePrefixFunc func(string) string -func init() { - // Taken from https://github.com/golang/lint/blob/3390df4df2787994aea98de825b964ac7944b817/lint.go#L732-L769 - var configuredInitialisms = map[string]bool{ - "ACL": true, - "API": true, - "ASCII": true, - "CPU": true, - "CSS": true, - "DNS": true, - "EOF": true, - "GUID": true, - "HTML": true, - "HTTPS": true, - "HTTP": true, - "ID": true, - "IP": true, - "IPv4": true, - "IPv6": true, - "JSON": true, - "LHS": true, - "OAI": true, - "QPS": true, - "RAM": true, - "RHS": true, - "RPC": true, - "SLA": true, - "SMTP": true, - "SQL": true, - "SSH": true, - "TCP": true, - "TLS": true, - "TTL": true, - "UDP": true, - "UI": true, - "UID": true, - "UUID": true, - "URI": true, - "URL": true, - "UTF8": true, - "VM": true, - "XML": true, - "XMPP": true, - "XSRF": true, - "XSS": true, +func prefixFunc(name, in string) string { + if GoNamePrefixFunc == nil { + return "X" + in } - // a thread-safe index of initialisms - commonInitialisms = newIndexOfInitialisms().load(configuredInitialisms) - initialisms = commonInitialisms.sorted() - - // a test function - isInitialism = commonInitialisms.isInitialism + return GoNamePrefixFunc(name) + in } const ( @@ -156,25 +105,9 @@ func SplitByFormat(data, format string) []string { return result } -type byInitialism []string - -func (s byInitialism) Len() int { - return len(s) -} -func (s byInitialism) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} -func (s byInitialism) Less(i, j int) bool { - if len(s[i]) != len(s[j]) { - return len(s[i]) < len(s[j]) - } - - return strings.Compare(s[i], s[j]) > 0 -} - // Removes leading whitespaces func trim(str string) string { - return strings.Trim(str, " ") + return strings.TrimSpace(str) } // Shortcut to strings.ToUpper() @@ -188,15 +121,20 @@ func lower(str string) string { } // Camelize an uppercased word -func Camelize(word string) (camelized string) { +func Camelize(word string) string { + camelized := poolOfBuffers.BorrowBuffer(len(word)) + defer func() { + poolOfBuffers.RedeemBuffer(camelized) + }() + for pos, ru := range []rune(word) { if pos > 0 { - camelized += string(unicode.ToLower(ru)) + camelized.WriteRune(unicode.ToLower(ru)) } else { - camelized += string(unicode.ToUpper(ru)) + camelized.WriteRune(unicode.ToUpper(ru)) } } - return + return camelized.String() } // ToFileName lowercases and underscores a go type name @@ -224,33 +162,40 @@ func ToCommandName(name string) string { // ToHumanNameLower represents a code name as a human series of words func ToHumanNameLower(name string) string { - in := newSplitter(withPostSplitInitialismCheck).split(name) - out := make([]string, 0, len(in)) + s := poolOfSplitters.BorrowSplitter(withPostSplitInitialismCheck) + in := s.split(name) + poolOfSplitters.RedeemSplitter(s) + out := make([]string, 0, len(*in)) - for _, w := range in { + for _, w := range *in { if !w.IsInitialism() { out = append(out, lower(w.GetOriginal())) } else { - out = append(out, w.GetOriginal()) + out = append(out, trim(w.GetOriginal())) } } + poolOfLexems.RedeemLexems(in) return strings.Join(out, " ") } // ToHumanNameTitle represents a code name as a human series of words with the first letters titleized func ToHumanNameTitle(name string) string { - in := newSplitter(withPostSplitInitialismCheck).split(name) + s := poolOfSplitters.BorrowSplitter(withPostSplitInitialismCheck) + in := s.split(name) + poolOfSplitters.RedeemSplitter(s) - out := make([]string, 0, len(in)) - for _, w := range in { - original := w.GetOriginal() + out := make([]string, 0, len(*in)) + for _, w := range *in { + original := trim(w.GetOriginal()) if !w.IsInitialism() { out = append(out, Camelize(original)) } else { out = append(out, original) } } + poolOfLexems.RedeemLexems(in) + return strings.Join(out, " ") } @@ -264,7 +209,7 @@ func ToJSONName(name string) string { out = append(out, lower(w)) continue } - out = append(out, Camelize(w)) + out = append(out, Camelize(trim(w))) } return strings.Join(out, "") } @@ -283,35 +228,70 @@ func ToVarName(name string) string { // ToGoName translates a swagger name which can be underscored or camel cased to a name that golint likes func ToGoName(name string) string { - lexems := newSplitter(withPostSplitInitialismCheck).split(name) + s := poolOfSplitters.BorrowSplitter(withPostSplitInitialismCheck) + lexems := s.split(name) + poolOfSplitters.RedeemSplitter(s) + defer func() { + poolOfLexems.RedeemLexems(lexems) + }() + lexemes := *lexems + + if len(lexemes) == 0 { + return "" + } + + result := poolOfBuffers.BorrowBuffer(len(name)) + defer func() { + poolOfBuffers.RedeemBuffer(result) + }() + + // check if not starting with a letter, upper case + firstPart := lexemes[0].GetUnsafeGoName() + if lexemes[0].IsInitialism() { + firstPart = upper(firstPart) + } + + if c := firstPart[0]; c < utf8.RuneSelf { + // ASCII + switch { + case 'A' <= c && c <= 'Z': + result.WriteString(firstPart) + case 'a' <= c && c <= 'z': + result.WriteByte(c - 'a' + 'A') + result.WriteString(firstPart[1:]) + default: + result.WriteString(prefixFunc(name, firstPart)) + // NOTE: no longer check if prefixFunc returns a string that starts with uppercase: + // assume this is always the case + } + } else { + // unicode + firstRune, _ := utf8.DecodeRuneInString(firstPart) + switch { + case !unicode.IsLetter(firstRune): + result.WriteString(prefixFunc(name, firstPart)) + case !unicode.IsUpper(firstRune): + result.WriteString(prefixFunc(name, firstPart)) + /* + result.WriteRune(unicode.ToUpper(firstRune)) + result.WriteString(firstPart[offset:]) + */ + default: + result.WriteString(firstPart) + } + } - result := "" - for _, lexem := range lexems { + for _, lexem := range lexemes[1:] { goName := lexem.GetUnsafeGoName() // to support old behavior if lexem.IsInitialism() { goName = upper(goName) } - result += goName + result.WriteString(goName) } - if len(result) > 0 { - // Only prefix with X when the first character isn't an ascii letter - first := []rune(result)[0] - if !unicode.IsLetter(first) || (first > unicode.MaxASCII && !unicode.IsUpper(first)) { - if GoNamePrefixFunc == nil { - return "X" + result - } - result = GoNamePrefixFunc(name) + result - } - first = []rune(result)[0] - if unicode.IsLetter(first) && !unicode.IsUpper(first) { - result = string(append([]rune{unicode.ToUpper(first)}, []rune(result)[1:]...)) - } - } - - return result + return result.String() } // ContainsStrings searches a slice of strings for a case-sensitive match @@ -343,7 +323,7 @@ type zeroable interface { func IsZero(data interface{}) bool { v := reflect.ValueOf(data) // check for nil data - switch v.Kind() { + switch v.Kind() { //nolint:exhaustive case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: if v.IsNil() { return true @@ -356,7 +336,7 @@ func IsZero(data interface{}) bool { } // continue with slightly more complex reflection - switch v.Kind() { + switch v.Kind() { //nolint:exhaustive case reflect.String: return v.Len() == 0 case reflect.Bool: @@ -376,16 +356,6 @@ func IsZero(data interface{}) bool { } } -// AddInitialisms add additional initialisms -func AddInitialisms(words ...string) { - for _, word := range words { - // commonInitialisms[upper(word)] = true - commonInitialisms.add(upper(word)) - } - // sort again - initialisms = commonInitialisms.sorted() -} - // CommandLineOptionsGroup represents a group of user-defined command line options type CommandLineOptionsGroup struct { ShortDescription string diff --git a/go-controller/vendor/github.com/go-openapi/swag/yaml.go b/go-controller/vendor/github.com/go-openapi/swag/yaml.go index f09ee609f3..f59e025932 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/yaml.go +++ b/go-controller/vendor/github.com/go-openapi/swag/yaml.go @@ -16,8 +16,11 @@ package swag import ( "encoding/json" + "errors" "fmt" "path/filepath" + "reflect" + "sort" "strconv" "github.com/mailru/easyjson/jlexer" @@ -48,7 +51,7 @@ func BytesToYAMLDoc(data []byte) (interface{}, error) { return nil, err } if document.Kind != yaml.DocumentNode || len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { - return nil, fmt.Errorf("only YAML documents that are objects are supported") + return nil, errors.New("only YAML documents that are objects are supported") } return &document, nil } @@ -147,7 +150,7 @@ func yamlScalar(node *yaml.Node) (interface{}, error) { case yamlTimestamp: return node.Value, nil case yamlNull: - return nil, nil + return nil, nil //nolint:nilnil default: return nil, fmt.Errorf("YAML tag %q is not supported", node.LongTag()) } @@ -245,7 +248,27 @@ func (s JSONMapSlice) MarshalYAML() (interface{}, error) { return yaml.Marshal(&n) } +func isNil(input interface{}) bool { + if input == nil { + return true + } + kind := reflect.TypeOf(input).Kind() + switch kind { //nolint:exhaustive + case reflect.Ptr, reflect.Map, reflect.Slice, reflect.Chan: + return reflect.ValueOf(input).IsNil() + default: + return false + } +} + func json2yaml(item interface{}) (*yaml.Node, error) { + if isNil(item) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Value: "null", + }, nil + } + switch val := item.(type) { case JSONMapSlice: var n yaml.Node @@ -265,7 +288,14 @@ func json2yaml(item interface{}) (*yaml.Node, error) { case map[string]interface{}: var n yaml.Node n.Kind = yaml.MappingNode - for k, v := range val { + keys := make([]string, 0, len(val)) + for k := range val { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + v := val[k] childNode, err := json2yaml(v) if err != nil { return nil, err @@ -318,8 +348,9 @@ func json2yaml(item interface{}) (*yaml.Node, error) { Tag: yamlBoolScalar, Value: strconv.FormatBool(val), }, nil + default: + return nil, fmt.Errorf("unhandled type: %T", val) } - return nil, nil } // JSONMapItem represents the value of a key in a JSON object held by JSONMapSlice diff --git a/go-controller/vendor/github.com/imdario/mergo/CONTRIBUTING.md b/go-controller/vendor/github.com/imdario/mergo/CONTRIBUTING.md new file mode 100644 index 0000000000..0a1ff9f94d --- /dev/null +++ b/go-controller/vendor/github.com/imdario/mergo/CONTRIBUTING.md @@ -0,0 +1,112 @@ + +# Contributing to mergo + +First off, thanks for taking the time to contribute! ❤️ + +All types of contributions are encouraged and valued. See the [Table of Contents](#table-of-contents) for different ways to help and details about how this project handles them. Please make sure to read the relevant section before making your contribution. It will make it a lot easier for us maintainers and smooth out the experience for all involved. The community looks forward to your contributions. 🎉 + +> And if you like the project, but just don't have time to contribute, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about: +> - Star the project +> - Tweet about it +> - Refer this project in your project's readme +> - Mention the project at local meetups and tell your friends/colleagues + + +## Table of Contents + +- [Code of Conduct](#code-of-conduct) +- [I Have a Question](#i-have-a-question) +- [I Want To Contribute](#i-want-to-contribute) +- [Reporting Bugs](#reporting-bugs) +- [Suggesting Enhancements](#suggesting-enhancements) + +## Code of Conduct + +This project and everyone participating in it is governed by the +[mergo Code of Conduct](https://github.com/imdario/mergoblob/master/CODE_OF_CONDUCT.md). +By participating, you are expected to uphold this code. Please report unacceptable behavior +to <>. + + +## I Have a Question + +> If you want to ask a question, we assume that you have read the available [Documentation](https://pkg.go.dev/github.com/imdario/mergo). + +Before you ask a question, it is best to search for existing [Issues](https://github.com/imdario/mergo/issues) that might help you. In case you have found a suitable issue and still need clarification, you can write your question in this issue. It is also advisable to search the internet for answers first. + +If you then still feel the need to ask a question and need clarification, we recommend the following: + +- Open an [Issue](https://github.com/imdario/mergo/issues/new). +- Provide as much context as you can about what you're running into. +- Provide project and platform versions (nodejs, npm, etc), depending on what seems relevant. + +We will then take care of the issue as soon as possible. + +## I Want To Contribute + +> ### Legal Notice +> When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license. + +### Reporting Bugs + + +#### Before Submitting a Bug Report + +A good bug report shouldn't leave others needing to chase you up for more information. Therefore, we ask you to investigate carefully, collect information and describe the issue in detail in your report. Please complete the following steps in advance to help us fix any potential bug as fast as possible. + +- Make sure that you are using the latest version. +- Determine if your bug is really a bug and not an error on your side e.g. using incompatible environment components/versions (Make sure that you have read the [documentation](). If you are looking for support, you might want to check [this section](#i-have-a-question)). +- To see if other users have experienced (and potentially already solved) the same issue you are having, check if there is not already a bug report existing for your bug or error in the [bug tracker](https://github.com/imdario/mergoissues?q=label%3Abug). +- Also make sure to search the internet (including Stack Overflow) to see if users outside of the GitHub community have discussed the issue. +- Collect information about the bug: +- Stack trace (Traceback) +- OS, Platform and Version (Windows, Linux, macOS, x86, ARM) +- Version of the interpreter, compiler, SDK, runtime environment, package manager, depending on what seems relevant. +- Possibly your input and the output +- Can you reliably reproduce the issue? And can you also reproduce it with older versions? + + +#### How Do I Submit a Good Bug Report? + +> You must never report security related issues, vulnerabilities or bugs including sensitive information to the issue tracker, or elsewhere in public. Instead sensitive bugs must be sent by email to . + + +We use GitHub issues to track bugs and errors. If you run into an issue with the project: + +- Open an [Issue](https://github.com/imdario/mergo/issues/new). (Since we can't be sure at this point whether it is a bug or not, we ask you not to talk about a bug yet and not to label the issue.) +- Explain the behavior you would expect and the actual behavior. +- Please provide as much context as possible and describe the *reproduction steps* that someone else can follow to recreate the issue on their own. This usually includes your code. For good bug reports you should isolate the problem and create a reduced test case. +- Provide the information you collected in the previous section. + +Once it's filed: + +- The project team will label the issue accordingly. +- A team member will try to reproduce the issue with your provided steps. If there are no reproduction steps or no obvious way to reproduce the issue, the team will ask you for those steps and mark the issue as `needs-repro`. Bugs with the `needs-repro` tag will not be addressed until they are reproduced. +- If the team is able to reproduce the issue, it will be marked `needs-fix`, as well as possibly other tags (such as `critical`), and the issue will be left to be implemented by someone. + +### Suggesting Enhancements + +This section guides you through submitting an enhancement suggestion for mergo, **including completely new features and minor improvements to existing functionality**. Following these guidelines will help maintainers and the community to understand your suggestion and find related suggestions. + + +#### Before Submitting an Enhancement + +- Make sure that you are using the latest version. +- Read the [documentation]() carefully and find out if the functionality is already covered, maybe by an individual configuration. +- Perform a [search](https://github.com/imdario/mergo/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one. +- Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on/plugin library. + + +#### How Do I Submit a Good Enhancement Suggestion? + +Enhancement suggestions are tracked as [GitHub issues](https://github.com/imdario/mergo/issues). + +- Use a **clear and descriptive title** for the issue to identify the suggestion. +- Provide a **step-by-step description of the suggested enhancement** in as many details as possible. +- **Describe the current behavior** and **explain which behavior you expected to see instead** and why. At this point you can also tell which alternatives do not work for you. +- You may want to **include screenshots and animated GIFs** which help you demonstrate the steps or point out the part which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. +- **Explain why this enhancement would be useful** to most mergo users. You may also want to point out the other projects that solved it better and which could serve as inspiration. + + +## Attribution +This guide is based on the **contributing-gen**. [Make your own](https://github.com/bttger/contributing-gen)! diff --git a/go-controller/vendor/github.com/imdario/mergo/README.md b/go-controller/vendor/github.com/imdario/mergo/README.md index aa8cbd7ce6..ffbbb62c70 100644 --- a/go-controller/vendor/github.com/imdario/mergo/README.md +++ b/go-controller/vendor/github.com/imdario/mergo/README.md @@ -1,18 +1,20 @@ # Mergo - -[![GoDoc][3]][4] [![GitHub release][5]][6] [![GoCard][7]][8] -[![Build Status][1]][2] -[![Coverage Status][9]][10] +[![Test status][1]][2] +[![OpenSSF Scorecard][21]][22] +[![OpenSSF Best Practices][19]][20] +[![Coverage status][9]][10] [![Sourcegraph][11]][12] -[![FOSSA Status][13]][14] +[![FOSSA status][13]][14] -[![GoCenter Kudos][15]][16] +[![GoDoc][3]][4] +[![Become my sponsor][15]][16] +[![Tidelift][17]][18] -[1]: https://travis-ci.org/imdario/mergo.png -[2]: https://travis-ci.org/imdario/mergo +[1]: https://github.com/imdario/mergo/workflows/tests/badge.svg?branch=master +[2]: https://github.com/imdario/mergo/actions/workflows/tests.yml [3]: https://godoc.org/github.com/imdario/mergo?status.svg [4]: https://godoc.org/github.com/imdario/mergo [5]: https://img.shields.io/github/release/imdario/mergo.svg @@ -25,8 +27,14 @@ [12]: https://sourcegraph.com/github.com/imdario/mergo?badge [13]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield [14]: https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield -[15]: https://search.gocenter.io/api/ui/badge/github.com%2Fimdario%2Fmergo -[16]: https://search.gocenter.io/github.com/imdario/mergo +[15]: https://img.shields.io/github/sponsors/imdario +[16]: https://github.com/sponsors/imdario +[17]: https://tidelift.com/badges/package/go/github.com%2Fimdario%2Fmergo +[18]: https://tidelift.com/subscription/pkg/go-github.com-imdario-mergo +[19]: https://bestpractices.coreinfrastructure.org/projects/7177/badge +[20]: https://bestpractices.coreinfrastructure.org/projects/7177 +[21]: https://api.securityscorecards.dev/projects/github.com/imdario/mergo/badge +[22]: https://api.securityscorecards.dev/projects/github.com/imdario/mergo A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements. @@ -36,11 +44,11 @@ Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the ## Status -It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc](https://github.com/imdario/mergo#mergo-in-the-wild). +It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, Microsoft, etc](https://github.com/imdario/mergo#mergo-in-the-wild). ### Important note -Please keep in mind that a problematic PR broke [0.3.9](//github.com/imdario/mergo/releases/tag/0.3.9). I reverted it in [0.3.10](//github.com/imdario/mergo/releases/tag/0.3.10), and I consider it stable but not bug-free. Also, this version adds suppot for go modules. +Please keep in mind that a problematic PR broke [0.3.9](//github.com/imdario/mergo/releases/tag/0.3.9). I reverted it in [0.3.10](//github.com/imdario/mergo/releases/tag/0.3.10), and I consider it stable but not bug-free. Also, this version adds support for go modules. Keep in mind that in [0.3.2](//github.com/imdario/mergo/releases/tag/0.3.2), Mergo changed `Merge()`and `Map()` signatures to support [transformers](#transformers). I added an optional/variadic argument so that it won't break the existing code. @@ -51,9 +59,8 @@ If you were using Mergo before April 6th, 2015, please check your project works If Mergo is useful to you, consider buying me a coffee, a beer, or making a monthly donation to allow me to keep building great free software. :heart_eyes: Buy Me a Coffee at ko-fi.com -[![Beerpay](https://beerpay.io/imdario/mergo/badge.svg)](https://beerpay.io/imdario/mergo) -[![Beerpay](https://beerpay.io/imdario/mergo/make-wish.svg)](https://beerpay.io/imdario/mergo) Donate using Liberapay +Become my sponsor ### Mergo in the wild @@ -98,6 +105,8 @@ If Mergo is useful to you, consider buying me a coffee, a beer, or making a mont - [jnuthong/item_search](https://github.com/jnuthong/item_search) - [bukalapak/snowboard](https://github.com/bukalapak/snowboard) - [containerssh/containerssh](https://github.com/containerssh/containerssh) +- [goreleaser/goreleaser](https://github.com/goreleaser/goreleaser) +- [tjpnz/structbot](https://github.com/tjpnz/structbot) ## Install @@ -168,7 +177,7 @@ func main() { Note: if test are failing due missing package, please execute: - go get gopkg.in/yaml.v2 + go get gopkg.in/yaml.v3 ### Transformers @@ -218,7 +227,6 @@ func main() { } ``` - ## Contact me If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): [@im_dario](https://twitter.com/im_dario) @@ -227,21 +235,8 @@ If I can help you, you have an idea or you are using Mergo in your projects, don Written by [Dario Castañé](http://dario.im). -## Top Contributors - -[![0](https://sourcerer.io/fame/imdario/imdario/mergo/images/0)](https://sourcerer.io/fame/imdario/imdario/mergo/links/0) -[![1](https://sourcerer.io/fame/imdario/imdario/mergo/images/1)](https://sourcerer.io/fame/imdario/imdario/mergo/links/1) -[![2](https://sourcerer.io/fame/imdario/imdario/mergo/images/2)](https://sourcerer.io/fame/imdario/imdario/mergo/links/2) -[![3](https://sourcerer.io/fame/imdario/imdario/mergo/images/3)](https://sourcerer.io/fame/imdario/imdario/mergo/links/3) -[![4](https://sourcerer.io/fame/imdario/imdario/mergo/images/4)](https://sourcerer.io/fame/imdario/imdario/mergo/links/4) -[![5](https://sourcerer.io/fame/imdario/imdario/mergo/images/5)](https://sourcerer.io/fame/imdario/imdario/mergo/links/5) -[![6](https://sourcerer.io/fame/imdario/imdario/mergo/images/6)](https://sourcerer.io/fame/imdario/imdario/mergo/links/6) -[![7](https://sourcerer.io/fame/imdario/imdario/mergo/images/7)](https://sourcerer.io/fame/imdario/imdario/mergo/links/7) - - ## License [BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE). - [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_large) diff --git a/go-controller/vendor/github.com/imdario/mergo/SECURITY.md b/go-controller/vendor/github.com/imdario/mergo/SECURITY.md new file mode 100644 index 0000000000..a5de61f77b --- /dev/null +++ b/go-controller/vendor/github.com/imdario/mergo/SECURITY.md @@ -0,0 +1,14 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 0.3.x | :white_check_mark: | +| < 0.3 | :x: | + +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure. diff --git a/go-controller/vendor/github.com/imdario/mergo/map.go b/go-controller/vendor/github.com/imdario/mergo/map.go index a13a7ee46c..b50d5c2a4e 100644 --- a/go-controller/vendor/github.com/imdario/mergo/map.go +++ b/go-controller/vendor/github.com/imdario/mergo/map.go @@ -44,7 +44,7 @@ func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, conf } } // Remember, remember... - visited[h] = &visit{addr, typ, seen} + visited[h] = &visit{typ, seen, addr} } zeroValue := reflect.Value{} switch dst.Kind() { @@ -58,7 +58,7 @@ func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, conf } fieldName := field.Name fieldName = changeInitialCase(fieldName, unicode.ToLower) - if v, ok := dstMap[fieldName]; !ok || (isEmptyValue(reflect.ValueOf(v)) || overwrite) { + if v, ok := dstMap[fieldName]; !ok || (isEmptyValue(reflect.ValueOf(v), !config.ShouldNotDereference) || overwrite) { dstMap[fieldName] = src.Field(i).Interface() } } @@ -142,7 +142,7 @@ func MapWithOverwrite(dst, src interface{}, opts ...func(*Config)) error { func _map(dst, src interface{}, opts ...func(*Config)) error { if dst != nil && reflect.ValueOf(dst).Kind() != reflect.Ptr { - return ErrNonPointerAgument + return ErrNonPointerArgument } var ( vDst, vSrc reflect.Value diff --git a/go-controller/vendor/github.com/imdario/mergo/merge.go b/go-controller/vendor/github.com/imdario/mergo/merge.go index 8c2a8fcd90..0ef9b2138c 100644 --- a/go-controller/vendor/github.com/imdario/mergo/merge.go +++ b/go-controller/vendor/github.com/imdario/mergo/merge.go @@ -38,10 +38,11 @@ func isExportedComponent(field *reflect.StructField) bool { } type Config struct { + Transformers Transformers Overwrite bool + ShouldNotDereference bool AppendSlice bool TypeCheck bool - Transformers Transformers overwriteWithEmptyValue bool overwriteSliceWithEmptyValue bool sliceDeepCopy bool @@ -76,10 +77,10 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co } } // Remember, remember... - visited[h] = &visit{addr, typ, seen} + visited[h] = &visit{typ, seen, addr} } - if config.Transformers != nil && !isEmptyValue(dst) { + if config.Transformers != nil && !isReflectNil(dst) && dst.IsValid() { if fn := config.Transformers.Transformer(dst.Type()); fn != nil { err = fn(dst, src) return @@ -95,7 +96,7 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co } } } else { - if dst.CanSet() && (isReflectNil(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc) { + if dst.CanSet() && (isReflectNil(dst) || overwrite) && (!isEmptyValue(src, !config.ShouldNotDereference) || overwriteWithEmptySrc) { dst.Set(src) } } @@ -110,7 +111,7 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co } if src.Kind() != reflect.Map { - if overwrite { + if overwrite && dst.CanSet() { dst.Set(src) } return @@ -162,7 +163,7 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co dstSlice = reflect.ValueOf(dstElement.Interface()) } - if (!isEmptyValue(src) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice && !sliceDeepCopy { + if (!isEmptyValue(src, !config.ShouldNotDereference) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst, !config.ShouldNotDereference)) && !config.AppendSlice && !sliceDeepCopy { if typeCheck && srcSlice.Type() != dstSlice.Type() { return fmt.Errorf("cannot override two slices with different type (%s, %s)", srcSlice.Type(), dstSlice.Type()) } @@ -194,22 +195,38 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co dst.SetMapIndex(key, dstSlice) } } - if dstElement.IsValid() && !isEmptyValue(dstElement) && (reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Map || reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Slice) { - continue + + if dstElement.IsValid() && !isEmptyValue(dstElement, !config.ShouldNotDereference) { + if reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Slice { + continue + } + if reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Map && reflect.TypeOf(dstElement.Interface()).Kind() == reflect.Map { + continue + } } - if srcElement.IsValid() && ((srcElement.Kind() != reflect.Ptr && overwrite) || !dstElement.IsValid() || isEmptyValue(dstElement)) { + if srcElement.IsValid() && ((srcElement.Kind() != reflect.Ptr && overwrite) || !dstElement.IsValid() || isEmptyValue(dstElement, !config.ShouldNotDereference)) { if dst.IsNil() { dst.Set(reflect.MakeMap(dst.Type())) } dst.SetMapIndex(key, srcElement) } } + + // Ensure that all keys in dst are deleted if they are not in src. + if overwriteWithEmptySrc { + for _, key := range dst.MapKeys() { + srcElement := src.MapIndex(key) + if !srcElement.IsValid() { + dst.SetMapIndex(key, reflect.Value{}) + } + } + } case reflect.Slice: if !dst.CanSet() { break } - if (!isEmptyValue(src) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice && !sliceDeepCopy { + if (!isEmptyValue(src, !config.ShouldNotDereference) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst, !config.ShouldNotDereference)) && !config.AppendSlice && !sliceDeepCopy { dst.Set(src) } else if config.AppendSlice { if src.Type() != dst.Type() { @@ -244,12 +261,18 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co if src.Kind() != reflect.Interface { if dst.IsNil() || (src.Kind() != reflect.Ptr && overwrite) { - if dst.CanSet() && (overwrite || isEmptyValue(dst)) { + if dst.CanSet() && (overwrite || isEmptyValue(dst, !config.ShouldNotDereference)) { dst.Set(src) } } else if src.Kind() == reflect.Ptr { - if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil { - return + if !config.ShouldNotDereference { + if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil { + return + } + } else { + if overwriteWithEmptySrc || (overwrite && !src.IsNil()) || dst.IsNil() { + dst.Set(src) + } } } else if dst.Elem().Type() == src.Type() { if err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil { @@ -262,7 +285,7 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co } if dst.IsNil() || overwrite { - if dst.CanSet() && (overwrite || isEmptyValue(dst)) { + if dst.CanSet() && (overwrite || isEmptyValue(dst, !config.ShouldNotDereference)) { dst.Set(src) } break @@ -275,7 +298,7 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co break } default: - mustSet := (isEmptyValue(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc) + mustSet := (isEmptyValue(dst, !config.ShouldNotDereference) || overwrite) && (!isEmptyValue(src, !config.ShouldNotDereference) || overwriteWithEmptySrc) if mustSet { if dst.CanSet() { dst.Set(src) @@ -326,6 +349,12 @@ func WithOverrideEmptySlice(config *Config) { config.overwriteSliceWithEmptyValue = true } +// WithoutDereference prevents dereferencing pointers when evaluating whether they are empty +// (i.e. a non-nil pointer is never considered empty). +func WithoutDereference(config *Config) { + config.ShouldNotDereference = true +} + // WithAppendSlice will make merge append slices instead of overwriting it. func WithAppendSlice(config *Config) { config.AppendSlice = true @@ -344,7 +373,7 @@ func WithSliceDeepCopy(config *Config) { func merge(dst, src interface{}, opts ...func(*Config)) error { if dst != nil && reflect.ValueOf(dst).Kind() != reflect.Ptr { - return ErrNonPointerAgument + return ErrNonPointerArgument } var ( vDst, vSrc reflect.Value diff --git a/go-controller/vendor/github.com/imdario/mergo/mergo.go b/go-controller/vendor/github.com/imdario/mergo/mergo.go index 3cc926c7f6..0a721e2d85 100644 --- a/go-controller/vendor/github.com/imdario/mergo/mergo.go +++ b/go-controller/vendor/github.com/imdario/mergo/mergo.go @@ -17,10 +17,10 @@ import ( var ( ErrNilArguments = errors.New("src and dst must not be nil") ErrDifferentArgumentsTypes = errors.New("src and dst must be of same type") - ErrNotSupported = errors.New("only structs and maps are supported") + ErrNotSupported = errors.New("only structs, maps, and slices are supported") ErrExpectedMapAsDestination = errors.New("dst was expected to be a map") ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct") - ErrNonPointerAgument = errors.New("dst must be a pointer") + ErrNonPointerArgument = errors.New("dst must be a pointer") ) // During deepMerge, must keep track of checks that are @@ -28,13 +28,13 @@ var ( // checks in progress are true when it reencounters them. // Visited are stored in a map indexed by 17 * a1 + a2; type visit struct { - ptr uintptr typ reflect.Type next *visit + ptr uintptr } // From src/pkg/encoding/json/encode.go. -func isEmptyValue(v reflect.Value) bool { +func isEmptyValue(v reflect.Value, shouldDereference bool) bool { switch v.Kind() { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: return v.Len() == 0 @@ -50,7 +50,10 @@ func isEmptyValue(v reflect.Value) bool { if v.IsNil() { return true } - return isEmptyValue(v.Elem()) + if shouldDereference { + return isEmptyValue(v.Elem(), shouldDereference) + } + return false case reflect.Func: return v.IsNil() case reflect.Invalid: @@ -65,7 +68,7 @@ func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) { return } vDst = reflect.ValueOf(dst).Elem() - if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map { + if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map && vDst.Kind() != reflect.Slice { err = ErrNotSupported return } diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/LICENSE b/go-controller/vendor/github.com/metallb/frr-k8s/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frr_node_state_types.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frr_node_state_types.go new file mode 100644 index 0000000000..61e5dd2b65 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frr_node_state_types.go @@ -0,0 +1,61 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// FRRNodeStateSpec defines the desired state of FRRNodeState. +type FRRNodeStateSpec struct { +} + +// FRRNodeStateStatus defines the observed state of FRRNodeState. +type FRRNodeStateStatus struct { + // RunningConfig represents the current FRR running config, which is the configuration the FRR instance is currently running with. + RunningConfig string `json:"runningConfig,omitempty"` + // LastConversionResult is the status of the last translation between the `FRRConfiguration`s resources and FRR's configuration, contains "success" or an error. + LastConversionResult string `json:"lastConversionResult,omitempty"` + // LastReloadResult represents the status of the last configuration update operation by FRR, contains "success" or an error. + LastReloadResult string `json:"lastReloadResult,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster + +// FRRNodeState exposes the status of the FRR instance running on each node. +type FRRNodeState struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FRRNodeStateSpec `json:"spec,omitempty"` + Status FRRNodeStateStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// FRRNodeStateList contains a list of FRRNodeStatus. +type FRRNodeStateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FRRNodeState `json:"items"` +} + +func init() { + SchemeBuilder.Register(&FRRNodeState{}, &FRRNodeStateList{}) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go new file mode 100644 index 0000000000..421f38a1e9 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go @@ -0,0 +1,391 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// FRRConfigurationSpec defines the desired state of FRRConfiguration. +type FRRConfigurationSpec struct { + // BGP is the configuration related to the BGP protocol. + // +optional + BGP BGPConfig `json:"bgp,omitempty"` + + // Raw is a snippet of raw frr configuration that gets appended to the + // one rendered translating the type safe API. + // +optional + Raw RawConfig `json:"raw,omitempty"` + // NodeSelector limits the nodes that will attempt to apply this config. + // When specified, the configuration will be considered only on nodes + // whose labels match the specified selectors. + // When it is not specified all nodes will attempt to apply this config. + // +optional + NodeSelector metav1.LabelSelector `json:"nodeSelector,omitempty"` +} + +// RawConfig is a snippet of raw frr configuration that gets appended to the +// rendered configuration. +type RawConfig struct { + // Priority is the order with this configuration is appended to the + // bottom of the rendered configuration. A higher value means the + // raw config is appended later in the configuration file. + Priority int `json:"priority,omitempty"` + + // Config is a raw FRR configuration to be appended to the configuration + // rendered via the k8s api. + Config string `json:"rawConfig,omitempty"` +} + +// BGPConfig is the configuration related to the BGP protocol. +type BGPConfig struct { + // Routers is the list of routers we want FRR to configure (one per VRF). + // +optional + Routers []Router `json:"routers"` + // BFDProfiles is the list of bfd profiles to be used when configuring the neighbors. + // +optional + BFDProfiles []BFDProfile `json:"bfdProfiles,omitempty"` +} + +// Router represent a neighbor router we want FRR to connect to. +type Router struct { + // ASN is the AS number to use for the local end of the session. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=4294967295 + ASN uint32 `json:"asn"` + // ID is the BGP router ID + // +optional + ID string `json:"id,omitempty"` + // VRF is the host vrf used to establish sessions from this router. + // +optional + VRF string `json:"vrf,omitempty"` + // Neighbors is the list of neighbors we want to establish BGP sessions with. + // +optional + Neighbors []Neighbor `json:"neighbors,omitempty"` + // Prefixes is the list of prefixes we want to advertise from this router instance. + // +optional + Prefixes []string `json:"prefixes,omitempty"` + + // Imports is the list of imported VRFs we want for this router / vrf. + // +optional + Imports []Import `json:"imports,omitempty"` +} + +// Import represents the possible imported VRFs to a given router. +type Import struct { + // Vrf is the vrf we want to import from + // +optional + VRF string `json:"vrf,omitempty"` +} + +// Neighbor represents a BGP Neighbor we want FRR to connect to. +type Neighbor struct { + // ASN is the AS number to use for the local end of the session. + // ASN and DynamicASN are mutually exclusive and one of them must be specified. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=4294967295 + // +optional + ASN uint32 `json:"asn,omitempty"` + + // DynamicASN detects the AS number to use for the local end of the session + // without explicitly setting it via the ASN field. Limited to: + // internal - if the neighbor's ASN is different than the router's the connection is denied. + // external - if the neighbor's ASN is the same as the router's the connection is denied. + // ASN and DynamicASN are mutually exclusive and one of them must be specified. + // +kubebuilder:validation:Enum=internal;external + // +optional + DynamicASN DynamicASNMode `json:"dynamicASN,omitempty"` + + // SourceAddress is the IPv4 or IPv6 source address to use for the BGP + // session to this neighbour, may be specified as either an IP address + // directly or as an interface name + // +optional + SourceAddress string `json:"sourceaddress,omitempty"` + + // Address is the IP address to establish the session with. + Address string `json:"address"` + + // Port is the port to dial when establishing the session. + // Defaults to 179. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=16384 + Port *uint16 `json:"port,omitempty"` + + // Password to be used for establishing the BGP session. + // Password and PasswordSecret are mutually exclusive. + // +optional + Password string `json:"password,omitempty"` + + // PasswordSecret is name of the authentication secret for the neighbor. + // the secret must be of type "kubernetes.io/basic-auth", and created in the + // same namespace as the frr-k8s daemon. The password is stored in the + // secret as the key "password". + // Password and PasswordSecret are mutually exclusive. + // +optional + PasswordSecret SecretReference `json:"passwordSecret,omitempty"` + + // HoldTime is the requested BGP hold time, per RFC4271. + // Defaults to 180s. + // +optional + HoldTime *metav1.Duration `json:"holdTime,omitempty"` + + // KeepaliveTime is the requested BGP keepalive time, per RFC4271. + // Defaults to 60s. + // +optional + KeepaliveTime *metav1.Duration `json:"keepaliveTime,omitempty"` + + // Requested BGP connect time, controls how long BGP waits between connection attempts to a neighbor. + // +kubebuilder:validation:XValidation:message="connect time should be between 1 seconds to 65535",rule="duration(self).getSeconds() >= 1 && duration(self).getSeconds() <= 65535" + // +kubebuilder:validation:XValidation:message="connect time should contain a whole number of seconds",rule="duration(self).getMilliseconds() % 1000 == 0" + // +optional + ConnectTime *metav1.Duration `json:"connectTime,omitempty"` + + // EBGPMultiHop indicates if the BGPPeer is multi-hops away. + // +optional + EBGPMultiHop bool `json:"ebgpMultiHop,omitempty"` + + // BFDProfile is the name of the BFD Profile to be used for the BFD session associated + // to the BGP session. If not set, the BFD session won't be set up. + // +optional + BFDProfile string `json:"bfdProfile,omitempty"` + + // EnableGracefulRestart allows BGP peer to continue to forward data packets along + // known routes while the routing protocol information is being restored. If + // the session is already established, the configuration will have effect + // after reconnecting to the peer + // +optional + EnableGracefulRestart bool `json:"enableGracefulRestart,omitempty"` + + // ToAdvertise represents the list of prefixes to advertise to the given neighbor + // and the associated properties. + // +optional + ToAdvertise Advertise `json:"toAdvertise,omitempty"` + + // ToReceive represents the list of prefixes to receive from the given neighbor. + // +optional + ToReceive Receive `json:"toReceive,omitempty"` + + // To set if we want to disable MP BGP that will separate IPv4 and IPv6 route exchanges into distinct BGP sessions. + // +optional + // +kubebuilder:default:=false + DisableMP bool `json:"disableMP,omitempty"` +} + +// Advertise represents a list of prefixes to advertise to the given neighbor. + +type Advertise struct { + + // Allowed is is the list of prefixes allowed to be propagated to + // this neighbor. They must match the prefixes defined in the router. + Allowed AllowedOutPrefixes `json:"allowed,omitempty"` + + // PrefixesWithLocalPref is a list of prefixes that are associated to a local + // preference when being advertised. The prefixes associated to a given local pref + // must be in the prefixes allowed to be advertised. + // +optional + PrefixesWithLocalPref []LocalPrefPrefixes `json:"withLocalPref,omitempty"` + + // PrefixesWithCommunity is a list of prefixes that are associated to a + // bgp community when being advertised. The prefixes associated to a given local pref + // must be in the prefixes allowed to be advertised. + // +optional + PrefixesWithCommunity []CommunityPrefixes `json:"withCommunity,omitempty"` +} + +// Receive represents a list of prefixes to receive from the given neighbor. +type Receive struct { + // Allowed is the list of prefixes allowed to be received from + // this neighbor. + // +optional + Allowed AllowedInPrefixes `json:"allowed,omitempty"` +} + +// PrefixSelector is a filter of prefixes to receive. +type PrefixSelector struct { + // +kubebuilder:validation:Format="cidr" + Prefix string `json:"prefix,omitempty"` + // The prefix length modifier. This selector accepts any matching prefix with length + // less or equal the given value. + // +kubebuilder:validation:Maximum:=128 + // +kubebuilder:validation:Minimum:=1 + LE uint32 `json:"le,omitempty"` + // The prefix length modifier. This selector accepts any matching prefix with length + // greater or equal the given value. + // +kubebuilder:validation:Maximum:=128 + // +kubebuilder:validation:Minimum:=1 + GE uint32 `json:"ge,omitempty"` +} + +type AllowedInPrefixes struct { + Prefixes []PrefixSelector `json:"prefixes,omitempty"` + // Mode is the mode to use when handling the prefixes. + // When set to "filtered", only the prefixes in the given list will be allowed. + // When set to "all", all the prefixes configured on the router will be allowed. + // +kubebuilder:default:=filtered + Mode AllowMode `json:"mode,omitempty"` +} + +type AllowedOutPrefixes struct { + Prefixes []string `json:"prefixes,omitempty"` + // Mode is the mode to use when handling the prefixes. + // When set to "filtered", only the prefixes in the given list will be allowed. + // When set to "all", all the prefixes configured on the router will be allowed. + // +kubebuilder:default:=filtered + Mode AllowMode `json:"mode,omitempty"` +} + +// LocalPrefPrefixes is a list of prefixes associated to a local preference. +type LocalPrefPrefixes struct { + // Prefixes is the list of prefixes associated to the local preference. + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:Format="cidr" + Prefixes []string `json:"prefixes,omitempty"` + // LocalPref is the local preference associated to the prefixes. + LocalPref uint32 `json:"localPref,omitempty"` +} + +// CommunityPrefixes is a list of prefixes associated to a community. +type CommunityPrefixes struct { + // Prefixes is the list of prefixes associated to the community. + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:Format="cidr" + Prefixes []string `json:"prefixes,omitempty"` + // Community is the community associated to the prefixes. + Community string `json:"community,omitempty"` +} + +// BFDProfile is the configuration related to the BFD protocol associated +// to a BGP session. +type BFDProfile struct { + // The name of the BFD Profile to be referenced in other parts + // of the configuration. + Name string `json:"name"` + + // The minimum interval that this system is capable of + // receiving control packets in milliseconds. + // Defaults to 300ms. + // +kubebuilder:validation:Maximum:=60000 + // +kubebuilder:validation:Minimum:=10 + // +optional + ReceiveInterval *uint32 `json:"receiveInterval,omitempty"` + + // The minimum transmission interval (less jitter) + // that this system wants to use to send BFD control packets in + // milliseconds. Defaults to 300ms + // +kubebuilder:validation:Maximum:=60000 + // +kubebuilder:validation:Minimum:=10 + // +optional + TransmitInterval *uint32 `json:"transmitInterval,omitempty"` + + // Configures the detection multiplier to determine + // packet loss. The remote transmission interval will be multiplied + // by this value to determine the connection loss detection timer. + // +kubebuilder:validation:Maximum:=255 + // +kubebuilder:validation:Minimum:=2 + // +optional + DetectMultiplier *uint32 `json:"detectMultiplier,omitempty"` + + // Configures the minimal echo receive transmission + // interval that this system is capable of handling in milliseconds. + // Defaults to 50ms + // +kubebuilder:validation:Maximum:=60000 + // +kubebuilder:validation:Minimum:=10 + // +optional + EchoInterval *uint32 `json:"echoInterval,omitempty"` + + // Enables or disables the echo transmission mode. + // This mode is disabled by default, and not supported on multi + // hops setups. + // +optional + EchoMode *bool `json:"echoMode,omitempty"` + + // Mark session as passive: a passive session will not + // attempt to start the connection and will wait for control packets + // from peer before it begins replying. + // +optional + PassiveMode *bool `json:"passiveMode,omitempty"` + + // For multi hop sessions only: configure the minimum + // expected TTL for an incoming BFD control packet. + // +kubebuilder:validation:Maximum:=254 + // +kubebuilder:validation:Minimum:=1 + // +optional + MinimumTTL *uint32 `json:"minimumTtl,omitempty"` +} + +// FRRConfigurationStatus defines the observed state of FRRConfiguration. +type FRRConfigurationStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//nolint +//+genclient + +// FRRConfiguration is a piece of FRR configuration. +type FRRConfiguration struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FRRConfigurationSpec `json:"spec,omitempty"` + Status FRRConfigurationStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// FRRConfigurationList contains a list of FRRConfiguration. +type FRRConfigurationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FRRConfiguration `json:"items"` +} + +//nolint +//+structType=atomic + +// SecretReference represents a Secret Reference. It has enough information to retrieve secret +// in any namespace. +type SecretReference struct { + // name is unique within a namespace to reference a secret resource. + // +optional + Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"` + // namespace defines the space within which the secret name must be unique. + // +optional + Namespace string `json:"namespace,omitempty" protobuf:"bytes,2,opt,name=namespace"` +} + +func init() { + SchemeBuilder.Register(&FRRConfiguration{}, &FRRConfigurationList{}) +} + +// +kubebuilder:validation:Enum=all;filtered +type AllowMode string + +const ( + AllowAll AllowMode = "all" + AllowRestricted AllowMode = "filtered" +) + +type DynamicASNMode string + +const ( + InternalASNMode DynamicASNMode = "internal" + ExternalASNMode DynamicASNMode = "external" +) diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/groupversion_info.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/groupversion_info.go new file mode 100644 index 0000000000..987dda84a0 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/groupversion_info.go @@ -0,0 +1,42 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the frrk8s v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=frrk8s.metallb.io +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "frrk8s.metallb.io", Version: "v1beta1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +var SchemeGroupVersion = schema.GroupVersion{Group: "frrk8s.metallb.io", Version: "v1beta1"} + +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..982781e7a5 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,528 @@ +//go:build !ignore_autogenerated + +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Advertise) DeepCopyInto(out *Advertise) { + *out = *in + in.Allowed.DeepCopyInto(&out.Allowed) + if in.PrefixesWithLocalPref != nil { + in, out := &in.PrefixesWithLocalPref, &out.PrefixesWithLocalPref + *out = make([]LocalPrefPrefixes, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PrefixesWithCommunity != nil { + in, out := &in.PrefixesWithCommunity, &out.PrefixesWithCommunity + *out = make([]CommunityPrefixes, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Advertise. +func (in *Advertise) DeepCopy() *Advertise { + if in == nil { + return nil + } + out := new(Advertise) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AllowedInPrefixes) DeepCopyInto(out *AllowedInPrefixes) { + *out = *in + if in.Prefixes != nil { + in, out := &in.Prefixes, &out.Prefixes + *out = make([]PrefixSelector, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllowedInPrefixes. +func (in *AllowedInPrefixes) DeepCopy() *AllowedInPrefixes { + if in == nil { + return nil + } + out := new(AllowedInPrefixes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AllowedOutPrefixes) DeepCopyInto(out *AllowedOutPrefixes) { + *out = *in + if in.Prefixes != nil { + in, out := &in.Prefixes, &out.Prefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllowedOutPrefixes. +func (in *AllowedOutPrefixes) DeepCopy() *AllowedOutPrefixes { + if in == nil { + return nil + } + out := new(AllowedOutPrefixes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BFDProfile) DeepCopyInto(out *BFDProfile) { + *out = *in + if in.ReceiveInterval != nil { + in, out := &in.ReceiveInterval, &out.ReceiveInterval + *out = new(uint32) + **out = **in + } + if in.TransmitInterval != nil { + in, out := &in.TransmitInterval, &out.TransmitInterval + *out = new(uint32) + **out = **in + } + if in.DetectMultiplier != nil { + in, out := &in.DetectMultiplier, &out.DetectMultiplier + *out = new(uint32) + **out = **in + } + if in.EchoInterval != nil { + in, out := &in.EchoInterval, &out.EchoInterval + *out = new(uint32) + **out = **in + } + if in.EchoMode != nil { + in, out := &in.EchoMode, &out.EchoMode + *out = new(bool) + **out = **in + } + if in.PassiveMode != nil { + in, out := &in.PassiveMode, &out.PassiveMode + *out = new(bool) + **out = **in + } + if in.MinimumTTL != nil { + in, out := &in.MinimumTTL, &out.MinimumTTL + *out = new(uint32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BFDProfile. +func (in *BFDProfile) DeepCopy() *BFDProfile { + if in == nil { + return nil + } + out := new(BFDProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BGPConfig) DeepCopyInto(out *BGPConfig) { + *out = *in + if in.Routers != nil { + in, out := &in.Routers, &out.Routers + *out = make([]Router, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.BFDProfiles != nil { + in, out := &in.BFDProfiles, &out.BFDProfiles + *out = make([]BFDProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BGPConfig. +func (in *BGPConfig) DeepCopy() *BGPConfig { + if in == nil { + return nil + } + out := new(BGPConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommunityPrefixes) DeepCopyInto(out *CommunityPrefixes) { + *out = *in + if in.Prefixes != nil { + in, out := &in.Prefixes, &out.Prefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommunityPrefixes. +func (in *CommunityPrefixes) DeepCopy() *CommunityPrefixes { + if in == nil { + return nil + } + out := new(CommunityPrefixes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRConfiguration) DeepCopyInto(out *FRRConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRConfiguration. +func (in *FRRConfiguration) DeepCopy() *FRRConfiguration { + if in == nil { + return nil + } + out := new(FRRConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FRRConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRConfigurationList) DeepCopyInto(out *FRRConfigurationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FRRConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRConfigurationList. +func (in *FRRConfigurationList) DeepCopy() *FRRConfigurationList { + if in == nil { + return nil + } + out := new(FRRConfigurationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FRRConfigurationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRConfigurationSpec) DeepCopyInto(out *FRRConfigurationSpec) { + *out = *in + in.BGP.DeepCopyInto(&out.BGP) + out.Raw = in.Raw + in.NodeSelector.DeepCopyInto(&out.NodeSelector) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRConfigurationSpec. +func (in *FRRConfigurationSpec) DeepCopy() *FRRConfigurationSpec { + if in == nil { + return nil + } + out := new(FRRConfigurationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRConfigurationStatus) DeepCopyInto(out *FRRConfigurationStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRConfigurationStatus. +func (in *FRRConfigurationStatus) DeepCopy() *FRRConfigurationStatus { + if in == nil { + return nil + } + out := new(FRRConfigurationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRNodeState) DeepCopyInto(out *FRRNodeState) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRNodeState. +func (in *FRRNodeState) DeepCopy() *FRRNodeState { + if in == nil { + return nil + } + out := new(FRRNodeState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FRRNodeState) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRNodeStateList) DeepCopyInto(out *FRRNodeStateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FRRNodeState, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRNodeStateList. +func (in *FRRNodeStateList) DeepCopy() *FRRNodeStateList { + if in == nil { + return nil + } + out := new(FRRNodeStateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FRRNodeStateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRNodeStateSpec) DeepCopyInto(out *FRRNodeStateSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRNodeStateSpec. +func (in *FRRNodeStateSpec) DeepCopy() *FRRNodeStateSpec { + if in == nil { + return nil + } + out := new(FRRNodeStateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRNodeStateStatus) DeepCopyInto(out *FRRNodeStateStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRNodeStateStatus. +func (in *FRRNodeStateStatus) DeepCopy() *FRRNodeStateStatus { + if in == nil { + return nil + } + out := new(FRRNodeStateStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Import) DeepCopyInto(out *Import) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Import. +func (in *Import) DeepCopy() *Import { + if in == nil { + return nil + } + out := new(Import) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalPrefPrefixes) DeepCopyInto(out *LocalPrefPrefixes) { + *out = *in + if in.Prefixes != nil { + in, out := &in.Prefixes, &out.Prefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalPrefPrefixes. +func (in *LocalPrefPrefixes) DeepCopy() *LocalPrefPrefixes { + if in == nil { + return nil + } + out := new(LocalPrefPrefixes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Neighbor) DeepCopyInto(out *Neighbor) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(uint16) + **out = **in + } + out.PasswordSecret = in.PasswordSecret + if in.HoldTime != nil { + in, out := &in.HoldTime, &out.HoldTime + *out = new(v1.Duration) + **out = **in + } + if in.KeepaliveTime != nil { + in, out := &in.KeepaliveTime, &out.KeepaliveTime + *out = new(v1.Duration) + **out = **in + } + if in.ConnectTime != nil { + in, out := &in.ConnectTime, &out.ConnectTime + *out = new(v1.Duration) + **out = **in + } + in.ToAdvertise.DeepCopyInto(&out.ToAdvertise) + in.ToReceive.DeepCopyInto(&out.ToReceive) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Neighbor. +func (in *Neighbor) DeepCopy() *Neighbor { + if in == nil { + return nil + } + out := new(Neighbor) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrefixSelector) DeepCopyInto(out *PrefixSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrefixSelector. +func (in *PrefixSelector) DeepCopy() *PrefixSelector { + if in == nil { + return nil + } + out := new(PrefixSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RawConfig) DeepCopyInto(out *RawConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RawConfig. +func (in *RawConfig) DeepCopy() *RawConfig { + if in == nil { + return nil + } + out := new(RawConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Receive) DeepCopyInto(out *Receive) { + *out = *in + in.Allowed.DeepCopyInto(&out.Allowed) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Receive. +func (in *Receive) DeepCopy() *Receive { + if in == nil { + return nil + } + out := new(Receive) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Router) DeepCopyInto(out *Router) { + *out = *in + if in.Neighbors != nil { + in, out := &in.Neighbors, &out.Neighbors + *out = make([]Neighbor, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Prefixes != nil { + in, out := &in.Prefixes, &out.Prefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Imports != nil { + in, out := &in.Imports, &out.Imports + *out = make([]Import, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Router. +func (in *Router) DeepCopy() *Router { + if in == nil { + return nil + } + out := new(Router) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretReference) DeepCopyInto(out *SecretReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference. +func (in *SecretReference) DeepCopy() *SecretReference { + if in == nil { + return nil + } + out := new(SecretReference) + in.DeepCopyInto(out) + return out +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/clientset.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/clientset.go new file mode 100644 index 0000000000..5c92e69505 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/clientset.go @@ -0,0 +1,106 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + "fmt" + "net/http" + + apiv1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1" + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + ApiV1beta1() apiv1beta1.ApiV1beta1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + *discovery.DiscoveryClient + apiV1beta1 *apiv1beta1.ApiV1beta1Client +} + +// ApiV1beta1 retrieves the ApiV1beta1Client +func (c *Clientset) ApiV1beta1() apiv1beta1.ApiV1beta1Interface { + return c.apiV1beta1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.apiV1beta1, err = apiv1beta1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.apiV1beta1 = apiv1beta1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/clientset_generated.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 0000000000..d6e24860a6 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + clientset "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" + apiv1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1" + fakeapiv1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +// +// DEPRECATED: NewClientset replaces this with support for field management, which significantly improves +// server side apply testing. NewClientset is only available when apply configurations are generated (e.g. +// via --with-applyconfig). +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// ApiV1beta1 retrieves the ApiV1beta1Client +func (c *Clientset) ApiV1beta1() apiv1beta1.ApiV1beta1Interface { + return &fakeapiv1beta1.FakeApiV1beta1{Fake: &c.Fake} +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/doc.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/doc.go new file mode 100644 index 0000000000..1284e3407a --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/doc.go @@ -0,0 +1,6 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/register.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/register.go new file mode 100644 index 0000000000..7e347a2be6 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/register.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + apiv1beta1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme/doc.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme/doc.go new file mode 100644 index 0000000000..1f081673bd --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme/doc.go @@ -0,0 +1,6 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme/register.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme/register.go new file mode 100644 index 0000000000..ed900d3f47 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme/register.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + apiv1beta1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/api_client.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/api_client.go new file mode 100644 index 0000000000..6aebb4137d --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/api_client.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "net/http" + + v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type ApiV1beta1Interface interface { + RESTClient() rest.Interface + FRRConfigurationsGetter +} + +// ApiV1beta1Client is used to interact with features provided by the api group. +type ApiV1beta1Client struct { + restClient rest.Interface +} + +func (c *ApiV1beta1Client) FRRConfigurations(namespace string) FRRConfigurationInterface { + return newFRRConfigurations(c, namespace) +} + +// NewForConfig creates a new ApiV1beta1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*ApiV1beta1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new ApiV1beta1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*ApiV1beta1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &ApiV1beta1Client{client}, nil +} + +// NewForConfigOrDie creates a new ApiV1beta1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *ApiV1beta1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new ApiV1beta1Client for the given RESTClient. +func New(c rest.Interface) *ApiV1beta1Client { + return &ApiV1beta1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1beta1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *ApiV1beta1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/doc.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/doc.go new file mode 100644 index 0000000000..0fb653f41c --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/doc.go @@ -0,0 +1,6 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1beta1 diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/doc.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/doc.go new file mode 100644 index 0000000000..8a9a842070 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/doc.go @@ -0,0 +1,6 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go new file mode 100644 index 0000000000..21b5048ff3 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeApiV1beta1 struct { + *testing.Fake +} + +func (c *FakeApiV1beta1) FRRConfigurations(namespace string) v1beta1.FRRConfigurationInterface { + return &FakeFRRConfigurations{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeApiV1beta1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrconfiguration.go new file mode 100644 index 0000000000..70a9e8d0e9 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrconfiguration.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeFRRConfigurations implements FRRConfigurationInterface +type FakeFRRConfigurations struct { + Fake *FakeApiV1beta1 + ns string +} + +var frrconfigurationsResource = v1beta1.SchemeGroupVersion.WithResource("frrconfigurations") + +var frrconfigurationsKind = v1beta1.SchemeGroupVersion.WithKind("FRRConfiguration") + +// Get takes name of the fRRConfiguration, and returns the corresponding fRRConfiguration object, and an error if there is any. +func (c *FakeFRRConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.FRRConfiguration, err error) { + emptyResult := &v1beta1.FRRConfiguration{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(frrconfigurationsResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1beta1.FRRConfiguration), err +} + +// List takes label and field selectors, and returns the list of FRRConfigurations that match those selectors. +func (c *FakeFRRConfigurations) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.FRRConfigurationList, err error) { + emptyResult := &v1beta1.FRRConfigurationList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(frrconfigurationsResource, frrconfigurationsKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1beta1.FRRConfigurationList{ListMeta: obj.(*v1beta1.FRRConfigurationList).ListMeta} + for _, item := range obj.(*v1beta1.FRRConfigurationList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested fRRConfigurations. +func (c *FakeFRRConfigurations) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(frrconfigurationsResource, c.ns, opts)) + +} + +// Create takes the representation of a fRRConfiguration and creates it. Returns the server's representation of the fRRConfiguration, and an error, if there is any. +func (c *FakeFRRConfigurations) Create(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.CreateOptions) (result *v1beta1.FRRConfiguration, err error) { + emptyResult := &v1beta1.FRRConfiguration{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(frrconfigurationsResource, c.ns, fRRConfiguration, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1beta1.FRRConfiguration), err +} + +// Update takes the representation of a fRRConfiguration and updates it. Returns the server's representation of the fRRConfiguration, and an error, if there is any. +func (c *FakeFRRConfigurations) Update(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.UpdateOptions) (result *v1beta1.FRRConfiguration, err error) { + emptyResult := &v1beta1.FRRConfiguration{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(frrconfigurationsResource, c.ns, fRRConfiguration, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1beta1.FRRConfiguration), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeFRRConfigurations) UpdateStatus(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.UpdateOptions) (result *v1beta1.FRRConfiguration, err error) { + emptyResult := &v1beta1.FRRConfiguration{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(frrconfigurationsResource, "status", c.ns, fRRConfiguration, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1beta1.FRRConfiguration), err +} + +// Delete takes name of the fRRConfiguration and deletes it. Returns an error if one occurs. +func (c *FakeFRRConfigurations) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(frrconfigurationsResource, c.ns, name, opts), &v1beta1.FRRConfiguration{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeFRRConfigurations) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(frrconfigurationsResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1beta1.FRRConfigurationList{}) + return err +} + +// Patch applies the patch and returns the patched fRRConfiguration. +func (c *FakeFRRConfigurations) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.FRRConfiguration, err error) { + emptyResult := &v1beta1.FRRConfiguration{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(frrconfigurationsResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1beta1.FRRConfiguration), err +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrconfiguration.go new file mode 100644 index 0000000000..8b098aa1cf --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrconfiguration.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "context" + + v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + scheme "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// FRRConfigurationsGetter has a method to return a FRRConfigurationInterface. +// A group's client should implement this interface. +type FRRConfigurationsGetter interface { + FRRConfigurations(namespace string) FRRConfigurationInterface +} + +// FRRConfigurationInterface has methods to work with FRRConfiguration resources. +type FRRConfigurationInterface interface { + Create(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.CreateOptions) (*v1beta1.FRRConfiguration, error) + Update(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.UpdateOptions) (*v1beta1.FRRConfiguration, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.UpdateOptions) (*v1beta1.FRRConfiguration, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.FRRConfiguration, error) + List(ctx context.Context, opts v1.ListOptions) (*v1beta1.FRRConfigurationList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.FRRConfiguration, err error) + FRRConfigurationExpansion +} + +// fRRConfigurations implements FRRConfigurationInterface +type fRRConfigurations struct { + *gentype.ClientWithList[*v1beta1.FRRConfiguration, *v1beta1.FRRConfigurationList] +} + +// newFRRConfigurations returns a FRRConfigurations +func newFRRConfigurations(c *ApiV1beta1Client, namespace string) *fRRConfigurations { + return &fRRConfigurations{ + gentype.NewClientWithList[*v1beta1.FRRConfiguration, *v1beta1.FRRConfigurationList]( + "frrconfigurations", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1beta1.FRRConfiguration { return &v1beta1.FRRConfiguration{} }, + func() *v1beta1.FRRConfigurationList { return &v1beta1.FRRConfigurationList{} }), + } +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go new file mode 100644 index 0000000000..864abd79f8 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go @@ -0,0 +1,7 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +type FRRConfigurationExpansion interface{} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/interface.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/interface.go new file mode 100644 index 0000000000..31f3cca1db --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/interface.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by informer-gen. DO NOT EDIT. + +package api + +import ( + v1beta1 "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1" + internalinterfaces "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1beta1 provides access to shared informers for resources in V1beta1. + V1beta1() v1beta1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1beta1 returns a new v1beta1.Interface. +func (g *group) V1beta1() v1beta1.Interface { + return v1beta1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrconfiguration.go new file mode 100644 index 0000000000..93414be862 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrconfiguration.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by informer-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "context" + time "time" + + apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + versioned "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" + internalinterfaces "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces" + v1beta1 "github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// FRRConfigurationInformer provides access to a shared informer and lister for +// FRRConfigurations. +type FRRConfigurationInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1beta1.FRRConfigurationLister +} + +type fRRConfigurationInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewFRRConfigurationInformer constructs a new informer for FRRConfiguration type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFRRConfigurationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredFRRConfigurationInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredFRRConfigurationInformer constructs a new informer for FRRConfiguration type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredFRRConfigurationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiV1beta1().FRRConfigurations(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiV1beta1().FRRConfigurations(namespace).Watch(context.TODO(), options) + }, + }, + &apiv1beta1.FRRConfiguration{}, + resyncPeriod, + indexers, + ) +} + +func (f *fRRConfigurationInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredFRRConfigurationInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *fRRConfigurationInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiv1beta1.FRRConfiguration{}, f.defaultInformer) +} + +func (f *fRRConfigurationInformer) Lister() v1beta1.FRRConfigurationLister { + return v1beta1.NewFRRConfigurationLister(f.Informer().GetIndexer()) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/interface.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/interface.go new file mode 100644 index 0000000000..f87efaa202 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/interface.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by informer-gen. DO NOT EDIT. + +package v1beta1 + +import ( + internalinterfaces "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // FRRConfigurations returns a FRRConfigurationInformer. + FRRConfigurations() FRRConfigurationInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// FRRConfigurations returns a FRRConfigurationInformer. +func (v *version) FRRConfigurations() FRRConfigurationInformer { + return &fRRConfigurationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/factory.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/factory.go new file mode 100644 index 0000000000..e0802fd32e --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/factory.go @@ -0,0 +1,248 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + reflect "reflect" + sync "sync" + time "time" + + versioned "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" + api "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api" + internalinterfaces "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + transform cache.TransformFunc + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// WithTransform sets a transform on all informers. +func WithTransform(transform cache.TransformFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.transform = transform + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.shuttingDown { + return + } + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + f.wg.Add(1) + // We need a new variable in each loop iteration, + // otherwise the goroutine would use the loop variable + // and that keeps changing. + informer := informer + go func() { + defer f.wg.Done() + informer.Run(stopCh) + }() + f.startedInformers[informerType] = true + } + } +} + +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() +} + +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + informer.SetTransform(f.transform) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.Background() +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + // Warning: Start does not block. When run in a go-routine, it will race with a later WaitForCacheSync. + Start(stopCh <-chan struct{}) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + + Api() api.Interface +} + +func (f *sharedInformerFactory) Api() api.Interface { + return api.New(f, f.namespace, f.tweakListOptions) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/generic.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/generic.go new file mode 100644 index 0000000000..9c85f3dd1d --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/generic.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + "fmt" + + v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=api, Version=v1beta1 + case v1beta1.SchemeGroupVersion.WithResource("frrconfigurations"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Api().V1beta1().FRRConfigurations().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 0000000000..349578797b --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + versioned "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/expansion_generated.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/expansion_generated.go new file mode 100644 index 0000000000..4b69446dc4 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/expansion_generated.go @@ -0,0 +1,13 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by lister-gen. DO NOT EDIT. + +package v1beta1 + +// FRRConfigurationListerExpansion allows custom methods to be added to +// FRRConfigurationLister. +type FRRConfigurationListerExpansion interface{} + +// FRRConfigurationNamespaceListerExpansion allows custom methods to be added to +// FRRConfigurationNamespaceLister. +type FRRConfigurationNamespaceListerExpansion interface{} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrconfiguration.go new file mode 100644 index 0000000000..f0799a932d --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrconfiguration.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by lister-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/listers" + "k8s.io/client-go/tools/cache" +) + +// FRRConfigurationLister helps list FRRConfigurations. +// All objects returned here must be treated as read-only. +type FRRConfigurationLister interface { + // List lists all FRRConfigurations in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1beta1.FRRConfiguration, err error) + // FRRConfigurations returns an object that can list and get FRRConfigurations. + FRRConfigurations(namespace string) FRRConfigurationNamespaceLister + FRRConfigurationListerExpansion +} + +// fRRConfigurationLister implements the FRRConfigurationLister interface. +type fRRConfigurationLister struct { + listers.ResourceIndexer[*v1beta1.FRRConfiguration] +} + +// NewFRRConfigurationLister returns a new FRRConfigurationLister. +func NewFRRConfigurationLister(indexer cache.Indexer) FRRConfigurationLister { + return &fRRConfigurationLister{listers.New[*v1beta1.FRRConfiguration](indexer, v1beta1.Resource("frrconfiguration"))} +} + +// FRRConfigurations returns an object that can list and get FRRConfigurations. +func (s *fRRConfigurationLister) FRRConfigurations(namespace string) FRRConfigurationNamespaceLister { + return fRRConfigurationNamespaceLister{listers.NewNamespaced[*v1beta1.FRRConfiguration](s.ResourceIndexer, namespace)} +} + +// FRRConfigurationNamespaceLister helps list and get FRRConfigurations. +// All objects returned here must be treated as read-only. +type FRRConfigurationNamespaceLister interface { + // List lists all FRRConfigurations in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1beta1.FRRConfiguration, err error) + // Get retrieves the FRRConfiguration from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1beta1.FRRConfiguration, error) + FRRConfigurationNamespaceListerExpansion +} + +// fRRConfigurationNamespaceLister implements the FRRConfigurationNamespaceLister +// interface. +type fRRConfigurationNamespaceLister struct { + listers.ResourceIndexer[*v1beta1.FRRConfiguration] +} diff --git a/go-controller/vendor/golang.org/x/time/rate/rate.go b/go-controller/vendor/golang.org/x/time/rate/rate.go index f0e0cf3cb1..8f6c7f493f 100644 --- a/go-controller/vendor/golang.org/x/time/rate/rate.go +++ b/go-controller/vendor/golang.org/x/time/rate/rate.go @@ -52,6 +52,8 @@ func Every(interval time.Duration) Limit { // or its associated context.Context is canceled. // // The methods AllowN, ReserveN, and WaitN consume n tokens. +// +// Limiter is safe for simultaneous use by multiple goroutines. type Limiter struct { mu sync.Mutex limit Limit diff --git a/go-controller/vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go b/go-controller/vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go index 333676b7cf..c6e7c0d442 100644 --- a/go-controller/vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go +++ b/go-controller/vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go @@ -13,6 +13,7 @@ import ( "golang.org/x/tools/internal/gocommand" ) +// TODO(adonovan): move back into go/packages. func GetSizesForArgsGolist(ctx context.Context, inv gocommand.Invocation, gocmdRunner *gocommand.Runner) (string, string, error) { inv.Verb = "list" inv.Args = []string{"-f", "{{context.GOARCH}} {{context.Compiler}}", "--", "unsafe"} diff --git a/go-controller/vendor/golang.org/x/tools/go/packages/doc.go b/go-controller/vendor/golang.org/x/tools/go/packages/doc.go index a8d7b06ac0..3531ac8f5f 100644 --- a/go-controller/vendor/golang.org/x/tools/go/packages/doc.go +++ b/go-controller/vendor/golang.org/x/tools/go/packages/doc.go @@ -198,14 +198,6 @@ Instead, ssadump no longer requests the runtime package, but seeks it among the dependencies of the user-specified packages, and emits an error if it is not found. -Overlays: The Overlay field in the Config allows providing alternate contents -for Go source files, by providing a mapping from file path to contents. -go/packages will pull in new imports added in overlay files when go/packages -is run in LoadImports mode or greater. -Overlay support for the go list driver isn't complete yet: if the file doesn't -exist on disk, it will only be recognized in an overlay if it is a non-test file -and the package would be reported even without the overlay. - Questions & Tasks - Add GOARCH/GOOS? diff --git a/go-controller/vendor/golang.org/x/tools/go/packages/external.go b/go-controller/vendor/golang.org/x/tools/go/packages/external.go index 4335c1eb14..c2b4b711b5 100644 --- a/go-controller/vendor/golang.org/x/tools/go/packages/external.go +++ b/go-controller/vendor/golang.org/x/tools/go/packages/external.go @@ -34,8 +34,8 @@ type DriverRequest struct { // Tests specifies whether the patterns should also return test packages. Tests bool `json:"tests"` - // Overlay maps file paths (relative to the driver's working directory) to the byte contents - // of overlay files. + // Overlay maps file paths (relative to the driver's working directory) + // to the contents of overlay files (see Config.Overlay). Overlay map[string][]byte `json:"overlay"` } @@ -119,7 +119,19 @@ func findExternalDriver(cfg *Config) driver { stderr := new(bytes.Buffer) cmd := exec.CommandContext(cfg.Context, tool, words...) cmd.Dir = cfg.Dir - cmd.Env = cfg.Env + // The cwd gets resolved to the real path. On Darwin, where + // /tmp is a symlink, this breaks anything that expects the + // working directory to keep the original path, including the + // go command when dealing with modules. + // + // os.Getwd stdlib has a special feature where if the + // cwd and the PWD are the same node then it trusts + // the PWD, so by setting it in the env for the child + // process we fix up all the paths returned by the go + // command. + // + // (See similar trick in Invocation.run in ../../internal/gocommand/invoke.go) + cmd.Env = append(slicesClip(cfg.Env), "PWD="+cfg.Dir) cmd.Stdin = bytes.NewReader(req) cmd.Stdout = buf cmd.Stderr = stderr @@ -138,3 +150,7 @@ func findExternalDriver(cfg *Config) driver { return &response, nil } } + +// slicesClip removes unused capacity from the slice, returning s[:len(s):len(s)]. +// TODO(adonovan): use go1.21 slices.Clip. +func slicesClip[S ~[]E, E any](s S) S { return s[:len(s):len(s)] } diff --git a/go-controller/vendor/golang.org/x/tools/go/packages/golist.go b/go-controller/vendor/golang.org/x/tools/go/packages/golist.go index 22305d9c90..d9be410aa1 100644 --- a/go-controller/vendor/golang.org/x/tools/go/packages/golist.go +++ b/go-controller/vendor/golang.org/x/tools/go/packages/golist.go @@ -841,6 +841,7 @@ func (state *golistState) cfgInvocation() gocommand.Invocation { Env: cfg.Env, Logf: cfg.Logf, WorkingDir: cfg.Dir, + Overlay: cfg.goListOverlayFile, } } @@ -849,26 +850,6 @@ func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer, cfg := state.cfg inv := state.cfgInvocation() - - // For Go versions 1.16 and above, `go list` accepts overlays directly via - // the -overlay flag. Set it, if it's available. - // - // The check for "list" is not necessarily required, but we should avoid - // getting the go version if possible. - if verb == "list" { - goVersion, err := state.getGoVersion() - if err != nil { - return nil, err - } - if goVersion >= 16 { - filename, cleanup, err := state.writeOverlays() - if err != nil { - return nil, err - } - defer cleanup() - inv.Overlay = filename - } - } inv.Verb = verb inv.Args = args gocmdRunner := cfg.gocmdRunner @@ -1015,67 +996,6 @@ func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer, return stdout, nil } -// OverlayJSON is the format overlay files are expected to be in. -// The Replace map maps from overlaid paths to replacement paths: -// the Go command will forward all reads trying to open -// each overlaid path to its replacement path, or consider the overlaid -// path not to exist if the replacement path is empty. -// -// From golang/go#39958. -type OverlayJSON struct { - Replace map[string]string `json:"replace,omitempty"` -} - -// writeOverlays writes out files for go list's -overlay flag, as described -// above. -func (state *golistState) writeOverlays() (filename string, cleanup func(), err error) { - // Do nothing if there are no overlays in the config. - if len(state.cfg.Overlay) == 0 { - return "", func() {}, nil - } - dir, err := os.MkdirTemp("", "gopackages-*") - if err != nil { - return "", nil, err - } - // The caller must clean up this directory, unless this function returns an - // error. - cleanup = func() { - os.RemoveAll(dir) - } - defer func() { - if err != nil { - cleanup() - } - }() - overlays := map[string]string{} - for k, v := range state.cfg.Overlay { - // Create a unique filename for the overlaid files, to avoid - // creating nested directories. - noSeparator := strings.Join(strings.Split(filepath.ToSlash(k), "/"), "") - f, err := os.CreateTemp(dir, fmt.Sprintf("*-%s", noSeparator)) - if err != nil { - return "", func() {}, err - } - if _, err := f.Write(v); err != nil { - return "", func() {}, err - } - if err := f.Close(); err != nil { - return "", func() {}, err - } - overlays[k] = f.Name() - } - b, err := json.Marshal(OverlayJSON{Replace: overlays}) - if err != nil { - return "", func() {}, err - } - // Write out the overlay file that contains the filepath mappings. - filename = filepath.Join(dir, "overlay.json") - if err := os.WriteFile(filename, b, 0665); err != nil { - return "", func() {}, err - } - return filename, cleanup, nil -} - func containsGoFile(s []string) bool { for _, f := range s { if strings.HasSuffix(f, ".go") { diff --git a/go-controller/vendor/golang.org/x/tools/go/packages/packages.go b/go-controller/vendor/golang.org/x/tools/go/packages/packages.go index 3ea1b3fa46..34306ddd39 100644 --- a/go-controller/vendor/golang.org/x/tools/go/packages/packages.go +++ b/go-controller/vendor/golang.org/x/tools/go/packages/packages.go @@ -37,10 +37,20 @@ import ( // A LoadMode controls the amount of detail to return when loading. // The bits below can be combined to specify which fields should be // filled in the result packages. +// // The zero value is a special case, equivalent to combining // the NeedName, NeedFiles, and NeedCompiledGoFiles bits. +// // ID and Errors (if present) will always be filled. -// Load may return more information than requested. +// [Load] may return more information than requested. +// +// Unfortunately there are a number of open bugs related to +// interactions among the LoadMode bits: +// - https://github.com/golang/go/issues/48226 +// - https://github.com/golang/go/issues/56633 +// - https://github.com/golang/go/issues/56677 +// - https://github.com/golang/go/issues/58726 +// - https://github.com/golang/go/issues/63517 type LoadMode int const ( @@ -123,7 +133,14 @@ const ( // A Config specifies details about how packages should be loaded. // The zero value is a valid configuration. +// // Calls to Load do not modify this struct. +// +// TODO(adonovan): #67702: this is currently false: in fact, +// calls to [Load] do not modify the public fields of this struct, but +// may modify hidden fields, so concurrent calls to [Load] must not +// use the same Config. But perhaps we should reestablish the +// documented invariant. type Config struct { // Mode controls the level of information returned for each package. Mode LoadMode @@ -199,13 +216,23 @@ type Config struct { // setting Tests may have no effect. Tests bool - // Overlay provides a mapping of absolute file paths to file contents. - // If the file with the given path already exists, the parser will use the - // alternative file contents provided by the map. + // Overlay is a mapping from absolute file paths to file contents. + // + // For each map entry, [Load] uses the alternative file + // contents provided by the overlay mapping instead of reading + // from the file system. This mechanism can be used to enable + // editor-integrated tools to correctly analyze the contents + // of modified but unsaved buffers, for example. // - // Overlays provide incomplete support for when a given file doesn't - // already exist on disk. See the package doc above for more details. + // The overlay mapping is passed to the build system's driver + // (see "The driver protocol") so that it too can report + // consistent package metadata about unsaved files. However, + // drivers may vary in their level of support for overlays. Overlay map[string][]byte + + // goListOverlayFile is the JSON file that encodes the Overlay + // mapping, used by 'go list -overlay=...' + goListOverlayFile string } // Load loads and returns the Go packages named by the given patterns. @@ -213,6 +240,20 @@ type Config struct { // Config specifies loading options; // nil behaves the same as an empty Config. // +// The [Config.Mode] field is a set of bits that determine what kinds +// of information should be computed and returned. Modes that require +// more information tend to be slower. See [LoadMode] for details +// and important caveats. Its zero value is equivalent to +// NeedName | NeedFiles | NeedCompiledGoFiles. +// +// Each call to Load returns a new set of [Package] instances. +// The Packages and their Imports form a directed acyclic graph. +// +// If the [NeedTypes] mode flag was set, each call to Load uses a new +// [types.Importer], so [types.Object] and [types.Type] values from +// different calls to Load must not be mixed as they will have +// inconsistent notions of type identity. +// // If any of the patterns was invalid as defined by the // underlying build system, Load returns an error. // It may return an empty list of packages without an error, @@ -286,6 +327,17 @@ func defaultDriver(cfg *Config, patterns ...string) (*DriverResponse, bool, erro // (fall through) } + // go list fallback + // + // Write overlays once, as there are many calls + // to 'go list' (one per chunk plus others too). + overlay, cleanupOverlay, err := gocommand.WriteOverlays(cfg.Overlay) + if err != nil { + return nil, false, err + } + defer cleanupOverlay() + cfg.goListOverlayFile = overlay + response, err := callDriverOnChunks(goListDriver, cfg, chunks) if err != nil { return nil, false, err @@ -365,6 +417,9 @@ func mergeResponses(responses ...*DriverResponse) *DriverResponse { } // A Package describes a loaded Go package. +// +// It also defines part of the JSON schema of [DriverResponse]. +// See the package documentation for an overview. type Package struct { // ID is a unique identifier for a package, // in a syntax provided by the underlying build system. @@ -423,6 +478,13 @@ type Package struct { // to corresponding loaded Packages. Imports map[string]*Package + // Module is the module information for the package if it exists. + // + // Note: it may be missing for std and cmd; see Go issue #65816. + Module *Module + + // -- The following fields are not part of the driver JSON schema. -- + // Types provides type information for the package. // The NeedTypes LoadMode bit sets this field for packages matching the // patterns; type information for dependencies may be missing or incomplete, @@ -431,15 +493,15 @@ type Package struct { // Each call to [Load] returns a consistent set of type // symbols, as defined by the comment at [types.Identical]. // Avoid mixing type information from two or more calls to [Load]. - Types *types.Package + Types *types.Package `json:"-"` // Fset provides position information for Types, TypesInfo, and Syntax. // It is set only when Types is set. - Fset *token.FileSet + Fset *token.FileSet `json:"-"` // IllTyped indicates whether the package or any dependency contains errors. // It is set only when Types is set. - IllTyped bool + IllTyped bool `json:"-"` // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles. // @@ -449,26 +511,28 @@ type Package struct { // // Syntax is kept in the same order as CompiledGoFiles, with the caveat that nils are // removed. If parsing returned nil, Syntax may be shorter than CompiledGoFiles. - Syntax []*ast.File + Syntax []*ast.File `json:"-"` // TypesInfo provides type information about the package's syntax trees. // It is set only when Syntax is set. - TypesInfo *types.Info + TypesInfo *types.Info `json:"-"` // TypesSizes provides the effective size function for types in TypesInfo. - TypesSizes types.Sizes + TypesSizes types.Sizes `json:"-"` + + // -- internal -- // forTest is the package under test, if any. forTest string // depsErrors is the DepsErrors field from the go list response, if any. depsErrors []*packagesinternal.PackageError - - // module is the module information for the package if it exists. - Module *Module } // Module provides module information for a package. +// +// It also defines part of the JSON schema of [DriverResponse]. +// See the package documentation for an overview. type Module struct { Path string // module path Version string // module version @@ -601,6 +665,7 @@ func (p *Package) UnmarshalJSON(b []byte) error { OtherFiles: flat.OtherFiles, EmbedFiles: flat.EmbedFiles, EmbedPatterns: flat.EmbedPatterns, + IgnoredFiles: flat.IgnoredFiles, ExportFile: flat.ExportFile, } if len(flat.Imports) > 0 { diff --git a/go-controller/vendor/golang.org/x/tools/internal/gocommand/invoke.go b/go-controller/vendor/golang.org/x/tools/internal/gocommand/invoke.go index eb7a8282f9..af0ee6c614 100644 --- a/go-controller/vendor/golang.org/x/tools/internal/gocommand/invoke.go +++ b/go-controller/vendor/golang.org/x/tools/internal/gocommand/invoke.go @@ -8,12 +8,14 @@ package gocommand import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" "log" "os" "os/exec" + "path/filepath" "reflect" "regexp" "runtime" @@ -167,7 +169,9 @@ type Invocation struct { // TODO(rfindley): remove, in favor of Args. ModFile string - // If Overlay is set, the go command is invoked with -overlay=Overlay. + // Overlay is the name of the JSON overlay file that describes + // unsaved editor buffers; see [WriteOverlays]. + // If set, the go command is invoked with -overlay=Overlay. // TODO(rfindley): remove, in favor of Args. Overlay string @@ -255,12 +259,15 @@ func (i *Invocation) run(ctx context.Context, stdout, stderr io.Writer) error { waitDelay.Set(reflect.ValueOf(30 * time.Second)) } - // On darwin the cwd gets resolved to the real path, which breaks anything that - // expects the working directory to keep the original path, including the + // The cwd gets resolved to the real path. On Darwin, where + // /tmp is a symlink, this breaks anything that expects the + // working directory to keep the original path, including the // go command when dealing with modules. - // The Go stdlib has a special feature where if the cwd and the PWD are the - // same node then it trusts the PWD, so by setting it in the env for the child - // process we fix up all the paths returned by the go command. + // + // os.Getwd has a special feature where if the cwd and the PWD + // are the same node then it trusts the PWD, so by setting it + // in the env for the child process we fix up all the paths + // returned by the go command. if !i.CleanEnv { cmd.Env = os.Environ() } @@ -351,6 +358,7 @@ func runCmdContext(ctx context.Context, cmd *exec.Cmd) (err error) { } } + startTime := time.Now() err = cmd.Start() if stdoutW != nil { // The child process has inherited the pipe file, @@ -377,7 +385,7 @@ func runCmdContext(ctx context.Context, cmd *exec.Cmd) (err error) { case err := <-resChan: return err case <-timer.C: - HandleHangingGoCommand(cmd.Process) + HandleHangingGoCommand(startTime, cmd) case <-ctx.Done(): } } else { @@ -411,7 +419,7 @@ func runCmdContext(ctx context.Context, cmd *exec.Cmd) (err error) { return <-resChan } -func HandleHangingGoCommand(proc *os.Process) { +func HandleHangingGoCommand(start time.Time, cmd *exec.Cmd) { switch runtime.GOOS { case "linux", "darwin", "freebsd", "netbsd": fmt.Fprintln(os.Stderr, `DETECTED A HANGING GO COMMAND @@ -444,7 +452,7 @@ See golang/go#54461 for more details.`) panic(fmt.Sprintf("running %s: %v", listFiles, err)) } } - panic(fmt.Sprintf("detected hanging go command (pid %d): see golang/go#54461 for more details", proc.Pid)) + panic(fmt.Sprintf("detected hanging go command (golang/go#54461); waited %s\n\tcommand:%s\n\tpid:%d", time.Since(start), cmd, cmd.Process.Pid)) } func cmdDebugStr(cmd *exec.Cmd) string { @@ -468,3 +476,73 @@ func cmdDebugStr(cmd *exec.Cmd) string { } return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v GOPROXY=%v PWD=%v %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["GOPROXY"], env["PWD"], strings.Join(args, " ")) } + +// WriteOverlays writes each value in the overlay (see the Overlay +// field of go/packages.Config) to a temporary file and returns the name +// of a JSON file describing the mapping that is suitable for the "go +// list -overlay" flag. +// +// On success, the caller must call the cleanup function exactly once +// when the files are no longer needed. +func WriteOverlays(overlay map[string][]byte) (filename string, cleanup func(), err error) { + // Do nothing if there are no overlays in the config. + if len(overlay) == 0 { + return "", func() {}, nil + } + + dir, err := os.MkdirTemp("", "gocommand-*") + if err != nil { + return "", nil, err + } + + // The caller must clean up this directory, + // unless this function returns an error. + // (The cleanup operand of each return + // statement below is ignored.) + defer func() { + cleanup = func() { + os.RemoveAll(dir) + } + if err != nil { + cleanup() + cleanup = nil + } + }() + + // Write each map entry to a temporary file. + overlays := make(map[string]string) + for k, v := range overlay { + // Use a unique basename for each file (001-foo.go), + // to avoid creating nested directories. + base := fmt.Sprintf("%d-%s.go", 1+len(overlays), filepath.Base(k)) + filename := filepath.Join(dir, base) + err := os.WriteFile(filename, v, 0666) + if err != nil { + return "", nil, err + } + overlays[k] = filename + } + + // Write the JSON overlay file that maps logical file names to temp files. + // + // OverlayJSON is the format overlay files are expected to be in. + // The Replace map maps from overlaid paths to replacement paths: + // the Go command will forward all reads trying to open + // each overlaid path to its replacement path, or consider the overlaid + // path not to exist if the replacement path is empty. + // + // From golang/go#39958. + type OverlayJSON struct { + Replace map[string]string `json:"replace,omitempty"` + } + b, err := json.Marshal(OverlayJSON{Replace: overlays}) + if err != nil { + return "", nil, err + } + filename = filepath.Join(dir, "overlay.json") + if err := os.WriteFile(filename, b, 0666); err != nil { + return "", nil, err + } + + return filename, nil, nil +} diff --git a/go-controller/vendor/golang.org/x/tools/internal/versions/types_go122.go b/go-controller/vendor/golang.org/x/tools/internal/versions/types_go122.go index e8180632a5..aac5db62c9 100644 --- a/go-controller/vendor/golang.org/x/tools/internal/versions/types_go122.go +++ b/go-controller/vendor/golang.org/x/tools/internal/versions/types_go122.go @@ -12,7 +12,7 @@ import ( "go/types" ) -// FileVersions returns a file's Go version. +// FileVersion returns a file's Go version. // The reported version is an unknown Future version if a // version cannot be determined. func FileVersion(info *types.Info, file *ast.File) string { diff --git a/go-controller/vendor/modules.txt b/go-controller/vendor/modules.txt index 3b94d4f30f..59d96dc841 100644 --- a/go-controller/vendor/modules.txt +++ b/go-controller/vendor/modules.txt @@ -86,7 +86,7 @@ github.com/cpuguy83/go-md2man/v2/md2man # github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc ## explicit github.com/davecgh/go-spew/spew -# github.com/emicklei/go-restful/v3 v3.11.0 +# github.com/emicklei/go-restful/v3 v3.12.1 ## explicit; go 1.13 github.com/emicklei/go-restful/v3 github.com/emicklei/go-restful/v3/log @@ -107,18 +107,19 @@ github.com/gaissmai/cidrtree ## explicit; go 1.18 github.com/go-logr/logr github.com/go-logr/logr/funcr +github.com/go-logr/logr/testr # github.com/go-logr/stdr v1.2.2 ## explicit; go 1.16 github.com/go-logr/stdr -# github.com/go-openapi/jsonpointer v0.19.6 -## explicit; go 1.13 +# github.com/go-openapi/jsonpointer v0.21.0 +## explicit; go 1.20 github.com/go-openapi/jsonpointer -# github.com/go-openapi/jsonreference v0.20.2 -## explicit; go 1.13 +# github.com/go-openapi/jsonreference v0.21.0 +## explicit; go 1.20 github.com/go-openapi/jsonreference github.com/go-openapi/jsonreference/internal -# github.com/go-openapi/swag v0.22.4 -## explicit; go 1.18 +# github.com/go-openapi/swag v0.23.0 +## explicit; go 1.20 github.com/go-openapi/swag # github.com/go-task/slim-sprig/v3 v3.0.0 ## explicit; go 1.20 @@ -177,7 +178,7 @@ github.com/gorilla/mux # github.com/gorilla/websocket v1.5.0 ## explicit; go 1.12 github.com/gorilla/websocket -# github.com/imdario/mergo v0.3.12 +# github.com/imdario/mergo v0.3.16 ## explicit; go 1.13 github.com/imdario/mergo # github.com/josharian/intern v1.0.0 @@ -264,6 +265,19 @@ github.com/mdlayher/packet # github.com/mdlayher/socket v0.2.1 ## explicit; go 1.17 github.com/mdlayher/socket +# github.com/metallb/frr-k8s v0.0.15 +## explicit; go 1.22.0 +github.com/metallb/frr-k8s/api/v1beta1 +github.com/metallb/frr-k8s/pkg/client/clientset/versioned +github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake +github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme +github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1 +github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake +github.com/metallb/frr-k8s/pkg/client/informers/externalversions +github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api +github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1 +github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces +github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1 # github.com/miekg/dns v1.1.31 ## explicit; go 1.12 github.com/miekg/dns @@ -455,11 +469,11 @@ go.opencensus.io/trace/tracestate golang.org/x/crypto/cryptobyte golang.org/x/crypto/cryptobyte/asn1 golang.org/x/crypto/ed25519 -# golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 +# golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 ## explicit; go 1.20 golang.org/x/exp/constraints golang.org/x/exp/maps -# golang.org/x/mod v0.17.0 +# golang.org/x/mod v0.18.0 ## explicit; go 1.18 golang.org/x/mod/semver # golang.org/x/net v0.26.0 @@ -522,10 +536,10 @@ golang.org/x/text/secure/bidirule golang.org/x/text/transform golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm -# golang.org/x/time v0.3.0 -## explicit +# golang.org/x/time v0.5.0 +## explicit; go 1.18 golang.org/x/time/rate -# golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d +# golang.org/x/tools v0.22.0 ## explicit; go 1.19 golang.org/x/tools/cmd/stringer golang.org/x/tools/cover @@ -1155,7 +1169,7 @@ k8s.io/klog/v2/internal/severity k8s.io/klog/v2/internal/sloghandler k8s.io/klog/v2/internal/verbosity k8s.io/klog/v2/textlogger -# k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 +# k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a ## explicit; go 1.20 k8s.io/kube-openapi/pkg/cached k8s.io/kube-openapi/pkg/common diff --git a/helm/basic-deploy.sh b/helm/basic-deploy.sh index c4f14c2009..3aa7603062 100755 --- a/helm/basic-deploy.sh +++ b/helm/basic-deploy.sh @@ -1,67 +1,97 @@ #!/usr/bin/env bash - -# check docker -docker info >/dev/null 2>&1 -if [ "$?" -eq 127 ]; then - echo "docker not found" - exit -fi -# check kubectl -kubectl cluster-info >/dev/null 2>&1 -if [ "$?" -eq 127 ]; then - echo "kubectl not found" - exit -fi +# Helper usage +function usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "This script deploys a kind cluster and configures it to use OVN Kubernetes CNI." + echo "" + echo "Options:" + echo " BUILD_IMAGE=${BUILD_IMAGE:-false} Set to true to build the Docker image instead of pulling it." + echo " OVN_INTERCONNECT=${OVN_INTERCONNECT:-true} Set to false to use a non-interconnect deployment (values-no-ic.yaml)." + echo "" + echo "Example: BUILD_IMAGE=true OVN_INTERCONNECT=false $0" + exit 1 +} -# check kind -kind version >/dev/null 2>&1 -if [ "$?" -eq 127 ]; then - echo "kind not found" - exit -fi +# Default values for flags +BUILD_IMAGE=${BUILD_IMAGE:-false} +OVN_INTERCONNECT=${OVN_INTERCONNECT:-true} -# check go -go version >/dev/null 2>&1 -if [ "$?" -eq 127 ]; then - echo "go not found" - exit +# Determine the values file based on OVN_INTERCONNECT +if [[ "$OVN_INTERCONNECT" == "true" ]]; then + VALUES_FILE="values-single-node-zone.yaml" +else + VALUES_FILE="values-no-ic.yaml" fi -# Ensure that kind network does not have ip6 enabled -enabled_ipv6=$(docker network inspect kind -f '{{.EnableIPv6}}' 2>/dev/null) -[ "${enabled_ipv6}" = "false" ] || { - docker network rm kind 2>/dev/null || : - docker network create kind -o "com.docker.network.bridge.enable_ip_masquerade"="true" -o "com.docker.network.driver.mtu"="1500" +# Verify dependencies +check_command() { + command -v "$1" >/dev/null 2>&1 || { echo "$1 not found, please install it."; exit 1; } } -[ "${enabled_ipv6}" = "false" ] || { 2&>1 echo the kind network is not what we expected ; exit 1; } +check_command docker +check_command kubectl +check_command kind + +export DIR="$( cd -- "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR + +if [[ "$1" == "--help" || "$1" == "-h" ]]; then + usage +fi + +IMG_PREFIX='ghcr.io/ovn-kubernetes/ovn-kubernetes/ovn-kube-ubuntu' +TAG='master' +IMG="${IMG_PREFIX}:${TAG}" + +if [[ "$BUILD_IMAGE" == "true" ]]; then + check_command go # Only check for Go when building the image + # Build image + echo "Building Docker image..." + pushd ../dist/images + make ubuntu + popd + docker tag ovn-kube-ubuntu:latest $IMG +else + # Pull image from GitHub + echo "Pulling Docker image..." + docker pull $IMG +fi + +# Configure system parameters set -euxo pipefail -# increate fs.inotify.max_user_watches sudo sysctl fs.inotify.max_user_watches=524288 -# increase fs.inotify.max_user_instances sudo sysctl fs.inotify.max_user_instances=512 -# build image -cd ../dist/images -make ubuntu -docker tag ovn-kube-ubuntu:latest ghcr.io/ovn-org/ovn-kubernetes/ovn-kube-ubuntu:master -# create a cluster, with 1 controller and 1 worker node + +# Create a kind cluster kind_cluster_name=ovn-helm -cat < /tmp/kind.yaml +kind delete clusters $kind_cluster_name || true +cat <=", 2), "test requires >= 2 Ready nodes") + serverPodConfig.namespace = f.Namespace.Name + clientPodConfig.namespace = f.Namespace.Name + runUDNPod(cs, f.Namespace.Name, serverPodConfig, nil) + runUDNPod(cs, f.Namespace.Name, clientPodConfig, nil) + serverIP, err := podIPsForUserDefinedPrimaryNetwork(cs, f.Namespace.Name, serverPodConfig.name, namespacedName(f.Namespace.Name, netConfig.name), 0) + Expect(err).ShouldNot(HaveOccurred(), "UDN pod IP must be retrieved") + By("restart OVNKube node pods on client and server Nodes and ensure connectivity") + serverPod := getPod(f, serverPodConfig.name) + clientPod := getPod(f, clientPodConfig.name) + for _, testPod := range []*v1.Pod{clientPod, serverPod} { + By(fmt.Sprintf("asserting the server pod IP %v is reachable from client before restart of OVNKube node pod on Node %s", serverIP, testPod.Spec.Hostname)) + Expect(reachToServerPodFromClient(cs, serverPodConfig, clientPodConfig, serverIP, port)).ShouldNot(HaveOccurred(), "must have connectivity to server pre OVN Kube node Pod restart") + By(fmt.Sprintf("restarting OVNKube node Pod located on Node %s which hosts test Pod %s/%s", testPod.Spec.NodeName, testPod.Namespace, testPod.Name)) + Expect(restartOVNKubeNodePod(cs, ovnNamespace, testPod.Spec.NodeName)).ShouldNot(HaveOccurred(), "restart of OVNKube node pod must succeed") + By(fmt.Sprintf("asserting the server pod IP %v is reachable from client post restart", serverIP)) + Expect(reachToServerPodFromClient(cs, serverPodConfig, clientPodConfig, serverIP, port)).ShouldNot(HaveOccurred(), "must have connectivity to server post restart") + } + }, + Entry( + "L3", + networkAttachmentConfigParams{ + name: nadName, + topology: "layer3", + cidr: correctCIDRFamily(userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), + role: "primary", + }, + *podConfig( + "client-pod", + withNodeSelector(map[string]string{nodeHostnameKey: workerOneNodeName}), + ), + *podConfig( + "server-pod", + withCommand(func() []string { + return httpServerContainerCmd(port) + }), + withNodeSelector(map[string]string{nodeHostnameKey: workerTwoNodeName}), + ), + ), + Entry( + "L2", + networkAttachmentConfigParams{ + name: nadName, + topology: "layer2", + cidr: correctCIDRFamily(userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), + role: "primary", + }, + *podConfig( + "client-pod", + withNodeSelector(map[string]string{nodeHostnameKey: workerOneNodeName}), + ), + *podConfig( + "server-pod", + withCommand(func() []string { + return httpServerContainerCmd(port) + }), + withNodeSelector(map[string]string{nodeHostnameKey: workerTwoNodeName}), + ), + ), + ) + }) }) +// randomNetworkMetaName return pseudo random name for network related objects (NAD,UDN,CUDN). +// CUDN is cluster-scoped object, in case tests running in parallel, having random names avoids +// conflicting with other tests. +func randomNetworkMetaName() string { + return fmt.Sprintf("test-net-%s", rand.String(5)) +} + var nadToUdnParams = map[string]string{ "primary": "Primary", "secondary": "Secondary", @@ -1533,12 +1641,12 @@ func generateLayer3Subnets(cidrs string) []string { } func waitForUserDefinedNetworkReady(namespace, name string, timeout time.Duration) error { - _, err := e2ekubectl.RunKubectl(namespace, "wait", "userdefinednetwork", name, "--for", "condition=NetworkReady=True", "--timeout", timeout.String()) + _, err := e2ekubectl.RunKubectl(namespace, "wait", "userdefinednetwork", name, "--for", "condition=NetworkCreated=True", "--timeout", timeout.String()) return err } func waitForClusterUserDefinedNetworkReady(name string, timeout time.Duration) error { - _, err := e2ekubectl.RunKubectl("", "wait", "clusteruserdefinednetwork", name, "--for", "condition=NetworkReady=True", "--timeout", timeout.String()) + _, err := e2ekubectl.RunKubectl("", "wait", "clusteruserdefinednetwork", name, "--for", "condition=NetworkCreated=True", "--timeout", timeout.String()) return err } @@ -1599,7 +1707,7 @@ func assertUDNStatusReportsConsumers(udnNamesapce, udnName, expectedPodName stri found := false for _, condition := range conditions { if found, _ = Equal(metav1.Condition{ - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "SyncError", Message: expectedMsg, @@ -1658,9 +1766,9 @@ func assertClusterUDNStatusReportsActiveNamespaces(cudnName string, expectedActi c := conditions[0] // equality matcher cannot be used since condition message namespaces order is inconsistent - ExpectWithOffset(1, c.Type).Should(Equal("NetworkReady")) + ExpectWithOffset(1, c.Type).Should(Equal("NetworkCreated")) ExpectWithOffset(1, c.Status).Should(Equal(metav1.ConditionTrue)) - ExpectWithOffset(1, c.Reason).Should(Equal("NetworkAttachmentDefinitionReady")) + ExpectWithOffset(1, c.Reason).Should(Equal("NetworkAttachmentDefinitionCreated")) ExpectWithOffset(1, c.Message).To(ContainSubstring("NetworkAttachmentDefinition has been created in following namespaces:")) for _, ns := range expectedActiveNsNames { @@ -1679,7 +1787,7 @@ func assertClusterUDNStatusReportConsumers(conditionsJSON, udnName, udnNamespace udnNamespace, udnName, expectedPodName) ExpectWithOffset(1, conditions).To(Equal([]metav1.Condition{ { - Type: "NetworkReady", + Type: "NetworkCreated", Status: "False", Reason: "NetworkAttachmentDefinitionSyncError", Message: expectedMsg, @@ -1725,8 +1833,7 @@ spec: topology: Layer3 layer3: role: Primary - subnets: [{cidr: "10.100.0.0/16"}] -` + subnets: ` + generateCIDRforClusterUDN() } func newL2SecondaryUDNManifest(name string) string { @@ -1771,7 +1878,16 @@ func generateCIDRforUDN() string { ` } return cidr +} +func generateCIDRforClusterUDN() string { + cidr := `[{cidr: "10.100.0.0/16"}]` + if isIPv6Supported() && isIPv4Supported() { + cidr = `[{cidr: "10.100.0.0/16"},{cidr: "2014:100:200::0/60"}]` + } else if isIPv6Supported() { + cidr = `[{cidr: "2014:100:200::0/60"}]` + } + return cidr } type podOption func(*podConfiguration) diff --git a/test/e2e/network_segmentation_policy.go b/test/e2e/network_segmentation_policy.go index d420c3767c..7b1c48882f 100644 --- a/test/e2e/network_segmentation_policy.go +++ b/test/e2e/network_segmentation_policy.go @@ -30,7 +30,6 @@ var _ = ginkgo.Describe("Network Segmentation: Network Policies", func() { workerOneNodeName = "ovn-worker" workerTwoNodeName = "ovn-worker2" port = 9000 - netPrefixLengthPerNode = 24 randomStringLength = 5 nameSpaceYellowSuffix = "yellow" namespaceBlueSuffix = "blue" @@ -99,8 +98,7 @@ var _ = ginkgo.Describe("Network Segmentation: Network Policies", func() { i, ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - const netPrefixLengthPerNode = 24 - ginkgo.By(fmt.Sprintf("asserting the server pod IP %v is from the configured range %v/%v", serverIP, cidr, netPrefixLengthPerNode)) + ginkgo.By(fmt.Sprintf("asserting the server pod IP %v is from the configured range %v", serverIP, cidr)) subnet, err := getNetCIDRSubnet(cidr) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(inRange(subnet, serverIP)).To(gomega.Succeed()) @@ -210,21 +208,24 @@ var _ = ginkgo.Describe("Network Segmentation: Network Policies", func() { runUDNPod(cs, namespaceBlue, clientPodConfig, nil) ginkgo.By("asserting the server pods have an IP from the configured range") - allowServerPodIP, err := podIPsForUserDefinedPrimaryNetwork(cs, namespaceYellow, allowServerPodConfig.name, - namespacedName(namespaceYellow, netConfName), 0) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("asserting the allow server pod IP %v is from the configured range %v/%v", allowServerPodIP, - userDefinedNetworkIPv4Subnet, netPrefixLengthPerNode)) - subnet, err := getNetCIDRSubnet(userDefinedNetworkIPv4Subnet) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(inRange(subnet, allowServerPodIP)).To(gomega.Succeed()) - denyServerPodIP, err := podIPsForUserDefinedPrimaryNetwork(cs, namespaceYellow, denyServerPodConfig.name, - namespacedName(namespaceYellow, netConfName), 0) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("asserting the deny server pod IP %v is from the configured range %v/%v", denyServerPodIP, - userDefinedNetworkIPv4Subnet, netPrefixLengthPerNode)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(inRange(subnet, denyServerPodIP)).To(gomega.Succeed()) + var allowServerPodIP, denyServerPodIP string + for i, cidr := range strings.Split(nad.cidr, ",") { + if cidr == "" { + continue + } + subnet, err := getNetCIDRSubnet(cidr) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + allowServerPodIP, err = podIPsForUserDefinedPrimaryNetwork(cs, namespaceYellow, allowServerPodConfig.name, + namespacedName(namespaceYellow, netConfName), i) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("asserting the allow server pod IP %v is from the configured range %v", allowServerPodIP, cidr)) + gomega.Expect(inRange(subnet, allowServerPodIP)).To(gomega.Succeed()) + denyServerPodIP, err = podIPsForUserDefinedPrimaryNetwork(cs, namespaceYellow, denyServerPodConfig.name, + namespacedName(namespaceYellow, netConfName), i) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("asserting the deny server pod IP %v is from the configured range %v", denyServerPodIP, cidr)) + gomega.Expect(inRange(subnet, denyServerPodIP)).To(gomega.Succeed()) + } ginkgo.By("asserting the *client* pod can contact the allow server pod exposed endpoint") gomega.Eventually(func() error { @@ -237,7 +238,7 @@ var _ = ginkgo.Describe("Network Segmentation: Network Policies", func() { }, 2*time.Minute, 6*time.Second).Should(gomega.Succeed()) ginkgo.By("creating a \"default deny\" network policy") - _, err = makeDenyAllPolicy(f, namespaceYellow, "deny-all") + _, err := makeDenyAllPolicy(f, namespaceYellow, "deny-all") gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("asserting the *client* pod can not contact the allow server pod exposed endpoint") diff --git a/test/scripts/traffic-flow-tests.sh b/test/scripts/traffic-flow-tests.sh index c33acd80ef..9fb88acfb4 100755 --- a/test/scripts/traffic-flow-tests.sh +++ b/test/scripts/traffic-flow-tests.sh @@ -5,10 +5,10 @@ set -euo pipefail # Set default values export KUBECONFIG="${KUBECONFIG:-${HOME}/ovn.conf}" export OCI_BIN="${KIND_EXPERIMENTAL_PROVIDER:-docker}" -export TFT_TEST_IMAGE="ghcr.io/wizhaoredhat/ocp-traffic-flow-tests:latest" -export TRAFFIC_FLOW_TESTS_DIRNAME="ocp-traffic-flow-tests" -export TRAFFIC_FLOW_TESTS_REPO="https://github.com/wizhaoredhat/ocp-traffic-flow-tests.git" -export TRAFFIC_FLOW_TESTS_COMMIT="eb46da7a3ee1c3cfab5e141f69a3ccd1cdbfbabd" +export TFT_TEST_IMAGE="ghcr.io/ovn-kubernetes/kubernetes-traffic-flow-tests:latest" +export TRAFFIC_FLOW_TESTS_DIRNAME="kubernetes-traffic-flow-tests" +export TRAFFIC_FLOW_TESTS_REPO="https://github.com/ovn-kubernetes/kubernetes-traffic-flow-tests.git" +export TRAFFIC_FLOW_TESTS_COMMIT="3e092f2c448bdffcddd5d025b97df53145ccea62" export TRAFFIC_FLOW_TESTS="${TRAFFIC_FLOW_TESTS:-1,2,3}" export SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"