diff --git a/.ci-operator.yaml b/.ci-operator.yaml index e307e5af66..284a910090 100644 --- a/.ci-operator.yaml +++ b/.ci-operator.yaml @@ -1,4 +1,4 @@ build_root_image: name: release namespace: openshift - tag: rhel-9-release-golang-1.24-openshift-4.21 + tag: rhel-9-release-golang-1.24-openshift-4.22 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 612abb681e..e28f96a64e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,7 +2,7 @@ name: ovn-docker-images on: push: - branches: [ master,release-1.0,release-1.1 ] + branches: [ master,release-1.0,release-1.1,release-1.2 ] permissions: contents: read @@ -14,12 +14,26 @@ env: REPOSITORY: ovn-kubernetes FEDORA_IMAGE_NAME: ovn-kube-fedora UBUNTU_IMAGE_NAME: ovn-kube-ubuntu - BUILDER_IMAGE: quay.io/lib/golang:1.24 + BUILDER_IMAGE: quay.io/projectquay/golang:1.24 jobs: - build: - name: Build Images - runs-on: ubuntu-latest + # Build Fedora image for each platform + build-fedora: + name: Build Fedora (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: true + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -39,8 +53,8 @@ jobs: with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up environment run: | export GOPATH=$(go env GOPATH) @@ -64,23 +78,19 @@ jobs: pushd dist/images echo "ref: ${BRANCH} commit: ${COMMIT}" > git_info popd - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: all - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - name: Extract metadata (tags, labels) for fedora ovn-k image - id: meta-fedora + id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.FEDORA_IMAGE_NAME }} - name: Build and push Fedora based Docker image + id: build uses: docker/build-push-action@v5 with: builder: ${{ steps.buildx.outputs.name }} @@ -89,23 +99,201 @@ jobs: push: true build-args: | BUILDER_IMAGE=${{ env.BUILDER_IMAGE }} - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta-fedora.outputs.tags }} - labels: ${{ steps.meta-fedora.outputs.labels }} + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=fedora-${{ env.PLATFORM_PAIR }} + cache-to: type=gha,mode=max,scope=fedora-${{ env.PLATFORM_PAIR }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.FEDORA_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-fedora-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # Merge Fedora multi-platform images + merge-fedora: + name: Merge Fedora + runs-on: ubuntu-latest + needs: build-fedora + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-fedora-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the GH Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for fedora ovn-k image + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.FEDORA_IMAGE_NAME }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.FEDORA_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.FEDORA_IMAGE_NAME }}:${{ steps.meta.outputs.version }} + + # Build Ubuntu image for each platform + build-ubuntu: + name: Build Ubuntu (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: true + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go-controller/go.mod' + # Disabling cache to avoid warnings until these two issues are fixed + # https://github.com/actions/setup-go/issues/424 + # https://github.com/actions/setup-go/issues/403 + # cache-dependency-path: "**/*.sum" + cache: false + id: go + + - name: Log in to the GH Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up environment + run: | + export GOPATH=$(go env GOPATH) + echo "GOPATH=$GOPATH" >> $GITHUB_ENV + echo "$GOPATH/bin" >> $GITHUB_PATH + + - name: Build ovnkube-binaries copy to context + run: | + pushd go-controller + make + popd + + pushd dist/images + cp -r ../../go-controller/_output/go/bin/* . + popd + + - name: Generate git-info to write to image + run: | + BRANCH=$(git rev-parse --short "$GITHUB_SHA") + COMMIT=$(git rev-parse HEAD) + pushd dist/images + echo "ref: ${BRANCH} commit: ${COMMIT}" > git_info + popd + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 - name: Extract metadata (tags, labels) for ubuntu ovn-k image - id: meta-ubuntu + id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.UBUNTU_IMAGE_NAME }} - name: Build and push Ubuntu based Docker image + id: build uses: docker/build-push-action@v5 with: builder: ${{ steps.buildx.outputs.name }} context: ./dist/images file: ./dist/images/Dockerfile.ubuntu push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta-ubuntu.outputs.tags }} - labels: ${{ steps.meta-ubuntu.outputs.labels }} + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=ubuntu-${{ env.PLATFORM_PAIR }} + cache-to: type=gha,mode=max,scope=ubuntu-${{ env.PLATFORM_PAIR }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.UBUNTU_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-ubuntu-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # Merge Ubuntu multi-platform images + merge-ubuntu: + name: Merge Ubuntu + runs-on: ubuntu-latest + needs: build-ubuntu + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-ubuntu-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the GH Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for ubuntu ovn-k image + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.UBUNTU_IMAGE_NAME }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.UBUNTU_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ env.REPOSITORY }}/${{ env.UBUNTU_IMAGE_NAME }}:${{ steps.meta.outputs.version }} \ No newline at end of file diff --git a/.github/workflows/performance-test.yml b/.github/workflows/performance-test.yml new file mode 100644 index 0000000000..9369e659a2 --- /dev/null +++ b/.github/workflows/performance-test.yml @@ -0,0 +1,483 @@ +name: performance-test +on: + pull_request: + branches: [ master ] + issue_comment: + types: [created] + schedule: + - cron: '0 6 * * *' # Daily at 6 AM UTC + workflow_dispatch: + +env: + K8S_VERSION: v1.34.0 + KIND_CLUSTER_NAME: ovn + KIND_INSTALL_INGRESS: true + KIND_ALLOW_SYSTEM_WRITES: true + # This skips tests tagged as Serial for most lanes + # Serial tests are run in a dedicated lane + PARALLEL: true + + # This must be a directory + CI_IMAGE_CACHE: tmp/image_cache/ + CI_IMAGE_BASE_TAR: image-base.tar + CI_IMAGE_PR_TAR: image-pr.tar + CI_DIST_IMAGES_OUTPUT: dist/images/_output/ + + # To run CI over custom OVN + # OVN_REPO: https://github.com/ovn-org/ovn + # OVN_GITREF: main + +jobs: + performance-job: + name: performance + runs-on: [oracle-vm-32cpu-128gb-x86-64] + timeout-minutes: 240 + needs: [build-pr] + permissions: + contents: read + pull-requests: write + issues: write + strategy: + fail-fast: false + matrix: + # Valid options are: + # target: ["shard-conformance", "control-plane", "multi-homing", "multi-node-zones", "node-ip-mac-migration", "compact-mode", "serial"] + # shard-conformance: hybrid-overlay = multicast-enable = emptylb-enable = false + # control-plane: hybrid-overlay = multicast-enable = emptylb-enable = true + # perf-test: ["all","kubelet-density-cni", "udn-density-l2-noPods", "cudn-density-l2-noPods"] + # ha: ["HA", "noHA"] + # gateway-mode: ["local", "shared"] + # ipfamily: ["ipv4", "ipv6", "dualstack"] + # disable-snat-multiple-gws: ["noSnatGW", "snatGW"] + # second-bridge: ["2br", "1br"] + # ic: ["ic-disabled", "ic-single-node-zones", "ic-multi-node-zones"] + # num-workers : "" + # num-nodes-per-zone : "" + # forwarding : ["", "disable-forwarding"] + # dns-name-resolver : ["", "enable-dns-name-resolver"] + # network-segmentation : ["", "enable-network-segmentation"] + # traffic-flow-tests : "" + include: + - {"target": "control-plane", perf-test: "kubelet-density-cni", "ha": "HA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "num-workers": "3", "network-segmentation": ""} + - {"target": "control-plane", perf-test: "udn-density-l2-noPods", "ha": "HA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "num-workers": "3", "network-segmentation": ""} + - {"target": "control-plane", perf-test: "cudn-density-l2-noPods", "ha": "HA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "num-workers": "3", "network-segmentation": ""} + - {"target": "control-plane", perf-test: "udn-density-l2-pods", "ha": "HA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "num-workers": "3", "network-segmentation": ""} + env: + ES_SERVER: "${{ secrets.PERF_DATASTORE }}" + JOB_NAME: "${{ matrix.target }}-${{ matrix.ha }}-${{ matrix.gateway-mode }}-${{ matrix.ipfamily }}-${{ matrix.disable-snat-multiple-gws }}-${{ matrix.second-bridge }}-${{ matrix.ic }}" + OVN_HYBRID_OVERLAY_ENABLE: ${{ (matrix.target == 'control-plane' || matrix.target == 'control-plane-helm') && (matrix.ipfamily == 'ipv4' || matrix.ipfamily == 'dualstack' ) }} + OVN_MULTICAST_ENABLE: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || matrix.target == 'network-segmentation' || matrix.target == 'bgp' || matrix.target == 'bgp-loose-isolation' }}" + OVN_EMPTY_LB_EVENTS: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || matrix.target == 'bgp' || matrix.target == 'bgp-loose-isolation' }}" + OVN_HA: "${{ matrix.ha == 'HA' }}" + OVN_DISABLE_SNAT_MULTIPLE_GWS: "${{ matrix.disable-snat-multiple-gws == 'noSnatGW' }}" + KIND_INSTALL_METALLB: "false" + OVN_GATEWAY_MODE: "${{ matrix.gateway-mode }}" + OVN_SECOND_BRIDGE: "${{ matrix.second-bridge == '2br' }}" + ENABLE_MULTI_NET: true + ENABLE_NETWORK_SEGMENTATION: true + PLATFORM_IPV4_SUPPORT: "${{ matrix.ipfamily == 'IPv4' || matrix.ipfamily == 'dualstack' }}" + PLATFORM_IPV6_SUPPORT: "${{ matrix.ipfamily == 'IPv6' || matrix.ipfamily == 'dualstack' }}" + KIND_INSTALL_KUBEVIRT: "${{ matrix.target == 'kv-live-migration' }}" + OVN_COMPACT_MODE: "${{ matrix.target == 'compact-mode' }}" + OVN_DUMMY_GATEWAY_BRIDGE: "${{ matrix.target == 'compact-mode' }}" + OVN_ENABLE_INTERCONNECT: "${{ matrix.ic == 'ic-single-node-zones' || matrix.ic == 'ic-multi-node-zones'}}" + KIND_NUM_WORKER: "${{ matrix.num-workers }}" + KIND_NUM_NODES_PER_ZONE: "${{ matrix.num-nodes-per-zone }}" + OVN_DISABLE_FORWARDING: "${{ matrix.forwarding == 'disable-forwarding' }}" + USE_HELM: "${{ matrix.target == 'control-plane-helm' || matrix.target == 'multi-homing-helm' }}" + OVN_ENABLE_DNSNAMERESOLVER: "${{ matrix.dns-name-resolver == 'enable-dns-name-resolver' }}" + OVN_NETWORK_QOS_ENABLE: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' }}" + TRAFFIC_FLOW_TESTS: "${{ matrix.traffic-flow-tests }}" + ENABLE_ROUTE_ADVERTISEMENTS: "${{ matrix.routeadvertisements != '' }}" + ADVERTISE_DEFAULT_NETWORK: "${{ matrix.routeadvertisements == 'advertise-default' }}" + ENABLE_PRE_CONF_UDN_ADDR: "${{ matrix.ic == 'ic-single-node-zones' && (matrix.target == 'network-segmentation' || matrix.network-segmentation == 'enable-network-segmentation') }}" + ENABLE_NETWORK_CONNECT: "${{ matrix.target == 'network-segmentation' }}" + ADVERTISED_UDN_ISOLATION_MODE: "${{ matrix.advertised-udn-isolation-mode }}" + # Override PARALLEL=true for Serial tests target to run Serial tests + PARALLEL: "${{ matrix.target != 'serial' }}" + OVN_UNPRIVILEGED_MODE: "${{ matrix.cni-mode == 'unprivileged' }}" + MULTI_POD_SUBNET: true + # Performance test specific settings + KIND_NUM_INFRA: "2" + KIND_INSTALL_PROMETHEUS: "true" + KIND_PROMETHEUS_INFRA_ONLY: "true" + METRICS_IP: "127.0.0.1" + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + steps: + - uses: actions/checkout@v4 + + # Debug session for the performance test + #- name: Setup tmate session + # id: tmate + # uses: mxschmitt/action-tmate@v3 + # with: + # detached: true + + - name: Get PR info for issue comment + if: github.event_name == 'issue_comment' + id: pr_info + run: | + PR_NUMBER=${{ github.event.issue.number }} + PR_INFO=$(curl -s -H "Authorization: token ${{ github.token }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER") + PR_SHA=$(echo "$PR_INFO" | jq -r .head.sha) + echo "sha=$PR_SHA" >> $GITHUB_OUTPUT + + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'issue_comment' && steps.pr_info.outputs.sha || github.sha }} + + - name: Runner Diagnostics + if: always() + uses: ./.github/actions/diagnostics + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go-controller/go.mod' + cache: false + id: go + + - name: Set up environment + run: | + export GOPATH=$(go env GOPATH) + echo "GOPATH=$GOPATH" >> $GITHUB_ENV + echo "$GOPATH/bin" >> $GITHUB_PATH + if [ $OVN_SECOND_BRIDGE == "true" ]; then + # must be "greater" lexigraphically than network "kind", therefore external gateway is named xgw + echo OVN_TEST_EX_GW_NETWORK=xgw >> $GITHUB_ENV + echo OVN_ENABLE_EX_GW_NETWORK_BRIDGE=true >> $GITHUB_ENV + fi + if [ "$ADVERTISE_DEFAULT_NETWORK" == "true" ]; then + echo "ADVERTISE_DEFAULT_NETWORK=true" >> $GITHUB_ENV + + # Use proper variable declaration with default values + if [ $MULTI_POD_SUBNET == "true" ]; then + NET_CIDR_IPV4=${NET_CIDR_IPV4:-10.243.0.0/23,10.244.0.0/16} + NET_CIDR_IPV6=${NET_CIDR_IPV6:-fd00:10:243::/63,fd00:10:244::/48} + else + NET_CIDR_IPV4=${NET_CIDR_IPV4:-10.244.0.0/16} + NET_CIDR_IPV6=${NET_CIDR_IPV6:-fd00:10:244::/48} + fi + sudo ip a + sudo ip r + + # Add masquerade rules for both IPv4 and IPv6 networks + echo "Adding masquerade rule for $NET_CIDR_IPV4" + IFS=',' read -r -a ELEMENTS <<< "$NET_CIDR_IPV4" + for element in "${ELEMENTS[@]}"; do + sudo iptables -t nat -A POSTROUTING -s $element -o eth0 -j MASQUERADE + done + + echo "Adding masquerade rule for $NET_CIDR_IPV6" + IFS=',' read -r -a ELEMENTS <<< "$NET_CIDR_IPV6" + for element in "${ELEMENTS[@]}"; do + sudo ip6tables -t nat -A POSTROUTING -s $element -o eth0 -j MASQUERADE + done + + # Verify the rules were added + echo "IPv4 POSTROUTING rules:" + sudo iptables -t nat -L POSTROUTING -v + + echo "IPv6 POSTROUTING rules:" + sudo ip6tables -t nat -L POSTROUTING -v + fi + + - name: Free up disk space + uses: ./.github/actions/free-disk-space + + - name: Setup /mnt/runner directory + run: | + sudo mkdir -pv /mnt/runner + sudo chown $USER:$USER /mnt/runner + + - name: Setup /mnt/docker-data as docker storage + run: | + sudo mkdir -pv /mnt/docker-data + sudo systemctl stop docker.socket docker + [ -s "/etc/docker/daemon.json" ] && { + cat "/etc/docker/daemon.json" | jq '. + {"data-root": "/mnt/docker-data"}' | sudo tee /etc/docker/daemon.$$ + } || { + echo '{"data-root": "/mnt/docker-data"}' | sudo tee /etc/docker/daemon.$$ + } + sudo mv -f /etc/docker/daemon.$$ /etc/docker/daemon.json + sudo systemctl start docker docker.socket + docker system info + + - name: Disable ufw + run: | + sudo ufw disable + + - name: Download test-image-pr + uses: actions/download-artifact@v4 + with: + name: test-image-pr + + - name: Load container image + run: | + echo "Loading image with docker..." + docker load --input ${CI_IMAGE_PR_TAR} && rm -rf ${CI_IMAGE_PR_TAR} + + - name: kind setup with infra nodes + timeout-minutes: 45 + run: | + export OVN_IMAGE="ovn-daemonset-fedora:pr" + # Set enhanced timeouts for large cluster performance testing + export KIND_CLUSTER_LOGLEVEL=2 + + # Use the existing kind infrastructure but with enhanced kubelet health settings + make -C test install-kind + + kind get kubeconfig > kconfig + export KUBECONFIG=${PWD}/kconfig + + # Wait for kubelet health on all nodes with retries + echo "Verifying kubelet health on all nodes..." + for i in {1..10}; do + echo "Health check attempt $i/10..." + if kubectl get nodes --no-headers | awk '{print $2}' | grep -v Ready; then + echo "Some nodes not ready, waiting 30s..." + sleep 30 + else + echo "All nodes are ready!" + break + fi + + if [ $i -eq 10 ]; then + echo "Cluster health check failed after 10 attempts" + echo "=== Node Status ===" + kubectl get nodes -o wide + echo "=== Node Conditions ===" + kubectl describe nodes + echo "=== Kubelet Logs ===" + for node in $(kind get nodes --name ${KIND_CLUSTER_NAME}); do + echo "--- Kubelet logs for $node ---" + docker exec $node journalctl -u kubelet --no-pager -l --since "10 minutes ago" | tail -50 + done + exit 1 + fi + done + + - name: Label 2 random worker nodes for Prometheus + run: | + + kind get kubeconfig > kconfig + export KUBECONFIG=${PWD}/kconfig + + workers=$(kubectl get nodes --no-headers | grep -v control-plane | awk '{print $1}' | shuf -n 2) + for worker in $workers; do + kubectl label node $worker prometheus-node=true + echo "Labeled node: $worker" + done + + - name: Label remaining worker nodes + run: | + kind get kubeconfig > kconfig + export KUBECONFIG=${PWD}/kconfig + + # Get all worker nodes that don't have the prometheus-node label + workers_without_prometheus=$(kubectl get nodes --no-headers -l '!prometheus-node' | grep -v control-plane | awk '{print $1}') + for worker in $workers_without_prometheus; do + kubectl label node $worker node-role.kubernetes.io/worker="" + echo "Labeled worker node: $worker" + done + + - name: Install Prometheus on infra nodes + run: | + kind get kubeconfig > kconfig + export KUBECONFIG=${PWD}/kconfig + ./contrib/install-prometheus-infra.sh + + - name: Upload Prometheus installation logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: prometheus-install-logs-${{ matrix.perf-test }}-${{ github.run_id }} + path: prometheus-install.log + retention-days: 7 + + - name: Download and setup kube-burner v2.1.0 + run: | + curl -L https://github.com/kube-burner/kube-burner/releases/download/v2.1.0/kube-burner-V2.1.0-linux-x86_64.tar.gz | tar xz + chmod +x kube-burner + sudo mv kube-burner /usr/local/bin/ + + - name: "Run kube-burner ${{ matrix.perf-test }} workload" + timeout-minutes: 120 + run: | + kind get kubeconfig > kconfig + export KUBECONFIG=${PWD}/kconfig + + # Port-Forward so we can scrape metrics. + kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090 & + + # Make sure the port-forward is up prior to running the workload + sleep 30 + + cd contrib/perf + + #Generate metadata. + envsubst < performance-meta.yml > perf-meta.yml + + if [[ -z "${ES_SERVER}" ]]; then + kube-burner init --config workloads/${{ matrix.perf-test }}.yml -e metric-endpoint-local.yml --user-metadata perf-meta.yml + else + kube-burner init --config workloads/${{ matrix.perf-test }}.yml -e metric-endpoint.yml --user-metadata perf-meta.yml + fi + + mkdir -p /tmp/${{ matrix.perf-test }}/pprof-data + mkdir -p /tmp/${{ matrix.perf-test }}/perf + cp -r pprof-data/* /tmp/${{ matrix.perf-test }}/pprof-data + cp -r * /tmp/${{ matrix.perf-test }}/perf + + - name: Generate performance report + if: ${{ matrix.perf-test == 'kubelet-density-cni' || matrix.perf-test == 'udn-density-l2-noPods' || matrix.perf-test == 'cudn-density-l2-noPods' || matrix.perf-test == 'udn-density-l2-pods' }} + run: | + cd contrib/perf + # Generate the performance report + python3 generate_perf_report.py \ + --workload ${{ matrix.perf-test }} \ + --metrics-dir /tmp/${{ matrix.perf-test }}/perf/metrics/ \ + --output /tmp/${{ matrix.perf-test }}/performance_report.md \ + --title "OVN-Kubernetes Performance Test Results - Run ${{ github.run_id }}" \ + --pr-number ${{ github.event_name == 'issue_comment' && github.event.issue.number || github.event_name == 'pull_request' && github.event.pull_request.number || github.event_name == 'workflow_dispatch' && '' }} \ + --github-comment + + echo "Performance report generated successfully" + cat /tmp/${{ matrix.perf-test }}/performance_report.md + + - name: Upload performance report + if: ${{ matrix.perf-test == 'kubelet-density-cni' || matrix.perf-test == 'udn-density-l2-noPods' || matrix.perf-test == 'cudn-density-l2-noPods' || matrix.perf-test == 'udn-density-l2-pods' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.perf-test }}-performance-report-${{ github.run_id }} + path: /tmp/${{ matrix.perf-test }}/performance_report.md + + - name: Upload pprof data + if: ${{ matrix.perf-test == 'kubelet-density-cni' || matrix.perf-test == 'udn-density-l2-noPods' || matrix.perf-test == 'cudn-density-l2-noPods' || matrix.perf-test == 'udn-density-l2-pods' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.perf-test }}-pprof-${{ github.run_id }} + path: /tmp/${{ matrix.perf-test }}/pprof-data + + - name: Upload performance test data + if: ${{ matrix.perf-test == 'kubelet-density-cni' || matrix.perf-test == 'udn-density-l2-noPods' || matrix.perf-test == 'cudn-density-l2-noPods' || matrix.perf-test == 'udn-density-l2-pods' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.perf-test }}-performance-test-data-${{ github.run_id }} + path: /tmp/${{ matrix.perf-test }} + + - name: Export kind logs + if: always() + run: | + mkdir -p /tmp/${{ matrix.perf-test }}/logs + kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/${{ matrix.perf-test }}/logs + + - name: Upload kind logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.perf-test }}-kind-logs-${{ github.run_id }} + path: /tmp/${{ matrix.perf-test }}/logs + + build-pr: + name: Build-PR + runs-on: ubuntu-24.04 + if: | + (github.event_name != 'issue_comment') || + (github.event.issue.pull_request && + contains(github.event.comment.body, '/perf-test kubelet-density-cni')) || + (github.event.issue.pull_request && + contains(github.event.comment.body, '/perf-test udn-density-l2-noPods')) || + (github.event.issue.pull_request && + contains(github.event.comment.body, '/perf-test cudn-density-l2-noPods')) || + (github.event_name == 'workflow_dispatch') + steps: + - name: Restore PR image cache + id: image_cache_pr + uses: actions/cache@v4 + with: + path: | + ${{ env.CI_IMAGE_CACHE }} + key: ${{ github.run_id }}-image-cache-pr + + - name: Check if PR image build is needed + id: is_pr_image_build_needed + continue-on-error: true + run: | + set -x + if [ -f ${CI_IMAGE_CACHE}/${CI_IMAGE_PR_TAR}.gz ]; then + mkdir -p ${CI_DIST_IMAGES_OUTPUT} + cp ${CI_IMAGE_CACHE}/${CI_IMAGE_PR_TAR}.gz ${CI_DIST_IMAGES_OUTPUT}/${CI_IMAGE_PR_TAR}.gz + gunzip ${CI_DIST_IMAGES_OUTPUT}/${CI_IMAGE_PR_TAR}.gz + echo "PR_IMAGE_RESTORED=true" >> "$GITHUB_OUTPUT" + fi + + - name: Get PR info for issue comment + if: github.event_name == 'issue_comment' && steps.is_pr_image_build_needed.outputs.PR_IMAGE_RESTORED != 'true' && success() + id: pr_info + run: | + PR_NUMBER=${{ github.event.issue.number }} + PR_INFO=$(curl -s -H "Authorization: token ${{ github.token }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER") + PR_SHA=$(echo "$PR_INFO" | jq -r .head.sha) + echo "sha=$PR_SHA" >> $GITHUB_OUTPUT + + - name: Check out code into the Go module directory + if: steps.is_pr_image_build_needed.outputs.PR_IMAGE_RESTORED != 'true' && success() + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'issue_comment' && steps.pr_info.outputs.sha || github.sha }} + + - name: Set up Go + if: steps.is_pr_image_build_needed.outputs.PR_IMAGE_RESTORED != 'true' && success() + uses: actions/setup-go@v5 + with: + go-version-file: 'go-controller/go.mod' + cache: false + id: go + + - name: Build + if: steps.is_pr_image_build_needed.outputs.PR_IMAGE_RESTORED != 'true' && success() + run: | + set -x + pushd go-controller + make gofmt + make verify-go-mod-vendor + make + make windows + popd + + - name: Build docker image + if: steps.is_pr_image_build_needed.outputs.PR_IMAGE_RESTORED != 'true' && success() + run: | + pushd dist/images + IMAGE=ovn-daemonset-fedora:pr + make IMAGE=${IMAGE} \ + OVN_REPO=${{ env.OVN_REPO }} \ + OVN_GITREF=${{ env.OVN_GITREF }} \ + fedora-image + mkdir _output + docker save ${IMAGE} > _output/${CI_IMAGE_PR_TAR} + popd + + - name: Cache PR image + if: steps.is_pr_image_build_needed.outputs.PR_IMAGE_RESTORED != 'true' && success() + continue-on-error: true + run: | + set -x + if [ -f ${CI_IMAGE_CACHE}/${CI_IMAGE_PR_TAR} ]; then + rm -f ${CI_IMAGE_CACHE}/${CI_IMAGE_PR_TAR} + fi + if [ -f ${CI_IMAGE_CACHE}/${CI_IMAGE_PR_TAR}.gz ]; then + rm -f ${CI_IMAGE_CACHE}/${CI_IMAGE_PR_TAR}.gz + fi + mkdir -p ${CI_IMAGE_CACHE}/ + cp ${CI_DIST_IMAGES_OUTPUT}/${CI_IMAGE_PR_TAR} ${CI_IMAGE_CACHE}/${CI_IMAGE_PR_TAR} + gzip ${CI_IMAGE_CACHE}/${CI_IMAGE_PR_TAR} + + - uses: actions/upload-artifact@v4 + with: + name: test-image-pr + path: ${{ env.CI_DIST_IMAGES_OUTPUT }}/${{ env.CI_IMAGE_PR_TAR }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b8ce5ce3d7..4955bfffc2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -341,9 +341,6 @@ jobs: echo "GOPATH=$GOPATH" >> $GITHUB_ENV echo "$GOPATH/bin" >> $GITHUB_PATH - - name: Check out code into the Go module directory - from PR branch - uses: actions/checkout@v4 - - name: Free up disk space uses: ./.github/actions/free-disk-space @@ -367,11 +364,13 @@ jobs: export OVN_IMAGE="ovn-daemonset-fedora:dev" make -C test install-kind + - name: Check out code into the Go module directory - from PR branch + uses: actions/checkout@v4 + - name: Export kind logs if: always() run: | - mkdir -p /tmp/kind/logs - kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/kind/logs + ./contrib/export-kind-logs.sh set -x docker ps -a docker exec ovn-control-plane crictl images @@ -417,9 +416,7 @@ jobs: - name: Export kind logs if: always() - run: | - mkdir -p /tmp/kind/logs-kind-pr-branch - kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/kind/logs-kind-pr-branch + run: ./contrib/export-kind-logs.sh /tmp/kind/logs-kind-pr-branch - name: Upload kind logs if: always() @@ -457,6 +454,8 @@ jobs: - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "shard-conformance", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default"} - {"target": "shard-conformance", "ha": "HA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default"} + - {"target": "shard-conformance", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "no-overlay": "true"} + - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "no-overlay": "true"} - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "control-plane", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-disabled"} @@ -483,11 +482,13 @@ jobs: - {"target": "control-plane", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "forwarding": "disable-forwarding"} - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "forwarding": "disable-forwarding"} - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "cni-mode": "unprivileged"} + - {"target": "network-segmentation-dynamic", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "cni-mode": "unprivileged"} - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "bgp", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation", "dns-name-resolver": "enable-dns-name-resolver"} - {"target": "bgp", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation", "dns-name-resolver": "enable-dns-name-resolver"} - {"target": "bgp", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv6", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation"} + - {"target": "bgp-no-overlay-helm", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation", "no-overlay": "true"} + - {"target": "bgp-no-overlay", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation", "no-overlay": "true"} - {"target": "bgp-loose-isolation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation", "advertised-udn-isolation-mode": "loose"} - {"target": "traffic-flow-test-only","ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "traffic-flow-tests": "1-24", "network-segmentation": "enable-network-segmentation"} - {"target": "tools", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "network-segmentation": "enable-network-segmentation"} @@ -496,15 +497,15 @@ jobs: env: JOB_NAME: "${{ matrix.target }}-${{ matrix.ha }}-${{ matrix.gateway-mode }}-${{ matrix.ipfamily }}-${{ matrix.disable-snat-multiple-gws }}-${{ matrix.second-bridge }}-${{ matrix.ic }}" OVN_HYBRID_OVERLAY_ENABLE: ${{ (matrix.target == 'control-plane' || matrix.target == 'control-plane-helm') && (matrix.ipfamily == 'ipv4' || matrix.ipfamily == 'dualstack' ) }} - OVN_MULTICAST_ENABLE: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || matrix.target == 'network-segmentation' || matrix.target == 'bgp' || matrix.target == 'bgp-loose-isolation' }}" - OVN_EMPTY_LB_EVENTS: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || matrix.target == 'bgp' || matrix.target == 'bgp-loose-isolation' }}" + OVN_MULTICAST_ENABLE: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || startsWith(matrix.target, 'network-segmentation') || startsWith(matrix.target, 'bgp') }}" + OVN_EMPTY_LB_EVENTS: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || startsWith(matrix.target, 'bgp') }}" OVN_HA: "${{ matrix.ha == 'HA' }}" OVN_DISABLE_SNAT_MULTIPLE_GWS: "${{ matrix.disable-snat-multiple-gws == 'noSnatGW' }}" - KIND_INSTALL_METALLB: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || matrix.target == 'network-segmentation' }}" + KIND_INSTALL_METALLB: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || startsWith(matrix.target, 'network-segmentation') }}" OVN_GATEWAY_MODE: "${{ matrix.gateway-mode }}" OVN_SECOND_BRIDGE: "${{ matrix.second-bridge == '2br' }}" - ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' || matrix.target == 'kv-live-migration' || matrix.target == 'network-segmentation' || matrix.target == 'tools' || matrix.target == 'multi-homing-helm' || matrix.target == 'traffic-flow-test-only' || matrix.routeadvertisements != '' }}" - ENABLE_NETWORK_SEGMENTATION: "${{ matrix.target == 'network-segmentation' || matrix.network-segmentation == 'enable-network-segmentation' }}" + ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' || matrix.target == 'kv-live-migration' || startsWith(matrix.target, 'network-segmentation') || matrix.target == 'tools' || matrix.target == 'multi-homing-helm' || matrix.target == 'traffic-flow-test-only' || matrix.routeadvertisements != '' }}" + ENABLE_NETWORK_SEGMENTATION: "${{ startsWith(matrix.target, 'network-segmentation') || matrix.network-segmentation == 'enable-network-segmentation' }}" PLATFORM_IPV4_SUPPORT: "${{ matrix.ipfamily == 'IPv4' || matrix.ipfamily == 'dualstack' }}" PLATFORM_IPV6_SUPPORT: "${{ matrix.ipfamily == 'IPv6' || matrix.ipfamily == 'dualstack' }}" KIND_INSTALL_KUBEVIRT: "${{ matrix.target == 'kv-live-migration' }}" @@ -514,19 +515,22 @@ jobs: KIND_NUM_WORKER: "${{ matrix.num-workers }}" KIND_NUM_NODES_PER_ZONE: "${{ matrix.num-nodes-per-zone }}" OVN_DISABLE_FORWARDING: "${{ matrix.forwarding == 'disable-forwarding' }}" - USE_HELM: "${{ matrix.target == 'control-plane-helm' || matrix.target == 'multi-homing-helm' }}" + USE_HELM: "${{ matrix.target == 'control-plane-helm' || matrix.target == 'multi-homing-helm' || matrix.target == 'bgp-no-overlay-helm' }}" OVN_ENABLE_DNSNAMERESOLVER: "${{ matrix.dns-name-resolver == 'enable-dns-name-resolver' }}" OVN_NETWORK_QOS_ENABLE: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' }}" TRAFFIC_FLOW_TESTS: "${{ matrix.traffic-flow-tests }}" ENABLE_ROUTE_ADVERTISEMENTS: "${{ matrix.routeadvertisements != '' }}" + ENABLE_EVPN: "${{ matrix.routeadvertisements != '' }}" ADVERTISE_DEFAULT_NETWORK: "${{ matrix.routeadvertisements == 'advertise-default' }}" - ENABLE_PRE_CONF_UDN_ADDR: "${{ matrix.ic == 'ic-single-node-zones' && (matrix.target == 'network-segmentation' || matrix.network-segmentation == 'enable-network-segmentation') }}" - ENABLE_NETWORK_CONNECT: "${{ matrix.target == 'network-segmentation' }}" + ENABLE_NETWORK_CONNECT: "${{ startsWith(matrix.target, 'network-segmentation') }}" + ENABLE_PRE_CONF_UDN_ADDR: "${{ matrix.ic == 'ic-single-node-zones' && (startsWith(matrix.target, 'network-segmentation') || matrix.network-segmentation == 'enable-network-segmentation') }}" ADVERTISED_UDN_ISOLATION_MODE: "${{ matrix.advertised-udn-isolation-mode }}" # Override PARALLEL=true for Serial tests target to run Serial tests PARALLEL: "${{ matrix.target != 'serial' }}" OVN_UNPRIVILEGED_MODE: "${{ matrix.cni-mode == 'unprivileged' }}" MULTI_POD_SUBNET: true + DYNAMIC_UDN_ALLOCATION: "${{ matrix.target == 'network-segmentation-dynamic' }}" + ENABLE_NO_OVERLAY: "${{ matrix.no-overlay == 'true' }}" steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -654,7 +658,7 @@ jobs: # set 3 hours for control-plane tests as these might take a while # give 10m extra to give ginkgo chance to timeout before github so that we # get its output - timeout-minutes: ${{ matrix.target == 'bgp-loose-isolation' && 190 || matrix.target == 'bgp' && 190 || matrix.target == 'control-plane' && 190 || matrix.target == 'control-plane-helm' && 190 || matrix.target == 'external-gateway' && 190 || 130 }} + timeout-minutes: ${{ startsWith(matrix.target, 'bgp') && 190 || matrix.target == 'control-plane' && 190 || matrix.target == 'control-plane-helm' && 190 || matrix.target == 'external-gateway' && 190 || startsWith(matrix.target, 'network-segmentation') && 190 || 130 }} run: | # used by e2e diagnostics package export OVN_IMAGE="ovn-daemonset-fedora:pr" @@ -676,10 +680,10 @@ jobs: if [ "${{ matrix.ipfamily }}" != "ipv6" ]; then make -C test conformance fi - elif [ "${{ matrix.target }}" == "network-segmentation" ]; then + elif [[ "${{ matrix.target }}" == network-segmentation* ]]; then make -C test control-plane WHAT="Network Segmentation" make -C test control-plane WHAT="ClusterNetworkConnect" - elif [ "${{ matrix.target }}" == "bgp" ] || [ "${{ matrix.target }}" == "bgp-loose-isolation" ]; then + elif [[ "${{ matrix.target }}" == bgp* ]]; then make -C test control-plane elif [ "${{ matrix.target }}" == "serial" ]; then # Run only Serial tests with ginkgo focus @@ -710,8 +714,7 @@ jobs: - name: Export kind logs if: always() run: | - mkdir -p /tmp/kind/logs - kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/kind/logs + ./contrib/export-kind-logs.sh if [ -n "${TRAFFIC_FLOW_TESTS}" ]; then mv -v /tmp/{,kind/logs/}traffic_flow_test_result.json ||: fi @@ -823,9 +826,7 @@ jobs: - name: Export kind logs if: always() - run: | - mkdir -p /tmp/kind/logs - kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/kind/logs + run: ./contrib/export-kind-logs.sh - name: Upload kind logs if: always() diff --git a/.gitignore b/.gitignore index 513e0d3421..bc6961cd32 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ contrib/bin ovn-kubernetes-anp-test-report.yaml **/ginkgo.report + +.gocache diff --git a/ADOPTERS.md b/ADOPTERS.md index 39fe56aa3b..0254f998c3 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -4,6 +4,7 @@ 1. Red Hat, Inc. (Uses OVN-Kubernetes as their default CNI in OpenShift product) 2. NVIDIA (Uses OVN-Kubernetes in their production environments) +3. Internet Initiative Japan Inc. (Uses OVN-Kubernetes in their on-premise Kubernetes platform) ## Projects diff --git a/Dockerfile b/Dockerfile index 4ab32a018e..2a1047d354 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,15 +5,17 @@ # The standard name for this image is ovn-kube # Build RHEL-9 binaries -FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.24-openshift-4.21 AS builder +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.24-openshift-4.22 AS builder WORKDIR /go/src/github.com/openshift/ovn-kubernetes COPY . . RUN cd go-controller; CGO_ENABLED=1 make RUN cd go-controller; CGO_ENABLED=0 make windows +RUN cd openshift; CGO_ENABLED=0 ./hack/build-tests-ext.sh && \ + gzip ./bin/ovn-kubernetes-tests-ext # Build RHEL-8 binaries (for upgrades from 4.12 and earlier) -FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.24-openshift-4.21 AS rhel8 +FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.24-openshift-4.22 AS rhel8 WORKDIR /go/src/github.com/openshift/ovn-kubernetes COPY . . RUN cd go-controller; CGO_ENABLED=1 make @@ -26,7 +28,7 @@ RUN cd go-controller; CGO_ENABLED=1 make # - creating directories required by ovn-kubernetes # - git commit number # - ovnkube.sh script -FROM registry.ci.openshift.org/ocp/4.21:ovn-kubernetes-base +FROM registry.ci.openshift.org/ocp/4.22:ovn-kubernetes-base USER root @@ -58,6 +60,7 @@ COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_o COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/ovnkube-trace /usr/bin/ COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/hybrid-overlay-node /usr/bin/ COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/ovnkube-observ /usr/bin/ +COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/openshift/bin/ovn-kubernetes-tests-ext.gz /usr/bin/ # Copy RHEL-8 and RHEL-9 shim binaries where the CNO's ovnkube-node container startup script can find them RUN mkdir -p /usr/libexec/cni/rhel9 @@ -73,7 +76,7 @@ RUN stat /usr/bin/oc LABEL io.k8s.display-name="ovn kubernetes" \ io.k8s.description="This is a component of OpenShift Container Platform that provides an overlay network using ovn." \ summary="This is a component of OpenShift Container Platform that provides an overlay network using ovn." \ - io.openshift.tags="openshift" \ + io.openshift.tags="openshift,networking" \ maintainer="Tim Rozet " WORKDIR /root diff --git a/Dockerfile.base b/Dockerfile.base index 522251dcf0..10019149cf 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -5,7 +5,7 @@ # The standard name for this image is ovn-kubernetes-base # build base image shared by both OpenShift and MicroShift -FROM registry.ci.openshift.org/ocp/4.21:base-rhel9 +FROM registry.ci.openshift.org/ocp/4.22:base-rhel9 # install selinux-policy first to avoid a race RUN dnf --setopt=retries=2 --setopt=timeout=2 install -y --nodocs \ @@ -13,12 +13,12 @@ RUN dnf --setopt=retries=2 --setopt=timeout=2 install -y --nodocs \ dnf clean all ARG ovsver=3.5 -ARG ovnver=25.03 +ARG ovnver=25.09 # NOTE: Ensure that the versions of OVS and OVN are overriden for OKD in each of the subsequent layers. # Centos and RHEL releases for ovn are built out of sync, so please make sure to bump for OKD with # the corresponding Centos version when updating the OCP version. ARG ovsver_okd=3.5 -ARG ovnver_okd=25.03 +ARG ovnver_okd=25.09 RUN INSTALL_PKGS="iptables nftables" && \ source /etc/os-release && \ diff --git a/Dockerfile.microshift b/Dockerfile.microshift index bb4309ce60..6d2b616c2a 100644 --- a/Dockerfile.microshift +++ b/Dockerfile.microshift @@ -12,7 +12,7 @@ # openvswitch-devel, openvswitch-ipsec, libpcap, iproute etc # ovn-kube-util, hybrid-overlay-node.exe, ovndbchecker and ovnkube-trace -FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.24-openshift-4.21 AS builder +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.24-openshift-4.22 AS builder WORKDIR /go/src/github.com/openshift/ovn-kubernetes COPY . . @@ -20,7 +20,7 @@ COPY . . # build the binaries RUN cd go-controller; CGO_ENABLED=0 make -FROM registry.ci.openshift.org/ocp/4.21:ovn-kubernetes-base +FROM registry.ci.openshift.org/ocp/4.22:ovn-kubernetes-base USER root diff --git a/contrib/export-kind-logs.sh b/contrib/export-kind-logs.sh new file mode 100755 index 0000000000..4db2230ee6 --- /dev/null +++ b/contrib/export-kind-logs.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Export kind cluster logs and collect coredump binaries +# Usage: ./export-kind-logs.sh [logs_dir] +# Default logs_dir: /tmp/kind/logs + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/kind-common.sh" + +# Don't create cluster or delete kubeconfig - we're just exporting logs +KIND_CREATE=false +set_common_default_params + +export_logs "$@" diff --git a/contrib/install-prometheus-infra.sh b/contrib/install-prometheus-infra.sh new file mode 100755 index 0000000000..d4ee32c6cd --- /dev/null +++ b/contrib/install-prometheus-infra.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +# Install Prometheus on infra nodes in kind cluster +# This script installs the kube-prometheus-stack helm chart on infra nodes + +set -euo pipefail + +# Returns the full directory name of the script +DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Check if kubectl is available +command_exists() { + command -v "$@" > /dev/null 2>&1 +} + +if ! command_exists kubectl; then + echo "Error: kubectl is not installed or not in PATH" + exit 1 +fi + +if ! command_exists helm; then + echo "Error: helm is not installed or not in PATH" + exit 1 +fi + +# Set default values +PROMETHEUS_NAMESPACE=${PROMETHEUS_NAMESPACE:-monitoring} +PROMETHEUS_RELEASE_NAME=${PROMETHEUS_RELEASE_NAME:-kube-prometheus-stack} +LOG_FILE=${LOG_FILE:-prometheus-install.log} + +echo "Installing Prometheus on nodes with prometheus-node=true label..." +echo "Namespace: ${PROMETHEUS_NAMESPACE}" +echo "Release name: ${PROMETHEUS_RELEASE_NAME}" +echo "Log file: ${LOG_FILE}" + +# Create log file and redirect all output +exec > >(tee -a "${LOG_FILE}") +exec 2>&1 + +# Wait for API server to be fully ready +echo "Waiting for Kubernetes API server to be ready..." +kubectl wait --for=condition=Ready nodes --all --timeout=300s || true +sleep 5 + +# Create namespace if it doesn't exist +kubectl create namespace "${PROMETHEUS_NAMESPACE}" --dry-run=client -o yaml | kubectl apply --validate=false -f - + +# Add prometheus-community helm repository +echo "Adding prometheus-community helm repository..." +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update + +# Check if there are nodes with prometheus-node=true label +PROMETHEUS_NODES=$(kubectl get nodes -l prometheus-node=true --no-headers 2>/dev/null | wc -l) + +if [ "${PROMETHEUS_NODES}" -gt 0 ]; then + echo "Found ${PROMETHEUS_NODES} prometheus node(s), installing Prometheus on nodes with prometheus-node=true label..." + # Install on prometheus nodes using values file with node selectors + helm upgrade --install "${PROMETHEUS_RELEASE_NAME}" prometheus-community/kube-prometheus-stack \ + --namespace "${PROMETHEUS_NAMESPACE}" \ + --values "${DIR}/prometheus-values.yaml" \ + --wait --timeout=10m +else + echo "No nodes with prometheus-node=true label found, installing Prometheus on any available nodes..." + # Install without node selector if no prometheus nodes are available + helm upgrade --install "${PROMETHEUS_RELEASE_NAME}" prometheus-community/kube-prometheus-stack \ + --namespace "${PROMETHEUS_NAMESPACE}" \ + --set prometheusOperator.tls.enabled=false \ + --set prometheusOperator.admissionWebhooks.enabled=false \ + --set prometheusOperator.admissionWebhooks.patch.enabled=false \ + --wait --timeout=10m +fi + +echo "Waiting for Prometheus pods to be ready..." +kubectl wait --for=condition=ready pod -l "release=${PROMETHEUS_RELEASE_NAME}" -n "${PROMETHEUS_NAMESPACE}" --timeout=300s + +# Mark nodes running prometheus pods as unschedulable +echo "Marking nodes running Prometheus as unschedulable..." +PROM_NODES=$(kubectl get pods -n "${PROMETHEUS_NAMESPACE}" -l "release=${PROMETHEUS_RELEASE_NAME}" -o jsonpath='{.items[*].spec.nodeName}' | tr ' ' '\n' | sort -u) +if [ -n "${PROM_NODES}" ]; then + for node in ${PROM_NODES}; do + echo "Marking node ${node} as unschedulable..." + kubectl cordon "${node}" + done + echo "Marked $(echo "${PROM_NODES}" | wc -w) node(s) running Prometheus as unschedulable" +else + echo "No nodes found running Prometheus pods" +fi + +echo "Prometheus installation completed successfully!" +echo "Access Prometheus at: kubectl port-forward -n ${PROMETHEUS_NAMESPACE} svc/${PROMETHEUS_RELEASE_NAME}-prometheus 9090:9090" +echo "Access Grafana at: kubectl port-forward -n ${PROMETHEUS_NAMESPACE} svc/${PROMETHEUS_RELEASE_NAME}-grafana 3000:80" +echo "Default Grafana credentials: admin / prom-operator" +echo "Installation logs saved to: ${LOG_FILE}" diff --git a/contrib/kind-common b/contrib/kind-common.sh similarity index 66% rename from contrib/kind-common rename to contrib/kind-common.sh index 208f6965c6..12ed53d7c2 100644 --- a/contrib/kind-common +++ b/contrib/kind-common.sh @@ -14,6 +14,9 @@ case $(uname -m) in aarch64) ARCH="arm64" ;; esac +# Directory for coredump collection (used by setup_coredumps and collect_coredump_binaries) +readonly COREDUMP_DIR="/tmp/kind/logs/coredumps" + if_error_exit() { ########################################################################### # Description: # @@ -33,12 +36,205 @@ if_error_exit() { } set_common_default_params() { + # KIND/cluster params + KIND_CREATE=${KIND_CREATE:-true} KIND_IMAGE=${KIND_IMAGE:-kindest/node} + KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ovn} K8S_VERSION=${K8S_VERSION:-v1.34.0} KIND_SETTLE_DURATION=${KIND_SETTLE_DURATION:-30} + KIND_CONFIG=${KIND_CONFIG:-${DIR}/kind.yaml.j2} + KIND_LOCAL_REGISTRY=${KIND_LOCAL_REGISTRY:-false} + KIND_INSTALL_INGRESS=${KIND_INSTALL_INGRESS:-false} + KIND_INSTALL_METALLB=${KIND_INSTALL_METALLB:-false} + KIND_INSTALL_PLUGINS=${KIND_INSTALL_PLUGINS:-false} + KIND_INSTALL_KUBEVIRT=${KIND_INSTALL_KUBEVIRT:-false} + KIND_REMOVE_TAINT=${KIND_REMOVE_TAINT:-true} + OCI_BIN=${KIND_EXPERIMENTAL_PROVIDER:-docker} + # Setup KUBECONFIG patch based on cluster-name + export KUBECONFIG=${KUBECONFIG:-${HOME}/${KIND_CLUSTER_NAME}.conf} + # Scrub any existing kubeconfigs at the path + if [ "${KIND_CREATE}" == true ]; then + rm -f "${KUBECONFIG}" + fi + + # Image/source code params + OVN_IMAGE=${OVN_IMAGE:-local} + OVN_REPO=${OVN_REPO:-""} + OVN_GITREF=${OVN_GITREF:-""} + + # Subnet params + # Input not currently validated. Modify outside script at your own risk. + # These are the same values defaulted to in KIND code (kind/default.go). + # NOTE: KIND NET_CIDR_IPV6 default use a /64 but OVN have a /64 per host + # so it needs to use a larger subnet + # Upstream - NET_CIDR_IPV6=fd00:10:244::/64 SVC_CIDR_IPV6=fd00:10:96::/112 + MASQUERADE_SUBNET_IPV4=${MASQUERADE_SUBNET_IPV4:-169.254.0.0/17} + MASQUERADE_SUBNET_IPV6=${MASQUERADE_SUBNET_IPV6:-fd69::/112} + NET_CIDR_IPV4=${NET_CIDR_IPV4:-10.244.0.0/16} + NET_CIDR_IPV6=${NET_CIDR_IPV6:-fd00:10:244::/48} + MULTI_POD_SUBNET=${MULTI_POD_SUBNET:-false} + if [ "$MULTI_POD_SUBNET" == true ]; then + NET_CIDR_IPV4="10.243.0.0/23/24,10.244.0.0/16" + NET_CIDR_IPV6="fd00:10:243::/63/64,fd00:10:244::/48" + fi + NET_SECOND_CIDR_IPV4=${NET_SECOND_CIDR_IPV4:-172.19.0.0/16} + SVC_CIDR_IPV4=${SVC_CIDR_IPV4:-10.96.0.0/16} + SVC_CIDR_IPV6=${SVC_CIDR_IPV6:-fd00:10:96::/112} + JOIN_SUBNET_IPV4=${JOIN_SUBNET_IPV4:-100.64.0.0/16} + JOIN_SUBNET_IPV6=${JOIN_SUBNET_IPV6:-fd98::/64} + TRANSIT_SUBNET_IPV4=${TRANSIT_SUBNET_IPV4:-100.88.0.0/16} + TRANSIT_SUBNET_IPV6=${TRANSIT_SUBNET_IPV6:-fd97::/64} + METALLB_CLIENT_NET_SUBNET_IPV4=${METALLB_CLIENT_NET_SUBNET_IPV4:-172.22.0.0/16} + METALLB_CLIENT_NET_SUBNET_IPV6=${METALLB_CLIENT_NET_SUBNET_IPV6:-fc00:f853:ccd:e792::/64} + PLATFORM_IPV4_SUPPORT=${PLATFORM_IPV4_SUPPORT:-true} + PLATFORM_IPV6_SUPPORT=${PLATFORM_IPV6_SUPPORT:-false} + + # Feature params + OVN_HYBRID_OVERLAY_ENABLE=${OVN_HYBRID_OVERLAY_ENABLE:-false} + OVN_MULTICAST_ENABLE=${OVN_MULTICAST_ENABLE:-false} + OVN_HA=${OVN_HA:-false} + ADVERTISE_DEFAULT_NETWORK=${ADVERTISE_DEFAULT_NETWORK:-false} + ADVERTISED_UDN_ISOLATION_MODE=${ADVERTISED_UDN_ISOLATION_MODE:-strict} + BGP_SERVER_NET_SUBNET_IPV4=${BGP_SERVER_NET_SUBNET_IPV4:-172.26.0.0/16} + BGP_SERVER_NET_SUBNET_IPV6=${BGP_SERVER_NET_SUBNET_IPV6:-fc00:f853:ccd:e796::/64} + OVN_OBSERV_ENABLE=${OVN_OBSERV_ENABLE:-false} + OVN_EMPTY_LB_EVENTS=${OVN_EMPTY_LB_EVENTS:-false} + OVN_NETWORK_QOS_ENABLE=${OVN_NETWORK_QOS_ENABLE:-false} + OVN_ENABLE_DNSNAMERESOLVER=${OVN_ENABLE_DNSNAMERESOLVER:-false} + ENABLE_COREDUMPS=${ENABLE_COREDUMPS:-false} + METRICS_IP=${METRICS_IP:-""} + OVN_COMPACT_MODE=${OVN_COMPACT_MODE:-false} + if [ "$OVN_COMPACT_MODE" == true ]; then + KIND_NUM_WORKER=0 + fi + + KIND_NUM_MASTER=1 + if [ "$OVN_HA" == true ]; then + KIND_NUM_MASTER=3 + KIND_NUM_WORKER=${KIND_NUM_WORKER:-0} + else + KIND_NUM_WORKER=${KIND_NUM_WORKER:-2} + fi + + OVN_ENABLE_INTERCONNECT=${OVN_ENABLE_INTERCONNECT:-true} + if [ "$OVN_COMPACT_MODE" == true ] && [ "$OVN_ENABLE_INTERCONNECT" != false ]; then + echo "Compact mode cannot be used together with Interconnect" + exit 1 + fi + if [ "$OVN_ENABLE_INTERCONNECT" == true ]; then + KIND_NUM_NODES_PER_ZONE=${KIND_NUM_NODES_PER_ZONE:-1} + + TOTAL_NODES=$((KIND_NUM_WORKER + KIND_NUM_MASTER)) + if [[ ${KIND_NUM_NODES_PER_ZONE} -gt 1 ]] && [[ $((TOTAL_NODES % KIND_NUM_NODES_PER_ZONE)) -ne 0 ]]; then + echo "(Total k8s nodes / number of nodes per zone) should be zero" + exit 1 + fi + else + KIND_NUM_NODES_PER_ZONE=0 + fi + + ENABLE_MULTI_NET=${ENABLE_MULTI_NET:-false} + ENABLE_NETWORK_SEGMENTATION=${ENABLE_NETWORK_SEGMENTATION:-false} + if [ "$ENABLE_NETWORK_SEGMENTATION" == true ] && [ "$ENABLE_MULTI_NET" != true ]; then + echo "Network segmentation (UDN) requires multi-network to be enabled (-mne)" + exit 1 + fi + + ENABLE_NETWORK_CONNECT=${ENABLE_NETWORK_CONNECT:-false} + if [[ $ENABLE_NETWORK_CONNECT == true && $ENABLE_NETWORK_SEGMENTATION != true ]]; then + echo "Network connect requires network-segmentation to be enabled (-nse)" + exit 1 + fi + + DYNAMIC_UDN_ALLOCATION=${DYNAMIC_UDN_ALLOCATION:-false} + if [[ $DYNAMIC_UDN_ALLOCATION == true && $ENABLE_NETWORK_SEGMENTATION != true ]]; then + echo "Dynamic UDN allocation requires network-segmentation to be enabled (-nse)" + exit 1 + fi + DYNAMIC_UDN_GRACE_PERIOD=${DYNAMIC_UDN_GRACE_PERIOD:-120s} + + ENABLE_PRE_CONF_UDN_ADDR=${ENABLE_PRE_CONF_UDN_ADDR:-false} + if [[ $ENABLE_PRE_CONF_UDN_ADDR == true && $ENABLE_NETWORK_SEGMENTATION != true ]]; then + echo "Preconfigured UDN addresses requires network-segmentation to be enabled (-nse)" + exit 1 + fi + if [[ $ENABLE_PRE_CONF_UDN_ADDR == true && $OVN_ENABLE_INTERCONNECT != true ]]; then + echo "Preconfigured UDN addresses requires interconnect to be enabled (-ic)" + exit 1 + fi + + ENABLE_ROUTE_ADVERTISEMENTS=${ENABLE_ROUTE_ADVERTISEMENTS:-false} + if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ] && [ "$ENABLE_MULTI_NET" != true ]; then + echo "Route advertisements requires multi-network to be enabled (-mne)" + exit 1 + fi + if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ] && [ "$OVN_ENABLE_INTERCONNECT" != true ]; then + echo "Route advertisements requires interconnect to be enabled (-ic)" + exit 1 + fi + + ENABLE_EVPN=${ENABLE_EVPN:-false} + if [ "$ENABLE_EVPN" == true ] && [ "$ENABLE_ROUTE_ADVERTISEMENTS" != true ]; then + echo "EVPN requires Route advertisements to be enabled (-rae)" + exit 1 + fi + + ENABLE_NO_OVERLAY=${ENABLE_NO_OVERLAY:-false} + if [ "$ENABLE_NO_OVERLAY" == true ] && [ "$ENABLE_ROUTE_ADVERTISEMENTS" != true ]; then + echo "No-overlay mode requires route advertisement to be enabled (-rae)" + exit 1 + fi + if [ "$ENABLE_NO_OVERLAY" == true ] && [ "$ADVERTISE_DEFAULT_NETWORK" != true ]; then + echo "No-overlay mode requires advertise the default network (-adv)" + exit 1 + fi + + if [ "$ENABLE_NO_OVERLAY" == true ]; then + # Set default MTU for no-overlay mode (1500) if not already set + OVN_MTU=${OVN_MTU:-1500} + else + # Set default MTU for overlay mode (1400) if not already set + OVN_MTU=${OVN_MTU:-1400} + fi +} + +set_ovn_image() { + if [ "${KIND_LOCAL_REGISTRY:-false}" == true ]; then + OVN_IMAGE="localhost:5000/ovn-daemonset-fedora:latest" + else + OVN_IMAGE="localhost/ovn-daemonset-fedora:dev" + fi +} + +build_ovn_image() { + local push_args="" + if [ "$OCI_BIN" == "podman" ]; then + # docker doesn't perform tls check by default only podman does, hence we need to disable it for podman. + push_args="--tls-verify=false" + fi + + if [ "$OVN_IMAGE" == local ]; then + set_ovn_image + + # Build image + make -C ${DIR}/../dist/images IMAGE="${OVN_IMAGE}" OVN_REPO="${OVN_REPO}" OVN_GITREF="${OVN_GITREF}" OCI_BIN="${OCI_BIN}" fedora-image + + # store in local registry + if [ "$KIND_LOCAL_REGISTRY" == true ];then + echo "Pushing built image to local $OCI_BIN registry" + $OCI_BIN push $push_args "$OVN_IMAGE" + fi + # We should push to local registry if image is not remote + elif [[ -n "${OVN_IMAGE}" && "${KIND_LOCAL_REGISTRY}" == true && "${OVN_IMAGE}" != */* ]]; then + local local_registry_ovn_image="localhost:5000/${OVN_IMAGE}" + $OCI_BIN tag "$OVN_IMAGE" $local_registry_ovn_image + OVN_IMAGE=$local_registry_ovn_image + $OCI_BIN push $push_args "$OVN_IMAGE" + fi } run_kubectl() { + kind export kubeconfig --name ${KIND_CLUSTER_NAME} local retries=0 local attempts=10 while true; do @@ -542,6 +738,71 @@ build_dnsnameresolver_images() { build_image /tmp/coredns-ocp-dnsnameresolver/operator ${DNSNAMERESOLVER_OPERATOR} Dockerfile } +check_common_dependencies() { + if ! command_exists curl ; then + echo "Dependency not met: Command not found 'curl'" + exit 1 + fi + + if ! command_exists kubectl ; then + echo "'kubectl' not found, installing" + setup_kubectl_bin + fi + + if ! command_exists kind ; then + echo "Dependency not met: Command not found 'kind'" + exit 1 + fi + + local kind_min="0.27.0" + local kind_cur + kind_cur=$(kind version -q) + if [ "$(echo -e "$kind_min\n$kind_cur" | sort -V | head -1)" != "$kind_min" ]; then + echo "Dependency not met: expected kind version >= $kind_min but have $kind_cur" + exit 1 + fi + + if ! command_exists jq ; then + echo "Dependency not met: Command not found 'jq'" + exit 1 + fi + + if ! command_exists awk ; then + echo "Dependency not met: Command not found 'awk'" + exit 1 + fi + + if ! command_exists jinjanate ; then + if ! command_exists pipx ; then + echo "Dependency not met: 'jinjanator' not installed and cannot install with 'pipx'" + exit 1 + fi + echo "'jinjanate' not found, installing with 'pipx'" + install_jinjanator_renderer + fi + + if ! command_exists docker && ! command_exists podman; then + echo "Dependency not met: Neither docker nor podman found" + exit 1 + fi + + if command_exists podman && ! command_exists skopeo; then + echo "Dependency not met: skopeo not installed. Run the following command to install it: 'sudo dnf install skopeo'" + exit 1 + fi +} + +install_jinjanator_renderer() { + # ensure jinjanator renderer installed + pipx install jinjanator[yaml] + pipx ensurepath --force >/dev/null + export PATH=~/.local/bin:$PATH +} + +install_ovn_image() { + install_image "${OVN_IMAGE}" +} + # install_image accepts the image name along with the tag as an argument and installs it. install_image() { # If local registry is being used push image there for consumption by kind cluster @@ -668,7 +929,7 @@ get_kubevirt_release_url() { echo "$kubevirt_release_url" } -readonly FRR_K8S_VERSION=v0.0.17 +readonly FRR_K8S_VERSION=v0.0.21 readonly FRR_TMP_DIR=$(mktemp -d -u) clone_frr() { @@ -678,7 +939,7 @@ clone_frr() { git clone --depth 1 --branch $FRR_K8S_VERSION https://github.com/metallb/frr-k8s # Download the patches - curl -Ls https://github.com/jcaamano/frr-k8s/archive/refs/heads/ovnk-bgp.tar.gz | tar xzvf - frr-k8s-ovnk-bgp/patches --strip-components 1 + curl -Ls https://github.com/jcaamano/frr-k8s/archive/refs/heads/ovnk-bgp-v0.0.21.tar.gz | tar xzvf - frr-k8s-ovnk-bgp-v0.0.21/patches --strip-components 1 # Change into the cloned repo directory before applying patches pushd frr-k8s @@ -822,13 +1083,13 @@ destroy_bgp() { fi } -install_ffr_k8s() { +install_frr_k8s() { echo "Installing frr-k8s ..." clone_frr # apply frr-k8s kubectl apply -f "${FRR_TMP_DIR}"/frr-k8s/config/all-in-one/frr-k8s.yaml - kubectl wait -n frr-k8s-system deployment frr-k8s-webhook-server --for condition=Available --timeout 2m + kubectl wait -n frr-k8s-system deployment frr-k8s-statuscleaner --for condition=Available --timeout 2m kubectl rollout status -n frr-k8s-system daemonset frr-k8s-daemon --timeout 2m # apply a BGP peer configration with the external gateway that does not @@ -841,8 +1102,8 @@ install_ffr_k8s() { if [ "$PLATFORM_IPV6_SUPPORT" == true ]; then # Find all line numbers where the IPv4 prefix is defined IPv6_LINE=" - prefix: ${BGP_SERVER_NET_SUBNET_IPV6}" - # Process each occurrence of the IPv4 prefix - for LINE_NUM in $(grep -n "prefix: ${BGP_SERVER_NET_SUBNET_IPV4}" receive_filtered.yaml | cut -d ':' -f 1); do + # Process each occurrence of the IPv4 prefix in reverse order to avoid line number shifting + for LINE_NUM in $(grep -n "prefix: ${BGP_SERVER_NET_SUBNET_IPV4}" receive_filtered.yaml | cut -d ':' -f 1 | sort -rn); do # Insert the IPv6 prefix after each IPv4 prefix line sed -i "${LINE_NUM}a\\${IPv6_LINE}" receive_filtered.yaml done @@ -923,18 +1184,18 @@ interconnect_arg_check() { setup_coredumps() { # Setup core dump collection # - # Core dumps will be saved on the HOST at /tmp/kind/logs/coredumps (not inside containers) + # Core dumps will be saved on the HOST at $COREDUMP_DIR (not inside containers) # because kernel.core_pattern is a kernel-level setting shared across all containers. # # - Using a pipe instead of a file path avoids needing to mount - # /tmp/kind/logs/coredumps into every container that might crash - # - The pipe executes in the host's namespace, so /tmp/kind/logs/coredumps + # $COREDUMP_DIR into every container that might crash + # - The pipe executes in the host's namespace, so $COREDUMP_DIR # automatically refers to the host path # - # Location: /tmp/kind/logs is used to ensure coredumps are exported in CI + # Location: COREDUMP_DIR is under /tmp/kind/logs to ensure coredumps are exported in CI # Use container exec to avoid asking for root permissions - mkdir -p "/tmp/kind/logs/coredumps" + mkdir -p "$COREDUMP_DIR" ulimit -c unlimited for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}"); do # Core dump filename pattern variables: @@ -942,6 +1203,257 @@ setup_coredumps() { # %e - executable filename # %h - hostname (container hostname) # %s - signal number that caused dump - ${OCI_BIN} exec "$node" sysctl -w kernel.core_pattern="|/bin/dd of=/tmp/kind/logs/coredumps/core.%P.%e.%h.%s bs=1M status=none" + ${OCI_BIN} exec "$node" sysctl -w kernel.core_pattern="|/bin/dd of=${COREDUMP_DIR}/core.%P.%e.%h.%s bs=1M status=none" + done +} + +wait_for_coredumps() { + # Wait for any in-progress coredump writes to complete + # The kernel pipes coredumps to dd processes, which can take 30+ seconds for large Go binaries + # + # Challenge: Go's crash handling (printing stack traces for all goroutines) takes + # several seconds BEFORE it calls abort() and the kernel starts the coredump. + # So we can't just check for dd processes - we need to wait for potential crashes + # to fully materialize. + + local max_wait=120 # Maximum wait time in seconds + local initial_wait=15 # Initial wait for Go crash handling to complete + local waited=0 + + if [ ! -d "$COREDUMP_DIR" ]; then + return 0 + fi + + # Record initial coredump count + local initial_count + initial_count=$(find "$COREDUMP_DIR" -maxdepth 1 -name "core.*" -type f 2>/dev/null | wc -l || echo 0) + echo "Checking for in-progress coredump writes (initial count: $initial_count)..." + + # Initial wait: Go's crash handling (printing goroutine stack traces) can take + # 10+ seconds before abort() is called and the kernel starts the coredump + echo "Waiting ${initial_wait}s for any pending crash handling to complete..." + sleep "$initial_wait" + waited=$initial_wait + + while [ $waited -lt $max_wait ]; do + # Check for dd processes writing to the coredump directory + local dd_procs + dd_procs=$(pgrep -f "dd of=${COREDUMP_DIR}" 2>/dev/null || true) + + # Check current coredump count + local current_count + current_count=$(find "$COREDUMP_DIR" -maxdepth 1 -name "core.*" -type f 2>/dev/null | wc -l || echo 0) + + if [ -z "$dd_procs" ]; then + # No dd processes running + if [ "$current_count" -gt "$initial_count" ]; then + echo "New coredumps detected (initial: $initial_count, current: $current_count) after ${waited}s" + fi + echo "No coredump writes in progress after ${waited}s" + return 0 + fi + + echo "Waiting for coredump writes... (${waited}s, dd PIDs: $dd_procs, coredumps: $current_count)" + sleep 5 + waited=$((waited + 5)) + done + + echo "Warning: Timed out waiting for coredump writes after ${max_wait}s" +} + +export_logs() { + # Export kind logs and collect coredump binaries + # Usage: export_logs [logs_dir] + # Default logs_dir: /tmp/kind/logs + + local logs_dir="${1:-/tmp/kind/logs}" + + mkdir -p "$logs_dir" + + # Wait for any in-progress coredump writes to complete before exporting + wait_for_coredumps + + kind export logs --name "${KIND_CLUSTER_NAME}" --verbosity 4 "$logs_dir" + collect_coredump_binaries +} + +# Helper function to try extracting a binary from a container +# Used by collect_coredump_binaries() +try_extract_binary() { + local node=$1 + local container_id=$2 + local exe=$3 + local binary_dir=$4 + + # Get container's PID to access its rootfs via /proc//root + local pid + pid=$(${OCI_BIN} exec "$node" crictl inspect "$container_id" 2>/dev/null | jq -r '.info.pid // empty') + if [ -z "$pid" ] || [ "$pid" = "null" ] || [ "$pid" = "0" ]; then + return 1 + fi + + # Common paths where binaries might be located + local binary_paths=("/usr/bin" "/bin" "/usr/sbin" "/sbin" "/usr/libexec/cni" "/usr/lib/frr") + + for path in "${binary_paths[@]}"; do + local full_path="/proc/${pid}/root${path}/${exe}" + if ${OCI_BIN} exec "$node" test -f "$full_path" 2>/dev/null; then + if ${OCI_BIN} exec "$node" cat "$full_path" > "${binary_dir}/${exe}" 2>/dev/null && [ -s "${binary_dir}/${exe}" ]; then + echo " Collected binary: ${exe} from container $container_id (pid $pid)" + return 0 + fi + fi + done + rm -f "${binary_dir}/${exe}" 2>/dev/null + return 1 +} + +collect_coredump_binaries() { + # Collect binaries that caused coredumps for post-mortem debugging + # Parses coredump filenames (core.%P.%e.%h.%s) to identify executables + # Binaries run inside pod containers, so we use crictl to access them + + local binary_dir="${COREDUMP_DIR}/binaries" + + if [ ! -d "$COREDUMP_DIR" ]; then + echo "No coredump directory found, skipping binary collection" + return 0 + fi + + local coredumps + coredumps=$(find "$COREDUMP_DIR" -maxdepth 1 -name "core.*" -type f 2>/dev/null) + if [ -z "$coredumps" ]; then + echo "No coredumps found, skipping binary collection" + return 0 + fi + + mkdir -p "$binary_dir" + + # Get all KIND nodes + local nodes + nodes=$(kind get nodes --name "${KIND_CLUSTER_NAME}" 2>/dev/null) + if [ -z "$nodes" ]; then + echo "Warning: No KIND nodes available, cannot collect binaries" + return 0 + fi + + # Process each coredump: extract exe name (%e, field 3) + # Filename format: core.%P.%e.%h.%s (see setup_coredumps) + for coredump in $coredumps; do + local filename + filename=$(basename "$coredump") + local exe + exe=$(echo "$filename" | cut -d. -f3) + + echo "Processing coredump: $filename (exe=$exe)" + + # Skip if we already collected this binary + if [ -f "${binary_dir}/${exe}" ]; then + echo " Binary $exe already collected, skipping" + continue + fi + + local found=false + + # Search all containers on all nodes for the binary + for node in $nodes; do + local containers + containers=$(${OCI_BIN} exec "$node" crictl ps -q 2>/dev/null) || true + for container_id in $containers; do + if try_extract_binary "$node" "$container_id" "$exe" "$binary_dir"; then + echo " Collected $exe from container $container_id on node $node" + found=true + break 2 + fi + done + done + + # Fallback: binary running directly on KIND node (not in container) + if [ "$found" = false ]; then + for node in $nodes; do + local bin_path + bin_path=$(${OCI_BIN} exec "$node" which "$exe" 2>/dev/null) || true + if [ -n "$bin_path" ]; then + echo " Collected $exe from node $node at $bin_path" + ${OCI_BIN} cp "${node}:${bin_path}" "${binary_dir}/${exe}" && found=true || true + break + fi + done + fi + + if [ "$found" = false ]; then + echo " WARNING: Could not find binary '$exe'" + fi + done + + echo "Binary collection complete:" + ls -la "$binary_dir" 2>/dev/null || true +} + +# Some environments (Fedora32,31 on desktop), have problems when the cluster +# is deleted directly with kind `kind delete cluster --name ovn`, it restarts the host. +# The root cause is unknown, this also can not be reproduced in Ubuntu 20.04 or +# with Fedora32 Cloud, but it does not happen if we clean first the ovn-kubernetes resources. +delete() { + OCI_BIN=${KIND_EXPERIMENTAL_PROVIDER:-docker} + + if [ "$KIND_INSTALL_METALLB" == true ]; then + destroy_metallb + fi + if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ]; then + destroy_bgp + fi + timeout 5 kubectl --kubeconfig "${KUBECONFIG}" delete namespace ovn-kubernetes || true + sleep 5 + kind delete cluster --name "${KIND_CLUSTER_NAME:-ovn}" +} + +create_kind_cluster() { + # Output of the jinjanate command + KIND_CONFIG_LCL=${DIR}/kind-${KIND_CLUSTER_NAME}.yaml + + ovn_ip_family=${IP_FAMILY} \ + ovn_ha=${OVN_HA} \ + net_cidr="${KIND_CIDR}" \ + svc_cidr=${SVC_CIDR} \ + use_local_registry=${KIND_LOCAL_REGISTRY} \ + dns_domain=${KIND_DNS_DOMAIN} \ + ovn_num_master=${KIND_NUM_MASTER} \ + ovn_num_worker=${KIND_NUM_WORKER} \ + kind_num_infra=${KIND_NUM_INFRA} \ + cluster_log_level=${KIND_CLUSTER_LOGLEVEL:-4} \ + kind_local_registry_port=${KIND_LOCAL_REGISTRY_PORT} \ + kind_local_registry_name=${KIND_LOCAL_REGISTRY_NAME} \ + jinjanate "${KIND_CONFIG}" -o "${KIND_CONFIG_LCL}" + + # Create KIND cluster. For additional debug, add '--verbosity ': 0 None .. 3 Debug + if kind get clusters | grep "${KIND_CLUSTER_NAME}"; then + delete + fi + + if [[ "${KIND_LOCAL_REGISTRY}" == true ]]; then + create_local_registry + fi + + kind create cluster --name "${KIND_CLUSTER_NAME}" --kubeconfig "${KUBECONFIG}" --image "${KIND_IMAGE}":"${K8S_VERSION}" --config=${KIND_CONFIG_LCL} --retain + + cat "${KUBECONFIG}" +} + +remove_no_schedule_taint() { + KIND_NODES=$(kind_get_nodes | sort) + for n in $KIND_NODES; do + # do not error if it fails to remove the taint + kubectl taint node "$n" node-role.kubernetes.io/control-plane:NoSchedule- || true + done +} + +label_ovn_ha() { + MASTER_NODES=$(kind get nodes --name "${KIND_CLUSTER_NAME}" | sort | head -n "${KIND_NUM_MASTER}") + # We want OVN HA not Kubernetes HA + # leverage the kubeadm well-known label node-role.kubernetes.io/control-plane= + # to choose the nodes where ovn master components will be placed + for n in $MASTER_NODES; do + kubectl label node "$n" k8s.ovn.org/ovnkube-db=true node-role.kubernetes.io/control-plane="" --overwrite done } diff --git a/contrib/kind-helm.sh b/contrib/kind-helm.sh index a6dc8d9c9c..85764d6d91 100755 --- a/contrib/kind-helm.sh +++ b/contrib/kind-helm.sh @@ -5,91 +5,21 @@ set -eo pipefail # Returns the full directory name of the script export DIR="$( cd -- "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export OCI_BIN=${KIND_EXPERIMENTAL_PROVIDER:-docker} - -# Source the kind-common file from the same directory where this script is located -source "${DIR}/kind-common" +# Source the kind-common.sh file from the same directory where this script is located +source "${DIR}/kind-common.sh" set_default_params() { set_common_default_params - # Set default values - export KIND_CONFIG=${KIND_CONFIG:-} - export KIND_INSTALL_INGRESS=${KIND_INSTALL_INGRESS:-false} - export KIND_INSTALL_METALLB=${KIND_INSTALL_METALLB:-false} - export KIND_INSTALL_PLUGINS=${KIND_INSTALL_PLUGINS:-false} - export KIND_INSTALL_KUBEVIRT=${KIND_INSTALL_KUBEVIRT:-false} - export OVN_HA=${OVN_HA:-false} - export OVN_MULTICAST_ENABLE=${OVN_MULTICAST_ENABLE:-false} - export OVN_HYBRID_OVERLAY_ENABLE=${OVN_HYBRID_OVERLAY_ENABLE:-false} - export OVN_OBSERV_ENABLE=${OVN_OBSERV_ENABLE:-false} - export OVN_EMPTY_LB_EVENTS=${OVN_EMPTY_LB_EVENTS:-false} - export KIND_REMOVE_TAINT=${KIND_REMOVE_TAINT:-true} - export ENABLE_MULTI_NET=${ENABLE_MULTI_NET:-false} - export ENABLE_NETWORK_SEGMENTATION=${ENABLE_NETWORK_SEGMENTATION:-false} - export ENABLE_NETWORK_CONNECT=${ENABLE_NETWORK_CONNECT:-false} - export ENABLE_PRE_CONF_UDN_ADDR=${ENABLE_PRE_CONF_UDN_ADDR:-false} - export OVN_NETWORK_QOS_ENABLE=${OVN_NETWORK_QOS_ENABLE:-false} - export KIND_NUM_WORKER=${KIND_NUM_WORKER:-2} - export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ovn} - export OVN_IMAGE=${OVN_IMAGE:-'ghcr.io/ovn-kubernetes/ovn-kubernetes/ovn-kube-ubuntu:helm'} - - # Setup KUBECONFIG patch based on cluster-name - export KUBECONFIG=${KUBECONFIG:-${HOME}/${KIND_CLUSTER_NAME}.conf} - - # Validated params that work - export MASQUERADE_SUBNET_IPV4=${MASQUERADE_SUBNET_IPV4:-169.254.0.0/17} - export MASQUERADE_SUBNET_IPV6=${MASQUERADE_SUBNET_IPV6:-fd69::/112} - - # Input not currently validated. Modify outside script at your own risk. - # These are the same values defaulted to in KIND code (kind/default.go). - # NOTE: KIND NET_CIDR_IPV6 default use a /64 but OVN have a /64 per host - # so it needs to use a larger subnet - # Upstream - NET_CIDR_IPV6=fd00:10:244::/64 SVC_CIDR_IPV6=fd00:10:96::/112 - export NET_CIDR_IPV4=${NET_CIDR_IPV4:-10.244.0.0/16} - if [ "$MULTI_POD_SUBNET" == true ]; then - NET_CIDR_IPV4="10.243.0.0/23/24,10.244.0.0/16" - fi - export NET_SECOND_CIDR_IPV4=${NET_SECOND_CIDR_IPV4:-172.19.0.0/16} - export SVC_CIDR_IPV4=${SVC_CIDR_IPV4:-10.96.0.0/16} - export NET_CIDR_IPV6=${NET_CIDR_IPV6:-fd00:10:244::/48} - export SVC_CIDR_IPV6=${SVC_CIDR_IPV6:-fd00:10:96::/112} - export JOIN_SUBNET_IPV4=${JOIN_SUBNET_IPV4:-100.64.0.0/16} - export JOIN_SUBNET_IPV6=${JOIN_SUBNET_IPV6:-fd98::/64} - export TRANSIT_SUBNET_IPV4=${TRANSIT_SUBNET_IPV4:-100.88.0.0/16} - export TRANSIT_SUBNET_IPV6=${TRANSIT_SUBNET_IPV6:-fd97::/64} - export METALLB_CLIENT_NET_SUBNET_IPV4=${METALLB_CLIENT_NET_SUBNET_IPV4:-172.22.0.0/16} - export METALLB_CLIENT_NET_SUBNET_IPV6=${METALLB_CLIENT_NET_SUBNET_IPV6:-fc00:f853:ccd:e792::/64} - - export KIND_NUM_MASTER=1 - if [ "$OVN_HA" == true ]; then - KIND_NUM_MASTER=3 + # Hard code ipv4 support until IPv6 is implemented + if [ "$PLATFORM_IPV6_SUPPORT" == true ]; then + echo "kind-helm.sh does not support IPv6 yet" + exit 1 fi - - OVN_ENABLE_INTERCONNECT=${OVN_ENABLE_INTERCONNECT:-true} - if [ "$OVN_COMPACT_MODE" == true ] && [ "$OVN_ENABLE_INTERCONNECT" != false ]; then - echo "Compact mode cannot be used together with Interconnect" - exit 1 + if [ "$PLATFORM_IPV4_SUPPORT" != true ]; then + echo "kind-helm.sh only supports IPv4, must set PLATFORM_IPV4_SUPPORT to true " + exit 1 fi - - - if [ "$OVN_ENABLE_INTERCONNECT" == true ]; then - KIND_NUM_NODES_PER_ZONE=${KIND_NUM_NODES_PER_ZONE:-1} - TOTAL_NODES=$((KIND_NUM_WORKER + KIND_NUM_MASTER)) - if [[ ${KIND_NUM_NODES_PER_ZONE} -gt 1 ]] && [[ $((TOTAL_NODES % KIND_NUM_NODES_PER_ZONE)) -ne 0 ]]; then - echo "(Total k8s nodes / number of nodes per zone) should be zero" - exit 1 - fi - else - KIND_NUM_NODES_PER_ZONE=0 - fi - - # Hard code ipv4 support until IPv6 is implemented - export PLATFORM_IPV4_SUPPORT=true - - export OVN_ENABLE_DNSNAMERESOLVER=${OVN_ENABLE_DNSNAMERESOLVER:-false} - export MULTI_POD_SUBNET=${MULTI_POD_SUBNET:-false} - export ENABLE_COREDUMPS=${ENABLE_COREDUMPS:-false} } usage() { @@ -108,11 +38,23 @@ usage() { echo " [ -nse | --network-segmentation-enable ]" echo " [ -nce | --network-connect-enable ]" echo " [ -uae | --preconfigured-udn-addresses-enable ]" + echo " [ -rae | --route-advertisements-enable ]" + echo " [ -evpn | --evpn-enable ]" + echo " [-dudn | --dynamic-udn-allocation]" + echo " [-dug | --dynamic-udn-removal-grace-period]" + echo " [-adv | --advertise-default-network]" + echo " [-rud | --routed-udn-isolation-disable]" echo " [ -nqe | --network-qos-enable ]" + echo " [ -noe | --no-overlay-enable ]" echo " [ -wk | --num-workers ]" echo " [ -ic | --enable-interconnect]" echo " [ -npz | --node-per-zone ]" + echo " [ -ov | --ovn-image ]" + echo " [ -ovr | --ovn-repo ]" + echo " [ -ovg | --ovn-gitref ]" echo " [ -cn | --cluster-name ]" + echo " [ -mip | --metrics-ip ]" + echo " [ -mtu ]" echo " [ --enable-coredumps ]" echo " [ -h ]" echo "" @@ -133,13 +75,25 @@ usage() { echo "-nse | --network-segmentation-enable Enable network segmentation. DEFAULT: Disabled" echo "-nce | --network-connect-enable Enable network connect (requires network segmentation). DEFAULT: Disabled" echo "-uae | --preconfigured-udn-addresses-enable Enable connecting workloads with preconfigured network to user-defined networks. DEFAULT: Disabled" + echo "-rae | --route-advertisements-enable Enable route advertisements" + echo "-evpn | --evpn-enable Enable EVPN" + echo "-dudn | --dynamic-udn-allocation Enable dynamic UDN allocation. DEFAULT: Disabled" + echo "-dug | --dynamic-udn-removal-grace-period Configure the grace period in seconds for dynamic UDN removal. DEFAULT: 120 seconds" + echo "-adv | --advertise-default-network Applies a RouteAdvertisements configuration to advertise the default network on all nodes" + echo "-rud | --routed-udn-isolation-disable Disable isolation across BGP-advertised UDNs (sets advertised-udn-isolation-mode=loose). DEFAULT: strict." echo "-nqe | --network-qos-enable Enable network QoS. DEFAULT: Disabled" + echo "-noe | --no-overlay-enable Enable no-overlay mode for the default network. DEFAULT: Disabled" echo "-ha | --ha-enabled Enable high availability. DEFAULT: HA Disabled" echo "-wk | --num-workers Number of worker nodes. DEFAULT: 2 workers" + echo "-ov | --ovn-image Use the specified docker image instead of building locally. DEFAULT: local build." + echo "-ovr | --ovn-repo Specify the repository to build OVN from" + echo "-ovg | --ovn-gitref Specify the branch, tag or commit id to build OVN from, it can be a pattern like 'branch-*' it will order results and use the first one" echo "-cn | --cluster-name Configure the kind cluster's name" + echo "-mip | --metrics-ip IP address to bind metrics endpoints. DEFAULT: K8S_NODE_IP or 0.0.0.0" + echo "-mtu Define the overlay mtu. DEFAULT: 1400 (1500 for no-overlay mode)" echo "--enable-coredumps Enable coredump collection on kind nodes. DEFAULT: Disabled" echo "-dns | --enable-dnsnameresolver Enable DNSNameResolver for resolving the DNS names used in the DNS rules of EgressFirewall." - echo "-ce | --enable-central Deploy with OVN Central (Legacy Architecture)" + echo "-ce | --enable-central [DEPRECATED] Deploy with OVN Central (Legacy Architecture)" echo "-npz | --nodes-per-zone Specify number of nodes per zone (Default 0, which means global zone; >0 means interconnect zone, where 1 for single-node zone, >1 for multi-node zone). If this value > 1, then (total k8s nodes (workers + 1) / num of nodes per zone) should be zero." echo "-mps | --multi-pod-subnet Use multiple subnets for the default cluster network" echo "" @@ -186,8 +140,31 @@ parse_args() { ;; -uae | --preconfigured-udn-addresses-enable) ENABLE_PRE_CONF_UDN_ADDR=true ;; + -rae | --route-advertisements-enable) ENABLE_ROUTE_ADVERTISEMENTS=true + ;; + -evpn | --evpn-enable) ENABLE_EVPN=true + ;; + -adv | --advertise-default-network) ADVERTISE_DEFAULT_NETWORK=true + ;; + -rud | --routed-udn-isolation-disable) ADVERTISED_UDN_ISOLATION_MODE=loose + ;; + -dudn | --dynamic-udn-allocation) DYNAMIC_UDN_ALLOCATION=true + ;; + -dug | --dynamic-udn-removal-grace-period) shift + if [[ -z "${1:-}" || "${1:-}" == -* ]]; then + echo "Missing value for --dynamic-udn-removal-grace-period" >&2 + usage + exit 1 + fi + DYNAMIC_UDN_GRACE_PERIOD=$1 + if [[ "$DYNAMIC_UDN_GRACE_PERIOD" =~ ^[0-9]+$ ]]; then + DYNAMIC_UDN_GRACE_PERIOD="${DYNAMIC_UDN_GRACE_PERIOD}s" + fi + ;; -nqe | --network-qos-enable ) OVN_NETWORK_QOS_ENABLE=true ;; + -noe | --no-overlay-enable ) ENABLE_NO_OVERLAY=true + ;; -ha | --ha-enabled ) OVN_HA=true KIND_NUM_MASTER=3 ;; @@ -199,6 +176,15 @@ parse_args() { fi KIND_NUM_WORKER=$1 ;; + -ov | --ovn-image ) shift + OVN_IMAGE=$1 + ;; + -ovr | --ovn-repo ) shift + OVN_REPO=$1 + ;; + -ovg | --ovn-gitref ) shift + OVN_GITREF=$1 + ;; -cn | --cluster-name ) shift KIND_CLUSTER_NAME=$1 # Setup KUBECONFIG @@ -206,7 +192,8 @@ parse_args() { ;; -dns | --enable-dnsnameresolver ) OVN_ENABLE_DNSNAMERESOLVER=true ;; - -ce | --enable-central ) OVN_ENABLE_INTERCONNECT=false + -ce | --enable-central ) echo "WARNING: --enable-central is deprecated. OVN Central (Legacy Architecture) will be removed in a future release." >&2 + OVN_ENABLE_INTERCONNECT=false CENTRAL_ARG_PROVIDED=true ;; -ic | --enable-interconnect ) OVN_ENABLE_INTERCONNECT=true @@ -222,6 +209,12 @@ parse_args() { ;; -mps| --multi-pod-subnet ) MULTI_POD_SUBNET=true ;; + -mip | --metrics-ip ) shift + METRICS_IP="$1" + ;; + -mtu ) shift + OVN_MTU=$1 + ;; --enable-coredumps ) ENABLE_COREDUMPS=true ;; * ) usage @@ -241,6 +234,7 @@ print_params() { echo "" echo "KIND_CONFIG_FILE = $KIND_CONFIG" echo "KUBECONFIG = $KUBECONFIG" + echo "OCI_BIN = $OCI_BIN" echo "KIND_INSTALL_INGRESS = $KIND_INSTALL_INGRESS" echo "KIND_INSTALL_METALLB = $KIND_INSTALL_METALLB" echo "KIND_INSTALL_PLUGINS = $KIND_INSTALL_PLUGINS" @@ -256,13 +250,23 @@ print_params() { echo "ENABLE_NETWORK_SEGMENTATION = $ENABLE_NETWORK_SEGMENTATION" echo "ENABLE_NETWORK_CONNECT = $ENABLE_NETWORK_CONNECT" echo "ENABLE_PRE_CONF_UDN_ADDR = $ENABLE_PRE_CONF_UDN_ADDR" + echo "ENABLE_ROUTE_ADVERTISEMENTS = $ENABLE_ROUTE_ADVERTISEMENTS" + echo "ENABLE_EVPN = $ENABLE_EVPN" + echo "ADVERTISE_DEFAULT_NETWORK = $ADVERTISE_DEFAULT_NETWORK" + echo "ADVERTISED_UDN_ISOLATION_MODE = $ADVERTISED_UDN_ISOLATION_MODE" echo "OVN_NETWORK_QOS_ENABLE = $OVN_NETWORK_QOS_ENABLE" + echo "ENABLE_NO_OVERLAY = $ENABLE_NO_OVERLAY" + echo "OVN_MTU = $OVN_MTU" echo "OVN_IMAGE = $OVN_IMAGE" + echo "OVN_REPO = $OVN_REPO" + echo "OVN_GITREF = $OVN_GITREF" echo "KIND_NUM_MASTER = $KIND_NUM_MASTER" echo "KIND_NUM_WORKER = $KIND_NUM_WORKER" echo "OVN_ENABLE_DNSNAMERESOLVER= $OVN_ENABLE_DNSNAMERESOLVER" echo "MULTI_POD_SUBNET= $MULTI_POD_SUBNET" echo "OVN_ENABLE_INTERCONNECT = $OVN_ENABLE_INTERCONNECT" + echo "DYNAMIC_UDN_ALLOCATION = $DYNAMIC_UDN_ALLOCATION" + echo "DYNAMIC_UDN_GRACE_PERIOD = $DYNAMIC_UDN_GRACE_PERIOD" if [[ $OVN_ENABLE_INTERCONNECT == true ]]; then echo "KIND_NUM_NODES_PER_ZONE = $KIND_NUM_NODES_PER_ZONE" if [ "${KIND_NUM_NODES_PER_ZONE}" -gt 1 ] && [ "${OVN_ENABLE_OVNKUBE_IDENTITY}" = "true" ]; then @@ -274,23 +278,11 @@ print_params() { } check_dependencies() { - if ! command_exists kubectl ; then - echo "'kubectl' not found, installing" - setup_kubectl_bin - fi - - for cmd in "$OCI_BIN" kind helm go ; do \ - if ! command_exists "$cmd" ; then - echo "Dependency not met: $cmd" - exit 1 - fi - done - - # check for currently unsupported features - if [ "${PLATFORM_IPV6_SUPPORT:-}" = "true" ]; then - echo "Fatal: PLATFORM_IPV6_SUPPORT support not implemented yet" - exit 1 - fi + check_common_dependencies + if ! command_exists helm ; then + echo "'helm' not found, exiting" + exit 1 + fi } helm_prereqs() { @@ -300,103 +292,6 @@ helm_prereqs() { sudo sysctl fs.inotify.max_user_instances=512 } -build_ovn_image() { - if [ "${SKIP_OVN_IMAGE_REBUILD}" == "true" ]; then - echo "Explicitly instructed not to rebuild ovn image: ${OVN_IMAGE}" - return - fi - - # Build ovn kube image - pushd ${DIR}/../dist/images - make fedora-image - popd -} - -get_image() { - local image_and_tag="${1:-$OVN_IMAGE}" # Use $1 if provided, otherwise use $OVN_IMAGE - local image="${image_and_tag%%:*}" # Extract everything before the first colon - echo "$image" -} - -get_tag() { - local image_and_tag="${1:-$OVN_IMAGE}" # Use $1 if provided, otherwise use $OVN_IMAGE - local tag="${image_and_tag##*:}" # Extract everything after the last colon - echo "$tag" -} - -create_kind_cluster() { - [ -n "${KIND_CONFIG}" ] || { - KIND_CONFIG='/tmp/kind.yaml' - - # Start of the kind configuration - cat < /tmp/kind.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - kubeadmConfigPatches: - - | - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" - authorization-mode: "AlwaysAllow" -EOT - } - - # Add control-plane nodes based on OVN_HA status. If there are 2 or more worker nodes, use - # 2 of them them to host databases instead of creating additional control plane nodes. - if [ "$OVN_HA" == true ] && [ "$KIND_NUM_WORKER" -lt 2 ]; then - for i in {2..3}; do # Have 3 control-plane nodes for HA - echo "- role: control-plane" >> /tmp/kind.yaml - done - fi - - # Add worker nodes based on KIND_NUM_WORKER - for i in $(seq 1 $KIND_NUM_WORKER); do - echo "- role: worker" >> /tmp/kind.yaml - done - # kind only allows single subnet for pod network, while ovn-kubernetes supports multiple subnets. - # So we pick the first subnet from the provided list for kind configuration and store it in KIND_CIDR. - # remove host subnet mask info for kind configuration (when the subnet is set as 10.0.0.0/16/14) - KIND_CIDR_IPV4=$(echo "${NET_CIDR_IPV4}"| cut -d',' -f1 | cut -d'/' -f1,2 ) - - # Add networking configuration - cat <> /tmp/kind.yaml -networking: - disableDefaultCNI: true - kubeProxyMode: none - podSubnet: $KIND_CIDR_IPV4 - serviceSubnet: $SVC_CIDR_IPV4 -EOT - - kind delete clusters $KIND_CLUSTER_NAME ||: - kind create cluster --name $KIND_CLUSTER_NAME --image "${KIND_IMAGE}":"${K8S_VERSION}" --config "${KIND_CONFIG}" --retain - kind load docker-image --name $KIND_CLUSTER_NAME $OVN_IMAGE - - # When using HA, label nodes to host db. - if [ "$OVN_HA" == true ]; then - kubectl label nodes k8s.ovn.org/ovnkube-db=true --overwrite \ - -l node-role.kubernetes.io/control-plane - if [ "$KIND_NUM_WORKER" -ge 2 ]; then - for n in ovn-worker ovn-worker2; do - # We want OVN HA not Kubernetes HA - # leverage the kubeadm well-known label node-role.kubernetes.io/control-plane= - # to choose the nodes where ovn master components will be placed - kubectl label node "$n" k8s.ovn.org/ovnkube-db=true node-role.kubernetes.io/control-plane="" --overwrite - done - fi - fi - - # Remove taint, so control-plane nodes can also schedule regular pods - if [ "$KIND_REMOVE_TAINT" == true ]; then - kubectl taint node "$n" node-role.kubernetes.io/master:NoSchedule- \ - -l node-role.kubernetes.io/control-plane ||: - kubectl taint node "$n" node-role.kubernetes.io/control-plane:NoSchedule- \ - -l node-role.kubernetes.io/control-plane ||: - fi -} - label_ovn_single_node_zones() { KIND_NODES=$(kind_get_nodes) for n in $KIND_NODES; do @@ -427,7 +322,6 @@ label_ovn_multiple_nodes_zones() { create_ovn_kubernetes() { cd ${DIR}/../helm/ovn-kubernetes - MASTER_REPLICAS=$(kubectl get node -l node-role.kubernetes.io/control-plane --no-headers | wc -l) if [[ $KIND_NUM_NODES_PER_ZONE == 1 ]]; then label_ovn_single_node_zones value_file="values-single-node-zone.yaml" @@ -437,10 +331,12 @@ create_ovn_kubernetes() { value_file="values-multi-node-zone.yaml" ovnkube_db_options="" else + label_ovn_ha value_file="values-no-ic.yaml" ovnkube_db_options="--set tags.ovnkube-db-raft=$(if [ "${OVN_HA}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set tags.ovnkube-db=$(if [ "${OVN_HA}" == "false" ]; then echo "true"; else echo "false"; fi)" fi + MASTER_REPLICAS=$(kubectl get node -l node-role.kubernetes.io/control-plane --no-headers | wc -l) echo "value_file=${value_file}" # For multi-pod-subnet case, NET_CIDR_IPV4 is a list of CIDRs separated by comma. # When Helm encounters a comma within a string value in a --set argument, it attempts to parse the comma as a separator @@ -452,20 +348,28 @@ helm install ovn-kubernetes . -f "${value_file}" \ --set k8sAPIServer=${API_URL} \ --set podNetwork="${ESCAPED_NET_CIDR_IPV4}" \ --set serviceNetwork=${SVC_CIDR_IPV4} \ + --set mtu=${OVN_MTU} \ --set ovnkube-master.replicas=${MASTER_REPLICAS} \ - --set global.image.repository=$(get_image) \ - --set global.image.tag=$(get_tag) \ + --set global.image.repository=${OVN_IMAGE%%:*} \ + --set global.image.tag=${OVN_IMAGE##*:} \ --set global.enableAdminNetworkPolicy=true \ --set global.enableMulticast=$(if [ "${OVN_MULTICAST_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.enableMultiNetwork=$(if [ "${ENABLE_MULTI_NET}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.enableNetworkSegmentation=$(if [ "${ENABLE_NETWORK_SEGMENTATION}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.enableNetworkConnect=$(if [ "${ENABLE_NETWORK_CONNECT}" == "true" ]; then echo "true"; else echo "false"; fi) \ + --set global.enableDynamicUDNAllocation=$(if [ "${DYNAMIC_UDN_ALLOCATION}" == "true" ]; then echo "true"; else echo "false"; fi) \ + $( [ -n "$DYNAMIC_UDN_GRACE_PERIOD" ] && echo "--set global.dynamicUDNGracePeriod=$DYNAMIC_UDN_GRACE_PERIOD" ) \ --set global.enablePreconfiguredUDNAddresses=$(if [ "${ENABLE_PRE_CONF_UDN_ADDR}" == "true" ]; then echo "true"; else echo "false"; fi) \ + --set global.enableRouteAdvertisements=$(if [ "${ENABLE_ROUTE_ADVERTISEMENTS}" == "true" ]; then echo "true"; else echo "false"; fi) \ + --set global.enableEVPN=$(if [ "${ENABLE_EVPN}" == "true" ]; then echo "true"; else echo "false"; fi) \ + --set global.advertiseDefaultNetwork=$(if [ "${ADVERTISE_DEFAULT_NETWORK}" == "true" ]; then echo "true"; else echo "false"; fi) \ + --set global.advertisedUDNIsolationMode="${ADVERTISED_UDN_ISOLATION_MODE}" \ --set global.enableHybridOverlay=$(if [ "${OVN_HYBRID_OVERLAY_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.enableObservability=$(if [ "${OVN_OBSERV_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.emptyLbEvents=$(if [ "${OVN_EMPTY_LB_EVENTS}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.enableDNSNameResolver=$(if [ "${OVN_ENABLE_DNSNAMERESOLVER}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.enableNetworkQos=$(if [ "${OVN_NETWORK_QOS_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \ + --set global.enableNoOverlay=$(if [ "${ENABLE_NO_OVERLAY}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.enableCoredumps=$(if [ "${ENABLE_COREDUMPS}" == "true" ]; then echo "true"; else echo "false"; fi) \ ${ovnkube_db_options} EOF @@ -474,14 +378,6 @@ EOF eval "${cmd}" } -delete() { - if [ "$KIND_INSTALL_METALLB" == true ]; then - destroy_metallb - fi - helm uninstall ovn-kubernetes && sleep 5 ||: - kind delete cluster --name "${KIND_CLUSTER_NAME:-ovn}" -} - install_online_ovn_kubernetes_crds() { # NOTE: When you update vendoring versions for the ANP & BANP APIs, we must update the version of the CRD we pull from in the below URL run_kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/network-policy-api/v0.1.5/config/crd/experimental/policy.networking.k8s.io_adminnetworkpolicies.yaml @@ -499,6 +395,7 @@ if [ "$ENABLE_COREDUMPS" == true ]; then setup_coredumps fi detect_apiserver_url +install_ovn_image docker_disable_ipv6 coredns_patch if [ "$OVN_ENABLE_DNSNAMERESOLVER" == true ]; then @@ -509,6 +406,13 @@ if [ "$OVN_ENABLE_DNSNAMERESOLVER" == true ]; then add_ocp_dnsnameresolver_to_coredns_config update_coredns_deployment_image fi +if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ]; then + deploy_frr_external_container + deploy_bgp_external_server +fi +if [ "$KIND_REMOVE_TAINT" == true ]; then + remove_no_schedule_taint +fi create_ovn_kubernetes install_online_ovn_kubernetes_crds @@ -544,4 +448,8 @@ if [ "$KIND_INSTALL_KUBEVIRT" == true ]; then install_kubevirt fi +if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ]; then + install_frr_k8s +fi + interconnect_arg_check diff --git a/contrib/kind.sh b/contrib/kind.sh index 5231c5381b..ea8ce6fc7c 100755 --- a/contrib/kind.sh +++ b/contrib/kind.sh @@ -3,26 +3,8 @@ # Returns the full directory name of the script DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -# Source the kind-common file from the same directory where this script is located -source "${DIR}/kind-common" - -# Some environments (Fedora32,31 on desktop), have problems when the cluster -# is deleted directly with kind `kind delete cluster --name ovn`, it restarts the host. -# The root cause is unknown, this also can not be reproduced in Ubuntu 20.04 or -# with Fedora32 Cloud, but it does not happen if we clean first the ovn-kubernetes resources. -delete() { - OCI_BIN=${KIND_EXPERIMENTAL_PROVIDER:-docker} - - if [ "$KIND_INSTALL_METALLB" == true ]; then - destroy_metallb - fi - if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ]; then - destroy_bgp - fi - timeout 5 kubectl --kubeconfig "${KUBECONFIG}" delete namespace ovn-kubernetes || true - sleep 5 - kind delete cluster --name "${KIND_CLUSTER_NAME:-ovn}" -} +# Source the kind-common.sh file from the same directory where this script is located +source "${DIR}/kind-common.sh" usage() { echo "usage: kind.sh [[[-cf |--config-file ] [-kt|--keep-taint] [-ha|--ha-enabled]" @@ -47,16 +29,20 @@ usage() { echo " [-dd |--dns-domain |" echo " [-ric | --run-in-container |" echo " [-cn | --cluster-name |" - echo " [-ehp|--egress-ip-healthcheck-port ]" + echo " [-ehp|--egress-ip-healthcheck-port ] [-mip|--metrics-ip ]" echo " [-is | --ipsec]" echo " [-cm | --compact-mode]" echo " [-ic | --enable-interconnect]" echo " [-nce | --network-connect-enable]" echo " [-uae | --preconfigured-udn-addresses-enable]" - echo " [-rae | --enable-route-advertisements]" + echo " [-rae | --route-advertisements-enable]" + echo " [-evpn | --evpn-enable]" echo " [-rud | --routed-udn-isolation-disable]" + echo " [-dudn | --dynamic-udn-allocation]" + echo " [-dug | --dynamic-udn-removal-grace-period ]" echo " [-adv | --advertise-default-network]" echo " [-nqe | --network-qos-enable]" + echo " [-noe | --no-overlay-enable]" echo " [--isolated]" echo " [--enable-coredumps]" echo " [-dns | --enable-dnsnameresolver]" @@ -90,6 +76,8 @@ echo "-n4 | --no-ipv4 Disable IPv4. DEFAULT: IPv4 echo "-i6 | --ipv6 Enable IPv6. DEFAULT: IPv6 Disabled." echo "-wk | --num-workers Number of worker nodes. DEFAULT: HA - 2 worker" echo " nodes and no HA - 0 worker nodes." +echo "-inf | --num-infra Number of infra nodes. DEFAULT: 0" +echo "-prom| --install-prometheus Install Prometheus on infra nodes" echo "-sw | --allow-system-writes Allow script to update system. Intended to allow" echo " github CI to be updated with IPv6 settings." echo " DEFAULT: Don't allow." @@ -113,10 +101,11 @@ echo "-dd | --dns-domain Configure a custom dnsDomain echo "-cn | --cluster-name Configure the kind cluster's name" echo "-ric | --run-in-container Configure the script to be run from a docker container, allowing it to still communicate with the kind controlplane" echo "-ehp | --egress-ip-healthcheck-port TCP port used for gRPC session by egress IP node check. DEFAULT: 9107 (Use "0" for legacy dial to port 9)." +echo "-mip | --metrics-ip IP address to bind metrics endpoints. DEFAULT: K8S_NODE_IP or 0.0.0.0" echo "-is | --ipsec Enable IPsec encryption (spawns ovn-ipsec pods)" echo "-sm | --scale-metrics Enable scale metrics" echo "-cm | --compact-mode Enable compact mode, ovnkube master and node run in the same process." -echo "-ce | --enable-central Deploy with OVN Central (Legacy Architecture)" +echo "-ce | --enable-central [DEPRECATED] Deploy with OVN Central (Legacy Architecture)" echo "-nqe | --network-qos-enable Enable network QoS. DEFAULT: Disabled." echo "--disable-ovnkube-identity Disable per-node cert and ovnkube-identity webhook" echo "-npz | --nodes-per-zone If interconnect is enabled, number of nodes per zone (Default 1). If this value > 1, then (total k8s nodes (workers + 1) / num of nodes per zone) should be zero." @@ -129,10 +118,14 @@ echo "--add-nodes Adds nodes to an existing cl echo "-dns | --enable-dnsnameresolver Enable DNSNameResolver for resolving the DNS names used in the DNS rules of EgressFirewall." echo "-obs | --observability Enable OVN Observability feature." echo "-uae | --preconfigured-udn-addresses-enable Enable connecting workloads with preconfigured network to user-defined networks" -echo "-rae | --enable-route-advertisements Enable route advertisements" +echo "-rae | --route-advertisements-enable Enable route advertisements" +echo "-evpn | --evpn-enable Enable EVPN" +echo "-dudn | --dynamic-udn-allocation Enable dynamic UDN allocation" +echo "-dug | --dynamic-udn-removal-grace-period Configure the grace period in seconds for dynamic UDN removal. DEFAULT: 120 seconds" echo "-adv | --advertise-default-network Applies a RouteAdvertisements configuration to advertise the default network on all nodes" echo "-rud | --routed-udn-isolation-disable Disable isolation across BGP-advertised UDNs (sets advertised-udn-isolation-mode=loose). DEFAULT: strict." echo "-mps | --multi-pod-subnet Use multiple subnets for the default cluster network" +echo "-noe | --no-overlay-enable Enable no overlay" echo "" } @@ -213,6 +206,16 @@ parse_args() { fi KIND_NUM_WORKER=$1 ;; + -inf | --num-infra ) shift + if ! [[ "$1" =~ ^[0-9]+$ ]]; then + echo "Invalid num-infra: $1" + usage + exit 1 + fi + KIND_NUM_INFRA=$1 + ;; + -prom | --install-prometheus ) KIND_INSTALL_PROMETHEUS=true + ;; -npz | --nodes-per-zone ) shift if ! [[ "$1" =~ ^[0-9]+$ ]]; then echo "Invalid num-nodes-per-zone: $1" @@ -306,6 +309,9 @@ parse_args() { fi OVN_EGRESSIP_HEALTHCHECK_PORT=$1 ;; + -mip | --metrics-ip ) shift + METRICS_IP="$1" + ;; -sm | --scale-metrics ) OVN_METRICS_SCALE_ENABLE=true ;; -cm | --compact-mode ) OVN_COMPACT_MODE=true @@ -324,16 +330,34 @@ parse_args() { ;; -rae | --route-advertisements-enable) ENABLE_ROUTE_ADVERTISEMENTS=true ;; + -evpn | --evpn-enable) ENABLE_EVPN=true + ;; -adv | --advertise-default-network) ADVERTISE_DEFAULT_NETWORK=true ;; -rud | --routed-udn-isolation-disable) ADVERTISED_UDN_ISOLATION_MODE=loose ;; - -ce | --enable-central ) OVN_ENABLE_INTERCONNECT=false + -ce | --enable-central ) echo "WARNING: --enable-central is deprecated. OVN Central (Legacy Architecture) will be removed in a future release." >&2 + OVN_ENABLE_INTERCONNECT=false CENTRAL_ARG_PROVIDED=true ;; + -dudn | --dynamic-udn-allocation) DYNAMIC_UDN_ALLOCATION=true + ;; + -dug | --dynamic-udn-removal-grace-period) shift + if [[ -z "${1:-}" || "${1:-}" == -* ]]; then + echo "Missing value for --dynamic-udn-removal-grace-period" >&2 + usage + exit 1 + fi + DYNAMIC_UDN_GRACE_PERIOD=$1 + if [[ "$DYNAMIC_UDN_GRACE_PERIOD" =~ ^[0-9]+$ ]]; then + DYNAMIC_UDN_GRACE_PERIOD="${DYNAMIC_UDN_GRACE_PERIOD}s" + fi + ;; -ic | --enable-interconnect ) OVN_ENABLE_INTERCONNECT=true IC_ARG_PROVIDED=true ;; + -noe | --no-overlay-enable) ENABLE_NO_OVERLAY=true + ;; --disable-ovnkube-identity) OVN_ENABLE_OVNKUBE_IDENTITY=false ;; -mtu ) shift @@ -379,6 +403,7 @@ print_params() { echo "KIND_INSTALL_PLUGINS = $KIND_INSTALL_PLUGINS" echo "KIND_INSTALL_KUBEVIRT = $KIND_INSTALL_KUBEVIRT" echo "KIND_OPT_OUT_KUBEVIRT_IPAM = $KIND_OPT_OUT_KUBEVIRT_IPAM" + echo "OCI_BIN = $OCI_BIN" echo "OVN_HA = $OVN_HA" echo "RUN_IN_CONTAINER = $RUN_IN_CONTAINER" echo "KIND_CLUSTER_NAME = $KIND_CLUSTER_NAME" @@ -425,6 +450,7 @@ print_params() { echo "OVN_ENABLE_EX_GW_NETWORK_BRIDGE = $OVN_ENABLE_EX_GW_NETWORK_BRIDGE" echo "OVN_EX_GW_NETWORK_INTERFACE = $OVN_EX_GW_NETWORK_INTERFACE" echo "OVN_EGRESSIP_HEALTHCHECK_PORT = $OVN_EGRESSIP_HEALTHCHECK_PORT" + echo "METRICS_IP = $METRICS_IP" echo "OVN_DEPLOY_PODS = $OVN_DEPLOY_PODS" echo "OVN_METRICS_SCALE_ENABLE = $OVN_METRICS_SCALE_ENABLE" echo "OVN_ISOLATED = $OVN_ISOLATED" @@ -432,9 +458,13 @@ print_params() { echo "ENABLE_NETWORK_SEGMENTATION= $ENABLE_NETWORK_SEGMENTATION" echo "ENABLE_NETWORK_CONNECT = $ENABLE_NETWORK_CONNECT" echo "ENABLE_ROUTE_ADVERTISEMENTS= $ENABLE_ROUTE_ADVERTISEMENTS" + echo "ENABLE_EVPN= $ENABLE_EVPN" echo "ADVERTISED_UDN_ISOLATION_MODE= $ADVERTISED_UDN_ISOLATION_MODE" echo "ADVERTISE_DEFAULT_NETWORK = $ADVERTISE_DEFAULT_NETWORK" echo "ENABLE_PRE_CONF_UDN_ADDR = $ENABLE_PRE_CONF_UDN_ADDR" + echo "DYNAMIC_UDN_ALLOCATION = $DYNAMIC_UDN_ALLOCATION" + echo "DYNAMIC_UDN_GRACE_PERIOD = $DYNAMIC_UDN_GRACE_PERIOD" + echo "ENABLE_NO_OVERLAY = $ENABLE_NO_OVERLAY" echo "OVN_ENABLE_INTERCONNECT = $OVN_ENABLE_INTERCONNECT" if [ "$OVN_ENABLE_INTERCONNECT" == true ]; then echo "KIND_NUM_NODES_PER_ZONE = $KIND_NUM_NODES_PER_ZONE" @@ -446,73 +476,14 @@ print_params() { echo "OVN_ENABLE_OVNKUBE_IDENTITY = $OVN_ENABLE_OVNKUBE_IDENTITY" echo "OVN_NETWORK_QOS_ENABLE = $OVN_NETWORK_QOS_ENABLE" echo "KIND_NUM_WORKER = $KIND_NUM_WORKER" + echo "KIND_NUM_INFRA = $KIND_NUM_INFRA" + echo "KIND_INSTALL_PROMETHEUS = $KIND_INSTALL_PROMETHEUS" echo "OVN_MTU= $OVN_MTU" echo "OVN_ENABLE_DNSNAMERESOLVER= $OVN_ENABLE_DNSNAMERESOLVER" echo "MULTI_POD_SUBNET= $MULTI_POD_SUBNET" echo "" } -install_jinjanator_renderer() { - # ensure jinjanator renderer installed - pipx install jinjanator[yaml] - pipx ensurepath --force >/dev/null - export PATH=~/.local/bin:$PATH -} - -check_dependencies() { - if ! command_exists curl ; then - echo "Dependency not met: Command not found 'curl'" - exit 1 - fi - - if ! command_exists kubectl ; then - echo "'kubectl' not found, installing" - setup_kubectl_bin - fi - - if ! command_exists kind ; then - echo "Dependency not met: Command not found 'kind'" - exit 1 - fi - - local kind_min="0.27.0" - local kind_cur - kind_cur=$(kind version -q) - if [ "$(echo -e "$kind_min\n$kind_cur" | sort -V | head -1)" != "$kind_min" ]; then - echo "Dependency not met: expected kind version >= $kind_min but have $kind_cur" - exit 1 - fi - - if ! command_exists jq ; then - echo "Dependency not met: Command not found 'jq'" - exit 1 - fi - - if ! command_exists awk ; then - echo "Dependency not met: Command not found 'awk'" - exit 1 - fi - - if ! command_exists jinjanate ; then - if ! command_exists pipx ; then - echo "Dependency not met: 'jinjanator' not installed and cannot install with 'pipx'" - exit 1 - fi - echo "'jinjanate' not found, installing with 'pipx'" - install_jinjanator_renderer - fi - - if ! command_exists docker && ! command_exists podman; then - echo "Dependency not met: Neither docker nor podman found" - exit 1 - fi - - if command_exists podman && ! command_exists skopeo; then - echo "Dependency not met: skopeo not installed. Run the following command to install it: 'sudo dnf install skopeo'" - exit 1 - fi -} - OPENSSL="" set_openssl_binary() { for s in openssl openssl3; do @@ -536,47 +507,24 @@ set_default_params() { # Set default values # Used for multi cluster setups - KIND_CREATE=${KIND_CREATE:-true} KIND_ADD_NODES=${KIND_ADD_NODES:-false} - KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ovn} - # Setup KUBECONFIG patch based on cluster-name - export KUBECONFIG=${KUBECONFIG:-${HOME}/${KIND_CLUSTER_NAME}.conf} - # Scrub any existing kubeconfigs at the path - if [ "${KIND_CREATE}" == true ]; then - rm -f ${KUBECONFIG} - fi MANIFEST_OUTPUT_DIR=${MANIFEST_OUTPUT_DIR:-${DIR}/../dist/yaml} if [ ${KIND_CLUSTER_NAME} != "ovn" ]; then MANIFEST_OUTPUT_DIR="${DIR}/../dist/yaml/${KIND_CLUSTER_NAME}" fi RUN_IN_CONTAINER=${RUN_IN_CONTAINER:-false} OVN_GATEWAY_MODE=${OVN_GATEWAY_MODE:-shared} - KIND_INSTALL_INGRESS=${KIND_INSTALL_INGRESS:-false} - KIND_INSTALL_METALLB=${KIND_INSTALL_METALLB:-false} - KIND_INSTALL_PLUGINS=${KIND_INSTALL_PLUGINS:-false} - KIND_INSTALL_KUBEVIRT=${KIND_INSTALL_KUBEVIRT:-false} KIND_OPT_OUT_KUBEVIRT_IPAM=${KIND_OPT_OUT_KUBEVIRT_IPAM:-false} - OVN_HA=${OVN_HA:-false} - KIND_LOCAL_REGISTRY=${KIND_LOCAL_REGISTRY:-false} KIND_LOCAL_REGISTRY_NAME=${KIND_LOCAL_REGISTRY_NAME:-kind-registry} KIND_LOCAL_REGISTRY_PORT=${KIND_LOCAL_REGISTRY_PORT:-5000} KIND_DNS_DOMAIN=${KIND_DNS_DOMAIN:-"cluster.local"} - KIND_CONFIG=${KIND_CONFIG:-${DIR}/kind.yaml.j2} - KIND_REMOVE_TAINT=${KIND_REMOVE_TAINT:-true} - PLATFORM_IPV4_SUPPORT=${PLATFORM_IPV4_SUPPORT:-true} - PLATFORM_IPV6_SUPPORT=${PLATFORM_IPV6_SUPPORT:-false} ENABLE_IPSEC=${ENABLE_IPSEC:-false} - OVN_HYBRID_OVERLAY_ENABLE=${OVN_HYBRID_OVERLAY_ENABLE:-false} OVN_DISABLE_SNAT_MULTIPLE_GWS=${OVN_DISABLE_SNAT_MULTIPLE_GWS:-false} OVN_DISABLE_FORWARDING=${OVN_DISABLE_FORWARDING:=false} OVN_ENCAP_PORT=${OVN_ENCAP_PORT:-""} OVN_DISABLE_PKT_MTU_CHECK=${OVN_DISABLE_PKT_MTU_CHECK:-false} - OVN_EMPTY_LB_EVENTS=${OVN_EMPTY_LB_EVENTS:-false} - OVN_MULTICAST_ENABLE=${OVN_MULTICAST_ENABLE:-false} KIND_ALLOW_SYSTEM_WRITES=${KIND_ALLOW_SYSTEM_WRITES:-false} - OVN_IMAGE=${OVN_IMAGE:-local} - OVN_REPO=${OVN_REPO:-""} - OVN_GITREF=${OVN_GITREF:-""} + MASTER_LOG_LEVEL=${MASTER_LOG_LEVEL:-5} NODE_LOG_LEVEL=${NODE_LOG_LEVEL:-5} DBCHECKER_LOG_LEVEL=${DBCHECKER_LOG_LEVEL:-5} @@ -591,63 +539,14 @@ set_default_params() { if [ "$OVN_ENABLE_EX_GW_NETWORK_BRIDGE" == true ]; then OVN_EX_GW_NETWORK_INTERFACE="eth1" fi - MULTI_POD_SUBNET=${MULTI_POD_SUBNET:-false} - # Input not currently validated. Modify outside script at your own risk. - # These are the same values defaulted to in KIND code (kind/default.go). - # NOTE: KIND NET_CIDR_IPV6 default use a /64 but OVN have a /64 per host - # so it needs to use a larger subnet - # Upstream - NET_CIDR_IPV6=fd00:10:244::/64 SVC_CIDR_IPV6=fd00:10:96::/112 - NET_CIDR_IPV4=${NET_CIDR_IPV4:-10.244.0.0/16} - NET_CIDR_IPV6=${NET_CIDR_IPV6:-fd00:10:244::/48} - if [ "$MULTI_POD_SUBNET" == true ]; then - NET_CIDR_IPV4="10.243.0.0/23/24,10.244.0.0/16" - NET_CIDR_IPV6="fd00:10:243::/63/64,fd00:10:244::/48" - fi - NET_SECOND_CIDR_IPV4=${NET_SECOND_CIDR_IPV4:-172.19.0.0/16} - SVC_CIDR_IPV4=${SVC_CIDR_IPV4:-10.96.0.0/16} - SVC_CIDR_IPV6=${SVC_CIDR_IPV6:-fd00:10:96::/112} - JOIN_SUBNET_IPV4=${JOIN_SUBNET_IPV4:-100.64.0.0/16} - JOIN_SUBNET_IPV6=${JOIN_SUBNET_IPV6:-fd98::/64} - MASQUERADE_SUBNET_IPV4=${MASQUERADE_SUBNET_IPV4:-169.254.0.0/17} - MASQUERADE_SUBNET_IPV6=${MASQUERADE_SUBNET_IPV6:-fd69::/112} - TRANSIT_SUBNET_IPV4=${TRANSIT_SUBNET_IPV4:-100.88.0.0/16} - TRANSIT_SUBNET_IPV6=${TRANSIT_SUBNET_IPV6:-fd97::/64} - METALLB_CLIENT_NET_SUBNET_IPV4=${METALLB_CLIENT_NET_SUBNET_IPV4:-172.22.0.0/16} - METALLB_CLIENT_NET_SUBNET_IPV6=${METALLB_CLIENT_NET_SUBNET_IPV6:-fc00:f853:ccd:e792::/64} - BGP_SERVER_NET_SUBNET_IPV4=${BGP_SERVER_NET_SUBNET_IPV4:-172.26.0.0/16} - BGP_SERVER_NET_SUBNET_IPV6=${BGP_SERVER_NET_SUBNET_IPV6:-fc00:f853:ccd:e796::/64} - KIND_NUM_MASTER=1 - OVN_ENABLE_INTERCONNECT=${OVN_ENABLE_INTERCONNECT:-true} OVN_ENABLE_OVNKUBE_IDENTITY=${OVN_ENABLE_OVNKUBE_IDENTITY:-true} - OVN_NETWORK_QOS_ENABLE=${OVN_NETWORK_QOS_ENABLE:-false} - - - if [ "$OVN_COMPACT_MODE" == true ] && [ "$OVN_ENABLE_INTERCONNECT" != false ]; then - echo "Compact mode cannot be used together with Interconnect" - exit 1 - fi - if [ "$OVN_HA" == true ]; then - KIND_NUM_MASTER=3 - KIND_NUM_WORKER=${KIND_NUM_WORKER:-0} - else - KIND_NUM_WORKER=${KIND_NUM_WORKER:-2} - fi - - if [ "$OVN_ENABLE_INTERCONNECT" == true ]; then - KIND_NUM_NODES_PER_ZONE=${KIND_NUM_NODES_PER_ZONE:-1} - - TOTAL_NODES=$((KIND_NUM_WORKER + KIND_NUM_MASTER)) - if [[ ${KIND_NUM_NODES_PER_ZONE} -gt 1 ]] && [[ $((TOTAL_NODES % KIND_NUM_NODES_PER_ZONE)) -ne 0 ]]; then - echo "(Total k8s nodes / number of nodes per zone) should be zero" - exit 1 - fi - fi + KIND_NUM_INFRA=${KIND_NUM_INFRA:-0} + KIND_INSTALL_PROMETHEUS=${KIND_INSTALL_PROMETHEUS:-false} OVN_HOST_NETWORK_NAMESPACE=${OVN_HOST_NETWORK_NAMESPACE:-ovn-host-network} OVN_EGRESSIP_HEALTHCHECK_PORT=${OVN_EGRESSIP_HEALTHCHECK_PORT:-9107} - OCI_BIN=${KIND_EXPERIMENTAL_PROVIDER:-docker} OVN_DEPLOY_PODS=${OVN_DEPLOY_PODS:-"ovnkube-identity ovnkube-zone-controller ovnkube-control-plane ovnkube-master ovnkube-node"} OVN_METRICS_SCALE_ENABLE=${OVN_METRICS_SCALE_ENABLE:-false} OVN_ISOLATED=${OVN_ISOLATED:-false} @@ -659,47 +558,6 @@ set_default_params() { if [ "$OVN_DUMMY_GATEWAY_BRIDGE" == true ]; then OVN_GATEWAY_OPTS="--allow-no-uplink --gateway-interface=br-ex" fi - ENABLE_MULTI_NET=${ENABLE_MULTI_NET:-false} - ENABLE_NETWORK_SEGMENTATION=${ENABLE_NETWORK_SEGMENTATION:-false} - if [ "$ENABLE_NETWORK_SEGMENTATION" == true ] && [ "$ENABLE_MULTI_NET" != true ]; then - echo "Network segmentation (UDN) requires multi-network to be enabled (-mne)" - exit 1 - fi - - ENABLE_ROUTE_ADVERTISEMENTS=${ENABLE_ROUTE_ADVERTISEMENTS:-false} - if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ] && [ "$ENABLE_MULTI_NET" != true ]; then - echo "Route advertisements requires multi-network to be enabled (-mne)" - exit 1 - fi - if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ] && [ "$OVN_ENABLE_INTERCONNECT" != true ]; then - echo "Route advertisements requires interconnect to be enabled (-ic)" - exit 1 - fi - - ENABLE_PRE_CONF_UDN_ADDR=${ENABLE_PRE_CONF_UDN_ADDR:-false} - if [[ $ENABLE_PRE_CONF_UDN_ADDR == true && $ENABLE_NETWORK_SEGMENTATION != true ]]; then - echo "Preconfigured UDN addresses requires network-segmentation to be enabled (-nse)" - exit 1 - fi - if [[ $ENABLE_PRE_CONF_UDN_ADDR == true && $OVN_ENABLE_INTERCONNECT != true ]]; then - echo "Preconfigured UDN addresses requires interconnect to be enabled (-ic)" - exit 1 - fi - ENABLE_NETWORK_CONNECT=${ENABLE_NETWORK_CONNECT:-false} - if [[ $ENABLE_NETWORK_CONNECT == true && $ENABLE_NETWORK_SEGMENTATION != true ]]; then - echo "Network connect requires network-segmentation to be enabled (-nse)" - exit 1 - fi - ADVERTISED_UDN_ISOLATION_MODE=${ADVERTISED_UDN_ISOLATION_MODE:-strict} - ADVERTISE_DEFAULT_NETWORK=${ADVERTISE_DEFAULT_NETWORK:-false} - OVN_COMPACT_MODE=${OVN_COMPACT_MODE:-false} - if [ "$OVN_COMPACT_MODE" == true ]; then - KIND_NUM_WORKER=0 - fi - OVN_MTU=${OVN_MTU:-1400} - OVN_ENABLE_DNSNAMERESOLVER=${OVN_ENABLE_DNSNAMERESOLVER:-false} - OVN_OBSERV_ENABLE=${OVN_OBSERV_ENABLE:-false} - ENABLE_COREDUMPS=${ENABLE_COREDUMPS:-false} } check_ipv6() { @@ -823,73 +681,6 @@ scale_kind_cluster() { fi } -create_kind_cluster() { - # Output of the jinjanate command - KIND_CONFIG_LCL=${DIR}/kind-${KIND_CLUSTER_NAME}.yaml - - ovn_ip_family=${IP_FAMILY} \ - ovn_ha=${OVN_HA} \ - net_cidr="${KIND_CIDR}" \ - svc_cidr=${SVC_CIDR} \ - use_local_registy=${KIND_LOCAL_REGISTRY} \ - dns_domain=${KIND_DNS_DOMAIN} \ - ovn_num_master=${KIND_NUM_MASTER} \ - ovn_num_worker=${KIND_NUM_WORKER} \ - cluster_log_level=${KIND_CLUSTER_LOGLEVEL:-4} \ - kind_local_registry_port=${KIND_LOCAL_REGISTRY_PORT} \ - kind_local_registry_name=${KIND_LOCAL_REGISTRY_NAME} \ - jinjanate "${KIND_CONFIG}" -o "${KIND_CONFIG_LCL}" - - # Create KIND cluster. For additional debug, add '--verbosity ': 0 None .. 3 Debug - if kind get clusters | grep "${KIND_CLUSTER_NAME}"; then - delete - fi - - if [[ "${KIND_LOCAL_REGISTRY}" == true ]]; then - create_local_registry - fi - - kind create cluster --name "${KIND_CLUSTER_NAME}" --kubeconfig "${KUBECONFIG}" --image "${KIND_IMAGE}":"${K8S_VERSION}" --config=${KIND_CONFIG_LCL} --retain - - cat "${KUBECONFIG}" -} - -set_ovn_image() { - # if we're using the local registry and still need to build, push to local registry - if [ "$KIND_LOCAL_REGISTRY" == true ];then - OVN_IMAGE="localhost:5000/ovn-daemonset-fedora:latest" - else - OVN_IMAGE="localhost/ovn-daemonset-fedora:dev" - fi -} - -build_ovn_image() { - local push_args="" - if [ "$OCI_BIN" == "podman" ]; then - # docker doesn't perform tls check by default only podman does, hence we need to disable it for podman. - push_args="--tls-verify=false" - fi - - if [ "$OVN_IMAGE" == local ]; then - set_ovn_image - - # Build image - make -C ${DIR}/../dist/images IMAGE="${OVN_IMAGE}" OVN_REPO="${OVN_REPO}" OVN_GITREF="${OVN_GITREF}" OCI_BIN="${OCI_BIN}" fedora-image - - # store in local registry - if [ "$KIND_LOCAL_REGISTRY" == true ];then - echo "Pushing built image to local $OCI_BIN registry" - $OCI_BIN push $push_args "$OVN_IMAGE" - fi - # We should push to local registry if image is not remote - elif [ "${OVN_IMAGE}" != "" -a "${KIND_LOCAL_REGISTRY}" == true ] && (echo "$OVN_IMAGE" | grep / -vq); then - local local_registry_ovn_image="localhost:5000/${OVN_IMAGE}" - $OCI_BIN tag "$OVN_IMAGE" $local_registry_ovn_image - OVN_IMAGE=$local_registry_ovn_image - $OCI_BIN push $push_args "$OVN_IMAGE" - fi -} - create_ovn_kube_manifests() { local ovnkube_image=${OVN_IMAGE} if [ "$KIND_LOCAL_REGISTRY" == true ];then @@ -953,10 +744,15 @@ create_ovn_kube_manifests() { --network-segmentation-enable="${ENABLE_NETWORK_SEGMENTATION}" \ --network-connect-enable="${ENABLE_NETWORK_CONNECT}" \ --preconfigured-udn-addresses-enable="${ENABLE_PRE_CONF_UDN_ADDR}" \ + --enable-dynamic-udn-allocation="${DYNAMIC_UDN_ALLOCATION}" \ + --udn-deletion-grace-period="${DYNAMIC_UDN_GRACE_PERIOD}" \ --route-advertisements-enable="${ENABLE_ROUTE_ADVERTISEMENTS}" \ + --evpn-enable="${ENABLE_EVPN}" \ --advertise-default-network="${ADVERTISE_DEFAULT_NETWORK}" \ --advertised-udn-isolation-mode="${ADVERTISED_UDN_ISOLATION_MODE}" \ + --no-overlay-enable="${ENABLE_NO_OVERLAY}" \ --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}" \ + --metrics-ip="${METRICS_IP}" \ --compact-mode="${OVN_COMPACT_MODE}" \ --enable-interconnect="${OVN_ENABLE_INTERCONNECT}" \ --enable-multi-external-gateway=true \ @@ -965,15 +761,10 @@ create_ovn_kube_manifests() { --network-qos-enable="${OVN_NETWORK_QOS_ENABLE}" \ --mtu="${OVN_MTU}" \ --enable-dnsnameresolver="${OVN_ENABLE_DNSNAMERESOLVER}" \ - --mtu="${OVN_MTU}" \ --enable-observ="${OVN_OBSERV_ENABLE}" popd } -install_ovn_image() { - install_image ${OVN_IMAGE} -} - install_ovn_global_zone() { if [ "$OVN_HA" == true ]; then run_kubectl apply -f ovnkube-db-raft.yaml @@ -1054,6 +845,7 @@ install_ovn() { if [ "$ENABLE_NETWORK_CONNECT" == true ]; then run_kubectl apply -f k8s.ovn.org_clusternetworkconnects.yaml fi + run_kubectl apply -f k8s.ovn.org_vteps.yaml # NOTE: When you update vendoring versions for the ANP & BANP APIs, we must update the version of the CRD we pull from in the below URL run_kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/network-policy-api/v0.1.5/config/crd/experimental/policy.networking.k8s.io_adminnetworkpolicies.yaml run_kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/network-policy-api/v0.1.5/config/crd/experimental/policy.networking.k8s.io_baselineadminnetworkpolicies.yaml @@ -1063,20 +855,12 @@ install_ovn() { run_kubectl apply -f rbac-ovnkube-master.yaml run_kubectl apply -f rbac-ovnkube-node.yaml run_kubectl apply -f rbac-ovnkube-db.yaml - MASTER_NODES=$(kind get nodes --name "${KIND_CLUSTER_NAME}" | sort | head -n "${KIND_NUM_MASTER}") - # We want OVN HA not Kubernetes HA - # leverage the kubeadm well-known label node-role.kubernetes.io/control-plane= - # to choose the nodes where ovn master components will be placed - for n in $MASTER_NODES; do - kubectl label node "$n" k8s.ovn.org/ovnkube-db=true node-role.kubernetes.io/control-plane="" --overwrite - if [ "$KIND_REMOVE_TAINT" == true ]; then - # do not error if it fails to remove the taint - # remove both master and control-plane taints until master is removed from 1.25 - # // https://github.com/kubernetes/kubernetes/pull/107533 - kubectl taint node "$n" node-role.kubernetes.io/master:NoSchedule- || true - kubectl taint node "$n" node-role.kubernetes.io/control-plane:NoSchedule- || true - fi - done + if [ "${OVN_HA}" == "true" ]; then + label_ovn_ha + fi + if [ "$KIND_REMOVE_TAINT" == true ]; then + remove_no_schedule_taint + fi run_kubectl apply -f ovs-node.yaml @@ -1211,7 +995,7 @@ add_dns_hostnames() { done } -check_dependencies +check_common_dependencies # In order to allow providing arguments with spaces, e.g. "-vconsole:info -vfile:info" # the original command was replaced by parse_args "$@" @@ -1304,7 +1088,7 @@ if [ "$KIND_INSTALL_KUBEVIRT" == true ]; then fi fi if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ]; then - install_ffr_k8s + install_frr_k8s fi interconnect_arg_check diff --git a/contrib/kind.yaml.j2 b/contrib/kind.yaml.j2 index ab1711fead..ee6d719ea9 100644 --- a/contrib/kind.yaml.j2 +++ b/contrib/kind.yaml.j2 @@ -14,7 +14,7 @@ networking: {%- if ovn_ip_family %} ipFamily: {{ ovn_ip_family }} {%- endif %} -{%- if use_local_registy == "true"%} +{%- if use_local_registry == "true"%} containerdConfigPatches: - |- [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:{{ kind_local_registry_port }}"] @@ -69,3 +69,8 @@ nodes: {%- for _ in range(ovn_num_worker | int) %} - role: worker {%- endfor %} +{%- if kind_num_infra is defined and kind_num_infra|int > 0 %} +{%- for i in range(kind_num_infra|int) %} + - role: worker +{%- endfor %} +{%- endif %} diff --git a/contrib/perf/README.md b/contrib/perf/README.md new file mode 100644 index 0000000000..6632df21e9 --- /dev/null +++ b/contrib/perf/README.md @@ -0,0 +1,35 @@ +# Performance tooling +``` +contrib/perf +├── generate_perf_report.py # Generates high-level report of OVNK Container CPU/Memory utilization during a workload +├── metrics.yml # Metrics we capture with kube-burner +├── performance-meta.yml # Additional Metadata we append to our OpenSearch documents +└── workloads # Workload definition folder + └── kubelet-density-cni.yml # kubelet-density-cni workload for kube-burner +``` + +## Workloads +### kubelet-density-cni +This simple workload launches a webserver, service and a curl client within a namespace. + +For our use-case we make some modifications to the base config shipped with kube-burner. + +``` + - name: kubelet-density-cni + jobIterations: 100 + qps: 10 + burst: 10 + namespacedIterations: false + namespace: kubelet-density-cni + waitWhenFinished: true + podWait: false + preLoadImages: true + preLoadPeriod: 2m + churnConfig: + percent: 10 + cycles: 10 + mode: objects +``` + +We enable churn, which will delete and recreate the objects we created in the namespace. We also lower the QPS to 10/10 since +we are testing on a kind cluster. diff --git a/contrib/perf/generate_perf_report.py b/contrib/perf/generate_perf_report.py new file mode 100644 index 0000000000..0e80018f65 --- /dev/null +++ b/contrib/perf/generate_perf_report.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +""" +Kubernetes Workload Metrics Report Generator + +This script generates a text report from JSON metrics files and optionally posts it as a GitHub comment for PR runs. +It focuses on podreadylatency as the main KPI and OVN container-level CPU/Memory usage as secondary metrics. +""" + +import json +import os +import sys +import subprocess +from datetime import datetime +from typing import Dict, List, Any, Optional +import argparse + + +class MetricsProcessor: + """Process and analyze metrics data from JSON files.""" + + def __init__(self, metrics_dir: str = ".", workload: str = "kubelet-density-cni"): + self.workload = workload + self.metrics_dir = metrics_dir + self.pod_latency_file = f"podLatencyMeasurement-{self.workload}.json" + self.container_cpu_file = "containerCPU.json" + self.container_memory_file = "containerMemory.json" + + def load_json_file(self, filename: str) -> List[Dict[str, Any]]: + """Load and parse JSON file.""" + filepath = os.path.join(self.metrics_dir, filename) + try: + with open(filepath, 'r') as f: + data = json.load(f) + print(f"✓ Loaded {len(data)} records from {filename}") + return data + except FileNotFoundError: + print(f"✗ Error: {filename} not found in {self.metrics_dir}") + return [] + except json.JSONDecodeError as e: + print(f"✗ Error parsing {filename}: {e}") + return [] + + def process_pod_latency_data(self, data: List[Dict[str, Any]]) -> Dict[str, Any]: + """Process pod latency data and calculate statistics.""" + valid_data = [ + d for d in data + if d.get('podReadyLatency') is not None and d.get('timestamp') + ] + + if not valid_data: + return {"data": [], "stats": {}} + + # Sort by timestamp + valid_data.sort(key=lambda x: x['timestamp']) + + # Calculate statistics + latencies = [d['podReadyLatency'] for d in valid_data] + stats = { + "total_pods": len(valid_data), + "avg_latency": sum(latencies) / len(latencies), + "max_latency": max(latencies), + "min_latency": min(latencies), + "start_time": valid_data[0]['timestamp'], + "end_time": valid_data[-1]['timestamp'] + } + + return { + "data": valid_data, + "stats": stats + } + + def process_ovn_data(self, data: List[Dict[str, Any]], metric_type: str) -> Dict[str, List[Dict[str, Any]]]: + """Process OVN container CPU/Memory data.""" + ovn_data = {} + + for record in data: + labels = record.get('labels', {}) + pod_name = labels.get('pod', '') + container_name = labels.get('container', '') + + # Filter for OVN containers + if not any(keyword in pod_name for keyword in ['ovnkube-', 'ovs-']): + continue + + # Categorize container type based on container name + container_type = self.get_container_type_from_container(container_name, pod_name) + + if container_type not in ovn_data: + ovn_data[container_type] = [] + + # Convert memory to MB if needed + value = record.get('value', 0) + if metric_type == 'memory': + value = value / (1024 * 1024) # Convert bytes to MB + + ovn_data[container_type].append({ + "timestamp": record.get('timestamp'), + "value": value, + "pod": pod_name, + "container": container_name, + "node": labels.get('node', 'unknown') + }) + + # Sort each container type by timestamp + for container_type in ovn_data: + ovn_data[container_type].sort(key=lambda x: x['timestamp']) + + return ovn_data + + def get_container_type_from_container(self, container_name: str, pod_name: str) -> str: + """Categorize OVN container types based on container name.""" + if container_name == 'ovnkube-cluster-manager': + return 'OVNKube Cluster Manager' + elif container_name == 'ovnkube-identity': + return 'OVNKube Identity' + elif container_name == 'ovnkube-controller': + return 'OVNKube Controller' + elif container_name == 'ovn-controller': + return 'OVN Controller' + elif container_name == 'ovn-northd': + return 'OVN Northd' + elif container_name in ['nb-ovsdb', 'sb-ovsdb']: + return f'OVSDB ({container_name})' + elif container_name == 'ovs-daemons': + return 'OVS Daemons' + elif container_name == 'ovs-metrics-exporter': + return 'OVS Metrics Exporter' + else: + # Fallback to pod-based categorization + if 'ovnkube-node' in pod_name: + return f'OVNKube Node ({container_name})' + elif 'ovnkube-control-plane' in pod_name: + return f'OVNKube Control Plane ({container_name})' + elif 'ovs-node' in pod_name: + return f'OVS Node ({container_name})' + else: + return f'Other OVN ({container_name})' + + +class ReportGenerator: + """Generate text report from processed metrics data.""" + + def __init__(self, title: str = "Kubernetes Workload Metrics Report", workload: str = "kubelet-density-cni"): + self.title = title + self.workload = workload + + def generate_report(self, pod_latency: Dict[str, Any], ovn_cpu: Dict[str, Any], + ovn_memory: Dict[str, Any] ) -> str: + """Generate complete text report.""" + + stats = pod_latency['stats'] + report_lines = [] + + # Header + report_lines.append("# 📊 Kubernetes Workload Metrics Report") + report_lines.append(f"## {self.workload} Performance Results") + report_lines.append("") + report_lines.append(f"**Generated on:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC')}") + report_lines.append("") + + # Main KPI: Pod Ready Latency + report_lines.append("## 🎯 Pod Ready Latency (Main KPI)") + if stats: + report_lines.append("| Metric | Value |") + report_lines.append("|--------|-------|") + report_lines.append(f"| Average Latency | **{stats.get('avg_latency', 0):.1f} ms** |") + report_lines.append(f"| Max Latency | **{stats.get('max_latency', 0):.0f} ms** |") + report_lines.append(f"| Min Latency | **{stats.get('min_latency', 0):.0f} ms** |") + report_lines.append(f"| Total Pods | **{stats.get('total_pods', 0)}** |") + report_lines.append(f"| Time Range | {self._format_time_range(stats.get('start_time'), stats.get('end_time'))} |") + else: + report_lines.append("⚠️ No pod latency data available") + report_lines.append("") + + # OVN Container Summary + report_lines.append("## 💻 OVN Container-Level Resource Usage") + + # CPU Summary + if ovn_cpu: + report_lines.append("### CPU Usage Summary") + report_lines.append("| Container Type | Avg CPU (%) | Max CPU (%) | Data Points |") + report_lines.append("|----------------|-------------|-------------|-------------|") + + for container_type, data in sorted(ovn_cpu.items()): + if data: + cpu_values = [d['value'] for d in data] + avg_cpu = sum(cpu_values) / len(cpu_values) + max_cpu = max(cpu_values) + report_lines.append(f"| {container_type} | {avg_cpu:.2f}% | {max_cpu:.2f}% | {len(data)} |") + else: + report_lines.append("### CPU Usage Summary") + report_lines.append("⚠️ No OVN container CPU data available") + report_lines.append("") + + # Memory Summary + if ovn_memory: + report_lines.append("### Memory Usage Summary") + report_lines.append("| Container Type | Avg Memory (MB) | Max Memory (MB) | Data Points |") + report_lines.append("|----------------|-----------------|-----------------|-------------|") + + for container_type, data in sorted(ovn_memory.items()): + if data: + memory_values = [d['value'] for d in data] + avg_memory = sum(memory_values) / len(memory_values) + max_memory = max(memory_values) + report_lines.append(f"| {container_type} | {avg_memory:.2f} MB | {max_memory:.2f} MB | {len(data)} |") + else: + report_lines.append("### Memory Usage Summary") + report_lines.append("⚠️ No OVN container memory data available") + + report_lines.append("") + report_lines.append("---") + report_lines.append("*Report generated by ovn-kubernetes performance testing*") + + return "\n".join(report_lines) + + def _format_time_range(self, start_time: Optional[str], end_time: Optional[str]) -> str: + """Format time range for display.""" + if not start_time or not end_time: + return "-" + + try: + start = datetime.fromisoformat(start_time.replace('Z', '+00:00')) + end = datetime.fromisoformat(end_time.replace('Z', '+00:00')) + + if start.date() == end.date(): + return f"{start.strftime('%m/%d/%Y')}" + else: + return f"{start.strftime('%m/%d')} - {end.strftime('%m/%d')}" + except: + return "-" + + def post_github_comment(self, report_content: str, pr_number: str) -> bool: + """Post report as GitHub comment using gh CLI.""" + try: + # Use gh CLI to post comment + cmd = ['gh', 'pr', 'comment', pr_number, '--body', report_content] + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0: + print(f"✓ Posted performance report as comment to PR #{pr_number}") + return True + else: + print(f"✗ Failed to post GitHub comment: {result.stderr}") + return False + except Exception as e: + print(f"✗ Error posting GitHub comment: {e}") + return False + + def save_report(self, report_content: str, output_file: str) -> None: + """Save report to file.""" + with open(output_file, 'w') as f: + f.write(report_content) + print(f"✓ Report saved to: {output_file}") + + +def detect_pr_environment() -> Optional[str]: + """Detect if running in a PR environment and return PR number.""" + # Check for GitHub Actions PR environment + github_event_name = os.environ.get('GITHUB_EVENT_NAME') + github_ref = os.environ.get('GITHUB_REF') + + if github_event_name == 'pull_request': + # Extract PR number from GITHUB_REF (e.g., "refs/pull/123/merge") + if github_ref and 'pull' in github_ref: + try: + pr_number = github_ref.split('/')[2] + return pr_number + except (IndexError, ValueError): + pass + + # Check for GITHUB_EVENT_PATH which contains PR info + event_path = os.environ.get('GITHUB_EVENT_PATH') + if event_path and os.path.exists(event_path): + try: + with open(event_path, 'r') as f: + event_data = json.load(f) + if 'pull_request' in event_data: + return str(event_data['pull_request']['number']) + except (json.JSONDecodeError, KeyError, FileNotFoundError): + pass + + # Check for manual PR number in environment + pr_number = os.environ.get('PR_NUMBER') + if pr_number: + return pr_number + + return None + + + + +def main(): + """Main function to generate the performance report.""" + parser = argparse.ArgumentParser(description='Generate Kubernetes workload metrics report') + parser.add_argument('--workload', default='kubelet-density-cni', + help='Workload name (default: kubelet-density-cni)') + parser.add_argument('--metrics-dir', default='.', + help='Directory containing JSON metrics files (default: current directory)') + parser.add_argument('--output', default='performance_report.md', + help='Output file name (default: performance_report.md)') + parser.add_argument('--title', default='Kubernetes Workload Metrics Report', + help='Report title') + parser.add_argument('--pr-number', + help='PR number for GitHub comment (overrides auto-detection)') + parser.add_argument('--github-comment', action='store_true', + help='Post report as GitHub comment if PR detected') + + args = parser.parse_args() + + print(f"🚀 Generating Kubernetes Workload Metrics Report") + print(f"📁 Metrics directory: {args.metrics_dir}") + print(f"📄 Output file: {args.output}") + print() + + # Initialize processor and generator + processor = MetricsProcessor(args.metrics_dir, args.workload) + generator = ReportGenerator(args.title, args.workload) + + # Load and process data + print("📊 Loading and processing metrics data...") + + # Process pod latency data (main KPI) + pod_latency_raw = processor.load_json_file(processor.pod_latency_file) + pod_latency_processed = processor.process_pod_latency_data(pod_latency_raw) + + # Process OVN CPU data + container_cpu_raw = processor.load_json_file(processor.container_cpu_file) + ovn_cpu_processed = processor.process_ovn_data(container_cpu_raw, 'cpu') + + # Process OVN Memory data + container_memory_raw = processor.load_json_file(processor.container_memory_file) + ovn_memory_processed = processor.process_ovn_data(container_memory_raw, 'memory') + + print() + print("📈 Data Processing Summary:") + print(f" Pod Latency Records: {len(pod_latency_processed['data'])}") + print(f" OVN CPU Container Types: {len(ovn_cpu_processed)}") + print(f" OVN Memory Container Types: {len(ovn_memory_processed)}") + + if pod_latency_processed['stats']: + stats = pod_latency_processed['stats'] + print(f" Average Pod Ready Latency: {stats['avg_latency']:.1f}ms") + print(f" Max Pod Ready Latency: {stats['max_latency']:.0f}ms") + + print() + + # Generate text report + print("📝 Generating performance report...") + report_content = generator.generate_report( + pod_latency_processed, + ovn_cpu_processed, + ovn_memory_processed + ) + + # Save report to file + generator.save_report(report_content, args.output) + + # Check for PR environment and post GitHub comment if requested + pr_number = args.pr_number or detect_pr_environment() + + if args.github_comment and pr_number: + print(f"\n💬 Detected PR #{pr_number}, posting GitHub comment...") + success = generator.post_github_comment(report_content, pr_number) + if not success: + print("⚠️ GitHub comment failed, but report was saved to file") + elif args.github_comment: + print("\n⚠️ --github-comment specified but no PR detected") + print(" Set PR_NUMBER environment variable or use --pr-number") + elif pr_number: + print(f"\n💡 PR #{pr_number} detected. Use --github-comment to post report as comment") + + print() + print("🎉 Report generation complete!") + print(f"📄 Report saved to: {args.output}") + if pr_number and args.github_comment: + print(f"💬 GitHub comment posted to PR #{pr_number}") + + +if __name__ == "__main__": + main() diff --git a/contrib/perf/metric-endpoint-local.yml b/contrib/perf/metric-endpoint-local.yml new file mode 100644 index 0000000000..374329832b --- /dev/null +++ b/contrib/perf/metric-endpoint-local.yml @@ -0,0 +1,6 @@ +- endpoint: http://localhost:9090 + metrics: + - metrics.yml + indexer: + type: local + metricsDirectory: metrics diff --git a/contrib/perf/metric-endpoint.yml b/contrib/perf/metric-endpoint.yml new file mode 100644 index 0000000000..bafaf18ebb --- /dev/null +++ b/contrib/perf/metric-endpoint.yml @@ -0,0 +1,15 @@ +- endpoint: http://localhost:9090 + metrics: + - metrics.yml + indexer: + type: local + metricsDirectory: metrics + +- endpoint: http://localhost:9090 + metrics: + - metrics.yml + indexer: + esServers: ["{{.ES_SERVER}}"] + insecureSkipVerify: true + defaultIndex: upstream-ovnk-kube-burner + type: opensearch diff --git a/contrib/perf/metrics.yml b/contrib/perf/metrics.yml new file mode 100644 index 0000000000..f667e252d1 --- /dev/null +++ b/contrib/perf/metrics.yml @@ -0,0 +1,123 @@ +# API server +- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 + metricName: API99thLatency + +- query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 + metricName: APIRequestRate + +- query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 + metricName: APIInflightRequests + +# Containers & pod metrics + +- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!~"POD"}[2m]) * 100) by (container, pod, namespace, node)) > 0 + metricName: containerCPU + +- query: sum(container_memory_rss{name!="",container!~"POD"}) by (container, pod, namespace, node) + metricName: containerMemory + +- query: sum(irate(container_cpu_usage_seconds_total{}[2m]) * 100) by (pod, namespace, node) + metricName: podCPU + +- query: sum(container_memory_rss{}) by (pod, namespace, node) + metricName: podMemory + +- query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerDiskUsage + +# Kubelet & CRI-O metrics +- query: sum(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} + metricName: kubeletCPU + +- query: sum(process_resident_memory_bytes{service="kubelet",job="kubelet"}) by (node) and on (node) kube_node_role{role="worker"} + metricName: kubeletMemory + +- query: sum(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} + metricName: crioCPU + +- query: sum(process_resident_memory_bytes{service="kubelet",job="crio"}) by (node) and on (node) kube_node_role{role="worker"} + metricName: crioMemory + +# Node metrics +- query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 + metricName: nodeCPU + +- query: avg(node_memory_MemAvailable_bytes) by (instance) + metricName: nodeMemoryAvailable + +- query: avg(node_memory_Active_bytes) by (instance) + metricName: nodeMemoryActive + +- query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) + metricName: nodeMemoryCached+nodeMemoryBuffers + +- query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: rxNetworkBytes + +- query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: txNetworkBytes + +- query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskWrittenBytes + +- query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskReadBytes + +- query: sum(rate(etcd_server_leader_changes_seen_total[2m])) + metricName: etcdLeaderChangesRate + +# Etcd metrics +- query: etcd_server_is_leader > 0 + metricName: etcdServerIsLeader + +- query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskBackendCommitDurationSeconds + +- query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskWalFsyncDurationSeconds + +- query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) + metricName: 99thEtcdRoundTripTimeSeconds + +- query: etcd_mvcc_db_total_size_in_bytes + metricName: etcdDBPhysicalSizeBytes + +- query: etcd_mvcc_db_total_size_in_use_in_bytes + metricName: etcdDBLogicalSizeBytes + +- query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 + metricName: etcdObjectCount + +- query: sum by (cluster_version)(etcd_cluster_version) + metricName: etcdVersion + instant: true + +# Cluster metrics +- query: sum(kube_namespace_status_phase) by (phase) > 0 + metricName: namespaceCount + +- query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount + +- query: count(kube_secret_info{}) + metricName: secretCount + +- query: count(kube_deployment_spec_replicas{}) + metricName: deploymentCount + +- query: count(kube_configmap_info{}) + metricName: configmapCount + +- query: count(kube_service_info{}) + metricName: serviceCount + +- query: kube_node_role + metricName: nodeRoles + instant: true + +- query: sum(kube_node_status_condition{status="true"}) by (condition) + metricName: nodeStatus + +- query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true diff --git a/contrib/perf/performance-meta.yml b/contrib/perf/performance-meta.yml new file mode 100644 index 0000000000..5b80bb930b --- /dev/null +++ b/contrib/perf/performance-meta.yml @@ -0,0 +1,3 @@ +github_ref: $GITHUB_HEAD_REF +github_ref_name: $GITHUB_REF_NAME +github_sha: $GITHUB_SHA diff --git a/contrib/perf/workloads/cudn-density-l2-noPods.yml b/contrib/perf/workloads/cudn-density-l2-noPods.yml new file mode 100644 index 0000000000..401199555a --- /dev/null +++ b/contrib/perf/workloads/cudn-density-l2-noPods.yml @@ -0,0 +1,46 @@ +--- +global: + measurements: + - name: podLatency + - name: pprof + pprofInterval: 1m + pprofDirectory: pprof-data + pprofTargets: + - name: ovnkube-controller + namespace: "ovn-kubernetes" + labelSelector: {app: ovnkube-node} + url: http://localhost:9410/debug/pprof/profile?seconds=30 + - name: ovnkube-control-plane + namespace: "ovn-kubernetes" + labelSelector: {name: ovnkube-control-plane} + url: http://localhost:9411/debug/pprof/profile?seconds=30 + - name: ovnkube-controller-heap + namespace: "ovn-kubernetes" + labelSelector: {app: ovnkube-node} + url: http://localhost:9410/debug/pprof/heap?seconds=30 + - name: ovnkube-control-plane-heap + namespace: "ovn-kubernetes" + labelSelector: {name: ovnkube-control-plane} + url: http://localhost:9411/debug/pprof/heap?seconds=30 +jobs: + - name: cudn-density-l2-nopods + jobIterations: 200 + qps: 10 + burst: 10 + namespacedIterations: true + namespace: cudn-density-l2-nopods + waitWhenFinished: true + podWait: false + preLoadImages: false + preLoadPeriod: 2m + # Disabling churn until https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5883 is resolved + #churnConfig: + # percent: 10 + # cycles: 10 + # mode: objects + objects: + - objectTemplate: workloads/templates/udn-density/cudn_l2.yml + replicas: 1 + - objectTemplate: workloads/templates/udn-density/cudn_ns.yml + replicas: 1 + diff --git a/contrib/perf/workloads/kubelet-density-cni.yml b/contrib/perf/workloads/kubelet-density-cni.yml new file mode 100644 index 0000000000..dd3f193343 --- /dev/null +++ b/contrib/perf/workloads/kubelet-density-cni.yml @@ -0,0 +1,49 @@ +--- +global: + measurements: + - name: podLatency + - name: pprof + pprofInterval: 1m + pprofDirectory: pprof-data + pprofTargets: + - name: ovnkube-controller + namespace: "ovn-kubernetes" + labelSelector: {app: ovnkube-node} + url: http://localhost:9410/debug/pprof/profile?seconds=30 + - name: ovnkube-control-plane + namespace: "ovn-kubernetes" + labelSelector: {name: ovnkube-control-plane} + url: http://localhost:9411/debug/pprof/profile?seconds=30 + - name: ovnkube-controller-heap + namespace: "ovn-kubernetes" + labelSelector: {app: ovnkube-node} + url: http://localhost:9410/debug/pprof/heap?seconds=30 + - name: ovnkube-control-plane-heap + namespace: "ovn-kubernetes" + labelSelector: {name: ovnkube-control-plane} + url: http://localhost:9411/debug/pprof/heap?seconds=30 +jobs: + - name: kubelet-density-cni + jobIterations: 100 + qps: 10 + burst: 10 + namespacedIterations: false + namespace: kubelet-density-cni + waitWhenFinished: true + podWait: false + preLoadImages: true + preLoadPeriod: 2m + churnConfig: + percent: 10 + cycles: 10 + mode: objects + objects: + + - objectTemplate: workloads/templates/kubelet-density-cni/webserver-deployment.yml + replicas: 1 + + - objectTemplate: workloads/templates/kubelet-density-cni/webserver-service.yml + replicas: 1 + + - objectTemplate: workloads/templates/kubelet-density-cni/curl-deployment.yml + replicas: 1 diff --git a/contrib/perf/workloads/templates/kubelet-density-cni/curl-deployment.yml b/contrib/perf/workloads/templates/kubelet-density-cni/curl-deployment.yml new file mode 100644 index 0000000000..d437769475 --- /dev/null +++ b/contrib/perf/workloads/templates/kubelet-density-cni/curl-deployment.yml @@ -0,0 +1,40 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: curl-{{.Replica}}-{{.Iteration}} +spec: + template: + metadata: + labels: + name: curl-{{.Replica}}-{{.Iteration}} + spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + containers: + - name: curlapp + image: quay.io/cloud-bulldozer/curl:latest + command: ["sleep", "inf"] + env: + - name: WEBSERVER_HOSTNAME + value: webserver-{{.Replica}}-{{.Iteration}} + - name: WEBSERVER_PORT + value: "8080" + imagePullPolicy: IfNotPresent + securityContext: + privileged: false + startupProbe: + exec: + command: + - "/bin/sh" + - "-c" + - "curl ${WEBSERVER_HOSTNAME}:${WEBSERVER_PORT}" + periodSeconds: 1 + timeoutSeconds: 1 + failureThreshold: 600 + restartPolicy: Always + replicas: 1 + selector: + matchLabels: + name: curl-{{.Replica}}-{{.Iteration}} + strategy: + type: RollingUpdate diff --git a/contrib/perf/workloads/templates/kubelet-density-cni/webserver-deployment.yml b/contrib/perf/workloads/templates/kubelet-density-cni/webserver-deployment.yml new file mode 100644 index 0000000000..b9c904154a --- /dev/null +++ b/contrib/perf/workloads/templates/kubelet-density-cni/webserver-deployment.yml @@ -0,0 +1,28 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: webserver-{{.Replica}}-{{.Iteration}} +spec: + template: + metadata: + labels: + name: webserver-{{.Replica}}-{{.Iteration}} + spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + containers: + - name: webserver + image: quay.io/cloud-bulldozer/sampleapp:latest + ports: + - containerPort: 8080 + protocol: TCP + imagePullPolicy: IfNotPresent + securityContext: + privileged: false + restartPolicy: Always + replicas: 1 + selector: + matchLabels: + name: webserver-{{.Replica}}-{{.Iteration}} + strategy: + type: RollingUpdate diff --git a/contrib/perf/workloads/templates/kubelet-density-cni/webserver-service.yml b/contrib/perf/workloads/templates/kubelet-density-cni/webserver-service.yml new file mode 100644 index 0000000000..a569151b82 --- /dev/null +++ b/contrib/perf/workloads/templates/kubelet-density-cni/webserver-service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: webserver-{{.Replica}}-{{.Iteration}} +spec: + selector: + name: webserver-{{.Replica}}-{{.Iteration}} + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + type: ClusterIP diff --git a/contrib/perf/workloads/templates/udn-density/cudn_l2.yml b/contrib/perf/workloads/templates/udn-density/cudn_l2.yml new file mode 100644 index 0000000000..81be6ac9da --- /dev/null +++ b/contrib/perf/workloads/templates/udn-density/cudn_l2.yml @@ -0,0 +1,14 @@ +--- +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l2-network-{{.Iteration}} +spec: + namespaceSelector: + matchLabels: + cudn-scale: "{{.JobName}}-{{.Iteration}}" + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.132.0.0/16"] \ No newline at end of file diff --git a/contrib/perf/workloads/templates/udn-density/cudn_ns.yml b/contrib/perf/workloads/templates/udn-density/cudn_ns.yml new file mode 100644 index 0000000000..8f6387a459 --- /dev/null +++ b/contrib/perf/workloads/templates/udn-density/cudn_ns.yml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: "{{.JobName}}-{{.Iteration}}" + labels: + k8s.ovn.org/primary-user-defined-network: "" + cudn-scale: "{{.JobName}}-{{.Iteration}}" \ No newline at end of file diff --git a/contrib/perf/workloads/templates/udn-density/deployment-client.yml b/contrib/perf/workloads/templates/udn-density/deployment-client.yml new file mode 100644 index 0000000000..d153adfabe --- /dev/null +++ b/contrib/perf/workloads/templates/udn-density/deployment-client.yml @@ -0,0 +1,57 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: client-{{.Replica}} +spec: + replicas: 1 + selector: + matchLabels: + name: client-{{.Replica}} + template: + metadata: + labels: + name: client-{{.Replica}} + app: client + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: client + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: Exists + - key: node-role.kubernetes.io/infra + operator: DoesNotExist + - key: node-role.kubernetes.io/workload + operator: DoesNotExist + containers: + - name: client-app + image: quay.io/cloud-bulldozer/curl:latest + command: ["sleep", "inf"] + resources: + requests: + memory: "10Mi" + cpu: "10m" + imagePullPolicy: IfNotPresent + securityContext: + privileged: false + volumeMounts: + - name: podinfo + mountPath: /etc/podlabels + volumes: + - name: podinfo + downwardAPI: + items: + - path: "labels" + fieldRef: + fieldPath: metadata.labels + restartPolicy: Always + strategy: + type: RollingUpdate diff --git a/contrib/perf/workloads/templates/udn-density/udn_l2.yml b/contrib/perf/workloads/templates/udn-density/udn_l2.yml new file mode 100644 index 0000000000..fe0c222dd6 --- /dev/null +++ b/contrib/perf/workloads/templates/udn-density/udn_l2.yml @@ -0,0 +1,10 @@ +--- +apiVersion: k8s.ovn.org/v1 +kind: UserDefinedNetwork +metadata: + name: l2-network-{{.Iteration}} +spec: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.132.0.0/16"] \ No newline at end of file diff --git a/contrib/perf/workloads/templates/udn-density/udn_l3.yml b/contrib/perf/workloads/templates/udn-density/udn_l3.yml new file mode 100644 index 0000000000..0a8de7688d --- /dev/null +++ b/contrib/perf/workloads/templates/udn-density/udn_l3.yml @@ -0,0 +1,13 @@ +--- +apiVersion: k8s.ovn.org/v1 +kind: UserDefinedNetwork +metadata: + name: l3-network-{{.Iteration}} +spec: + topology: Layer3 + layer3: + role: Primary + subnets: + - cidr: 10.132.0.0/16 + hostSubnet: 24 + mtu: 1300 diff --git a/contrib/perf/workloads/udn-density-l2-noPods.yml b/contrib/perf/workloads/udn-density-l2-noPods.yml new file mode 100644 index 0000000000..310f3c87dd --- /dev/null +++ b/contrib/perf/workloads/udn-density-l2-noPods.yml @@ -0,0 +1,48 @@ +--- +global: + measurements: + - name: podLatency + - name: pprof + pprofInterval: 1m + pprofDirectory: pprof-data + pprofTargets: + - name: ovnkube-controller + namespace: "ovn-kubernetes" + labelSelector: {app: ovnkube-node} + url: http://localhost:9410/debug/pprof/profile?seconds=30 + - name: ovnkube-control-plane + namespace: "ovn-kubernetes" + labelSelector: {name: ovnkube-control-plane} + url: http://localhost:9411/debug/pprof/profile?seconds=30 + - name: ovnkube-controller-heap + namespace: "ovn-kubernetes" + labelSelector: {app: ovnkube-node} + url: http://localhost:9410/debug/pprof/heap?seconds=30 + - name: ovnkube-control-plane-heap + namespace: "ovn-kubernetes" + labelSelector: {name: ovnkube-control-plane} + url: http://localhost:9411/debug/pprof/heap?seconds=30 +jobs: + - name: udn-density-l2-nopods + jobIterations: 200 + qps: 10 + burst: 10 + namespacedIterations: true + namespace: udn-density-l2-nopods + waitWhenFinished: true + podWait: false + preLoadImages: false + preLoadPeriod: 2m + churnConfig: + percent: 10 + cycles: 5 + delay: 2m + namespaceLabels: + security.openshift.io/scc.podSecurityLabelSync: false + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + k8s.ovn.org/primary-user-defined-network: "" + objects: + - objectTemplate: workloads/templates/udn-density/udn_l2.yml + replicas: 1 diff --git a/contrib/perf/workloads/udn-density-l2-pods.yml b/contrib/perf/workloads/udn-density-l2-pods.yml new file mode 100644 index 0000000000..d6ba0170cd --- /dev/null +++ b/contrib/perf/workloads/udn-density-l2-pods.yml @@ -0,0 +1,50 @@ +--- +global: + measurements: + - name: podLatency + - name: pprof + pprofInterval: 1m + pprofDirectory: pprof-data + pprofTargets: + - name: ovnkube-controller + namespace: "ovn-kubernetes" + labelSelector: {app: ovnkube-node} + url: http://localhost:9410/debug/pprof/profile?seconds=30 + - name: ovnkube-control-plane + namespace: "ovn-kubernetes" + labelSelector: {name: ovnkube-control-plane} + url: http://localhost:9411/debug/pprof/profile?seconds=30 + - name: ovnkube-controller-heap + namespace: "ovn-kubernetes" + labelSelector: {app: ovnkube-node} + url: http://localhost:9410/debug/pprof/heap?seconds=30 + - name: ovnkube-control-plane-heap + namespace: "ovn-kubernetes" + labelSelector: {name: ovnkube-control-plane} + url: http://localhost:9411/debug/pprof/heap?seconds=30 +jobs: + - name: udn-density-l2-pods + jobIterations: 100 + qps: 10 + burst: 10 + namespacedIterations: true + namespace: udn-density-l2-pods + waitWhenFinished: true + podWait: false + preLoadImages: false + preLoadPeriod: 2m + churnConfig: + percent: 10 + cycles: 5 + delay: 2m + namespaceLabels: + security.openshift.io/scc.podSecurityLabelSync: false + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + k8s.ovn.org/primary-user-defined-network: "" + objects: + - objectTemplate: workloads/templates/udn-density/udn_l2.yml + replicas: 1 + - objectTemplate: workloads/templates/udn-density/deployment-client.yml + replicas: 1 diff --git a/contrib/prometheus-values.yaml b/contrib/prometheus-values.yaml new file mode 100644 index 0000000000..ff703323d9 --- /dev/null +++ b/contrib/prometheus-values.yaml @@ -0,0 +1,31 @@ +prometheusOperator: + tls: + enabled: false + admissionWebhooks: + enabled: false + patch: + enabled: false + nodeSelector: + prometheus-node: "true" + +prometheus: + prometheusSpec: + nodeSelector: + prometheus-node: "true" + +alertmanager: + alertmanagerSpec: + nodeSelector: + prometheus-node: "true" + +grafana: + nodeSelector: + prometheus-node: "true" + +kube-state-metrics: + nodeSelector: + prometheus-node: "true" + +prometheus-node-exporter: + nodeSelector: + prometheus-node: "true" \ No newline at end of file diff --git a/dist/images/Dockerfile.fedora b/dist/images/Dockerfile.fedora index a2d51a3976..96b4fc7c30 100644 --- a/dist/images/Dockerfile.fedora +++ b/dist/images/Dockerfile.fedora @@ -22,7 +22,7 @@ ARG TARGETOS ARG TARGETARCH WORKDIR /workspace -RUN apt-get update && apt-get install -y -qq make git +RUN dnf install --best --refresh -y make git COPY ${OVN_KUBERNETES_DIR} ovn-kubernetes RUN cd ovn-kubernetes/dist/images && \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} make bld @@ -107,7 +107,7 @@ RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM" # Install koji, don't clean dnf cache we will install extra packages at # Final stage -RUN dnf install --best --refresh -y --setopt=tsflags=nodocs koji +RUN dnf install --best --refresh -y --setopt=tsflags=nodocs koji RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] || [ -z "$TARGETPLATFORM" ] ; then koji download-build $ovnver --arch=x86_64 ; \ else koji download-build $ovnver --arch=aarch64 ; fi diff --git a/dist/images/Makefile b/dist/images/Makefile index ee62016d97..9efc50433c 100644 --- a/dist/images/Makefile +++ b/dist/images/Makefile @@ -28,7 +28,7 @@ OVN_FROM := source OVN_GITSHA := $(shell git ls-remote "${OVN_REPO}" "${OVN_GITREF}" | sort -k2 -V |tail -1 | awk '{ print $$1 }') endif GO_VERSION ?= 1.24 -GO_IMAGE = quay.io/lib/golang:${GO_VERSION} +GO_IMAGE = quay.io/projectquay/golang:${GO_VERSION} OCI_BIN ?= docker diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh index cb8a0aa699..4430d29143 100755 --- a/dist/images/daemonset.sh +++ b/dist/images/daemonset.sh @@ -69,12 +69,16 @@ OVN_EGRESSFIREWALL_ENABLE= OVN_EGRESSQOS_ENABLE= OVN_EGRESSSERVICE_ENABLE= OVN_MULTI_NETWORK_ENABLE= -OVN_NETWORK_SEGMENTATION_ENABLE= +OVN_NETWORK_SEGMENTATION_ENABLE="false" OVN_NETWORK_CONNECT_ENABLE= OVN_PRE_CONF_UDN_ADDR_ENABLE= +OVN_DYNAMIC_UDN_ALLOCATION= +OVN_DYNAMIC_UDN_GRACE_PERIOD= OVN_ROUTE_ADVERTISEMENTS_ENABLE= +OVN_EVPN_ENABLE= OVN_ADVERTISE_DEFAULT_NETWORK= OVN_ADVERTISED_UDN_ISOLATION_MODE= +OVN_NO_OVERLAY_ENABLE= OVN_V4_JOIN_SUBNET="" OVN_V6_JOIN_SUBNET="" OVN_V4_MASQUERADE_SUBNET="" @@ -90,6 +94,7 @@ OVN_IPFIX_CACHE_ACTIVE_TIMEOUT="" OVN_HOST_NETWORK_NAMESPACE="" OVN_EX_GW_NETWORK_INTERFACE="" OVNKUBE_NODE_MGMT_PORT_NETDEV="" +OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME="" OVNKUBE_CONFIG_DURATION_ENABLE= OVNKUBE_METRICS_SCALE_ENABLE= OVN_STATELESS_NETPOL_ENABLE="false" @@ -108,6 +113,7 @@ IN_UPGRADE= OVN_NORTHD_BACKOFF_INTERVAL= OVN_OBSERV_ENABLE="false" ENABLE_COREDUMPS="false" +METRICS_IP="" # Parse parameters given as arguments to this script. while [ "$1" != "" ]; do @@ -279,15 +285,27 @@ while [ "$1" != "" ]; do --preconfigured-udn-addresses-enable) OVN_PRE_CONF_UDN_ADDR_ENABLE=$VALUE ;; + --enable-dynamic-udn-allocation) + OVN_DYNAMIC_UDN_ALLOCATION=$VALUE + ;; + --udn-deletion-grace-period) + OVN_DYNAMIC_UDN_GRACE_PERIOD=$VALUE + ;; --route-advertisements-enable) OVN_ROUTE_ADVERTISEMENTS_ENABLE=$VALUE ;; + --evpn-enable) + OVN_EVPN_ENABLE=$VALUE + ;; --advertise-default-network) OVN_ADVERTISE_DEFAULT_NETWORK=$VALUE ;; --advertised-udn-isolation-mode) OVN_ADVERTISED_UDN_ISOLATION_MODE=$VALUE ;; + --no-overlay-enable) + OVN_NO_OVERLAY_ENABLE=$VALUE + ;; --egress-service-enable) OVN_EGRESSSERVICE_ENABLE=$VALUE ;; @@ -339,12 +357,18 @@ while [ "$1" != "" ]; do --ovnkube-node-mgmt-port-dp-resource-name) OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME=$VALUE ;; + --mgmt-port-vfs-count) + MGMT_PORT_VFS_COUNT=$VALUE + ;; --ovnkube-config-duration-enable) OVNKUBE_CONFIG_DURATION_ENABLE=$VALUE ;; --ovnkube-metrics-scale-enable) OVNKUBE_METRICS_SCALE_ENABLE=$VALUE ;; + --metrics-ip) + METRICS_IP=$VALUE + ;; --in-upgrade) IN_UPGRADE=true ;; @@ -387,9 +411,33 @@ while [ "$1" != "" ]; do --no-hostsubnet-label) OVN_NOHOSTSUBNET_LABEL=$VALUE ;; - --ovn_disable_requestedchassis) + --ovn-disable-requestedchassis) OVN_DISABLE_REQUESTEDCHASSIS=$value ;; + --metrics-port) + METRICS_PORT=$value + ;; + --dpuhost-cluster-net-cidr) + DPUHOST_CLUSTER_NET_CIDR=$value + ;; + --dpuhost-cluster-svc-cidr) + DPUHOST_CLUSTER_SVC_CIDR=$value + ;; + --dpuhost-cluster-k8s-apiserver) + DPUHOST_CLUSTER_K8S_APISERVER=$value + ;; + --dpuhost-cluster-k8s-token) + DPUHOST_CLUSTER_K8S_TOKEN=$value + ;; + --dpuhost-cluster-k8s-cacert-data) + DPUHOST_CLUSTER_K8S_CACERT_DATA=$value + ;; + --dpuhost-cluster-k8s-token-file) + DPUHOST_CLUSTER_K8S_TOKEN_FILE=$value + ;; + --dpuhost-cluster-k8s-cacert) + DPUHOST_CLUSTER_K8S_CACERT=$value + ;; *) echo "WARNING: unknown parameter \"$PARAM\"" exit 1 @@ -484,10 +532,14 @@ ovn_pre_conf_udn_addr_enable=${OVN_PRE_CONF_UDN_ADDR_ENABLE} echo "ovn_pre_conf_udn_addr_enable: ${ovn_pre_conf_udn_addr_enable}" ovn_route_advertisements_enable=${OVN_ROUTE_ADVERTISEMENTS_ENABLE} echo "ovn_route_advertisements_enable: ${ovn_route_advertisements_enable}" +ovn_evpn_enable=${OVN_EVPN_ENABLE} +echo "ovn_evpn_enable: ${ovn_evpn_enable}" ovn_advertise_default_network=${OVN_ADVERTISE_DEFAULT_NETWORK} echo "ovn_advertise_default_network: ${ovn_advertise_default_network}" ovn_advertised_udn_isolation_mode=${OVN_ADVERTISED_UDN_ISOLATION_MODE} echo "ovn_advertised_udn_isolation_mode: ${ovn_advertised_udn_isolation_mode}" +ovn_no_overlay_enable=${OVN_NO_OVERLAY_ENABLE} +echo "ovn_no_overlay_enable: ${ovn_no_overlay_enable}" ovn_hybrid_overlay_net_cidr=${OVN_HYBRID_OVERLAY_NET_CIDR} echo "ovn_hybrid_overlay_net_cidr: ${ovn_hybrid_overlay_net_cidr}" ovn_disable_snat_multiple_gws=${OVN_DISABLE_SNAT_MULTIPLE_GWS} @@ -560,10 +612,16 @@ ovn_ex_gw_networking_interface=${OVN_EX_GW_NETWORK_INTERFACE} echo "ovn_ex_gw_networking_interface: ${ovn_ex_gw_networking_interface}" ovnkube_node_mgmt_port_netdev=${OVNKUBE_NODE_MGMT_PORT_NETDEV} echo "ovnkube_node_mgmt_port_netdev: ${ovnkube_node_mgmt_port_netdev}" +ovnkube_node_mgmt_port_dp_resource_name=${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME} +echo "ovnkube_node_mgmt_port_dp_resource_name: ${ovnkube_node_mgmt_port_dp_resource_name}" +mgmt_port_vfs_count=${MGMT_PORT_VFS_COUNT:-1} +echo "mgmt_port_vfs_count: ${mgmt_port_vfs_count}" ovnkube_config_duration_enable=${OVNKUBE_CONFIG_DURATION_ENABLE} echo "ovnkube_config_duration_enable: ${ovnkube_config_duration_enable}" ovnkube_metrics_scale_enable=${OVNKUBE_METRICS_SCALE_ENABLE} echo "ovnkube_metrics_scale_enable: ${ovnkube_metrics_scale_enable}" +metrics_ip=${METRICS_IP} +echo "metrics_ip: ${metrics_ip}" ovn_stateless_netpol_enable=${OVN_STATELESS_NETPOL_ENABLE} echo "ovn_stateless_netpol_enable: ${ovn_stateless_netpol_enable}" ovnkube_compact_mode_enable=${COMPACT_MODE:-"false"} @@ -572,6 +630,10 @@ ovn_enable_interconnect=${OVN_ENABLE_INTERCONNECT} echo "ovn_enable_interconnect: ${ovn_enable_interconnect}" ovn_enable_multi_external_gateway=${OVN_ENABLE_MULTI_EXTERNAL_GATEWAY} echo "ovn_enable_multi_external_gateway: ${ovn_enable_multi_external_gateway}" +ovn_enable_dynamic_udn_allocation=${OVN_DYNAMIC_UDN_ALLOCATION} +echo "ovn_enable_dynamic_udn_allocation: ${ovn_enable_dynamic_udn_allocation}" +ovn_dynamic_udn_grace_period=${OVN_DYNAMIC_UDN_GRACE_PERIOD} +echo "ovn_dynamic_udn_grace_period=${ovn_dynamic_udn_grace_period}" ovn_enable_ovnkube_identity=${OVN_ENABLE_OVNKUBE_IDENTITY} echo "ovn_enable_ovnkube_identity: ${ovn_enable_ovnkube_identity}" @@ -633,8 +695,12 @@ ovn_image=${ovnkube_image} \ ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ ovn_network_connect_enable=${ovn_network_connect_enable} \ ovn_pre_conf_udn_addr_enable=${ovn_pre_conf_udn_addr_enable} \ + ovn_enable_dynamic_udn_allocation=${ovn_enable_dynamic_udn_allocation} \ + ovn_dynamic_udn_grace_period=${ovn_dynamic_udn_grace_period} \ ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ + ovn_evpn_enable=${ovn_evpn_enable} \ ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ ovn_egress_service_enable=${ovn_egress_service_enable} \ ovn_ssl_en=${ovn_ssl_en} \ ovn_remote_probe_interval=${ovn_remote_probe_interval} \ @@ -657,6 +723,7 @@ ovn_image=${ovnkube_image} \ ovn_observ_enable=${ovn_observ_enable} \ ovn_network_qos_enable=${ovn_network_qos_enable} \ enable_coredumps=${enable_coredumps} \ + metrics_ip=${metrics_ip} \ ovnkube_app_name=ovnkube-node \ jinjanate ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node.yaml @@ -690,7 +757,11 @@ ovn_image=${ovnkube_image} \ ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ ovn_network_connect_enable=${ovn_network_connect_enable} \ ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ + ovn_evpn_enable=${ovn_evpn_enable} \ ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ + ovn_enable_dynamic_udn_allocation=${ovn_enable_dynamic_udn_allocation} \ + ovn_dynamic_udn_grace_period=${ovn_dynamic_udn_grace_period} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ ovn_egress_service_enable=${ovn_egress_service_enable} \ ovn_ssl_en=${ovn_ssl_en} \ ovn_remote_probe_interval=${ovn_remote_probe_interval} \ @@ -712,6 +783,7 @@ ovn_image=${ovnkube_image} \ ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \ ovn_observ_enable=${ovn_observ_enable} \ ovn_network_qos_enable=${ovn_network_qos_enable} \ + metrics_ip=${metrics_ip} \ ovnkube_app_name=ovnkube-node-dpu \ jinjanate ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node-dpu.yaml @@ -746,6 +818,8 @@ ovn_image=${image} \ ovn_egress_ip_healthcheck_port=${ovn_egress_ip_healthcheck_port} \ ovn_egress_service_enable=${ovn_egress_service_enable} \ ovn_netflow_targets=${ovn_netflow_targets} \ + ovn_enable_dynamic_udn_allocation=${ovn_enable_dynamic_udn_allocation} \ + ovn_dynamic_udn_grace_period=${ovn_dynamic_udn_grace_period} \ ovn_sflow_targets=${ovn_sflow_targets} \ ovn_ipfix_targets=${ovn_ipfix_targets} \ ovn_ipfix_sampling=${ovn_ipfix_sampling} \ @@ -753,8 +827,14 @@ ovn_image=${image} \ ovn_ipfix_cache_active_timeout=${ovn_ipfix_cache_active_timeout} \ ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \ ovnkube_node_mgmt_port_netdev=${ovnkube_node_mgmt_port_netdev} \ + ovnkube_node_mgmt_port_dp_resource_name=${ovnkube_node_mgmt_port_dp_resource_name} \ + mgmt_port_vfs_count=${mgmt_port_vfs_count} \ ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \ ovn_network_qos_enable=${ovn_network_qos_enable} \ + metrics_ip=${metrics_ip} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ + ovn_enable_interconnect=${ovn_enable_interconnect} \ + ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ ovnkube_app_name=ovnkube-node-dpu-host \ jinjanate ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node-dpu-host.yaml @@ -768,6 +848,7 @@ ovn_image=${ovnkube_image} \ ovnkube_libovsdb_client_logfile=${ovnkube_libovsdb_client_logfile} \ ovnkube_config_duration_enable=${ovnkube_config_duration_enable} \ ovnkube_metrics_scale_enable=${ovnkube_metrics_scale_enable} \ + metrics_ip=${metrics_ip} \ ovn_acl_logging_rate_limit=${ovn_acl_logging_rate_limit} \ ovn_hybrid_overlay_net_cidr=${ovn_hybrid_overlay_net_cidr} \ ovn_hybrid_overlay_enable=${ovn_hybrid_overlay_enable} \ @@ -790,7 +871,11 @@ ovn_image=${ovnkube_image} \ ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ ovn_network_connect_enable=${ovn_network_connect_enable} \ ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ + ovn_evpn_enable=${ovn_evpn_enable} \ ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ + ovn_enable_dynamic_udn_allocation=${ovn_enable_dynamic_udn_allocation} \ + ovn_dynamic_udn_grace_period=${ovn_dynamic_udn_grace_period} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ ovn_egress_service_enable=${ovn_egress_service_enable} \ ovn_ssl_en=${ovn_ssl_en} \ ovn_master_count=${ovn_master_count} \ @@ -811,6 +896,7 @@ ovn_image=${ovnkube_image} \ ovn_nohostsubnet_label=${ovn_nohostsubnet_label} \ ovn_disable_requestedchassis=${ovn_disable_requestedchassis} \ enable_coredumps=${enable_coredumps} \ + metrics_ip=${metrics_ip} \ jinjanate ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml ovn_image=${ovnkube_image} \ @@ -822,6 +908,7 @@ ovn_image=${ovnkube_image} \ ovnkube_logfile_maxage=${ovnkube_logfile_maxage} \ ovnkube_config_duration_enable=${ovnkube_config_duration_enable} \ ovnkube_metrics_scale_enable=${ovnkube_metrics_scale_enable} \ + metrics_ip=${metrics_ip} \ ovn_acl_logging_rate_limit=${ovn_acl_logging_rate_limit} \ ovn_hybrid_overlay_net_cidr=${ovn_hybrid_overlay_net_cidr} \ ovn_hybrid_overlay_enable=${ovn_hybrid_overlay_enable} \ @@ -843,7 +930,11 @@ ovn_image=${ovnkube_image} \ ovn_network_connect_enable=${ovn_network_connect_enable} \ ovn_pre_conf_udn_addr_enable=${ovn_pre_conf_udn_addr_enable} \ ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ + ovn_evpn_enable=${ovn_evpn_enable} \ ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ + ovn_enable_dynamic_udn_allocation=${ovn_enable_dynamic_udn_allocation} \ + ovn_dynamic_udn_grace_period=${ovn_dynamic_udn_grace_period} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ ovn_egress_service_enable=${ovn_egress_service_enable} \ ovn_ssl_en=${ovn_ssl_en} \ ovn_master_count=${ovn_master_count} \ @@ -859,6 +950,7 @@ ovn_image=${ovnkube_image} \ ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \ ovn_observ_enable=${ovn_observ_enable} \ enable_coredumps=${enable_coredumps} \ + metrics_ip=${metrics_ip} \ jinjanate ../templates/ovnkube-control-plane.yaml.j2 -o ${output_dir}/ovnkube-control-plane.yaml ovn_image=${image} \ @@ -907,6 +999,7 @@ ovn_image=${ovnkube_image} \ ovnkube_libovsdb_client_logfile=${ovnkube_libovsdb_client_logfile} \ ovnkube_config_duration_enable=${ovnkube_config_duration_enable} \ ovnkube_metrics_scale_enable=${ovnkube_metrics_scale_enable} \ + metrics_ip=${metrics_ip} \ ovn_hybrid_overlay_net_cidr=${ovn_hybrid_overlay_net_cidr} \ ovn_hybrid_overlay_enable=${ovn_hybrid_overlay_enable} \ ovn_disable_snat_multiple_gws=${ovn_disable_snat_multiple_gws} \ @@ -928,7 +1021,9 @@ ovn_image=${ovnkube_image} \ ovn_network_connect_enable=${ovn_network_connect_enable} \ ovn_pre_conf_udn_addr_enable=${ovn_pre_conf_udn_addr_enable} \ ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ + ovn_evpn_enable=${ovn_evpn_enable} \ ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ ovn_egress_service_enable=${ovn_egress_service_enable} \ ovn_ssl_en=${ovn_ssl_en} \ ovn_remote_probe_interval=${ovn_remote_probe_interval} \ @@ -941,6 +1036,8 @@ ovn_image=${ovnkube_image} \ ovn_sflow_targets=${ovn_sflow_targets} \ ovn_ipfix_targets=${ovn_ipfix_targets} \ ovn_ipfix_sampling=${ovn_ipfix_sampling} \ + ovn_enable_dynamic_udn_allocation=${ovn_enable_dynamic_udn_allocation} \ + ovn_dynamic_udn_grace_period=${ovn_dynamic_udn_grace_period} \ ovn_ipfix_cache_max_flows=${ovn_ipfix_cache_max_flows} \ ovn_ipfix_cache_active_timeout=${ovn_ipfix_cache_active_timeout} \ ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \ @@ -963,6 +1060,102 @@ ovn_image=${ovnkube_image} \ enable_coredumps=${enable_coredumps} \ jinjanate ../templates/ovnkube-single-node-zone.yaml.j2 -o ${output_dir}/ovnkube-single-node-zone.yaml +# ovnkube-single-node-zone-dpu +dpuhost_cluster_net_cidr=${DPUHOST_CLUSTER_NET_CIDR:-"10.244.0.0/16/24"} +dpuhost_cluster_svc_cidr=${DPUHOST_CLUSTER_SVC_CIDR:-"10.96.0.0/16"} +dpuhost_cluster_k8s_apiserver=${DPUHOST_CLUSTER_K8S_APISERVER:-"https://172.25.0.2:6443"} +dpuhost_cluster_k8s_token=${DPUHOST_CLUSTER_K8S_TOKEN:-""} +dpuhost_cluster_k8s_cacert_data=${DPUHOST_CLUSTER_K8S_CACERT_DATA:-""} +dpuhost_cluster_k8s_token_file=${DPUHOST_CLUSTER_K8S_TOKEN_FILE:-""} +dpuhost_cluster_k8s_cacert=${DPUHOST_CLUSTER_K8S_CACERT:-""} +mtu=${OVN_MTU:-1400} +metrics_port=${METRICS_PORT:-9476} +echo "dpuhost_cluster_net_cidr: ${dpuhost_cluster_net_cidr}" +echo "dpuhost_cluster_svc_cidr: ${dpuhost_cluster_svc_cidr}" +echo "dpuhost_cluster_k8s_apiserver: ${dpuhost_cluster_k8s_apiserver}" +echo "dpuhost_cluster_k8s_token: ${dpuhost_cluster_k8s_token}" +echo "dpuhost_cluster_k8s_cacert_data: ${dpuhost_cluster_k8s_cacert_data}" +echo "dpuhost_cluster_k8s_token_file: ${dpuhost_cluster_k8s_token_file}" +echo "dpuhost_cluster_k8s_cacert: ${dpuhost_cluster_k8s_cacert}" +echo "mtu: ${mtu}" +echo "metrics_port: ${metrics_port}" + +ovn_image=${ovnkube_image} \ + ovn_image_pull_policy=${image_pull_policy} \ + ovn_unprivileged_mode=${ovn_unprivileged_mode} \ + ovn_gateway_mode=${ovn_gateway_mode} \ + ovn_gateway_opts=${ovn_gateway_opts} \ + ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \ + ovn_northd_backoff_interval=${ovn_northd_backoff_interval} \ + ovn_loglevel_northd=${ovn_loglevel_northd} \ + ovnkube_node_loglevel=${node_loglevel} \ + ovn_loglevel_controller=${ovn_loglevel_controller} \ + ovnkube_logfile_maxsize=${ovnkube_logfile_maxsize} \ + ovnkube_logfile_maxbackups=${ovnkube_logfile_maxbackups} \ + ovnkube_logfile_maxage=${ovnkube_logfile_maxage} \ + ovnkube_libovsdb_client_logfile=${ovnkube_libovsdb_client_logfile} \ + ovnkube_config_duration_enable=${ovnkube_config_duration_enable} \ + ovnkube_metrics_scale_enable=${ovnkube_metrics_scale_enable} \ + metrics_ip=${metrics_ip} \ + ovn_hybrid_overlay_net_cidr=${ovn_hybrid_overlay_net_cidr} \ + ovn_hybrid_overlay_enable=${ovn_hybrid_overlay_enable} \ + ovn_disable_snat_multiple_gws=${ovn_disable_snat_multiple_gws} \ + ovn_disable_forwarding=${ovn_disable_forwarding} \ + ovn_encap_port=${ovn_encap_port} \ + ovn_disable_pkt_mtu_check=${ovn_disable_pkt_mtu_check} \ + ovn_v4_join_subnet=${ovn_v4_join_subnet} \ + ovn_v6_join_subnet=${ovn_v6_join_subnet} \ + ovn_v4_masquerade_subnet=${ovn_v4_masquerade_subnet} \ + ovn_v6_masquerade_subnet=${ovn_v6_masquerade_subnet} \ + ovn_multicast_enable=${ovn_multicast_enable} \ + ovn_admin_network_policy_enable=${ovn_admin_network_policy_enable} \ + ovn_egress_ip_enable=${ovn_egress_ip_enable} \ + ovn_egress_ip_healthcheck_port=${ovn_egress_ip_healthcheck_port} \ + ovn_egress_firewall_enable=${ovn_egress_firewall_enable} \ + ovn_egress_qos_enable=${ovn_egress_qos_enable} \ + ovn_multi_network_enable=${ovn_multi_network_enable} \ + ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ + ovn_network_connect_enable=${ovn_network_connect_enable} \ + ovn_pre_conf_udn_addr_enable=${ovn_pre_conf_udn_addr_enable} \ + ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ + ovn_egress_service_enable=${ovn_egress_service_enable} \ + ovn_ssl_en=${ovn_ssl_en} \ + ovn_remote_probe_interval=${ovn_remote_probe_interval} \ + ovn_monitor_all=${ovn_monitor_all} \ + ovn_ofctrl_wait_before_clear=${ovn_ofctrl_wait_before_clear} \ + ovn_enable_lflow_cache=${ovn_enable_lflow_cache} \ + ovn_lflow_cache_limit=${ovn_lflow_cache_limit} \ + ovn_lflow_cache_limit_kb=${ovn_lflow_cache_limit_kb} \ + ovn_netflow_targets=${ovn_netflow_targets} \ + ovn_sflow_targets=${ovn_sflow_targets} \ + ovn_ipfix_targets=${ovn_ipfix_targets} \ + ovn_ipfix_sampling=${ovn_ipfix_sampling} \ + ovn_ipfix_cache_max_flows=${ovn_ipfix_cache_max_flows} \ + ovn_ipfix_cache_max_flows=${ovn_ipfix_cache_max_flows} \ + ovn_ipfix_cache_active_timeout=${ovn_ipfix_cache_active_timeout} \ + ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \ + ovn_acl_logging_rate_limit=${ovn_acl_logging_rate_limit} \ + ovn_empty_lb_events=${ovn_empty_lb_events} \ + ovn_enable_interconnect=${ovn_enable_interconnect} \ + ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \ + ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \ + ovn_network_qos_enable=${ovn_network_qos_enable} \ + ovn_enable_persistent_ips=${ovn_enable_persistent_ips} \ + ovn_enable_svc_template_support=${ovn_enable_svc_template_support} \ + ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \ + ovn_observ_enable=${ovn_observ_enable} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ + mtu_value=${mtu} \ + metrics_port=${metrics_port} \ + dpuhost_cluster_net_cidr=${dpuhost_cluster_net_cidr} \ + dpuhost_cluster_svc_cidr=${dpuhost_cluster_svc_cidr} \ + dpuhost_cluster_k8s_apiserver=${dpuhost_cluster_k8s_apiserver} \ + dpuhost_cluster_k8s_token=${dpuhost_cluster_k8s_token} \ + dpuhost_cluster_k8s_cacert_data=${dpuhost_cluster_k8s_cacert_data} \ + dpuhost_cluster_k8s_token_file=${dpuhost_cluster_k8s_token_file} \ + dpuhost_cluster_k8s_cacert=${dpuhost_cluster_k8s_cacert} \ + jinjanate ../templates/ovnkube-single-node-zone-dpu.yaml.j2 -o ${output_dir}/ovnkube-single-node-zone-dpu.yaml + ovn_image=${ovnkube_image} \ ovn_image_pull_policy=${image_pull_policy} \ ovn_unprivileged_mode=${ovn_unprivileged_mode} \ @@ -977,6 +1170,7 @@ ovn_image=${ovnkube_image} \ ovnkube_libovsdb_client_logfile=${ovnkube_libovsdb_client_logfile} \ ovnkube_config_duration_enable=${ovnkube_config_duration_enable} \ ovnkube_metrics_scale_enable=${ovnkube_metrics_scale_enable} \ + metrics_ip=${metrics_ip} \ ovn_hybrid_overlay_net_cidr=${ovn_hybrid_overlay_net_cidr} \ ovn_hybrid_overlay_enable=${ovn_hybrid_overlay_enable} \ ovn_disable_snat_multiple_gws=${ovn_disable_snat_multiple_gws} \ @@ -997,8 +1191,12 @@ ovn_image=${ovnkube_image} \ ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ ovn_network_connect_enable=${ovn_network_connect_enable} \ ovn_pre_conf_udn_addr_enable=${ovn_pre_conf_udn_addr_enable} \ + ovn_enable_dynamic_udn_allocation=${ovn_enable_dynamic_udn_allocation} \ + ovn_dynamic_udn_grace_period=${ovn_dynamic_udn_grace_period} \ ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ + ovn_evpn_enable=${ovn_evpn_enable} \ ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ ovn_ssl_en=${ovn_ssl_en} \ ovn_remote_probe_interval=${ovn_remote_probe_interval} \ ovn_monitor_all=${ovn_monitor_all} \ @@ -1030,6 +1228,7 @@ ovn_image=${ovnkube_image} \ ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \ ovn_observ_enable=${ovn_observ_enable} \ enable_coredumps=${enable_coredumps} \ + metrics_ip=${metrics_ip} \ jinjanate ../templates/ovnkube-zone-controller.yaml.j2 -o ${output_dir}/ovnkube-zone-controller.yaml ovn_image=${image} \ @@ -1087,6 +1286,7 @@ net_cidr=${net_cidr} svc_cidr=${svc_cidr} \ host_network_namespace=${host_network_namespace} \ in_upgrade=${in_upgrade} \ advertise_default_network=${ovn_advertise_default_network} \ + ovn_no_overlay_enable=${ovn_no_overlay_enable} \ jinjanate ../templates/ovn-setup.yaml.j2 -o ${output_dir}/ovn-setup.yaml ovn_enable_interconnect=${ovn_enable_interconnect} \ @@ -1098,6 +1298,7 @@ ovn_network_segmentation_enable=${ovn_network_segmentation_enable} \ ovn_pre_conf_udn_addr_enable=${ovn_pre_conf_udn_addr_enable} \ ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \ ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ +ovn_evpn_enable=${ovn_evpn_enable} \ ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ jinjanate ../templates/rbac-ovnkube-cluster-manager.yaml.j2 -o ${output_dir}/rbac-ovnkube-cluster-manager.yaml @@ -1106,6 +1307,7 @@ ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \ ovn_route_advertisements_enable=${ovn_route_advertisements_enable} \ ovn_pre_conf_udn_addr_enable=${ovn_pre_conf_udn_addr_enable} \ ovn_advertised_udn_isolation_mode=${ovn_advertised_udn_isolation_mode} \ +ovn_enable_interconnect=${ovn_enable_interconnect} \ jinjanate ../templates/rbac-ovnkube-master.yaml.j2 -o ${output_dir}/rbac-ovnkube-master.yaml cp ../templates/rbac-ovnkube-identity.yaml.j2 ${output_dir}/rbac-ovnkube-identity.yaml @@ -1121,5 +1323,6 @@ cp ../templates/k8s.ovn.org_userdefinednetworks.yaml.j2 ${output_dir}/k8s.ovn.or cp ../templates/k8s.ovn.org_clusteruserdefinednetworks.yaml.j2 ${output_dir}/k8s.ovn.org_clusteruserdefinednetworks.yaml cp ../templates/k8s.ovn.org_routeadvertisements.yaml.j2 ${output_dir}/k8s.ovn.org_routeadvertisements.yaml cp ../templates/k8s.ovn.org_clusternetworkconnects.yaml.j2 ${output_dir}/k8s.ovn.org_clusternetworkconnects.yaml +cp ../templates/k8s.ovn.org_vteps.yaml.j2 ${output_dir}/k8s.ovn.org_vteps.yaml exit 0 diff --git a/dist/images/ovndb-raft-functions.sh b/dist/images/ovndb-raft-functions.sh index 87c5d8a393..0db584a020 100644 --- a/dist/images/ovndb-raft-functions.sh +++ b/dist/images/ovndb-raft-functions.sh @@ -2,14 +2,14 @@ #set -euo pipefail verify-ovsdb-raft() { - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" if [[ ${ovn_db_host} == "" ]]; then echo "failed to retrieve the IP address of the host $(hostname). Exiting..." exit 1 fi - replicas=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ + replicas=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${k8s_cacert} \ get statefulset -n ${ovn_kubernetes_namespace} ovnkube-db -o=jsonpath='{.spec.replicas}') if [[ ${replicas} -lt 3 || $((${replicas} % 2)) -eq 0 ]]; then echo "at least 3 nodes need to be configured, and it must be odd number of nodes" @@ -25,7 +25,7 @@ db_part_of_cluster() { local db=${2} local port=${3} echo "Checking if ${pod} is part of cluster" - init_ip=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ + init_ip=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${k8s_cacert} \ get pod -n ${ovn_kubernetes_namespace} ${pod} -o=jsonpath='{.status.podIP}') if [[ $? != 0 ]]; then echo "Unable to get ${pod} ip " @@ -51,7 +51,7 @@ cluster_exists() { local db=${1} local port=${2} - db_pods=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ + db_pods=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${k8s_cacert} \ get pod -n ${ovn_kubernetes_namespace} -o=jsonpath='{.items[*].metadata.name}' | egrep -o 'ovnkube-db[^ ]+') for db_pod in $db_pods; do @@ -62,7 +62,7 @@ cluster_exists() { done # if we get here there is no cluster, set init_ip and get out - init_ip="$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ + init_ip="$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${k8s_cacert} \ get pod -n ${ovn_kubernetes_namespace} ovnkube-db-0 -o=jsonpath='{.status.podIP}')" if [[ $? != 0 ]]; then return 1 @@ -89,7 +89,7 @@ check_and_apply_ovnkube_db_ep() { local port=${1} # return if ovn db service endpoint already exists - result=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ + result=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${k8s_cacert} \ get ep -n ${ovn_kubernetes_namespace} ovnkube-db 2>&1) test $? -eq 0 && return if ! echo ${result} | grep -q "NotFound"; then @@ -99,7 +99,7 @@ check_and_apply_ovnkube_db_ep() { # Get IPs of all ovnkube-db PODs ips=() for ((i = 0; i < ${replicas}; i++)); do - ip=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ + ip=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${k8s_cacert} \ get pod -n ${ovn_kubernetes_namespace} ovnkube-db-${i} -o=jsonpath='{.status.podIP}') if [[ ${ip} == "" ]]; then break diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index 6e71648346..8f01b2f458 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -10,7 +10,7 @@ fi . /root/ovndb-raft-functions.sh # This script is the entrypoint to the image. -# Supports version 1.1.0 daemonsets +# Supports version 1.2.0 daemonsets # Keep the daemonset versioning aligned with the ovnkube release versions # Commands ($1 values) # ovs-server Runs the ovs daemons - ovsdb-server and ovs-switchd (v3) @@ -28,7 +28,7 @@ fi # ovn_debug Displays ovn/ovs configuration and flows # NOTE: The script/image must be compatible with the daemonset. -# This script supports version 1.1.0 daemonsets +# This script supports version 1.2.0 daemonsets # When called, it starts all needed daemons. # Currently the version here is used to match with the image version # It must be updated during every release @@ -41,9 +41,11 @@ fi # OVN_KUBERNETES_NAMESPACE - k8s namespace - v3 # K8S_NODE - hostname of the node - v3 # -# OVN_DAEMONSET_VERSION - version match daemonset and image - v1.1.0 +# OVN_DAEMONSET_VERSION - version match daemonset and image - v1.2.0 # K8S_TOKEN - the apiserver token. Automatically detected when running in a pod - v3 # K8S_CACERT - the apiserver CA. Automatically detected when running in a pod - v3 +# K8S_TOKEN_FILE - the apiserver token file. Automatically detected when running in a pod - v3 +# K8S_CACERT_DATA - the apiserver CA data. # OVN_CONTROLLER_OPTS - the options for ovn-ctl # OVN_NORTHD_OPTS - the options for the ovn northbound db # OVN_GATEWAY_MODE - the gateway mode (shared or local) - v3 @@ -121,11 +123,11 @@ ovnkube_logfile_maxage=${OVNKUBE_LOGFILE_MAXAGE:-"5"} ovnkube_libovsdb_client_logfile=${OVNKUBE_LIBOVSDB_CLIENT_LOGFILE:-} # ovnkube.sh version (Update during each release) -ovnkube_version="1.1.0" +ovnkube_version="1.2.0" # The daemonset version must be compatible with this script. # The default when OVN_DAEMONSET_VERSION is not set is version 3 -ovn_daemonset_version=${OVN_DAEMONSET_VERSION:-"1.1.0"} +ovn_daemonset_version=${OVN_DAEMONSET_VERSION:-"1.2.0"} # hostname is the host's hostname when using host networking, # This is useful on the master @@ -145,7 +147,7 @@ else fi # certs and private keys for k8s and OVN -K8S_CACERT=${K8S_CACERT:-/var/run/secrets/kubernetes.io/serviceaccount/ca.crt} +k8s_cacert=${K8S_CACERT:-/var/run/secrets/kubernetes.io/serviceaccount/ca.crt} ovn_ca_cert=/ovn-cert/ca-cert.pem ovn_nb_pk=/ovn-cert/ovnnb-privkey.pem @@ -185,8 +187,8 @@ svc_cidr=${OVN_SVC_CIDR:-172.30.0.0/16} mtu=${OVN_MTU:-1400} routable_mtu=${OVN_ROUTABLE_MTU:-} -# set metrics endpoint bind to K8S_NODE_IP. -metrics_endpoint_ip=${K8S_NODE_IP:-0.0.0.0} +# set metrics endpoint to METRICS_IP. if METRICS_IP not set K8S_NODE_IP or default to 0.0.0.0 +metrics_endpoint_ip="${METRICS_IP:-${K8S_NODE_IP:-0.0.0.0}}" metrics_endpoint_ip=$(bracketify $metrics_endpoint_ip) # set metrics master port @@ -271,10 +273,16 @@ ovn_network_segmentation_enable=${OVN_NETWORK_SEGMENTATION_ENABLE:=false} ovn_network_connect_enable=${OVN_NETWORK_CONNECT_ENABLE:=false} #OVN_PRE_CONF_UDN_ADDR_ENABLE - enable connecting workloads with custom network configuration to UDNs ovn_pre_conf_udn_addr_enable=${OVN_PRE_CONF_UDN_ADDR_ENABLE:=false} -#OVN_NROUTE_ADVERTISEMENTS_ENABLE - enable route advertisements for ovn-kubernetes +#OVN_ROUTE_ADVERTISEMENTS_ENABLE - enable route advertisements for ovn-kubernetes ovn_route_advertisements_enable=${OVN_ROUTE_ADVERTISEMENTS_ENABLE:=false} +#OVN_EVPN_ENABLE - enable EVPN for ovn-kubernetes +ovn_evpn_enable=${OVN_EVPN_ENABLE:=false} #OVN_ADVERTISED_UDN_ISOLATION_MODE - pod network isolation between advertised UDN networks. ovn_advertised_udn_isolation_mode=${OVN_ADVERTISED_UDN_ISOLATION_MODE:=strict} +#OVN_DYNAMIC_UDN_ALLOCATION - dynamic UDN allocation when a node requires it (pod or egress IP) +ovn_enable_dynamic_udn_allocation=${OVN_DYNAMIC_UDN_ALLOCATION} +#OVN_DYNAMIC_UDN_GRACE_PERIOD - period of time before an inactive UDN will be garbage collected +ovn_dynamic_udn_grace_period=${OVN_DYNAMIC_UDN_GRACE_PERIOD:-} ovn_acl_logging_rate_limit=${OVN_ACL_LOGGING_RATE_LIMIT:-"20"} ovn_netflow_targets=${OVN_NETFLOW_TARGETS:-} ovn_sflow_targets=${OVN_SFLOW_TARGETS:-} @@ -325,7 +333,7 @@ ovn_observ_enable=${OVN_OBSERV_ENABLE:-false} # OVN_NOHOSTSUBNET_LABEL - node label indicating nodes managing their own network ovn_nohostsubnet_label=${OVN_NOHOSTSUBNET_LABEL:-""} # OVN_DISABLE_REQUESTEDCHASSIS - disable requested-chassis option during pod creation -# should be set to true when dpu nodes are in the cluster +# should be set to true when dpu nodes are in the cluster for OVN Central mode ovn_disable_requestedchassis=${OVN_DISABLE_REQUESTEDCHASSIS:-false} # external_ids:host-k8s-nodename is set on an Open_vSwitch enabled system if the ovnkube stack @@ -444,7 +452,7 @@ ready_to_start_node() { ovnkube_db_ep=$(get_ovnkube_zone_db_ep) echo "Getting the ${ovnkube_db_ep} ep" # See if ep is available ... - IFS=" " read -a ovn_db_hosts <<<"$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ + IFS=" " read -a ovn_db_hosts <<<"$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${k8s_cacert} \ get ep -n ${ovn_kubernetes_namespace} ${ovnkube_db_ep} -o=jsonpath='{range .subsets[0].addresses[*]}{.ip}{" "}')" if [[ ${#ovn_db_hosts[@]} == 0 ]]; then return 1 @@ -626,6 +634,32 @@ check_health() { return 1 } +get_dpu_gw_options() { + # If ovn_gateway_opts or ovn_gateway_router_subnet is not set as environment variable, gather them from ovs settings + if [[ ${ovn_gateway_opts} == "" ]]; then + # get the gateway interface + gw_iface=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-interface | tr -d \") + if [[ ${gw_iface} == "" ]]; then + echo "Couldn't get OVN Gateway Interface from ovs external_ids setting" + else + ovn_gateway_opts="--gateway-interface=${gw_iface} " + fi + + # get the gateway nexthop + gw_nexthop=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-nexthop | tr -d \") + if [[ ${gw_nexthop} == "" ]]; then + echo "Couldn't get OVN Gateway NextHop from ovs external_ids setting" + else + ovn_gateway_opts+="--gateway-nexthop=${gw_nexthop} " + fi + fi + + # this is only required if the DPU and DPU Host are in different subnets + if [[ ${ovn_gateway_router_subnet} == "" ]]; then + ovn_gateway_router_subnet=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-router-subnet | tr -d \") + fi +} + display_file() { if [[ -f $3 ]]; then echo "====================== $1 pid " @@ -831,7 +865,7 @@ set_ovnkube_db_ep() { ovnkube_db_ep=$(get_ovnkube_zone_db_ep) echo "=============== setting ${ovnkube_db_ep} endpoints to ${ips[@]}" # create a new endpoint for the headless onvkube-db service without selectors - kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} apply -f - </dev/null 2>&1; exit 0' TERM - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" rm -f ${OVN_RUNDIR}/ovn-northd.pid rm -f ${OVN_RUNDIR}/ovn-northd.*.ctl @@ -1141,10 +1219,10 @@ run-ovn-northd() { exit 8 } -# v1.1.0 - run ovnkube-identity +# v1.2.0 - run ovnkube-identity ovnkube-identity() { trap 'kill $(jobs -p); exit 0' TERM - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" rm -f ${OVN_RUNDIR}/ovnkube-identity.pid ovnkube_enable_interconnect_flag= @@ -1171,10 +1249,10 @@ ovnkube-identity() { exit 9 } -# v1.1.0 - run ovnkube --master (both cluster-manager and ovnkube-controller) +# v1.2.0 - run ovnkube --master (both cluster-manager and ovnkube-controller) ovn-master() { trap 'kill $(jobs -p); exit 0' TERM - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" rm -f ${OVN_RUNDIR}/ovnkube-master.pid echo "=============== ovn-master (wait for ready_to_start_node) ========== MASTER ONLY" @@ -1308,18 +1386,38 @@ ovn-master() { fi echo "route_advertisements_enabled_flag=${route_advertisements_enabled_flag}" + evpn_enabled_flag= + if [[ ${ovn_evpn_enable} == "true" ]]; then + evpn_enabled_flag="--enable-evpn" + fi + echo "evpn_enabled_flag=${evpn_enabled_flag}" + advertised_udn_isolation_flag= if [[ -n ${ovn_advertised_udn_isolation_mode} ]]; then advertised_udn_isolation_flag="--advertised-udn-isolation-mode=${ovn_advertised_udn_isolation_mode}" fi + dynamic_udn_allocation_flag= + if [[ ${ovn_enable_dynamic_udn_allocation} == "true" ]]; then + dynamic_udn_allocation_flag="--enable-dynamic-udn-allocation" + fi + echo "dynamic_udn_allocation_flag=${dynamic_udn_allocation_flag}" + + dynamic_udn_grace_period= + if [[ -n ${ovn_dynamic_udn_grace_period} ]]; then + dynamic_udn_grace_period="--udn-deletion-grace-period ${ovn_dynamic_udn_grace_period}" + fi + echo "dynamic_udn_grace_period=${dynamic_udn_grace_period}" + + ovnkube_config_file_flag="--config-file=/run/ovnkube-config/ovnkube.conf" + echo "ovnkube_config_file_flag=${ovnkube_config_file_flag}" + egressservice_enabled_flag= if [[ ${ovn_egressservice_enable} == "true" ]]; then egressservice_enabled_flag="--enable-egress-service" fi echo "egressservice_enabled_flag=${egressservice_enabled_flag}" - ovnkube_master_metrics_bind_address="${metrics_endpoint_ip}:9409" ovnkube_master_metrics_bind_address="${metrics_endpoint_ip}:${metrics_master_port}" local ovnkube_metrics_tls_opts="" if [[ ${OVNKUBE_METRICS_PK} != "" && ${OVNKUBE_METRICS_CERT} != "" ]]; then @@ -1420,7 +1518,9 @@ ovn-master() { ${multi_network_enabled_flag} \ ${network_segmentation_enabled_flag} \ ${route_advertisements_enabled_flag} \ + ${evpn_enabled_flag} \ ${advertised_udn_isolation_flag} \ + ${ovnkube_config_file_flag} \ ${ovn_acl_logging_rate_limit_flag} \ ${ovn_enable_svc_template_support_flag} \ ${ovn_observ_enable_flag} \ @@ -1440,6 +1540,8 @@ ovn-master() { ${nohostsubnet_label_option} \ ${ovn_stateless_netpol_enable_flag} \ ${ovn_disable_requestedchassis_flag} \ + ${dynamic_udn_allocation_flag} \ + ${dynamic_udn_grace_period} \ --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ --gateway-mode=${ovn_gateway_mode} ${ovn_gateway_opts} \ --host-network-namespace ${ovn_host_network_namespace} \ @@ -1463,10 +1565,10 @@ ovn-master() { exit 9 } -# v1.1.0 - run ovnkube --ovnkube-controller +# v1.2.0 - run ovnkube --ovnkube-controller ovnkube-controller() { trap 'kill $(jobs -p); exit 0' TERM - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" rm -f ${OVN_RUNDIR}/ovnkube-controller.pid echo "=============== ovnkube-controller (wait for ready_to_start_node) ==========" @@ -1623,18 +1725,39 @@ ovnkube-controller() { fi echo "pre_conf_udn_addr_enable_flag=${pre_conf_udn_addr_enable_flag}" + dynamic_udn_allocation_flag= + if [[ ${ovn_enable_dynamic_udn_allocation} == "true" ]]; then + dynamic_udn_allocation_flag="--enable-dynamic-udn-allocation" + fi + echo "dynamic_udn_allocation_flag=${dynamic_udn_allocation_flag}" + + dynamic_udn_grace_period= + if [[ -n ${ovn_dynamic_udn_grace_period} ]]; then + dynamic_udn_grace_period="--udn-deletion-grace-period ${ovn_dynamic_udn_grace_period}" + fi + echo "dynamic_udn_grace_period=${dynamic_udn_grace_period}" + route_advertisements_enabled_flag= if [[ ${ovn_route_advertisements_enable} == "true" ]]; then route_advertisements_enabled_flag="--enable-route-advertisements" fi echo "route_advertisements_enabled_flag=${route_advertisements_enabled_flag}" + evpn_enabled_flag= + if [[ ${ovn_evpn_enable} == "true" ]]; then + evpn_enabled_flag="--enable-evpn" + fi + echo "evpn_enabled_flag=${evpn_enabled_flag}" + advertised_udn_isolation_flag= if [[ -n ${ovn_advertised_udn_isolation_mode} ]]; then advertised_udn_isolation_flag="--advertised-udn-isolation-mode=${ovn_advertised_udn_isolation_mode}" fi echo "advertised_udn_isolation_flag=${advertised_udn_isolation_flag}" + ovnkube_config_file_flag="--config-file=/run/ovnkube-config/ovnkube.conf" + echo "ovnkube_config_file_flag=${ovnkube_config_file_flag}" + egressservice_enabled_flag= if [[ ${ovn_egressservice_enable} == "true" ]]; then egressservice_enabled_flag="--enable-egress-service" @@ -1752,7 +1875,9 @@ ovnkube-controller() { ${network_connect_enabled_flag} \ ${pre_conf_udn_addr_enable_flag} \ ${route_advertisements_enabled_flag} \ + ${evpn_enabled_flag} \ ${advertised_udn_isolation_flag} \ + ${ovnkube_config_file_flag} \ ${ovn_acl_logging_rate_limit_flag} \ ${ovn_dbs} \ ${ovn_enable_svc_template_support_flag} \ @@ -1771,6 +1896,8 @@ ovnkube-controller() { ${ovn_v6_masquerade_subnet_opt} \ ${network_qos_enabled_flag} \ ${ovn_enable_dnsnameresolver_flag} \ + ${dynamic_udn_allocation_flag} \ + ${dynamic_udn_grace_period} \ --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ --gateway-mode=${ovn_gateway_mode} \ --host-network-namespace ${ovn_host_network_namespace} \ @@ -1796,7 +1923,7 @@ ovnkube-controller-with-node() { # currently we the process to background, therefore wait until that process removes its pid file on exit. # if the pid file doesnt exist, we exit immediately. trap 'kill $(jobs -p) ; rm -f /etc/cni/net.d/10-ovn-kubernetes.conf ; wait_ovnkube_controller_with_node_done; exit 0' TERM - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" rm -f ${OVN_RUNDIR}/ovnkube-controller-with-node.pid if [[ ${ovnkube_node_mode} != "dpu-host" ]]; then @@ -1974,12 +2101,21 @@ ovnkube-controller-with-node() { fi echo "route_advertisements_enabled_flag=${route_advertisements_enabled_flag}" + evpn_enabled_flag= + if [[ ${ovn_evpn_enable} == "true" ]]; then + evpn_enabled_flag="--enable-evpn" + fi + echo "evpn_enabled_flag=${evpn_enabled_flag}" + advertised_udn_isolation_flag= if [[ -n ${ovn_advertised_udn_isolation_mode} ]]; then advertised_udn_isolation_flag="--advertised-udn-isolation-mode=${ovn_advertised_udn_isolation_mode}" fi echo "advertised_udn_isolation_flag=${advertised_udn_isolation_flag}" + ovnkube_config_file_flag="--config-file=/run/ovnkube-config/ovnkube.conf" + echo "ovnkube_config_file_flag=${ovnkube_config_file_flag}" + egressservice_enabled_flag= if [[ ${ovn_egressservice_enable} == "true" ]]; then egressservice_enabled_flag="--enable-egress-service" @@ -2067,6 +2203,11 @@ ovnkube-controller-with-node() { fi fi + # Get gateway options for DPUs + if [[ ${ovnkube_node_mode} == "dpu" ]]; then + get_dpu_gw_options + fi + if [[ ${ovnkube_node_mode} != "dpu-host" && ! ${ovn_gateway_opts} =~ "gateway-vlanid" ]]; then # get the gateway vlanid gw_vlanid=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-vlanid | tr -d \") @@ -2204,12 +2345,34 @@ ovnkube-controller-with-node() { ovn_stateless_netpol_enable_flag="--enable-stateless-netpol" fi + dynamic_udn_allocation_flag= + if [[ ${ovn_enable_dynamic_udn_allocation} == "true" ]]; then + dynamic_udn_allocation_flag="--enable-dynamic-udn-allocation" + fi + echo "dynamic_udn_allocation_flag=${dynamic_udn_allocation_flag}" + + dynamic_udn_grace_period= + if [[ -n ${ovn_dynamic_udn_grace_period} ]]; then + dynamic_udn_grace_period="--udn-deletion-grace-period ${ovn_dynamic_udn_grace_period}" + fi + echo "dynamic_udn_grace_period=${dynamic_udn_grace_period}" + ovn_disable_requestedchassis_flag= if [[ ${ovn_disable_requestedchassis} == "true" ]]; then ovn_disable_requestedchassis_flag="--disable-requestedchassis" fi echo "ovn_disable_requestedchassis_flag=${ovn_disable_requestedchassis_flag}" + # Pass DPU Host cluster access credentials provided via environment variables in case of DPU + cluster_access_opts="" + if [[ ${ovnkube_node_mode} == "dpu" ]]; then + [[ -n ${K8S_APISERVER} ]] && cluster_access_opts+="--k8s-apiserver=${K8S_APISERVER} " + [[ -n ${K8S_TOKEN} ]] && cluster_access_opts+="--k8s-token=${K8S_TOKEN} " + [[ -n ${K8S_TOKEN_FILE} ]] && cluster_access_opts+="--k8s-token-file=${K8S_TOKEN_FILE} " + [[ -n ${K8S_CACERT_DATA} ]] && cluster_access_opts+="--k8s-cacert-data=${K8S_CACERT_DATA} " + [[ -n ${K8S_CACERT} ]] && cluster_access_opts+="--k8s-cacert=${K8S_CACERT} " + fi + echo "=============== ovnkube-controller-with-node --init-ovnkube-controller-with-node==========" /usr/bin/ovnkube --init-ovnkube-controller ${K8S_NODE} --init-node ${K8S_NODE} \ ${anp_enabled_flag} \ @@ -2237,7 +2400,9 @@ ovnkube-controller-with-node() { ${network_connect_enabled_flag} \ ${pre_conf_udn_addr_enable_flag} \ ${route_advertisements_enabled_flag} \ + ${evpn_enabled_flag} \ ${advertised_udn_isolation_flag} \ + ${ovnkube_config_file_flag} \ ${netflow_targets} \ ${ofctrl_wait_before_clear} \ ${ovn_acl_logging_rate_limit_flag} \ @@ -2262,9 +2427,12 @@ ovnkube-controller-with-node() { ${routable_mtu_flag} \ ${sflow_targets} \ ${ssl_opts} \ + ${dynamic_udn_allocation_flag} \ + ${dynamic_udn_grace_period} \ ${network_qos_enabled_flag} \ ${ovn_enable_dnsnameresolver_flag} \ ${ovn_disable_requestedchassis_flag} \ + ${cluster_access_opts} \ --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ --export-ovs-metrics \ --gateway-mode=${ovn_gateway_mode} ${ovn_gateway_opts} \ @@ -2298,7 +2466,7 @@ ovnkube-controller-with-node() { # run ovnkube --cluster-manager. ovn-cluster-manager() { trap 'kill $(jobs -p); exit 0' TERM - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" ovn_encap_port_flag= if [[ -n "${ovn_encap_port}" ]]; then @@ -2422,11 +2590,20 @@ ovn-cluster-manager() { fi echo "route_advertisements_enabled_flag=${route_advertisements_enabled_flag}" + evpn_enabled_flag= + if [[ ${ovn_evpn_enable} == "true" ]]; then + evpn_enabled_flag="--enable-evpn" + fi + echo "evpn_enabled_flag=${evpn_enabled_flag}" + advertised_udn_isolation_flag= if [[ -n ${ovn_advertised_udn_isolation_mode} ]]; then advertised_udn_isolation_flag="--advertised-udn-isolation-mode=${ovn_advertised_udn_isolation_mode}" fi + ovnkube_config_file_flag="--config-file=/run/ovnkube-config/ovnkube.conf" + echo "ovnkube_config_file_flag=${ovnkube_config_file_flag}" + persistent_ips_enabled_flag= if [[ ${ovn_enable_persistent_ips} == "true" ]]; then persistent_ips_enabled_flag="--enable-persistent-ips" @@ -2475,6 +2652,18 @@ ovn-cluster-manager() { fi echo "ovn_enable_dnsnameresolver_flag=${ovn_enable_dnsnameresolver_flag}" + dynamic_udn_allocation_flag= + if [[ ${ovn_enable_dynamic_udn_allocation} == "true" ]]; then + dynamic_udn_allocation_flag="--enable-dynamic-udn-allocation" + fi + echo "dynamic_udn_allocation_flag=${dynamic_udn_allocation_flag}" + + dynamic_udn_grace_period= + if [[ -n ${ovn_dynamic_udn_grace_period} ]]; then + dynamic_udn_grace_period="--udn-deletion-grace-period ${ovn_dynamic_udn_grace_period}" + fi + echo "dynamic_udn_grace_period=${dynamic_udn_grace_period}" + echo "=============== ovn-cluster-manager ========== MASTER ONLY" /usr/bin/ovnkube --init-cluster-manager ${K8S_NODE} \ ${anp_enabled_flag} \ @@ -2491,7 +2680,9 @@ ovn-cluster-manager() { ${network_connect_enabled_flag} \ ${pre_conf_udn_addr_enable_flag} \ ${route_advertisements_enabled_flag} \ + ${evpn_enabled_flag} \ ${advertised_udn_isolation_flag} \ + ${ovnkube_config_file_flag} \ ${persistent_ips_enabled_flag} \ ${ovnkube_enable_interconnect_flag} \ ${ovnkube_enable_multi_external_gateway_flag} \ @@ -2504,6 +2695,8 @@ ovn-cluster-manager() { ${ovn_v4_transit_subnet_opt} \ ${ovn_v6_transit_subnet_opt} \ ${network_qos_enabled_flag} \ + ${dynamic_udn_allocation_flag} \ + ${dynamic_udn_grace_period} \ ${ovn_enable_dnsnameresolver_flag} \ --gateway-mode=${ovn_gateway_mode} \ --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ @@ -2526,7 +2719,7 @@ ovn-cluster-manager() { # ovn-controller - all nodes ovn-controller() { - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" rm -f ${OVN_RUNDIR}/ovn-controller.pid echo "=============== ovn-controller - (wait for ovs)" @@ -2569,20 +2762,18 @@ ovn-controller() { # ovn-node - all nodes ovn-node() { trap 'kill $(jobs -p) ; rm -f /etc/cni/net.d/10-ovn-kubernetes.conf ; exit 0' TERM - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" rm -f ${OVN_RUNDIR}/ovnkube.pid + # ready_to_start_node checks for the NB/SB readiness state. + # This is not available on the DPU host when interconnect is enabled, + # because the DBs will run locally on the DPU if [[ ${ovnkube_node_mode} != "dpu-host" ]]; then echo "=============== ovn-node - (wait for ovs)" wait_for_event ovs_ready - fi - - if [[ ${ovnkube_node_mode} == "dpu-host" ]] && [[ ${ovn_enable_interconnect} == "true" ]]; then - # ready_to_start_node checks for the NB/SB readiness state. - # This is not available on the DPU host when interconnect is enabled, - # because the DBs will run locally on the DPU - echo "skipping ready_to_start_node on DPU Host and when interconnect is true" - else + echo "=============== ovn-node - (wait for ready_to_start_node)" + wait_for_event ready_to_start_node + elif [[ ${ovn_enable_interconnect} != "true" ]]; then echo "=============== ovn-node - (wait for ready_to_start_node)" wait_for_event ready_to_start_node fi @@ -2679,11 +2870,19 @@ ovn-node() { route_advertisements_enabled_flag="--enable-route-advertisements" fi + evpn_enabled_flag= + if [[ ${ovn_evpn_enable} == "true" ]]; then + evpn_enabled_flag="--enable-evpn" + fi + advertised_udn_isolation_flag= if [[ -n ${ovn_advertised_udn_isolation_mode} ]]; then advertised_udn_isolation_flag="--advertised-udn-isolation-mode=${ovn_advertised_udn_isolation_mode}" fi + ovnkube_config_file_flag="--config-file=/run/ovnkube-config/ovnkube.conf" + echo "ovnkube_config_file_flag=${ovnkube_config_file_flag}" + netflow_targets= if [[ -n ${ovn_netflow_targets} ]]; then netflow_targets="--netflow-targets ${ovn_netflow_targets}" @@ -2773,31 +2972,9 @@ ovn-node() { ovnkube_node_mgmt_port_netdev_flag="--ovnkube-node-mgmt-port-dp-resource-name=${ovnkube_node_mgmt_port_dp_resource_name}" fi + # Get gateway options for DPUs if [[ ${ovnkube_node_mode} == "dpu" ]]; then - if [[ ${ovn_gateway_opts} == "" ]]; then - # get the gateway interface - gw_iface=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-interface | tr -d \") - if [[ ${gw_iface} == "" ]]; then - echo "Couldn't get the required OVN Gateway Interface. Exiting..." - exit 1 - fi - ovn_gateway_opts="--gateway-interface=${gw_iface} " - - # get the gateway nexthop - gw_nexthop=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-nexthop | tr -d \") - if [[ ${gw_nexthop} == "" ]]; then - echo "Couldn't get the required OVN Gateway NextHop. Exiting..." - exit 1 - fi - ovn_gateway_opts+="--gateway-nexthop=${gw_nexthop} " - fi - - # this is required if the DPU and DPU Host are in different subnets - if [[ ${ovn_gateway_router_subnet} == "" ]]; then - # get the gateway router subnet - ovn_gateway_router_subnet=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-router-subnet | tr -d \") - fi - + get_dpu_gw_options fi if [[ ${ovnkube_node_mode} != "dpu-host" && ! ${ovn_gateway_opts} =~ "gateway-vlanid" ]]; then @@ -2893,6 +3070,18 @@ ovn-node() { ovn_v6_masquerade_subnet_opt="--gateway-v6-masquerade-subnet=${ovn_v6_masquerade_subnet}" fi + dynamic_udn_allocation_flag= + if [[ ${ovn_enable_dynamic_udn_allocation} == "true" ]]; then + dynamic_udn_allocation_flag="--enable-dynamic-udn-allocation" + fi + echo "dynamic_udn_allocation_flag=${dynamic_udn_allocation_flag}" + + dynamic_udn_grace_period= + if [[ -n ${ovn_dynamic_udn_grace_period} ]]; then + dynamic_udn_grace_period="--udn-deletion-grace-period ${ovn_dynamic_udn_grace_period}" + fi + echo "dynamic_udn_grace_period=${dynamic_udn_grace_period}" + echo "=============== ovn-node --init-node" /usr/bin/ovnkube --init-node ${K8S_NODE} \ ${anp_enabled_flag} \ @@ -2916,7 +3105,9 @@ ovn-node() { ${network_connect_enabled_flag} \ ${pre_conf_udn_addr_enable_flag} \ ${route_advertisements_enabled_flag} \ + ${evpn_enabled_flag} \ ${advertised_udn_isolation_flag} \ + ${ovnkube_config_file_flag} \ ${netflow_targets} \ ${ofctrl_wait_before_clear} \ ${ovn_dbs} \ @@ -2935,6 +3126,8 @@ ovn-node() { ${ovn_unprivileged_flag} \ ${routable_mtu_flag} \ ${sflow_targets} \ + ${dynamic_udn_allocation_flag} \ + ${dynamic_udn_grace_period} \ ${network_qos_enabled_flag} \ --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ --export-ovs-metrics \ @@ -2967,7 +3160,7 @@ ovn-node() { # cleanup-ovn-node - all nodes cleanup-ovn-node() { - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" rm -f /etc/cni/net.d/10-ovn-kubernetes.conf @@ -2985,15 +3178,15 @@ cleanup-ovn-node() { echo "=============== time: $(date +%d-%m-%H:%M:%S:%N) cleanup-ovn-node --cleanup-node" /usr/bin/ovnkube --cleanup-node ${K8S_NODE} --gateway-mode=${ovn_gateway_mode} ${ovn_gateway_opts} \ - --k8s-token=${k8s_token} --k8s-apiserver=${K8S_APISERVER} --k8s-cacert=${K8S_CACERT} \ + --k8s-token=${k8s_token} --k8s-apiserver=${K8S_APISERVER} --k8s-cacert=${k8s_cacert} \ --loglevel=${ovnkube_loglevel} \ --logfile /var/log/ovn-kubernetes/ovnkube.log } -# v1.1.0 - Runs ovn-kube-util in daemon mode to export prometheus metrics related to OVS. +# v1.2.0 - Runs ovn-kube-util in daemon mode to export prometheus metrics related to OVS. ovs-metrics() { - check_ovn_daemonset_version "1.1.0" + check_ovn_daemonset_version "1.2.0" echo "=============== ovs-metrics - (wait for ovs_ready)" wait_for_event ovs_ready diff --git a/dist/images/run-ovn-dpu.sh b/dist/images/run-ovn-dpu.sh index 328754e226..08f0ccd1a9 100755 --- a/dist/images/run-ovn-dpu.sh +++ b/dist/images/run-ovn-dpu.sh @@ -1,7 +1,7 @@ docker run --pid host --network host --user=0 --name ovn -dit --cap-add=SYS_NICE -v /var/run/dbus:/var/run/dbus:ro -v \ /var/log/openvswitch:/var/log/openvswitch -v /var/log/openvswitch:/var/log/ovn -v \ /var/run/openvswitch:/var/run/openvswitch -v /var/run/openvswitch:/var/run/ovn -v $K8S_CACERT:$K8S_CACERT -v \ - /etc/ovn:/ovn-cert:ro -e OVN_DAEMONSET_VERSION=1.1.0 -e OVN_LOGLEVEL_CONTROLLER="-vconsole:info" \ + /etc/ovn:/ovn-cert:ro -e OVN_DAEMONSET_VERSION=1.2.0 -e OVN_LOGLEVEL_CONTROLLER="-vconsole:info" \ -e K8S_APISERVER=$K8S_APISERVER -e OVN_KUBERNETES_NAMESPACE=ovn-kubernetes -e OVN_SSL_ENABLE=no \ -e K8S_NODE=$K8S_NODE -e K8S_TOKEN=$K8S_TOKEN -e K8S_CACERT=$K8S_CACERT --entrypoint=/root/ovnkube.sh \ ovn-daemonset:latest "ovn-controller" diff --git a/dist/images/run-ovnkube-node-dpu.sh b/dist/images/run-ovnkube-node-dpu.sh index 5fb9ca3790..45a57c46d1 100755 --- a/dist/images/run-ovnkube-node-dpu.sh +++ b/dist/images/run-ovnkube-node-dpu.sh @@ -3,7 +3,7 @@ docker run --pid host --network host --user=0 --name ovn-node -dit --cap-add=NET -v /var/log/ovn-kubernetes:/var/log/ovn-kubernetes -v /var/run/openvswitch:/var/run/openvswitch/ \ -v /var/run/openvswitch:/var/run/ovn/ -v /var/run/ovn-kubernetes:/var/run/ovn-kubernetes \ -v /etc/ovn:/ovn-cert:ro -v /var/lib/openvswitch:/etc/openvswitch:ro -v /var/lib/openvswitch:/etc/ovn:ro \ - -e OVN_DAEMONSET_VERSION=1.1.0 -e OVN_LOGLEVEL_CONTROLLER="-vconsole:info" \ + -e OVN_DAEMONSET_VERSION=1.2.0 -e OVN_LOGLEVEL_CONTROLLER="-vconsole:info" \ -e OVN_NET_CIDR=$OVN_NET_CIDR -e OVN_SVC_CIDR=$OVN_SVC_CIDR -e K8S_NODE=$K8S_NODE \ -e OVN_GATEWAY_MODE="shared" -e OVN_GATEWAY_ROUTER_SUBNET=$OVN_GATEWAY_ROUTER_SUBNET \ -e OVN_REMOTE_PROBE_INTERVAL=100000 -e K8S_APISERVER=$K8S_APISERVER \ diff --git a/dist/templates/k8s.ovn.org_clusteruserdefinednetworks.yaml.j2 b/dist/templates/k8s.ovn.org_clusteruserdefinednetworks.yaml.j2 index bcb4af69ff..36806f773c 100644 --- a/dist/templates/k8s.ovn.org_clusteruserdefinednetworks.yaml.j2 +++ b/dist/templates/k8s.ovn.org_clusteruserdefinednetworks.yaml.j2 @@ -91,6 +91,156 @@ spec: network: description: Network is the user-defined-network spec properties: + evpn: + description: |- + EVPN contains configuration for EVPN mode. + This is only allowed when Transport is "EVPN". + properties: + ipVRF: + description: |- + IPVRF contains the IP-VRF configuration for Layer 3 EVPN. + This field is required for Layer3 topology and optional for Layer2 topology. + properties: + routeTarget: + description: |- + RouteTarget is the import/export route target for this VRF. + If not specified, it will be auto-generated as ":". + Auto-generation will use 2-byte AS if VNI > 65535, since 4-byte AS/IPv4 only allows 2-byte local admin. + + Follows FRR EVPN L3 Route-Target format (A.B.C.D:MN|EF:OPQR|GHJK:MN|*:OPQR|*:MN): + - EF:OPQR = 2-byte AS (1-65535) : local admin (4 bytes, 1-4294967295) + - GHJK:MN = 4-byte AS (65536-4294967295) : local admin (2 bytes, 1-65535) + - A.B.C.D:MN = IPv4 address : local admin (2 bytes, 1-65535) + - *:OPQR = wildcard AS : local admin (4 bytes, 1-4294967295) - for import matching + - *:MN = wildcard AS : local admin (2 bytes, 1-65535) - for import matching + + The 6-byte value constraint (RFC 4360) means AS size + local admin size = 6 bytes. + maxLength: 21 + type: string + x-kubernetes-validations: + - message: RT must contain exactly one colon + rule: self.split(':').size() == 2 + - message: RT global administrator must be either '*', + an IPv4 address, or a number + rule: self.split(':').size() != 2 || (self.startsWith('*:') + || isIP(self.split(':')[0]) || self.split(':')[0].matches('[0-9]+')) + - message: RT local administrator must be a number + rule: self.split(':').size() != 2 || self.split(':')[1].matches('[0-9]+') + - message: RT with wildcard global administrator must + have format *:OPQR where OPQR <= 4294967295 + rule: self.split(':').size() != 2 || !self.startsWith('*:') + || (self.split(':')[1].matches('[0-9]+') && uint(self.split(':')[1]) + <= 4294967295u) + - message: RT with IPv4 global administrator must have + format A.B.C.D:MN where MN <= 65535 + rule: self.split(':').size() != 2 || !self.split(':')[0].contains('.') + || (self.split(':')[1].matches('[0-9]+') && uint(self.split(':')[1]) + <= 65535u) + - message: RT with 4-byte ASN global administrator must + have format GHJK:MN where GHJK <= 4294967295 and MN + <= 65535 + rule: self.split(':').size() != 2 || self.startsWith('*:') + || self.split(':')[0].contains('.') || !self.split(':')[0].matches('[0-9]+') + || !self.split(':')[1].matches('[0-9]+') || uint(self.split(':')[0]) + <= 65535u || uint(self.split(':')[1]) <= 65535u + - message: RT with 2-byte ASN global administrator must + have format EF:OPQR where EF <= 65535 and OPQR <= + 4294967295 + rule: self.split(':').size() != 2 || self.startsWith('*:') + || self.split(':')[0].contains('.') || !self.split(':')[0].matches('[0-9]+') + || !self.split(':')[1].matches('[0-9]+') || uint(self.split(':')[0]) + > 65535u || uint(self.split(':')[1]) <= 4294967295u + vni: + description: |- + VNI is the Virtual Network Identifier for this VRF. + VNI is a 24-bit field in the VXLAN header (RFC 7348), allowing values from 1 to 16777215. + but in the future this could be having different limit for other dataplane implementations. + Must be unique across all EVPN configurations in the cluster. + format: int32 + maximum: 16777215 + minimum: 1 + type: integer + required: + - vni + type: object + macVRF: + description: |- + MACVRF contains the MAC-VRF configuration for Layer 2 EVPN. + This field is required for Layer2 topology and forbidden for Layer3 topology. + properties: + routeTarget: + description: |- + RouteTarget is the import/export route target for this VRF. + If not specified, it will be auto-generated as ":". + Auto-generation will use 2-byte AS if VNI > 65535, since 4-byte AS/IPv4 only allows 2-byte local admin. + + Follows FRR EVPN L3 Route-Target format (A.B.C.D:MN|EF:OPQR|GHJK:MN|*:OPQR|*:MN): + - EF:OPQR = 2-byte AS (1-65535) : local admin (4 bytes, 1-4294967295) + - GHJK:MN = 4-byte AS (65536-4294967295) : local admin (2 bytes, 1-65535) + - A.B.C.D:MN = IPv4 address : local admin (2 bytes, 1-65535) + - *:OPQR = wildcard AS : local admin (4 bytes, 1-4294967295) - for import matching + - *:MN = wildcard AS : local admin (2 bytes, 1-65535) - for import matching + + The 6-byte value constraint (RFC 4360) means AS size + local admin size = 6 bytes. + maxLength: 21 + type: string + x-kubernetes-validations: + - message: RT must contain exactly one colon + rule: self.split(':').size() == 2 + - message: RT global administrator must be either '*', + an IPv4 address, or a number + rule: self.split(':').size() != 2 || (self.startsWith('*:') + || isIP(self.split(':')[0]) || self.split(':')[0].matches('[0-9]+')) + - message: RT local administrator must be a number + rule: self.split(':').size() != 2 || self.split(':')[1].matches('[0-9]+') + - message: RT with wildcard global administrator must + have format *:OPQR where OPQR <= 4294967295 + rule: self.split(':').size() != 2 || !self.startsWith('*:') + || (self.split(':')[1].matches('[0-9]+') && uint(self.split(':')[1]) + <= 4294967295u) + - message: RT with IPv4 global administrator must have + format A.B.C.D:MN where MN <= 65535 + rule: self.split(':').size() != 2 || !self.split(':')[0].contains('.') + || (self.split(':')[1].matches('[0-9]+') && uint(self.split(':')[1]) + <= 65535u) + - message: RT with 4-byte ASN global administrator must + have format GHJK:MN where GHJK <= 4294967295 and MN + <= 65535 + rule: self.split(':').size() != 2 || self.startsWith('*:') + || self.split(':')[0].contains('.') || !self.split(':')[0].matches('[0-9]+') + || !self.split(':')[1].matches('[0-9]+') || uint(self.split(':')[0]) + <= 65535u || uint(self.split(':')[1]) <= 65535u + - message: RT with 2-byte ASN global administrator must + have format EF:OPQR where EF <= 65535 and OPQR <= + 4294967295 + rule: self.split(':').size() != 2 || self.startsWith('*:') + || self.split(':')[0].contains('.') || !self.split(':')[0].matches('[0-9]+') + || !self.split(':')[1].matches('[0-9]+') || uint(self.split(':')[0]) + > 65535u || uint(self.split(':')[1]) <= 4294967295u + vni: + description: |- + VNI is the Virtual Network Identifier for this VRF. + VNI is a 24-bit field in the VXLAN header (RFC 7348), allowing values from 1 to 16777215. + but in the future this could be having different limit for other dataplane implementations. + Must be unique across all EVPN configurations in the cluster. + format: int32 + maximum: 16777215 + minimum: 1 + type: integer + required: + - vni + type: object + vtep: + description: VTEP is the name of the VTEP CR that defines + VTEP IPs for EVPN. + minLength: 1 + type: string + required: + - vtep + type: object + x-kubernetes-validations: + - message: at least one of macVRF or ipVRF must be specified + rule: has(self.macVRF) || has(self.ipVRF) layer2: description: Layer2 is the Layer2 topology configuration. properties: @@ -614,9 +764,9 @@ spec: IPv6 subnet is used rule: '!has(self.subnets) || !has(self.mtu) || !self.subnets.exists_one(i, isCIDR(i) && cidr(i).ip().family() == 6) || self.mtu >= 1280' - noOverlayOptions: + noOverlay: description: |- - NoOverlayOptions contains configuration for no-overlay mode. + NoOverlay contains configuration for no-overlay mode. This is only allowed when Transport is "NoOverlay". properties: outboundSNAT: @@ -653,13 +803,15 @@ spec: transport: description: |- Transport describes the transport technology for pod-to-pod traffic. - Allowed values are "NoOverlay" and "Geneve". + Allowed values are "NoOverlay", "Geneve", and "EVPN". - "NoOverlay": The network operates in no-overlay mode. - "Geneve": The network uses Geneve overlay. + - "EVPN": The network uses EVPN transport. When omitted, the default behaviour is Geneve. enum: - NoOverlay - Geneve + - EVPN type: string required: - topology @@ -682,11 +834,34 @@ spec: rule: '!has(self.transport) || self.transport != ''NoOverlay'' || (self.topology == ''Layer3'' && has(self.layer3) && self.layer3.role == ''Primary'')' - - message: noOverlayOptions is required when transport is 'NoOverlay' + - message: spec.noOverlay is required when type transport is 'NoOverlay' rule: '!has(self.transport) || self.transport != ''NoOverlay'' || - has(self.noOverlayOptions)' - - message: noOverlayOptions is forbidden when transport is not 'NoOverlay' - rule: self.transport == 'NoOverlay' || !has(self.noOverlayOptions) + has(self.noOverlay)' + - message: spec.noOverlay is forbidden when transport type is not + 'NoOverlay' + rule: self.transport == 'NoOverlay' || !has(self.noOverlay) + - message: transport 'EVPN' is only supported for Layer2 or Layer3 + primary networks + rule: '!has(self.transport) || self.transport != ''EVPN'' || ((self.topology + == ''Layer2'' && has(self.layer2) && self.layer2.role == ''Primary'') + || (self.topology == ''Layer3'' && has(self.layer3) && self.layer3.role + == ''Primary''))' + - message: spec.evpn field is required when transport is 'EVPN' + rule: '!has(self.transport) || self.transport != ''EVPN'' || has(self.evpn)' + - message: spec.evpn field is forbidden when transport is not 'EVPN' + rule: self.transport == 'EVPN' || !has(self.evpn) + - message: spec.evpn.macVRF field is required for Layer2 topology + when transport is 'EVPN' + rule: '!has(self.transport) || self.transport != ''EVPN'' || self.topology + != ''Layer2'' || (has(self.evpn) && has(self.evpn.macVRF))' + - message: spec.evpn.ipVRF field is required for Layer3 topology when + transport is 'EVPN' + rule: '!has(self.transport) || self.transport != ''EVPN'' || self.topology + != ''Layer3'' || (has(self.evpn) && has(self.evpn.ipVRF))' + - message: spec.evpn.macVRF field is forbidden for Layer3 topology + when transport is 'EVPN' + rule: '!has(self.transport) || self.transport != ''EVPN'' || self.topology + != ''Layer3'' || !has(self.evpn) || !has(self.evpn.macVRF)' - message: Network spec is immutable rule: self == oldSelf required: diff --git a/dist/templates/k8s.ovn.org_vteps.yaml.j2 b/dist/templates/k8s.ovn.org_vteps.yaml.j2 new file mode 100644 index 0000000000..39bc5c19fd --- /dev/null +++ b/dist/templates/k8s.ovn.org_vteps.yaml.j2 @@ -0,0 +1,151 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: vteps.k8s.ovn.org +spec: + group: k8s.ovn.org + names: + kind: VTEP + listKind: VTEPList + plural: vteps + singular: vtep + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: VTEP defines VTEP (VXLAN Tunnel Endpoint) IP configuration for + EVPN. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the desired VTEP configuration. + properties: + cidrs: + description: |- + CIDRs is the list of IP ranges from which VTEP IPs are allocated. + Dual-stack clusters may set 2 CIDRs (one for each IP family), otherwise only 1 CIDR is allowed. + The format should match standard CIDR notation (for example, "100.64.0.0/24" or "fd00::/64"). + items: + description: CIDR represents a CIDR notation IP range. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be a valid network address + rule: isCIDR(self) && cidr(self) == cidr(self).masked() + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: When 2 CIDRs are set, they must be from different IP families + rule: size(self) != 2 || !isCIDR(self[0]) || !isCIDR(self[1]) || + cidr(self[0]).ip().family() != cidr(self[1]).ip().family() + mode: + allOf: + - enum: + - Managed + - Unmanaged + - enum: + - Managed + - Unmanaged + default: Managed + description: |- + Mode specifies how VTEP IPs are managed. + "Managed" means OVN-Kubernetes allocates and assigns VTEP IPs per node automatically. + "Unmanaged" means an external provider handles IP assignment; OVN-Kubernetes discovers existing IPs on nodes. + Defaults to "Managed". + type: string + required: + - cidrs + type: object + status: + description: Status contains the observed state of the VTEP. + properties: + conditions: + description: Conditions slice of condition objects indicating details + about VTEP status. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/dist/templates/ovn-setup.yaml.j2 b/dist/templates/ovn-setup.yaml.j2 index 981a362859..c2f07aa6b3 100644 --- a/dist/templates/ovn-setup.yaml.j2 +++ b/dist/templates/ovn-setup.yaml.j2 @@ -64,7 +64,6 @@ data: mtu: "{{ mtu_value }}" host_network_namespace: "{{ host_network_namespace }}" - --- # ovn-host-network-namespace.yaml # @@ -79,6 +78,26 @@ metadata: name: "{{ host_network_namespace }}" {%- endif %} +--- +# ovnkube-config ConfigMap +# +# Configuration for ovnkube binaries +kind: ConfigMap +apiVersion: v1 +metadata: + name: ovnkube-config + namespace: ovn-kubernetes +data: + ovnkube.conf: | +{%- if ovn_no_overlay_enable == "true" %} + [default] + transport = no-overlay + + [no-overlay] + outbound-snat = disabled + routing = unmanaged +{%- endif %} + {% if advertise_default_network == "true" -%} --- apiVersion: k8s.ovn.org/v1 diff --git a/dist/templates/ovnkube-control-plane.yaml.j2 b/dist/templates/ovnkube-control-plane.yaml.j2 index b85c3792df..c74c6f400a 100644 --- a/dist/templates/ovnkube-control-plane.yaml.j2 +++ b/dist/templates/ovnkube-control-plane.yaml.j2 @@ -78,6 +78,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: @@ -89,7 +92,7 @@ spec: value: "crash" {% endif -%} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: "{{ ovnkube_master_loglevel }}" - name: OVNKUBE_LOGFILE_MAXSIZE @@ -100,6 +103,8 @@ spec: value: "{{ ovnkube_logfile_maxage }}" - name: OVNKUBE_CONFIG_DURATION_ENABLE value: "{{ ovnkube_config_duration_enable }}" + - name: METRICS_IP + value: "{{ metrics_ip }}" - name: OVN_NET_CIDR valueFrom: configMapKeyRef: @@ -154,10 +159,18 @@ spec: value: "{{ ovn_network_connect_enable }}" - name: OVN_PRE_CONF_UDN_ADDR_ENABLE value: "{{ ovn_pre_conf_udn_addr_enable }}" + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: "{{ ovn_enable_dynamic_udn_allocation }}" + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: "{{ ovn_dynamic_udn_grace_period }}" - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE value: "{{ ovn_route_advertisements_enable }}" + - name: OVN_EVPN_ENABLE + value: "{{ ovn_evpn_enable }}" - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: "{{ ovn_advertised_udn_isolation_mode }}" + - name: OVN_NO_OVERLAY_ENABLE + value: "{{ ovn_no_overlay_enable }}" - name: OVN_HYBRID_OVERLAY_NET_CIDR value: "{{ ovn_hybrid_overlay_net_cidr }}" - name: OVN_DISABLE_SNAT_MULTIPLE_GWS @@ -215,5 +228,8 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + - name: ovnkube-config + configMap: + name: ovnkube-config tolerations: - operator: "Exists" diff --git a/dist/templates/ovnkube-db-raft.yaml.j2 b/dist/templates/ovnkube-db-raft.yaml.j2 index ea607e3c66..6c85475a91 100644 --- a/dist/templates/ovnkube-db-raft.yaml.j2 +++ b/dist/templates/ovnkube-db-raft.yaml.j2 @@ -139,7 +139,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NB value: "{{ ovn_loglevel_nb }}" - name: K8S_APISERVER @@ -218,7 +218,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_SB value: "{{ ovn_loglevel_sb }}" - name: K8S_APISERVER @@ -285,7 +285,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: "{{ ovn_dbchecker_loglevel }}" - name: OVNKUBE_LOGFILE_MAXSIZE diff --git a/dist/templates/ovnkube-db.yaml.j2 b/dist/templates/ovnkube-db.yaml.j2 index 558d104818..9c834f2537 100644 --- a/dist/templates/ovnkube-db.yaml.j2 +++ b/dist/templates/ovnkube-db.yaml.j2 @@ -104,7 +104,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NB value: "{{ ovn_loglevel_nb }}" - name: K8S_APISERVER @@ -175,7 +175,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_SB value: "{{ ovn_loglevel_sb }}" - name: K8S_APISERVER diff --git a/dist/templates/ovnkube-identity.yaml.j2 b/dist/templates/ovnkube-identity.yaml.j2 index 1c989088a9..2b458cb512 100644 --- a/dist/templates/ovnkube-identity.yaml.j2 +++ b/dist/templates/ovnkube-identity.yaml.j2 @@ -57,7 +57,7 @@ spec: value: "crash" {% endif -%} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: K8S_APISERVER valueFrom: configMapKeyRef: diff --git a/dist/templates/ovnkube-master.yaml.j2 b/dist/templates/ovnkube-master.yaml.j2 index cc0783ee02..b4fee83afa 100644 --- a/dist/templates/ovnkube-master.yaml.j2 +++ b/dist/templates/ovnkube-master.yaml.j2 @@ -102,7 +102,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NORTHD value: "{{ ovn_loglevel_northd }}" - name: K8S_APISERVER @@ -170,6 +170,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true {% if ovnkube_compact_mode_enable=="true" %} # Common mounts # for the iptables wrapper @@ -203,7 +206,7 @@ spec: value: "crash" {% endif -%} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: "{{ ovnkube_master_loglevel }}" - name: OVNKUBE_LOGFILE_MAXSIZE @@ -218,6 +221,8 @@ spec: value: "{{ ovnkube_config_duration_enable }}" - name: OVNKUBE_METRICS_SCALE_ENABLE value: "{{ ovnkube_metrics_scale_enable }}" + - name: METRICS_IP + value: "{{ metrics_ip }}" - name: OVNKUBE_COMPACT_MODE_ENABLE value: "{{ ovnkube_compact_mode_enable }}" - name: OVN_NET_CIDR @@ -265,8 +270,16 @@ spec: value: "{{ ovn_network_segmentation_enable }}" - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE value: "{{ ovn_route_advertisements_enable }}" + - name: OVN_EVPN_ENABLE + value: "{{ ovn_evpn_enable }}" - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: "{{ ovn_advertised_udn_isolation_mode }}" + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: "{{ ovn_enable_dynamic_udn_allocation }}" + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: "{{ ovn_dynamic_udn_grace_period }}" + - name: OVN_NO_OVERLAY_ENABLE + value: "{{ ovn_no_overlay_enable }}" - name: OVN_EGRESSSERVICE_ENABLE value: "{{ ovn_egress_service_enable }}" - name: OVN_HYBRID_OVERLAY_NET_CIDR @@ -338,6 +351,9 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + - name: ovnkube-config + configMap: + name: ovnkube-config {% if ovnkube_compact_mode_enable=="true" %} - name: host-slash hostPath: diff --git a/dist/templates/ovnkube-node.yaml.j2 b/dist/templates/ovnkube-node.yaml.j2 index d129ca35a1..1167451dd1 100644 --- a/dist/templates/ovnkube-node.yaml.j2 +++ b/dist/templates/ovnkube-node.yaml.j2 @@ -100,6 +100,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /etc/openvswitch/ name: host-etc-ovs readOnly: true @@ -110,19 +113,34 @@ spec: # ovnkube-node dpu-host mounts - mountPath: /var/run/ovn name: var-run-ovn + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + {%- if ovn_network_segmentation_enable=="true" %} + - mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp + name: host-devinfo-dp + readOnly: true + {%- endif %} {%- endif %} resources: requests: cpu: 100m memory: 300Mi + {%- if ovnkube_app_name=="ovnkube-node-dpu-host" and ovn_network_segmentation_enable=="true" and ovnkube_node_mgmt_port_dp_resource_name!="" %} + {{ ovnkube_node_mgmt_port_dp_resource_name }}: {{ mgmt_port_vfs_count | default(1) }} + {%- endif %} + limits: + {%- if ovnkube_app_name=="ovnkube-node-dpu-host" and ovn_network_segmentation_enable=="true" and ovnkube_node_mgmt_port_dp_resource_name!="" %} + {{ ovnkube_node_mgmt_port_dp_resource_name }}: {{ mgmt_port_vfs_count | default(1) }} + {%- endif %} env: {% if (enable_coredumps | default("false")) == "true" -%} - name: GOTRACEBACK value: "crash" {% endif -%} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: "{{ ovnkube_node_loglevel }}" - name: OVNKUBE_LOGFILE_MAXSIZE @@ -131,6 +149,8 @@ spec: value: "{{ ovnkube_logfile_maxbackups }}" - name: OVNKUBE_LOGFILE_MAXAGE value: "{{ ovnkube_logfile_maxage }}" + - name: METRICS_IP + value: "{{ metrics_ip }}" - name: OVN_NET_CIDR valueFrom: configMapKeyRef: @@ -223,6 +243,10 @@ spec: value: "{{ ovn_enable_ovnkube_identity }}" - name: OVN_NETWORK_QOS_ENABLE value: "{{ ovn_network_qos_enable }}" + - name: OVN_ENABLE_INTERCONNECT + value: "{{ ovn_enable_interconnect }}" + - name: OVN_NETWORK_SEGMENTATION_ENABLE + value: "{{ ovn_network_segmentation_enable }}" {% if ovnkube_app_name!="ovnkube-node-dpu-host" -%} - name: OVN_SSL_ENABLE value: "{{ ovn_ssl_en }}" @@ -240,22 +264,28 @@ spec: value: "{{ ovn_lflow_cache_limit_kb }}" - name: OVN_MULTI_NETWORK_ENABLE value: "{{ ovn_multi_network_enable }}" - - name: OVN_NETWORK_SEGMENTATION_ENABLE - value: "{{ ovn_network_segmentation_enable }}" - name: OVN_NETWORK_CONNECT_ENABLE value: "{{ ovn_network_connect_enable }}" - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE value: "{{ ovn_route_advertisements_enable }}" + - name: OVN_EVPN_ENABLE + value: "{{ ovn_evpn_enable }}" - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: "{{ ovn_advertised_udn_isolation_mode }}" - - name: OVN_ENABLE_INTERCONNECT - value: "{{ ovn_enable_interconnect }}" + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: "{{ ovn_enable_dynamic_udn_allocation }}" + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: "{{ ovn_dynamic_udn_grace_period }}" + - name: OVN_NO_OVERLAY_ENABLE + value: "{{ ovn_no_overlay_enable }}" - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY value: "{{ ovn_enable_multi_external_gateway }}" {% endif -%} {% if ovnkube_app_name=="ovnkube-node-dpu-host" -%} - name: OVNKUBE_NODE_MODE value: "dpu-host" + - name: OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME + value: "{{ ovnkube_node_mgmt_port_dp_resource_name }}" {% endif -%} {% if ovnkube_app_name=="ovnkube-node-dpu" -%} - name: OVNKUBE_NODE_MODE @@ -323,6 +353,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: @@ -330,7 +363,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_CONTROLLER value: "{{ ovn_loglevel_controller }}" - name: K8S_APISERVER @@ -389,11 +422,13 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: K8S_NODE_IP valueFrom: fieldRef: fieldPath: status.hostIP + - name: METRICS_IP + value: "{{ metrics_ip }}" # end of container {% endif -%} @@ -447,6 +482,9 @@ spec: - name: run-systemd hostPath: path: /run/systemd + - name: ovnkube-config + configMap: + name: ovnkube-config {%- if ovnkube_app_name!="ovnkube-node-dpu-host" %} # non DPU related volumes - name: host-var-log-ovs @@ -471,6 +509,11 @@ spec: {%- else %} - name: var-run-ovn emptyDir: {} + {%- if ovn_network_segmentation_enable=="true" %} + - name: host-devinfo-dp + hostPath: + path: /var/run/k8s.cni.cncf.io/devinfo/dp + {%- endif %} {%- endif %} tolerations: diff --git a/dist/templates/ovnkube-single-node-zone-dpu.yaml.j2 b/dist/templates/ovnkube-single-node-zone-dpu.yaml.j2 new file mode 100644 index 0000000000..4912f2cf7e --- /dev/null +++ b/dist/templates/ovnkube-single-node-zone-dpu.yaml.j2 @@ -0,0 +1,575 @@ +--- +# ovnkube-node-dpu +# daemonset version 3 +# starts node daemons for single node zone ovn stack, each in a separate container on DPU +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovnkube-node-dpu + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This DaemonSet launches the ovn-kubernetes networking components on dpus in IC mode. +spec: + selector: + matchLabels: + app: ovnkube-node-dpu + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: ovnkube-node-dpu + name: ovnkube-node-dpu + component: network + type: infra + kubernetes.io/os: "linux" + ovn-db-pod: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: ovnkube-node + hostNetwork: true + dnsPolicy: Default + {{ "hostPID: true" if ovn_unprivileged_mode=="no" }} + + containers: + # nb-ovsdb - v3 + - name: nb-ovsdb + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "local-nb-ovsdb"] + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-etc-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_LOGLEVEL_NB + value: "{{ ovn_loglevel_nb }}" + - name: OVN_NORTHD_BACKOFF_INTERVAL + value: "{{ ovn_northd_backoff_interval }}" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnnb-db"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of nb-ovsdb container + # sb-ovsdb - v3 + - name: sb-ovsdb + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "local-sb-ovsdb"] + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-etc-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_LOGLEVEL_SB + value: "{{ ovn_loglevel_sb }}" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnsb-db"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of sb-ovsdb container + # ovn-northd - v3 + - name: ovn-northd + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "run-ovn-northd"] + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Run directories where we need to be able to access sockets + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_LOGLEVEL_NORTHD + value: "{{ ovn_loglevel_northd }}" + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + - name: OVN_NORTH + value: "local" + - name: OVN_SOUTH + value: "local" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-northd"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of ovn-northd container + # ovnkube-controller + - name: ovnkube-controller + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "ovnkube-controller-with-node"] + securityContext: + runAsUser: 0 + {% if ovn_unprivileged_mode=="no" -%} + privileged: true + {% else -%} + capabilities: + add: + - NET_ADMIN + {% endif %} + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Common mounts + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true + - mountPath: /var/lib/kubelet + name: host-kubelet + readOnly: true + - mountPath: /host-kubernetes + name: host-kubeconfig + readOnly: true + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/ovn-kubernetes/ + name: host-var-log-ovnkube + # We mount our socket here + - mountPath: /var/run/ovn-kubernetes + name: host-var-run-ovn-kubernetes + # CNI related mounts which we take over + - mountPath: /opt/cni/bin + name: host-opt-cni-bin + - mountPath: /etc/cni/net.d + name: host-etc-cni-netd + - mountPath: /var/run/netns + name: host-netns + mountPropagation: Bidirectional + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + - mountPath: /etc/openvswitch/ + name: host-etc-ovs + readOnly: true + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + readOnly: true + - mountPath: /run/systemd/private + name: run-systemd + subPath: private + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_EGRESSSERVICE_ENABLE + value: "{{ ovn_egress_service_enable }}" + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_LOGLEVEL + value: "{{ ovnkube_node_loglevel }}" + - name: OVNKUBE_LOGFILE_MAXSIZE + value: "{{ ovnkube_logfile_maxsize }}" + - name: OVNKUBE_LOGFILE_MAXBACKUPS + value: "{{ ovnkube_logfile_maxbackups }}" + - name: OVNKUBE_LOGFILE_MAXAGE + value: "{{ ovnkube_logfile_maxage }}" + - name: OVNKUBE_LIBOVSDB_CLIENT_LOGFILE + value: "{{ ovnkube_libovsdb_client_logfile }}" + - name: OVNKUBE_CONFIG_DURATION_ENABLE + value: "{{ ovnkube_config_duration_enable }}" + - name: OVNKUBE_METRICS_SCALE_ENABLE + value: "{{ ovnkube_metrics_scale_enable }}" + - name: OVN_NET_CIDR + value: "{{ dpuhost_cluster_net_cidr }}" + - name: OVN_SVC_CIDR + value: "{{ dpuhost_cluster_svc_cidr }}" + - name: K8S_APISERVER + value: "{{ dpuhost_cluster_k8s_apiserver }}" + - name: K8S_TOKEN + value: "{{ dpuhost_cluster_k8s_token }}" + - name: K8S_CACERT_DATA + value: "{{ dpuhost_cluster_k8s_cacert_data }}" + - name: K8S_TOKEN_FILE + value: "{{ dpuhost_cluster_k8s_token_file }}" + - name: K8S_CACERT + value: "{{ dpuhost_cluster_k8s_cacert }}" + - name: OVN_MTU + value: "{{ mtu_value }}" + - name: OVN_GATEWAY_MODE + value: "{{ ovn_gateway_mode }}" + - name: OVN_GATEWAY_OPTS + value: "{{ ovn_gateway_opts }}" + - name: OVN_HYBRID_OVERLAY_ENABLE + value: "{{ ovn_hybrid_overlay_enable }}" + - name: OVN_ADMIN_NETWORK_POLICY_ENABLE + value: "{{ ovn_admin_network_policy_enable }}" + - name: OVN_EGRESSIP_ENABLE + value: "{{ ovn_egress_ip_enable }}" + - name: OVN_EGRESSIP_HEALTHCHECK_PORT + value: "{{ ovn_egress_ip_healthcheck_port }}" + - name: OVN_EGRESSFIREWALL_ENABLE + value: "{{ ovn_egress_firewall_enable }}" + - name: OVN_EGRESSQOS_ENABLE + value: "{{ ovn_egress_qos_enable }}" + - name: OVN_HYBRID_OVERLAY_NET_CIDR + value: "{{ ovn_hybrid_overlay_net_cidr }}" + - name: OVN_DISABLE_SNAT_MULTIPLE_GWS + value: "{{ ovn_disable_snat_multiple_gws }}" + - name: OVN_DISABLE_FORWARDING + value: "{{ ovn_disable_forwarding }}" + - name: OVN_ENCAP_PORT + value: "{{ ovn_encap_port }}" + - name: METRICS_IP + value: "{{ metrics_ip }}" + - name: OVN_DISABLE_PKT_MTU_CHECK + value: "{{ ovn_disable_pkt_mtu_check }}" + - name: OVN_NETFLOW_TARGETS + value: "{{ ovn_netflow_targets }}" + - name: OVN_SFLOW_TARGETS + value: "{{ ovn_sflow_targets }}" + - name: OVN_IPFIX_TARGETS + value: "{{ ovn_ipfix_targets }}" + - name: OVN_IPFIX_SAMPLING + value: "{{ ovn_ipfix_sampling }}" + - name: OVN_IPFIX_CACHE_MAX_FLOWS + value: "{{ ovn_ipfix_cache_max_flows }}" + - name: OVN_IPFIX_CACHE_ACTIVE_TIMEOUT + value: "{{ ovn_ipfix_cache_active_timeout }}" + - name: OVN_V4_JOIN_SUBNET + value: "{{ ovn_v4_join_subnet }}" + - name: OVN_V6_JOIN_SUBNET + value: "{{ ovn_v6_join_subnet }}" + - name: OVN_V4_MASQUERADE_SUBNET + value: "{{ ovn_v4_masquerade_subnet }}" + - name: OVN_V6_MASQUERADE_SUBNET + value: "{{ ovn_v6_masquerade_subnet }}" + - name: OVN_MULTICAST_ENABLE + value: "{{ ovn_multicast_enable }}" + - name: OVN_UNPRIVILEGED_MODE + value: "{{ ovn_unprivileged_mode }}" + - name: OVN_EX_GW_NETWORK_INTERFACE + value: "{{ ovn_ex_gw_networking_interface }}" + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + - name: OVN_REMOTE_PROBE_INTERVAL + value: "{{ ovn_remote_probe_interval }}" + - name: OVN_MONITOR_ALL + value: "{{ ovn_monitor_all }}" + - name: OVN_OFCTRL_WAIT_BEFORE_CLEAR + value: "{{ ovn_ofctrl_wait_before_clear }}" + - name: OVN_ENABLE_LFLOW_CACHE + value: "{{ ovn_enable_lflow_cache }}" + - name: OVN_LFLOW_CACHE_LIMIT + value: "{{ ovn_lflow_cache_limit }}" + - name: OVN_LFLOW_CACHE_LIMIT_KB + value: "{{ ovn_lflow_cache_limit_kb }}" + - name: OVN_MULTI_NETWORK_ENABLE + value: "{{ ovn_multi_network_enable }}" + - name: OVN_NETWORK_SEGMENTATION_ENABLE + value: "{{ ovn_network_segmentation_enable }}" + - name: OVN_NETWORK_CONNECT_ENABLE + value: "{{ ovn_network_connect_enable }}" + - name: OVN_PRE_CONF_UDN_ADDR_ENABLE + value: "{{ ovn_pre_conf_udn_addr_enable }}" + - name: OVN_ADVERTISED_UDN_ISOLATION_MODE + value: "{{ ovn_advertised_udn_isolation_mode }}" + - name: OVN_EMPTY_LB_EVENTS + value: "{{ ovn_empty_lb_events }}" + - name: OVN_ACL_LOGGING_RATE_LIMIT + value: "{{ ovn_acl_logging_rate_limit }}" + - name: OVN_NORTH + value: "local" + - name: OVN_SOUTH + value: "local" + - name: OVN_ENABLE_INTERCONNECT + value: "{{ ovn_enable_interconnect }}" + - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY + value: "{{ ovn_enable_multi_external_gateway }}" + - name: OVN_ENABLE_OVNKUBE_IDENTITY + value: "{{ ovn_enable_ovnkube_identity }}" + - name: OVN_ENABLE_SVC_TEMPLATE_SUPPORT + value: "{{ ovn_enable_svc_template_support }}" + - name: OVN_ENABLE_DNSNAMERESOLVER + value: "{{ ovn_enable_dnsnameresolver }}" + - name: OVN_OBSERV_ENABLE + value: "{{ ovn_observ_enable }}" + - name: OVN_NETWORK_QOS_ENABLE + value: "{{ ovn_network_qos_enable }}" + - name: OVN_NO_OVERLAY_ENABLE + value: "{{ ovn_no_overlay_enable }}" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + readinessProbe: + httpGet: + path: /metrics + port: {{ metrics_port }} + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + periodSeconds: 30 + # end of ovnkube-controller container + # ovn-controller + - name: ovn-controller + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "ovn-controller"] + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_LOGLEVEL_CONTROLLER + value: "{{ ovn_loglevel_controller }}" + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + - name: OVN_NORTH + value: "local" + - name: OVN_SOUTH + value: "local" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-controller"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # ovs-metrics-exporter - v3 + - name: ovs-metrics-exporter + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "ovs-metrics"] + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_NORTH + value: "local" + - name: OVN_SOUTH + value: "local" + - name: METRICS_IP + value: "{{ metrics_ip }}" + # end of container + nodeSelector: + kubernetes.io/os: "linux" + k8s.ovn.org/dpu: "" + volumes: + # Common volumes + - name: host-var-run-dbus + hostPath: + path: /var/run/dbus + - name: host-kubelet + hostPath: + path: /var/lib/kubelet + - name: host-kubeconfig + hostPath: + path: /etc/kubernetes/ + - name: host-var-log-ovnkube + hostPath: + path: /var/log/ovn-kubernetes + - name: host-var-run-ovn-kubernetes + hostPath: + path: /var/run/ovn-kubernetes + - name: host-opt-cni-bin + hostPath: + path: /opt/cni/bin + - name: host-etc-cni-netd + hostPath: + path: /etc/cni/net.d + - name: host-slash + hostPath: + path: / + - name: host-netns + hostPath: + path: /var/run/netns + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-run-ovs + hostPath: + path: /var/run/openvswitch + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate + - name: host-etc-ovs + hostPath: + path: /etc/openvswitch + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: run-systemd + hostPath: + path: /run/systemd + - name: ovnkube-config + configMap: + name: ovnkube-config + tolerations: + - operator: "Exists" diff --git a/dist/templates/ovnkube-single-node-zone.yaml.j2 b/dist/templates/ovnkube-single-node-zone.yaml.j2 index 866d5fe6e2..258b448e16 100644 --- a/dist/templates/ovnkube-single-node-zone.yaml.j2 +++ b/dist/templates/ovnkube-single-node-zone.yaml.j2 @@ -63,6 +63,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /var/run/ovn/ name: host-var-run-ovs - mountPath: /var/run/openvswitch/ @@ -74,7 +77,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NB value: "{{ ovn_loglevel_nb }}" - name: OVN_NORTHD_BACKOFF_INTERVAL @@ -137,6 +140,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /var/run/ovn/ name: host-var-run-ovs - mountPath: /var/run/openvswitch/ @@ -148,7 +154,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_SB value: "{{ ovn_loglevel_sb }}" - name: K8S_APISERVER @@ -215,7 +221,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NORTHD value: "{{ ovn_loglevel_northd }}" - name: K8S_APISERVER @@ -306,6 +312,9 @@ spec: - mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp name: host-devinfo-dp readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: @@ -319,7 +328,7 @@ spec: - name: OVN_EGRESSSERVICE_ENABLE value: "{{ ovn_egress_service_enable }}" - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: "{{ ovnkube_node_loglevel }}" - name: OVNKUBE_LOGFILE_MAXSIZE @@ -396,6 +405,8 @@ spec: value: "{{ ovn_disable_forwarding }}" - name: OVN_ENCAP_PORT value: "{{ ovn_encap_port }}" + - name: METRICS_IP + value: "{{ metrics_ip }}" - name: OVN_DISABLE_PKT_MTU_CHECK value: "{{ ovn_disable_pkt_mtu_check }}" - name: OVN_NETFLOW_TARGETS @@ -446,10 +457,18 @@ spec: value: "{{ ovn_network_connect_enable }}" - name: OVN_PRE_CONF_UDN_ADDR_ENABLE value: "{{ ovn_pre_conf_udn_addr_enable }}" + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: "{{ ovn_enable_dynamic_udn_allocation }}" + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: "{{ ovn_dynamic_udn_grace_period }}" - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE value: "{{ ovn_route_advertisements_enable }}" + - name: OVN_EVPN_ENABLE + value: "{{ ovn_evpn_enable }}" - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: "{{ ovn_advertised_udn_isolation_mode }}" + - name: OVN_NO_OVERLAY_ENABLE + value: "{{ ovn_no_overlay_enable }}" - name: OVNKUBE_NODE_MGMT_PORT_NETDEV value: "{{ ovnkube_node_mgmt_port_netdev }}" - name: OVN_EMPTY_LB_EVENTS @@ -521,7 +540,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_CONTROLLER value: "{{ ovn_loglevel_controller }}" - name: K8S_APISERVER @@ -577,7 +596,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: K8S_NODE_IP valueFrom: fieldRef: @@ -586,6 +605,8 @@ spec: value: "local" - name: OVN_SOUTH value: "local" + - name: METRICS_IP + value: "{{ metrics_ip }}" # end of container nodeSelector: @@ -639,6 +660,9 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + - name: ovnkube-config + configMap: + name: ovnkube-config - name: host-etc-ovs hostPath: path: /etc/openvswitch diff --git a/dist/templates/ovnkube-zone-controller.yaml.j2 b/dist/templates/ovnkube-zone-controller.yaml.j2 index 150f089b56..c984eab781 100644 --- a/dist/templates/ovnkube-zone-controller.yaml.j2 +++ b/dist/templates/ovnkube-zone-controller.yaml.j2 @@ -81,6 +81,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /var/run/ovn/ name: host-var-run-ovs - mountPath: /var/run/openvswitch/ @@ -92,7 +95,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NB value: "{{ ovn_loglevel_nb }}" - name: OVN_NORTHD_BACKOFF_INTERVAL @@ -151,6 +154,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /var/run/ovn/ name: host-var-run-ovs - mountPath: /var/run/openvswitch/ @@ -162,7 +168,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_SB value: "{{ ovn_loglevel_sb }}" - name: K8S_APISERVER @@ -229,7 +235,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NORTHD value: "{{ ovn_loglevel_northd }}" - name: K8S_APISERVER @@ -280,6 +286,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: @@ -291,7 +300,7 @@ spec: value: "crash" {% endif -%} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: "{{ ovnkube_local_loglevel }}" - name: OVNKUBE_LOGFILE_MAXSIZE @@ -304,6 +313,8 @@ spec: value: "{{ ovnkube_libovsdb_client_logfile }}" - name: OVNKUBE_CONFIG_DURATION_ENABLE value: "{{ ovnkube_config_duration_enable }}" + - name: METRICS_IP + value: "{{ metrics_ip }}" - name: OVN_NET_CIDR valueFrom: configMapKeyRef: @@ -353,10 +364,18 @@ spec: value: "{{ ovn_network_connect_enable }}" - name: OVN_PRE_CONF_UDN_ADDR_ENABLE value: "{{ ovn_pre_conf_udn_addr_enable }}" + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: "{{ ovn_enable_dynamic_udn_allocation }}" + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: "{{ ovn_dynamic_udn_grace_period }}" - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE value: "{{ ovn_route_advertisements_enable }}" + - name: OVN_EVPN_ENABLE + value: "{{ ovn_evpn_enable }}" - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: "{{ ovn_advertised_udn_isolation_mode }}" + - name: OVN_NO_OVERLAY_ENABLE + value: "{{ ovn_no_overlay_enable }}" - name: OVN_HYBRID_OVERLAY_NET_CIDR value: "{{ ovn_hybrid_overlay_net_cidr }}" - name: OVN_DISABLE_SNAT_MULTIPLE_GWS @@ -440,6 +459,9 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + - name: ovnkube-config + configMap: + name: ovnkube-config - name: host-var-lib-ovs hostPath: path: /var/lib/openvswitch diff --git a/dist/templates/ovs-node.yaml.j2 b/dist/templates/ovs-node.yaml.j2 index b6e33fa6c0..1cae41e037 100644 --- a/dist/templates/ovs-node.yaml.j2 +++ b/dist/templates/ovs-node.yaml.j2 @@ -85,12 +85,9 @@ spec: requests: cpu: 100m memory: 300Mi - limits: - cpu: 500m - memory: 500Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" lifecycle: preStop: exec: diff --git a/dist/templates/rbac-ovnkube-cluster-manager.yaml.j2 b/dist/templates/rbac-ovnkube-cluster-manager.yaml.j2 index 8aecfc43ca..a35983443e 100644 --- a/dist/templates/rbac-ovnkube-cluster-manager.yaml.j2 +++ b/dist/templates/rbac-ovnkube-cluster-manager.yaml.j2 @@ -78,6 +78,7 @@ rules: - routeadvertisements - networkqoses - clusternetworkconnects + - vteps verbs: [ "get", "list", "watch" ] - apiGroups: ["k8s.ovn.org"] resources: @@ -135,5 +136,5 @@ rules: - apiGroups: ["frrk8s.metallb.io"] resources: - frrconfigurations - verbs: [ "create", "delete", "list", "patch", "update", "watch" ] + verbs: [ "create", "delete", "get", "list", "patch", "update", "watch" ] {%- endif %} diff --git a/dist/templates/rbac-ovnkube-master.yaml.j2 b/dist/templates/rbac-ovnkube-master.yaml.j2 index 0096e8c2ab..cf8a283721 100644 --- a/dist/templates/rbac-ovnkube-master.yaml.j2 +++ b/dist/templates/rbac-ovnkube-master.yaml.j2 @@ -86,6 +86,7 @@ rules: - userdefinednetworks - clusteruserdefinednetworks - networkqoses + - vteps verbs: [ "get", "list", "watch" ] - apiGroups: ["k8s.cni.cncf.io"] resources: @@ -230,3 +231,44 @@ spec: operations: ["UPDATE"] resources: ["pods"] {%- endif %} + +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: egressip-mark-annotation-validation +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["k8s.ovn.org"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["egressips"] + validations: + # Prevent creating EgressIP with the annotation + - expression: 'request.operation != "CREATE" || !has(object.metadata.annotations) || !("k8s.ovn.org/egressip-mark" in object.metadata.annotations)' + message: 'EgressIP resources cannot be created with the "k8s.ovn.org/egressip-mark" annotation. This annotation is managed by the system.' + reason: Invalid + # Prevent modifying or removing the annotation once set + - expression: 'request.operation != "UPDATE" || !has(oldObject.metadata.annotations) || !("k8s.ovn.org/egressip-mark" in oldObject.metadata.annotations) || + (has(object.metadata.annotations) && ("k8s.ovn.org/egressip-mark" in object.metadata.annotations) && oldObject.metadata.annotations["k8s.ovn.org/egressip-mark"] == object.metadata.annotations["k8s.ovn.org/egressip-mark"])' + message: 'The "k8s.ovn.org/egressip-mark" annotation cannot be modified or removed once set. This annotation is managed by the system.' + reason: Invalid + # Only OVN controller service account can add the annotation + {% if ovn_enable_interconnect == "true" -%} + - expression: 'request.operation != "UPDATE" || !has(object.metadata.annotations) || !("k8s.ovn.org/egressip-mark" in object.metadata.annotations) || (has(oldObject.metadata.annotations) && ("k8s.ovn.org/egressip-mark" in oldObject.metadata.annotations)) || request.userInfo.username == "system:serviceaccount:ovn-kubernetes:ovnkube-cluster-manager"' + {% else %} + - expression: 'request.operation != "UPDATE" || !has(object.metadata.annotations) || !("k8s.ovn.org/egressip-mark" in object.metadata.annotations) || (has(oldObject.metadata.annotations) && ("k8s.ovn.org/egressip-mark" in oldObject.metadata.annotations)) || request.userInfo.username == "system:serviceaccount:ovn-kubernetes:ovnkube-master"' + {%- endif %} + message: 'A regular user must not add "k8s.ovn.org/egressip-mark" annotation to an EgressIP custom resource.' + reason: Invalid + +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: egressip-mark-annotation-validation-binding +spec: + policyName: egressip-mark-annotation-validation + validationActions: [Deny] diff --git a/dist/templates/rbac-ovnkube-node.yaml.j2 b/dist/templates/rbac-ovnkube-node.yaml.j2 index b153fa2cbb..40cebd2294 100644 --- a/dist/templates/rbac-ovnkube-node.yaml.j2 +++ b/dist/templates/rbac-ovnkube-node.yaml.j2 @@ -187,6 +187,7 @@ rules: - clusteruserdefinednetworks - routeadvertisements - networkqoses + - clusternetworkconnects verbs: [ "get", "list", "watch" ] {% if ovn_enable_ovnkube_identity == "true" -%} - apiGroups: ["certificates.k8s.io"] diff --git a/docs/api-reference/introduction.md b/docs/api-reference/introduction.md index 1a221898ec..8e93ae3fde 100644 --- a/docs/api-reference/introduction.md +++ b/docs/api-reference/introduction.md @@ -39,3 +39,4 @@ designed and implemented by OVN-Kubernetes * [AdminPolicyBasedExternalRoutes](https://ovn-kubernetes.io/api-reference/admin-epbr-api-spec/) * [UserDefinedNetwork](https://ovn-kubernetes.io/api-reference/userdefinednetwork-api-spec/) * [RouteAdvertisements](routeadvertisements-api-spec.md) +* [VTEP](vtep-api-spec.md) diff --git a/docs/api-reference/userdefinednetwork-api-spec.md b/docs/api-reference/userdefinednetwork-api-spec.md index 30fd2832d7..cb8e5ae10f 100644 --- a/docs/api-reference/userdefinednetwork-api-spec.md +++ b/docs/api-reference/userdefinednetwork-api-spec.md @@ -153,6 +153,24 @@ _Appears in:_ +#### EVPNConfig + + + +EVPNConfig contains configuration options for networks operating in EVPN mode. + + + +_Appears in:_ +- [NetworkSpec](#networkspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `vtep` _string_ | VTEP is the name of the VTEP CR that defines VTEP IPs for EVPN. | | MinLength: 1
Required: \{\}
| +| `macVRF` _[VRFConfig](#vrfconfig)_ | MACVRF contains the MAC-VRF configuration for Layer 2 EVPN.
This field is required for Layer2 topology and forbidden for Layer3 topology. | | | +| `ipVRF` _[VRFConfig](#vrfconfig)_ | IPVRF contains the IP-VRF configuration for Layer 3 EVPN.
This field is required for Layer3 topology and optional for Layer2 topology. | | | + + #### IP _Underlying type:_ _string_ @@ -339,8 +357,9 @@ _Appears in:_ | `layer3` _[Layer3Config](#layer3config)_ | Layer3 is the Layer3 topology configuration. | | | | `layer2` _[Layer2Config](#layer2config)_ | Layer2 is the Layer2 topology configuration. | | | | `localnet` _[LocalnetConfig](#localnetconfig)_ | Localnet is the Localnet topology configuration. | | | -| `transport` _[TransportOption](#transportoption)_ | Transport describes the transport technology for pod-to-pod traffic.
Allowed values are "NoOverlay" and "Geneve".
- "NoOverlay": The network operates in no-overlay mode.
- "Geneve": The network uses Geneve overlay.
When omitted, the default behaviour is Geneve. | | Enum: [NoOverlay Geneve]
| -| `noOverlayOptions` _[NoOverlayOptions](#nooverlayoptions)_ | NoOverlayOptions contains configuration for no-overlay mode.
This is only allowed when Transport is "NoOverlay". | | | +| `transport` _[TransportOption](#transportoption)_ | Transport describes the transport technology for pod-to-pod traffic.
Allowed values are "NoOverlay", "Geneve", and "EVPN".
- "NoOverlay": The network operates in no-overlay mode.
- "Geneve": The network uses Geneve overlay.
- "EVPN": The network uses EVPN transport.
When omitted, the default behaviour is Geneve. | | Enum: [NoOverlay Geneve EVPN]
| +| `noOverlay` _[NoOverlayConfig](#nooverlayconfig)_ | NoOverlay contains configuration for no-overlay mode.
This is only allowed when Transport is "NoOverlay". | | | +| `evpn` _[EVPNConfig](#evpnconfig)_ | EVPN contains configuration for EVPN mode.
This is only allowed when Transport is "EVPN". | | | #### NetworkTopology @@ -362,11 +381,11 @@ _Appears in:_ | `Layer3` | | -#### NoOverlayOptions +#### NoOverlayConfig -NoOverlayOptions contains configuration options for networks operating in no-overlay mode. +NoOverlayConfig contains configuration options for networks operating in no-overlay mode. @@ -379,6 +398,38 @@ _Appears in:_ | `routing` _[RoutingOption](#routingoption)_ | Routing specifies whether the pod network routing is managed by OVN-Kubernetes or users. | | Enum: [Managed Unmanaged]
| +#### RouteTargetString + +_Underlying type:_ _string_ + +RouteTargetString represents the 6-byte value of a BGP extended community route target (RFC 4360). +BGP Extended Communities are 8 bytes total: 2-byte type field + 6-byte value field. +This string encodes the 6-byte value, split between a global administrator (Autonomous System or IPv4) and a local administrator. + +When auto-generated, the local administrator is set to the VNI, creating a natural mapping +between Route Targets and VXLAN network segments (e.g., "65000:100" for AS 65000 and VNI 100). +When explicitly specified, the local administrator can be any value within the type constraints. + +FRR EVPN L3 Route-Targets use format (A.B.C.D:MN|EF:OPQR|GHJK:MN|*:OPQR|*:MN) where: + - EF:OPQR = 2-byte AS (1-65535) : local administrator (4 bytes, 0-4294967295) + - GHJK:MN = 4-byte AS (65536-4294967295) : local administrator (2 bytes, 0-65535) + - A.B.C.D:MN = IPv4 address (4 bytes) : local administrator (2 bytes, 0-65535) + - *:OPQR = wildcard AS : local administrator (4 bytes, 0-4294967295) - for import matching + - *:MN = wildcard AS : local administrator (2 bytes, 0-65535) - for import matching + +The 6-byte constraint means: if AS is 4 bytes, local admin can only be 2 bytes, and vice versa. +Wildcard (*) matches any AS and is useful for import rules in Downstream VNI scenarios. +Note: VNI is 24-bit (max 16777215), so auto-generation with 4-byte AS or IPv4 only works if VNI <= 65535. +See: https://docs.frrouting.org/en/stable-8.5/bgp.html#evpn-l3-route-targets + +_Validation:_ +- MaxLength: 21 + +_Appears in:_ +- [VRFConfig](#vrfconfig) + + + #### RoutingOption _Underlying type:_ _string_ @@ -388,7 +439,7 @@ _Underlying type:_ _string_ _Appears in:_ -- [NoOverlayOptions](#nooverlayoptions) +- [NoOverlayConfig](#nooverlayconfig) | Field | Description | | --- | --- | @@ -405,7 +456,7 @@ _Underlying type:_ _string_ _Appears in:_ -- [NoOverlayOptions](#nooverlayoptions) +- [NoOverlayConfig](#nooverlayconfig) | Field | Description | | --- | --- | @@ -428,6 +479,7 @@ _Appears in:_ | --- | --- | | `NoOverlay` | | | `Geneve` | | +| `EVPN` | | #### UserDefinedNetwork @@ -536,3 +588,20 @@ _Appears in:_ | `Access` | | +#### VRFConfig + + + +VRFConfig contains configuration for a VRF in EVPN. + + + +_Appears in:_ +- [EVPNConfig](#evpnconfig) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `vni` _integer_ | VNI is the Virtual Network Identifier for this VRF.
VNI is a 24-bit field in the VXLAN header (RFC 7348), allowing values from 1 to 16777215.
but in the future this could be having different limit for other dataplane implementations.
Must be unique across all EVPN configurations in the cluster. | | Maximum: 1.6777215e+07
Minimum: 1
Required: \{\}
| +| `routeTarget` _[RouteTargetString](#routetargetstring)_ | RouteTarget is the import/export route target for this VRF.
If not specified, it will be auto-generated as ":".
Auto-generation will use 2-byte AS if VNI > 65535, since 4-byte AS/IPv4 only allows 2-byte local admin.
Follows FRR EVPN L3 Route-Target format (A.B.C.D:MN\|EF:OPQR\|GHJK:MN\|*:OPQR\|*:MN):
- EF:OPQR = 2-byte AS (1-65535) : local admin (4 bytes, 1-4294967295)
- GHJK:MN = 4-byte AS (65536-4294967295) : local admin (2 bytes, 1-65535)
- A.B.C.D:MN = IPv4 address : local admin (2 bytes, 1-65535)
- *:OPQR = wildcard AS : local admin (4 bytes, 1-4294967295) - for import matching
- *:MN = wildcard AS : local admin (2 bytes, 1-65535) - for import matching
The 6-byte value constraint (RFC 4360) means AS size + local admin size = 6 bytes. | | MaxLength: 21
| + + diff --git a/docs/api-reference/vtep-api-spec.md b/docs/api-reference/vtep-api-spec.md new file mode 100644 index 0000000000..d5b94d307f --- /dev/null +++ b/docs/api-reference/vtep-api-spec.md @@ -0,0 +1,135 @@ +# API Reference + +## Packages +- [k8s.ovn.org/v1](#k8sovnorgv1) + + +## k8s.ovn.org/v1 + +Package v1 contains API Schema definitions for the network v1 API group + +### Resource Types +- [VTEP](#vtep) +- [VTEPList](#vteplist) + + + +#### CIDR + +_Underlying type:_ _string_ + +CIDR represents a CIDR notation IP range. + +_Validation:_ +- MaxLength: 43 + +_Appears in:_ +- [DualStackCIDRs](#dualstackcidrs) + + + +#### DualStackCIDRs + +_Underlying type:_ _[CIDR](#cidr)_ + +DualStackCIDRs is a list of CIDRs that supports dual-stack (IPv4 and IPv6). + +_Validation:_ +- MaxItems: 2 +- MaxLength: 43 +- MinItems: 1 + +_Appears in:_ +- [VTEPSpec](#vtepspec) + + + +#### VTEP + + + +VTEP defines VTEP (VXLAN Tunnel Endpoint) IP configuration for EVPN. + + + +_Appears in:_ +- [VTEPList](#vteplist) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `k8s.ovn.org/v1` | | | +| `kind` _string_ | `VTEP` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[VTEPSpec](#vtepspec)_ | Spec defines the desired VTEP configuration. | | Required: \{\}
| +| `status` _[VTEPStatus](#vtepstatus)_ | Status contains the observed state of the VTEP. | | | + + +#### VTEPList + + + +VTEPList contains a list of VTEP. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `k8s.ovn.org/v1` | | | +| `kind` _string_ | `VTEPList` | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[VTEP](#vtep) array_ | | | | + + +#### VTEPMode + +_Underlying type:_ _string_ + +VTEPMode defines the mode of VTEP IP allocation. + +_Validation:_ +- Enum: [Managed Unmanaged] + +_Appears in:_ +- [VTEPSpec](#vtepspec) + +| Field | Description | +| --- | --- | +| `Managed` | VTEPModeManaged means OVN-Kubernetes allocates and assigns VTEP IPs per node automatically.
| +| `Unmanaged` | VTEPModeUnmanaged means an external provider handles IP assignment;
OVN-Kubernetes discovers existing IPs on nodes.
| + + +#### VTEPSpec + + + +VTEPSpec defines the desired state of VTEP. + + + +_Appears in:_ +- [VTEP](#vtep) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `cidrs` _[DualStackCIDRs](#dualstackcidrs)_ | CIDRs is the list of IP ranges from which VTEP IPs are allocated.
Dual-stack clusters may set 2 CIDRs (one for each IP family), otherwise only 1 CIDR is allowed.
The format should match standard CIDR notation (for example, "100.64.0.0/24" or "fd00::/64"). | | MaxItems: 2
MaxLength: 43
MinItems: 1
Required: \{\}
| +| `mode` _[VTEPMode](#vtepmode)_ | Mode specifies how VTEP IPs are managed.
"Managed" means OVN-Kubernetes allocates and assigns VTEP IPs per node automatically.
"Unmanaged" means an external provider handles IP assignment; OVN-Kubernetes discovers existing IPs on nodes.
Defaults to "Managed". | Managed | Enum: [Managed Unmanaged]
| + + +#### VTEPStatus + + + +VTEPStatus contains the observed state of the VTEP. + + + +_Appears in:_ +- [VTEP](#vtep) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#condition-v1-meta) array_ | Conditions slice of condition objects indicating details about VTEP status. | | | + + diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 10ce9e7813..6fb97f1557 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -3,15 +3,17 @@ There are two deployment modes for ovn-kubernetes depending on which the architecture is drastically different: -* default mode (centralized control plane architecture) -* interconnect mode (distributed control plane architecture) +* central mode (centralized control plane architecture) -- is DEPRECATED starting 1.2 release +* interconnect mode (distributed control plane architecture) -- is the default mode -End users can pick either of these modes depending on their use -cases and what suits them well. Let's look at both these modes -in depth so that you are empowered to make your choice between -these two modes of deployment. +End users are recommended to pick interconnect mode to deploy +new clusters since central mode will be removed in future releases. +Many of the new features already don't have support on central mode +and are only available in interconnect mode. -## OVN-Kubernetes Components - Default Mode +Let's look at these two modes in depth from architecture standpoint: + +## OVN-Kubernetes Components - Central Mode (DEPRECATED!!) ![ovn-kubernetes-centralized-components](../images/ovnkube-centralized-components.png) @@ -63,9 +65,9 @@ namespace which are running on all your nodes in the cluster. * OVS daemon and database running as a container * virtual switch that pushes the network plumbing to the edge on the node -## Default Mode Architecture +## Central Mode Architecture (DEPRECATED!!) -Now that we know the pods and components running in the default mode, let's tie up +Now that we know the pods and components running in the central mode, let's tie up loose ends and show how these components run on a standard HA Kubernetes cluster. ### Control Plane Nodes: @@ -76,7 +78,7 @@ loose ends and show how these components run on a standard HA Kubernetes cluster ![ovn-kubernetes-centralized-components-data-plane](../images/ovnkube-centralized-arch-dp.png) -## OVN-Kubernetes Components - Interconnect mode +## OVN-Kubernetes Components - Interconnect mode (DEFAULT) ![ovn-kubernetes-distributed-components](../images/ovnkube-distributed-components.png) @@ -137,7 +139,7 @@ and more distributed. As we can see, the databases, northd and ovn-kubernetes controller components now run per zone rather than only on the control-plane. -## Interconnect Mode Architecture +## Interconnect Mode Architecture (DEFAULT) ### What is Interconnect? @@ -190,8 +192,8 @@ stack is now lighter-weight. database is now contained within each node, overall cross-node and cross-cluster (HostedControlPlane, ManagedSaaS) chatter is decreased and traffic security can be increased. -## Default Mode versus Interconnect Mode +## Central Mode versus Interconnect Mode -* When you want your databases to stay centralized and don't mind much about linear scaling of number of nodes in your cluster, choose the default mode -* Note that there is no different to OVS between the two deployment modes. +* When you want your databases to stay centralized and do not require linear scaling with node count, choose Central Mode +* Note that there is no difference to OVS between the two deployment modes. * FIXME: This section needs to be written well \ No newline at end of file diff --git a/docs/design/topology.md b/docs/design/topology.md index d3353a4487..20aea5a695 100644 --- a/docs/design/topology.md +++ b/docs/design/topology.md @@ -3,13 +3,13 @@ Like we saw earlier in the architecture section there are two modes of deployment in OVN-Kubernetes: -* default mode (centralized control plane architecture) -* interconnect mode (distributed control plane architecture) +* central mode (centralized control plane architecture) -- deprecated starting 1.2 release +* interconnect mode (distributed control plane architecture) -- default mode Based on the mode, there are subtle differences in network topology running on each node in the cluster -## OVN-Kubernetes Network Topology - Default Mode +## OVN-Kubernetes Network Topology - Central Mode (DEPRECATED!!!) The centralized architecture in OVN-K looks like this today: @@ -32,7 +32,7 @@ the service traffic * node-local-external-switch: connects the gateway router to the external bridge -## OVN-Kubernetes Network Topology - Distributed (Interconnect) +## OVN-Kubernetes Network Topology - Distributed (Interconnect) (DEFAULT) The interconnect architecture in OVN-K looks like this today (we assume each node is in a zone of their own): diff --git a/docs/developer-guide/debugging.md b/docs/developer-guide/debugging.md index e69de29bb2..5ca9ecd72f 100644 --- a/docs/developer-guide/debugging.md +++ b/docs/developer-guide/debugging.md @@ -0,0 +1,87 @@ +# Debugging + +This document covers debugging techniques for ovn-kubernetes. + +## Coredump Analysis + +When ovn-kubernetes processes crash, coredumps and their corresponding binaries are +automatically collected in CI. This allows post-mortem debugging to investigate +what went wrong. + +### How It Works + +1. **Coredump collection is enabled** via `ENABLE_COREDUMPS=true` in KIND clusters. + This sets up: + - `kernel.core_pattern` to pipe coredumps to `/tmp/kind/logs/coredumps/` + - `GOTRACEBACK=crash` environment variable for Go binaries (required for Go to + generate coredumps on crashes) + +2. **When a process crashes**, the kernel writes a coredump file with the pattern: + ``` + core.... + ``` + +3. **Binary collection** happens during log export. The `export-kind-logs.sh` script + searches all containers for the crashed binary and copies it alongside the coredump. + +4. **Artifacts are uploaded** to GitHub Actions and can be downloaded from the job's + artifacts section. + +### Downloading Artifacts + +After a CI job completes, download the `kind-logs-*` artifact from the GitHub Actions +job page. Extract it to find: + +``` +/tmp/kind/logs/coredumps/ +├── core.29132.ovnkube.ovn-worker.6 # Coredump file +└── binaries/ + └── ovnkube # Matching binary +``` + +### Debugging with Delve + +Use the [Delve](https://github.com/go-delve/delve) debugger for post-mortem analysis. + +1. **Create a path substitution file** (`dlv.init`) to map build paths to your local + source checkout: + + ``` + config substitute-path /workspace/ovn-kubernetes/go-controller /path/to/your/ovn-kubernetes/go-controller + config substitute-path /usr/local/go /path/to/your/go/installation + ``` + + The build paths can be found by running `dlv core` without the init file and + using the `list` command - it will show the paths it's looking for. + +2. **Start the debugger**: + + ```bash + dlv core ./binaries/ovnkube ./core.29132.ovnkube.ovn-worker.6 --init dlv.init + ``` + +3. **Explore the crash**: + + ``` + (dlv) goroutines # List all goroutines + (dlv) goroutine # Switch to a specific goroutine + (dlv) bt # Show backtrace + (dlv) frame # Select stack frame + (dlv) list # Show source code at current location + (dlv) locals # Show local variables + (dlv) print # Print variable value + ``` + +### Local Development + +To enable coredump collection in a local KIND cluster: + +```bash +ENABLE_COREDUMPS=true ./contrib/kind.sh +``` + +To manually export logs with coredump binaries: + +```bash +./contrib/export-kind-logs.sh /path/to/output +``` diff --git a/docs/developer-guide/developer.md b/docs/developer-guide/developer.md index 3dc7d0dfb8..48514d6021 100644 --- a/docs/developer-guide/developer.md +++ b/docs/developer-guide/developer.md @@ -39,3 +39,52 @@ the `types.go` has been created according to sig-apimachinery docs, the develope `make codegen` to be able to generate all the clientgen, listers and informers for the new CRD along with the deep-copy methods and actual yaml files which get created in `_output/crd` folder and are copied over to `dist/templates` to then be used when creating a KIND cluster. + +## Level-Driven Controllers + +### Background + +OVN-Kubernetes has scale issues with network controllers today. We spin +up 1 network controller per UDN, which incurs heavy cost when handling +resource object events. The cost for example of unmarshaling a node +annotation can be O(n) where n is the number of UDN controllers that are +parsing the object. + +To fix this scale problem, the project has started to move to single +controller instances that are aware of all UDNs. Therefore, handling and +parsing of resource objects are done once. A few controllers have +already moved in this direction, and the remaining components that are +within a UDN controller (such as pod, node, namespace) will be migrated +incrementally. + +Furthermore, as we move to per-resource, multi-network aware +controllers, each controller type needs to be able to get network +information. One obvious way to do this is to add a level-driven controller +for NADs in each main controller. +However, this too has a performance cost, because upon each NAD event, +each controller will need to parse the NAD. Since we already have +networkManager (NAD Controller), it is parsing and updating its cache +with the NAD. In order to solve this problem, NetworkManager has been +extended with a "RegisterNADReconciler" function, which is a callback that +controllers can register with NAD Controller to be informed when a NAD +event happens. Controllers can then query NAD Controller to access its +cache as the source of truth. + +For example there is commonly used GetActiveNetworkForNamespace, and a +new API, GetPrimaryNADForNamespace is added in #5623, as well as +GetNetInfoForNADKey. + +Controllers should be using the pkg/controller Reconciler framework to +implement their level-driven controllers, which are fed keys externally +from NAD Controller. + +### Guidelines + +We prefer level-driven controllers built with `pkg/controller/controller.go`. When adding a new controller: + +- Use the shared controller framework rather than bespoke loops or per-network controllers. +- Design controllers to be User Defined Network (UDN) aware: a single controller instance should reconcile objects across all networks instead of spinning up one instance per network. +- If the controller is network-aware, do **not** create a separate NAD Controller, use a Reconciler. +- The network manager is the source of truth for NADs; register a lightweight, non-blocking handler via `RegisterNADReconciler` that will queue keys to the Reconciler. +- RegisterNADReconciler **before** starting controller workers to avoid missing events during startup. +- For a concrete example, see `go-controller/pkg/ovn/controller/egressfirewall/egressfirewall.go`. diff --git a/docs/developer-guide/release.md b/docs/developer-guide/release.md index 6452f93cf8..568c39dff1 100644 --- a/docs/developer-guide/release.md +++ b/docs/developer-guide/release.md @@ -25,7 +25,8 @@ Each new release of OVN-Kubernetes is defined with a "version" that represents t ## Release Cadence -* We will do two major releases each year. Ex: 1.0.0 at June 2024 and 1.1.0 at Dec 2024 +* We will do three major releases each year. Ex: 1.x.0 in April 2026 and 1.y.0 in August 2026 and 1.z.0 in December 2026 +* The release timings have been fixed roughly to be 3 or so months after a major Kubernetes release (allowing time for that to stabilize and for us to consume that kube release before we cut our own release) * At a given time we will maintain only two active major releases (So when 1.2.0 is released we will stop maintaining and backporting fixes into 1.0.0) * For a supported major release we will continue to do backports for backfixes and offer @@ -46,4 +47,10 @@ Each new release of OVN-Kubernetes is defined with a "version" that represents t * If a PR needs to be backported to an older release that should be requested by adding the `needs-backport` label. -* Reach out to the maintainers on slack or by tagging them directly on the PR. +* Reach out to the maintainers on slack or by tagging them directly on the PR or come to the community meetings to discuss this. + +## Information on Past Releases + +* [v1.0.0](https://github.com/ovn-kubernetes/ovn-kubernetes/tree/release-1.0) - [release-notes](https://github.com/ovn-kubernetes/ovn-kubernetes/releases/tag/v1.0.0) - not maintained anymore. +* [v1.1.0](https://github.com/ovn-kubernetes/ovn-kubernetes/tree/release-1.1) - [release-notes](https://github.com/ovn-kubernetes/ovn-kubernetes/releases/tag/v1.1.0) - actively maintained +* [v1.2.0](https://github.com/ovn-kubernetes/ovn-kubernetes/tree/release-1.2) - [release-notes](https://github.com/ovn-kubernetes/ovn-kubernetes/releases/tag/v1.2.0) - actively maintained diff --git a/docs/features/multiple-networks/multi-homing.md b/docs/features/multiple-networks/multi-homing.md index eee72f47df..f864c764e1 100644 --- a/docs/features/multiple-networks/multi-homing.md +++ b/docs/features/multiple-networks/multi-homing.md @@ -304,6 +304,37 @@ spec: > specifying a static IP address for the pod is only possible when the attachment configuration does **not** feature subnets. +### Multiple interfaces on the same network +OVN-Kubernetes supports attaching a pod to the same non-primary user-defined +network multiple times, allowing the pod to have multiple interfaces connected +to the same network. This is useful for workloads that require multiple network +interfaces on the same network for advanced networking scenarios. Note that only +layer 2 and layer 3 networks are supported. + +To request multiple interfaces on the same network, specify the network +multiple times in the `k8s.v1.cni.cncf.io/networks` annotation: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + annotations: + k8s.v1.cni.cncf.io/networks: l3-network,l3-network + name: multi-nic-pod + namespace: ns1 +spec: + containers: + - args: + - pause + image: registry.k8s.io/e2e-test-images/agnhost:2.36 + imagePullPolicy: IfNotPresent + name: agnhost-container +``` + +In this example, the pod will have two interfaces both connected to the +`l3-network`. Each interface will receive its own IP address from the +network's subnet. + ### Persistent IP addresses for virtualization workloads OVN-Kubernetes provides persistent IP addresses for virtualization workloads, allowing VMs to have the same IP addresses when they migrate, when they restart, @@ -395,8 +426,6 @@ overridden with the following command line options: ## Limitations OVN-Kubernetes currently does **not** support: -- the same attachment configured multiple times in the same pod - i.e. - `k8s.v1.cni.cncf.io/networks: l3-network,l3-network` is invalid. - updates to the network selection elements lists - i.e. `k8s.v1.cni.cncf.io/networks` annotation - external IPAM - i.e. the user can't define the IPAM attribute in the configuration. They must use the subnets attribute. diff --git a/docs/features/ovs-dynamic-cpu-affinity.md b/docs/features/ovs-dynamic-cpu-affinity.md new file mode 100644 index 0000000000..d18bd33568 --- /dev/null +++ b/docs/features/ovs-dynamic-cpu-affinity.md @@ -0,0 +1,378 @@ +# OVS Dynamic CPU Affinity + +## Introduction + +OVS Dynamic CPU Affinity is a feature that enables ovnkube-node pod to manage the CPU +affinity of `ovs-vswitchd` and `ovsdb-server` processes dynamically. When enabled, +ovnkube-controller container (running in the ovnkube-node pod) continuously monitors the available (non-exclusively-pinned) CPUs on +the node and aligns the OVS daemon processes to use those CPUs. This allows OVS +daemons to access more CPU cycles when needed to cope with network load spikes. + +## Motivation + +In Kubernetes clusters with performance-sensitive workloads, the kubelet can be +configured with the static CPU Manager policy to provide exclusive CPU allocation +to Guaranteed QoS pods. + +When using static CPU Manager, administrators configure `reservedSystemCPUs` which +are dedicated for housekeeping tasks of the system, such as systemd services. +The `ovs-vswitchd` and `ovsdb-server` daemons are such housekeeping processes. +However, giving them access only to `reservedSystemCPUs` may not be sufficient +when network load increases and OVS requires more CPU cycles. + +This feature addresses two complementary needs: + +1. **Expanding CPU access for OVS**: When the network load rises, OVS daemons can + span across all non-pinned CPUs (not just reserved ones), giving them access to + more processing power when needed. + +2. **Protecting Guaranteed workloads**: When a new Guaranteed QoS pod is admitted + and assigned to a specific CPU set, the OVS daemons must be moved off those CPUs + to avoid interrupting the guaranteed workload's exclusive CPU access. + +### User-Stories/Use-Cases + +#### Story 1: Dynamic CPU scaling for OVS under load + +As a cluster administrator running workloads with static CPU Manager policy, +I want OVS daemons to automatically have access to all available non-pinned CPUs, +so that OVS can handle network load spikes without being constrained to a fixed +CPU set. + +#### Story 2: Per-node enablement control + +As a cluster administrator, I want to enable OVS dynamic CPU affinity on specific +nodes where I expect high network load, so that I can selectively apply this +feature where it's most beneficial without affecting the entire cluster. + +#### Story 3: Runtime feature toggle + +As a cluster administrator, I want to be able to enable or disable this feature +at runtime by creating or deleting a file on the node, so that I can quickly +respond to changing workload requirements without restarting pods. + +## How to enable this feature on an OVN-Kubernetes cluster? + +This feature is enabled on a **per-node basis** by creating a non-empty file at +the following path on the host filesystem: + +```text +/etc/openvswitch/enable_dynamic_cpu_affinity +``` + +### Enabling the feature + +To enable the feature on a specific node: + +```bash +# On the node (or via SSH/kubectl debug) +# The file must be non-empty to enable the feature +echo 1 > /etc/openvswitch/enable_dynamic_cpu_affinity +``` + +After creating the file, the ovnkube-node pod watches for this specific file and activates the feature. + +### Disabling the feature + +To disable the feature: + +```bash +# Remove or empty the file +rm /etc/openvswitch/enable_dynamic_cpu_affinity +# or +truncate -s 0 /etc/openvswitch/enable_dynamic_cpu_affinity +``` + +The feature uses `fsnotify` to watch for changes to this file. When the file +is removed or emptied, the CPU affinity updates will stop (though existing +affinity settings on OVS processes remain until changed by another mechanism). + +### Prerequisites + +1. **Linux platform**: This feature is only supported on Linux nodes. + +2. **Kubelet Pod Resources API**: The kubelet must expose the Pod Resources API + socket at `/var/lib/kubelet/pod-resources/kubelet.sock`. This is enabled by + default in Kubernetes. + +3. **Static CPU Manager policy** (recommended): While not strictly required, this + feature is most beneficial when the kubelet is configured with the static CPU + Manager policy: + + ```yaml + # In kubelet configuration + cpuManagerPolicy: static + reservedSystemCPUs: "0-1" # Example: reserve CPUs 0 and 1 for system + ``` + +4. **Host filesystem access**: The ovnkube-node pod must have access to: + - `/etc/openvswitch/` (for the enable file) + - `/var/lib/kubelet/pod-resources/` (for the Pod Resources API) + - `/host/etc/kubernetes/kubelet.conf` (for reading kubelet configuration) + +## Workflow Description + +The following diagram illustrates how the OVS Dynamic CPU Affinity feature works: + +```text +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Kubernetes Node │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────────────┐ ┌───────────────────────────────────────────┐ │ +│ │ ovnkube-node │ │ Kubelet │ │ +│ │ (ovspinning) │ │ │ │ +│ │ │ │ ┌─────────────────────────────────────┐ │ │ +│ │ 1. Check enabler │ │ │ Pod Resources API │ │ │ +│ │ file on startup │ │ │ /var/lib/kubelet/pod-resources/ │ │ │ +│ │ │ │ │ │ │ │ +│ │ 2. Read reserved │◄────┼──┤ - GetAllocatableResources() │ │ │ +│ │ CPUs from │ │ │ - ListPodResources() │ │ │ +│ │ kubelet.conf │ │ │ │ │ │ +│ │ │ │ └─────────────────────────────────────┘ │ │ +│ │ 3. Every second: │ │ │ │ +│ │ - Get non-pinned │ │ cpuManagerPolicy: static │ │ +│ │ CPUs from │ │ reservedSystemCPUs: "0-1" │ │ +│ │ PodResources │ │ │ │ +│ │ API │ └───────────────────────────────────────────┘ │ +│ │ - Add reserved │ │ +│ │ CPUs │ │ +│ │ - Set affinity │ │ +│ │ on OVS procs │ │ +│ │ │ │ +│ └──────────┬───────────┘ │ +│ │ │ +│ │ sched_setaffinity() │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ OVS Daemons │ │ +│ │ │ │ +│ │ ┌─────────────────┐ ┌─────────────────┐ │ │ +│ │ │ ovs-vswitchd │ │ ovsdb-server │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ CPU Affinity: │ │ CPU Affinity: │ │ │ +│ │ │ 0-1,4-7 │ │ 0-1,4-7 │ │ │ +│ │ │ (non-pinned + │ │ (non-pinned + │ │ │ +│ │ │ reserved) │ │ reserved) │ │ │ +│ │ └─────────────────┘ └─────────────────┘ │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ CPU Allocation Example │ │ +│ │ │ │ +│ │ CPU 0-1: Reserved for system (reservedSystemCPUs) │ │ +│ │ CPU 2-3: Exclusively pinned to Guaranteed QoS pod │ │ +│ │ CPU 4-7: Available for BestEffort/Burstable pods + OVS │ │ +│ │ │ │ +│ │ OVS affinity mask = {0,1,4,5,6,7} = reserved ∪ non-pinned │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Workflow Steps + +1. **Feature enablement check**: On startup, ovnkube-node checks if + `/etc/openvswitch/enable_dynamic_cpu_affinity` exists and is non-empty. + +2. **Reserved CPUs detection**: The feature reads `reservedSystemCPUs` from + the kubelet configuration file (`/host/etc/kubernetes/kubelet.conf`). If + this fails, it falls back to calculating reserved CPUs as the difference + between online CPUs and allocatable CPUs from the Pod Resources API. + This process is done once at startup. If both methods fail, the feature logs a + warning and exits without applying CPU affinity. + +3. **Continuous monitoring**: A ticker runs every second to: + - Query the Pod Resources API for allocatable CPUs and currently pinned CPUs. + - Calculate non-pinned CPUs as: `allocatable - used_by_guaranteed_containers`. + - Add reserved system CPUs to the set. + - Apply this CPU set as the affinity mask for both `ovs-vswitchd` and + `ovsdb-server` processes (including all their threads). + +4. **File watcher**: An `fsnotify` watcher monitors the enabler file for changes. + If the file is removed or emptied, affinity updates stop. + +## Implementation Details + +### OVN-Kubernetes Implementation Details + +The feature is implemented in the `ovspinning` package under +`go-controller/pkg/node/ovspinning/`. It runs as a goroutine started by the +`DefaultNodeNetworkController` during node initialization. + +#### Key Components + +- **ovspinning_linux.go**: Main implementation for Linux systems +- **ovspinning_noop.go**: No-op implementation for non-Linux platforms +- **podresourcesapi package**: Client for the Kubelet Pod Resources API + +#### CPU Set Calculation + +The CPU affinity for OVS daemons is calculated as: + +```text +OVS CPU Affinity = (Allocatable CPUs - Exclusively Pinned CPUs) ∪ Reserved System CPUs +``` + +Where: + +- **Allocatable CPUs**: CPUs available for pod scheduling (from Kubelet perspective) +- **Exclusively Pinned CPUs**: CPUs assigned to QoS class guaranteed containers with exclusive CPU access. +- **Reserved System CPUs**: CPUs reserved for system processes and housekeeping tasks (from kubelet config). + +#### Thread-Level Affinity + +The feature sets CPU affinity at the thread level, iterating through all threads +of each OVS daemon process (`/proc//task/`) to ensure both the main process +and all spawned threads are properly pinned. + +#### Code Flow + +```go +// Simplified flow from ovspinning_linux.go + +func Run(ctx context.Context, stopCh <-chan struct{}, podResCli podresourcesapi.PodResourcesListerClient) { + // Check if feature is enabled + if !isFileNotEmpty("/etc/openvswitch/enable_dynamic_cpu_affinity") { + return + } + + // Get reserved CPUs (from kubelet config or fallback) + reservedCPUs := getReservedCPUs(kubeletConfigFilePath) + + // Main loop - runs every second + for { + // Get non-pinned CPUs from Pod Resources API + cpus := getNonPinnedCPUs(ctx, podResCli) + + // Add reserved CPUs + cpus = cpus.Union(reservedCPUs) + + // Set affinity on OVS processes + setOvsVSwitchdCPUAffinity(&cpus) + setOvsDBServerCPUAffinity(&cpus) + } +} +``` + +## Troubleshooting + +### Verifying the feature is enabled + +Check the ovnkube-node logs for the following messages: + +```bash +kubectl logs -n ovn-kubernetes | grep -i "ovspinning\|cpu pinning" +``` + +Feature enabled: + +```text +I0115 10:00:00.000000 Starting OVS daemon CPU pinning +I0115 10:00:00.000000 OVS CPU dynamic pinning reservedSystemCPUs set: 0-1 +``` + +Feature disabled: + +```text +I0115 10:00:00.000000 OVS CPU affinity pinning disabled +``` + +### Checking current OVS CPU affinity + +On the node, check the CPU affinity of OVS processes: + +```bash +# Get ovs-vswitchd PID +OVS_PID=$(pidof ovs-vswitchd) + +# Check current affinity +taskset -cp $OVS_PID + +# Example output: +# pid 1234's current affinity list: 0-1,4-7 +``` + +### Common issues + +#### Issue: "Failed to get reservedSystemCPUs from kubelet config file" + +This warning indicates the kubelet configuration file couldn't be read. The +feature falls back to detecting reserved CPUs from the Pod Resources API. +Ensure `/host/etc/kubernetes/kubelet.conf` is accessible from the pod. + +#### Issue: "GetAllocatableResources failed" + +Verify the Pod Resources API is available: + +```bash +ls -la /var/lib/kubelet/pod-resources/kubelet.sock +``` + +Ensure the ovnkube-node pod has the socket mounted. + +#### Issue: CPU affinity not being updated + +1. Verify the enabler file exists and is non-empty: + + ```bash + ls -la /etc/openvswitch/enable_dynamic_cpu_affinity + cat /etc/openvswitch/enable_dynamic_cpu_affinity + ``` + +2. Check for errors in the logs related to setting affinity + +3. Verify ovs-vswitchd and ovsdb-server are running: + + ```bash + pidof ovs-vswitchd ovsdb-server + ``` + +### Metrics and alerts + +Currently, this feature does not expose dedicated Prometheus metrics. The CPU +affinity changes are logged and can be observed in the ovnkube-node logs. + +## Best Practices + +1. **Use with static CPU Manager policy**: This feature is most beneficial when + the kubelet is configured with `cpuManagerPolicy: static` and you have + Guaranteed QoS pods with exclusive CPU access. + +2. **Set appropriate reservedSystemCPUs**: Ensure your kubelet configuration + includes `reservedSystemCPUs` to guarantee OVS always has access to some CPUs. + +3. **Enable selectively**: Only enable this feature on nodes where you expect + high network load or have many pods with exclusive CPU allocation. + +4. **Monitor OVS performance**: After enabling, monitor OVS packet drops and + latency to verify the feature is having the desired effect. + +## Known Limitations + +- **Linux only**: This feature is only supported on Linux nodes. On other + platforms, the feature logs a message and exits gracefully. + +- **No Windows/macOS support**: Due to the use of Linux-specific syscalls + (`sched_setaffinity`), this feature is not portable to other operating systems. + +- **Affinity persists after disabling**: When the feature is disabled (by + removing the enabler file), the existing CPU affinity on OVS processes is not + reset. The affinity will remain until changed by another mechanism or process + restart. + +- **Brief interference window**: The affinity is updated every second, plus up to + 5 seconds for the CPU Manager reconciliation loop to update the exclusive CPUs + set. During this window, OVS processes may still run on CPUs that have been + assigned to a newly admitted Guaranteed pod, potentially causing brief + interruptions to sensitive workloads. + +## References + +- [Kubelet CPU Manager Policy](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/) +- [Pod Resources API](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/#monitoring-device-plugin-resources) +- [Linux sched_setaffinity](https://man7.org/linux/man-pages/man2/sched_setaffinity.2.html) +- [Original PR #3542: OVS Dynamic CPU Affinity](https://github.com/ovn-kubernetes/ovn-kubernetes/pull/3542) +- [PR #5270: Using PodResourcesAPI to get affinity mask](https://github.com/ovn-kubernetes/ovn-kubernetes/pull/5270) diff --git a/docs/features/user-defined-networks/images/L2DeepDive-2segments.jpg b/docs/features/user-defined-networks/images/L2DeepDive-2segments.jpg new file mode 100644 index 0000000000..59f88c7ce0 Binary files /dev/null and b/docs/features/user-defined-networks/images/L2DeepDive-2segments.jpg differ diff --git a/docs/features/user-defined-networks/images/L2DeepDive-2segments.png b/docs/features/user-defined-networks/images/L2DeepDive-2segments.png deleted file mode 100644 index b04dda3f11..0000000000 Binary files a/docs/features/user-defined-networks/images/L2DeepDive-2segments.png and /dev/null differ diff --git a/docs/installation/launching-ovn-kubernetes-with-dpu.md b/docs/installation/launching-ovn-kubernetes-with-dpu.md new file mode 100644 index 0000000000..b7ad6c3613 --- /dev/null +++ b/docs/installation/launching-ovn-kubernetes-with-dpu.md @@ -0,0 +1,189 @@ +# Launching OVN-Kubernetes in DPU-Accelerated environment in interconnect mode + +## OVN K8s cluster setup + +OVN K8s CNI in a DPU-Accelerated environment is deployed using two Kubernetes clusters, one for the hosts and other for the DPUs. + +DPUs in the DPU cluster will watch DPU Host cluster for K8s resources such as Pods, Namespaces, NetworkAttachmentDefinitions, Services, and Endpoints and act on updates to those resources. Hence they require credentials to access DPU host cluster. Each DPU will have a setting denoting the DPU host to which it is associated. + +Refer [DPU support](https://github.com/ovn-kubernetes/ovn-kubernetes/blob/master/docs/features/hardware-offload/dpu-support.md) for more details on the setup. + +## SR-IOV settings on DPU Host + +Follow [OVS Acceleration with Kernel datapath](https://github.com/ovn-kubernetes/ovn-kubernetes/blob/master/docs/features/hardware-offload/ovs-kernel.md) or [OVS Acceleration with DOCA datapath](https://github.com/ovn-kubernetes/ovn-kubernetes/blob/master/docs/features/hardware-offload/ovs-doca.md) to enable Open vSwitch hardware offloading feature on DPU hosts. + +A single VF net-device or a group of VF net-devices (configured as SR-IOV device plugin resource pool) need to be setup separately to create management port(s). + +## K8s Settings on DPU Host + +The following node labels must be set on the DPU Host prior to installing OVN K8s CNI + +```yaml +k8s.ovn.org/dpu-host= +k8s.ovn.org/zone-name="dpu-host node name" +``` + +## Launching OVN K8s DPU Host cluster using helm +OVN K8s CNI can be deployed using helm charts provided under [OVN K8s Helm Charts](https://github.com/ovn-kubernetes/ovn-kubernetes/tree/master/helm/ovn-kubernetes). Refer [Launching OVN-Kubernetes using Helm Charts](https://github.com/ovn-kubernetes/ovn-kubernetes/blob/master/docs/installation/launching-ovn-kubernetes-with-helm.md) for general instructions on using helm charts and explanation of common values used in various subcharts. + +For DPU Hosts cluster use values-single-node-zone.yaml by setting the following fields as specified. The other fields in the file can be set as needed. + +```yaml +tags: + ovnkube-node-dpu-host: true # Removing this line will also enable applying ovnkube-node-dpu-host subchart + ovs-node: false # Disable ovs-node subchart, as OVS is already provided by the corresponding DPU +global: + enableOvnKubeIdentity: false # This feature is not supported currently for clusters with DPU/DPU-Hosts +``` + +ovn-kubernetes image to be used in the containers should be provided in the image section +```yaml +global: + image: + repository: ghcr.io/ovn-kubernetes/ovn-kubernetes/ovn-kube-fedora + tag: master +``` + +Management port netdevice information should be provided in values.yaml file under helm/ovn-kubernetes/charts/ovnkube-node-dpu-host. For example, +```yaml +nodeMgmtPortNetdev: "enp1s0f0v0" # Single VF net-device to be used for management port or +mgmtPortVFResourceName: "mgmtport_vfs" # SR-IOV device plugin resource pool from which VF net-device(s) can be selected. +mgmtPortVFsCount: 2 # If using UDNs, the number of VFs required to handle management ports, which depends on the number of primary UDNs needed should be specified. +``` + +mgmtPortVFResourceName will be prioritized over nodeMgmtPortNetdev if both are specified. +If using UDNs, mgmtPortVFResourceName and mgmtPortVFsCount should be specified. + +Launch OVN K8s using +``` +helm install ovn-kubernetes . -f values-single-node-zone.yaml +``` + +## Generating credentials for accessing this cluster from DPU + +After deploying the CNI, create a secret in this cluster for service account ovnkube-node by applying the following +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: ovnkube-node-sa-for-dpu + namespace: ovn-kubernetes + annotations: + kubernetes.io/service-account.name: ovnkube-node +type: kubernetes.io/service-account-token +``` + +Get the value of ca.crt and token, which will be used in the DPU cluster. The token should be base64 decoded, but the encoded ca.crt should be used as is. + +## K8s Settings on DPU + +The following node label is required on DPUs prior to installing OVN K8s CNI +```yaml +k8s.ovn.org/dpu= +``` + +## OVS settings on DPU +Some OVS settings are required on the DPU to enable hardware offloads, connect to the right DPU-host in the DPU-host cluster and correctly steer traffic flows. + +Consider an example with ovs bridge configuration on DPU and network settings on DPU and DPU Host as below. + +``` +ovs-vsctl show + Bridge brp0 + fail_mode: standalone + Port pf0hpf + tag: 3 + Interface pf0hpf + type: system + Port p0 + Interface p0 + type: system + Port vtep0 + tag: 2 + Interface vtep0 + type: internal + Port brp0 + Interface brp0 + type: internal +``` + +``` +$ ip addr show dev brp0 +4: brp0: mtu 1500 qdisc noqueue state UP group default qlen 1000 + link/ether 52:54:00:a1:b2:c3 brd ff:ff:ff:ff:ff:ff + inet 192.0.2.10/24 brd 192.0.2.255 scope global brp0 + valid_lft forever preferred_lft forever + +$ ip addr show dev vtep0 +5: vtep0: mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000 + link/ether 52:54:00:d4:e5:f6 brd ff:ff:ff:ff:ff:ff + inet 198.51.100.10/24 brd 198.51.100.255 scope global vtep0 + valid_lft forever preferred_lft forever +``` + +On the DPU host with node name dpu-host, the IP address is set as + +``` +$ ip addr show dev enp1s0f0 +2: enp1s0f0: mtu 1500 qdisc fq_codel state UP group default qlen 1000 + link/ether 52:54:00:12:34:56 brd ff:ff:ff:ff:ff:ff + inet 203.0.113.10/24 brd 203.0.113.255 scope global eth0 + valid_lft forever preferred_lft forever + +$ ip route show default +default via 203.0.113.1 dev enp1s0f0 proto static +``` + +Router subnet is 203.0.113.0/24 + +The required OVS settings are as below. The values provided are taken from the above example. + +``` +other_config:hw-offload=true - enable hardware offloading +external_ids:host-k8s-nodename="dpu-host" - name of DPU-Host node +external_ids:hostname="dpu" - OVN Chassis hostname of the DPU +external_ids:ovn-encap-ip="198.51.100.10" - encapsulation IP of the DPU +external_ids:ovn-encap-type="geneve" - supported encapsulation type +external_ids:ovn-gw-interface="brp0" - interface on the DPU that serves as gateway interface +external_ids:ovn-gw-nexthop="203.0.113.1" - default gateway address for the DPU-Host network +external_ids:ovn-gw-router-subnet="203.0.113.0/24" - subnet to be used for the gateway router if DPU is in a different subnet than DPU-Host network +external_ids:ovn-gw-vlanid="3" - optional setting if VLAN id of gateway is not on native VLAN +``` + +## Launching OVN K8s DPU cluster + +Once the DPU-host cluster is deployed, the credentials to access that cluster is needed for DPU cluster deployment. It also requires additional information regarding OVN K8s configuration. + +Use values-single-node-zone-dpu.yaml for deploying the DPU cluster. Only the ovnkube-single-node-zone-dpu chart has to be installed and is enabled by default. The rest of the charts are disabled by setting them to false under the tags section and it should not be changed. + +Set the following field as specified. +```yaml +global: + enableOvnKubeIdentity: false # This feature is not supported currently for clusters with DPU/DPU-Hosts +``` + +The following DPU Host cluster related information must be provided. +```yaml +global: + dpuHostClusterK8sAPIServer: "https://172.25.0.2:6443" # Endpoint of DPU Host cluster's K8s API server + dpuHostClusterK8sToken: "" # DPU Host cluster's K8s Access Token base64 decoded + dpuHostClusterK8sCACertData: "" # DPU Host cluster's encoded K8s Access Certs Data + dpuHostClusterNetworkCIDR: "10.244.0.0/16/24" # DPU Host cluster's Network CIDR + dpuHostClusterServiceCIDR: "10.96.0.0/16" # DPU Host cluster's Service CIDR + mtu: "1400" # MTU of network interface in K8s pod +``` + +ovn-kubernetes image to be used in the containers should be provided in the dpuImage section. It should be built for arm64 architecture. +```yaml +global: + dpuImage: + repository: ghcr.io/ovn-kubernetes/ovn-kubernetes/ovn-kube-ubuntu + tag: master +``` + +The rest of the fields can be set as needed. + +Launch OVN K8s using +``` +helm install ovn-kubernetes . -f values-single-node-zone-dpu.yaml +``` diff --git a/docs/okeps/okep-5674-dpu-healthcheck.md b/docs/okeps/okep-5674-dpu-healthcheck.md new file mode 100644 index 0000000000..c07264f124 --- /dev/null +++ b/docs/okeps/okep-5674-dpu-healthcheck.md @@ -0,0 +1,169 @@ +# OKEP-5674: DPU Healthcheck + +* Issue: [#5674](https://github.com/ovn-org/ovn-kubernetes/issues/5674) + +## Problem Statement + +With the DPU (trusted) architecture, we have ovnkube running on the host (DPU-Host mode), and ovnkube running in the +DPU (DPU mode). Kubelet talks on the host to dpu-host mode, which annotates the VF rep information so that DPU ovnkube +can plug the port into OVS and handle all the OVN configuration. The DPU component is managed outside the host +kubernetes cluster. + +The purpose of this addition is to be able to detect when the DPU side goes down, and indicate that the CNI is no longer +ready on the node. + +## Goals + +* Decrease CNI failure time by detecting when the DPU is unhealthy +* Avoid Kubernetes scheduling new pods via to nodes where the DPU is unhealthy + +## Non-Goals + +* Tainting the node +* Distinguishing between different types of failure in the DPU + +## Introduction + +As a refresher, the architecture of the DPU architecture in "trusted" mode is shown below: + +```mermaid +flowchart TB + + subgraph HOST[Host] + direction TB + H1["Kubelet"] + H2["ovn-kubernetes (DPU-Host mode)"] + end + + subgraph DPU[DPU] + direction TB + D1["ovn-kubernetes (DPU mode)"] + D2[OVS] + D3[OVN] + D1 --> D2 + D1 --> D3 + end + + HOST <--> DPU +``` + +Note, "trusted" mode here means the DPU side has the kubeconfig of the host cluster, and runs OVNK in the DPU. There +is an "untrusted" mode, but it is outside the scope of this enhancement and will not use this feature. + +As can be seen in the diagram, the ovn-kubernetes DPU mode is running down in the DPU, managed outside the purview of +the host Kubernetes cluster. Therefore, the Host Kubernetes cluster has no observability or failure detection of this pod. +When a DPU-accelerated pod is created, the following happens: + +1. Kubelet invokes Multus which checks that the UDN/NAD has a resourceName field +2. SR-IOV plugin is called and reserves a VF, passes the PCI address in the CNI ADD +3. OVN-Kubernetes DPU-Host receives the CNI ADD, and annotates the pod with the connection details. +4. OVN-Kubernetes DPU waits for the annotation, plugs the VF representor into OVS, and annotates the pod with typical +k8s.ovn.org annotations. +5. OVN-Kubernetes DPU-Host sees annotations, and then moves the VF into the pod and configures it +6. DPU-Host sends CNI ADD reply + +Now, if the DPU side goes down, DPU-Host will keep waiting on the CNI ADD for 2 minutes. All CNI ADDs will keep failing. +From a user's perspective on the Host Kubernetes cluster, they may have no insight into what is running in the DPU from +KAPI. We can make this better by: + +1. Detecting when the DPU side has gone down. +2. Failing CNI operations fast as we know when the DPU is down. +3. Triggering Kubelet to report the node as not ready to avoid future pods from being scheduled to this node. + +## User-Stories/Use-Cases + +Story 1: Make DPU operations more robust + +As a cluster admin, I want to leverage DPUs for hardware offload. However, I do not want to compromise my +visibility into the cluster and be stuck with components that are failing outside my view. I'd like to have DPU +issues propagate up the stack so that I can see when a node has failed. Additionally, I would like Kubelet to no longer +schedule pods to this node until it has come back online. + +## Proposed Solution + +Kubernetes uses node leases as a mechanism to maintain health within a cluster. This solution proposes that a custom +node lease is created and used by OVN-Kubernetes to detect when a DPU is no longer functioning. The DPU OVN-Kubernetes +component will update this node lease periodically. DPU-Host will monitor this node lease, and when it expires it will: + +1. Trigger CNI operations to fail fast, CNI is down. +2. Implement the CNI STATUS verb to signal CNI issues to the container runtime. + +The container runtime such as CRI-O or containerd will call CNI STATUS. When CNI STATUS returns an error, the runtime +will report to Kubelet that `NetworkReady=false`, and then Kubelet will set the Kubernetes node status as `NotReady`. +Kubernetes will stop scheduling new pods to this node, and after a grace period (default 300 seconds), Kube Node Controller +will taint the node, triggering eviction for pods that do not tolerate the taint. + +### API Details + +A configuration knob will be introduced in OVN-Kubernetes to set the "--dpu-node-lease-renew-interval" time in seconds. +The default setting will be 10 seconds. Set this value to 0 to disable. +In addition, another config knob "--dpu-node-lease-duration" will be exposed that takes a time value in seconds. It will +be used as the max time before the DPU is considered dead. The default time will be 40 seconds. +These values are similar to what Kubernetes uses for Kubelet node leases. + +### Implementation Details + +OVN-Kubernetes default node network controller (DNNC) will be updated to update the node lease when the mode is configured +to be DPU mode, and the --dpu-node-lease-renew-interval is > 0. OVN-Kubernetes CNI server will be updated to track +this lease, and when the DPU node lease expires it will trigger CNI to: + +- Immediately fail CNI ADD, rather than annotating the pod with the VF and waiting 2 minutes. The CNI ADD will fail +with a new message "DPU Not Ready". +- Report STATUS according to the [CNI specification](https://www.cni.dev/docs/spec/#status-check-plugin-status). + +#### DPU Custom Node Lease + +OVN-Kubernetes RBAC will be updated to allow for a RoleBinding to allow ovnkube to interact with the Kubernetes `lease` +resource in the ovn-kubernetes namespace. +DPU-Host will be responsible for creating the lease, and the owner reference will be set to the Kubernetes +Node object. This will allow automatic clean up if the node is deleted in Kubernetes. Example: + +```yaml +apiVersion: coordination.k8s.io/v1 +kind: Lease +metadata: + name: ovn-dpu-worker-node-1 + namespace: ovn-kubernetes + ownerReferences: + - apiVersion: v1 + kind: Node + name: worker-node-1 + uid: # Critical for GC to work +spec: + holderIdentity: "ovnkube-dpu-node" + leaseDurationSeconds: 40 + renewTime: "2023-10-27T10:00:00Z" +``` + +ovnkube-node-dpu will be given update and read RBAC permissions in order to update the heartbeat every interval. + +### Testing Details + +* Unit tests will be added to make sure CNI ADD/STATUS function correctly under different DPU conditions. +* Unit tests will be added to ensure that the DPU custom node lease functions as expected. +* E2E tests will be added where there is no DPU mode, and DPU-Host mode is just launched. Checking that +the node condition gets updated correctly and pods are not scheduled to the node. +* Future E2E tests will be added upstream once there is a full DPU CI environment to do more comprehensive testing. + +### Documentation Details + +Documentation will be added describing this feature as well as the overall OVN-Kubernetes Trusted DPU architecture. + +## Risks, Known Limitations and Mitigations + +Only applicable for Trusted DPU Architecture. Zero-trust will be handled later. + +## OVN-Kubernetes Version Skew + +Targeting version 1.2. + +## Alternatives + +Instead of using the node condition to indicate the CNI is down, cluster-manager could listen for the lease expiry and +then taint the node. We have tried tainting the node in the past, and it led to a bunch of issues causing us to revert it: +https://github.com/ovn-kubernetes/ovn-kubernetes/pull/2459. The SDN should refrain from tainting nodes, as it can cause +system critical pods to not be able to start, including OVN-Kubernetes itself. + +## References + +None diff --git a/go-controller/.mockery.yaml b/go-controller/.mockery.yaml index 7acd04759d..3192b73207 100644 --- a/go-controller/.mockery.yaml +++ b/go-controller/.mockery.yaml @@ -26,6 +26,9 @@ packages: github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/managementport: interfaces: Interface: + k8s.io/kubelet/pkg/apis/podresources/v1: + interfaces: + PodResourcesListerClient: github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set: config: all: true @@ -79,4 +82,3 @@ packages: PodTemplateNamespaceLister: PodTemplateNamespaceListerExpansion: NodeLister: - diff --git a/go-controller/Makefile b/go-controller/Makefile index 38b22aa0d2..cb5142d708 100644 --- a/go-controller/Makefile +++ b/go-controller/Makefile @@ -22,7 +22,7 @@ else CONTAINER_RUNTIME?=docker endif CONTAINER_RUNNABLE ?= $(shell $(CONTAINER_RUNTIME) -v > /dev/null 2>&1; echo $$?) -OVN_SCHEMA_VERSION ?= v25.03.1 +OVN_SCHEMA_VERSION ?= v25.09.2 OVS_VERSION ?= v2.17.0 ifeq ($(NOROOT),TRUE) C_ARGS = -e NOROOT=TRUE diff --git a/go-controller/cmd/ovn-kube-util/app/ovs-exporter.go b/go-controller/cmd/ovn-kube-util/app/ovs-exporter.go index aae2ce91b0..7a89ceab1b 100644 --- a/go-controller/cmd/ovn-kube-util/app/ovs-exporter.go +++ b/go-controller/cmd/ovn-kube-util/app/ovs-exporter.go @@ -2,10 +2,9 @@ package app import ( "context" - "net/http" + "sync" "time" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/urfave/cli/v2" "k8s.io/klog/v2" @@ -16,8 +15,6 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) -var metricsScrapeInterval int - var OvsExporterCommand = cli.Command{ Name: "ovs-exporter", Usage: "", @@ -26,15 +23,11 @@ var OvsExporterCommand = cli.Command{ Name: "metrics-bind-address", Usage: `The IP address and port for the metrics server to serve on (default ":9310")`, }, - &cli.IntFlag{ - Name: "metrics-interval", - Usage: "The interval in seconds at which ovs metrics are collected", - Value: 30, - Destination: &metricsScrapeInterval, - }, }, Action: func(ctx *cli.Context) error { - stopChan := make(chan struct{}) + innerCtx, cancel := context.WithCancel(ctx.Context) + defer cancel() + bindAddress := ctx.String("metrics-bind-address") if bindAddress == "" { bindAddress = "0.0.0.0:9310" @@ -45,31 +38,38 @@ var OvsExporterCommand = cli.Command{ } // start the ovsdb client for ovs metrics monitoring - ovsClient, err := libovsdb.NewOVSClient(stopChan) + ovsClient, err := libovsdb.NewOVSClient(innerCtx.Done()) if err != nil { klog.Errorf("Error initializing ovs client: %v", err) + return err } - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) + wg := &sync.WaitGroup{} - // register ovs metrics that will be served off of /metrics path - metrics.RegisterStandaloneOvsMetrics(ovsClient, metricsScrapeInterval, stopChan) + opts := metrics.MetricServerOptions{ + BindAddress: bindAddress, + EnableOVSMetrics: true, + OnFatalError: cancel, + } + + _ = metrics.StartOVNMetricsServer(opts, ovsClient, nil, innerCtx.Done(), wg) - server := &http.Server{Addr: bindAddress, Handler: mux} + // run until cancelled (by OS signal or fatal error) + <-innerCtx.Done() + klog.Info("Shutdown signal received, stopping metrics server...") + + // Wait for all goroutines to finish with a timeout + done := make(chan struct{}) go func() { - if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - klog.Exitf("Metrics server exited with error: %v", err) - } + wg.Wait() + close(done) }() - // run until cancelled - <-ctx.Context.Done() - close(stopChan) - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := server.Shutdown(shutdownCtx); err != nil { - klog.Errorf("Error stopping metrics server: %v", err) + select { + case <-done: + klog.Info("Metrics server stopped gracefully") + case <-time.After(10 * time.Second): + klog.Warning("Timeout waiting for metrics server to stop") } return nil diff --git a/go-controller/cmd/ovnkube/ovnkube.go b/go-controller/cmd/ovnkube/ovnkube.go index c05ce1ab7e..7a4ab3bc9a 100644 --- a/go-controller/cmd/ovnkube/ovnkube.go +++ b/go-controller/cmd/ovnkube/ovnkube.go @@ -17,6 +17,7 @@ import ( "github.com/urfave/cli/v2" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" @@ -611,7 +612,6 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util // start the prometheus server to serve OVS and OVN Metrics (default port: 9476) // Note: for ovnkube node mode dpu-host no metrics is required as ovs/ovn is not running on the node. if config.OvnKubeNode.Mode != types.NodeModeDPUHost && config.Metrics.OVNMetricsBindAddress != "" { - metricsScrapeInterval := 30 if ovsClient == nil { ovsClient, err = libovsdb.NewOVSClient(ctx.Done()) @@ -621,13 +621,44 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util } } if ovsClient != nil { - if config.Metrics.ExportOVSMetrics { - metrics.RegisterOvsMetricsWithOvnMetrics(ovsClient, metricsScrapeInterval, ctx.Done()) + opts := metrics.MetricServerOptions{ + BindAddress: config.Metrics.OVNMetricsBindAddress, + CertFile: config.Metrics.NodeServerCert, + KeyFile: config.Metrics.NodeServerPrivKey, + EnableOVSMetrics: config.Metrics.ExportOVSMetrics, + EnableOVNControllerMetrics: true, + EnableOVNNorthdMetrics: true, + EnableOVNDBMetrics: true, + } + + if !config.OVNKubernetesFeature.EnableInterconnect { + // In Central mode, OVNKube Node doesn't need to register OVN Northd and DB metrics unless + // OVNKube Master Pod is running on this node. + opts.EnableOVNNorthdMetrics = false + opts.EnableOVNDBMetrics = false + } + + metricsServer := metrics.StartOVNMetricsServer(opts, ovsClient, ovnClientset.KubeClient, ctx.Done(), wg) + + if !config.OVNKubernetesFeature.EnableInterconnect { + // In Central mode, check if the OVNKube Master Pod is running on this node; + // and if it is, the OVN Northd and DB Metrics will be registered. + wg.Add(1) + go func() { + defer wg.Done() + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(_ context.Context) (bool, error) { + return metrics.CheckPodRunsOnGivenNode(ovnClientset.KubeClient, []string{"ovn-db-pod=true"}, runMode.identity, true) + }) + if err != nil { + klog.Infof("Not registering OVN Northd and DB Metrics, because OVNKube Master Pod is not running on this node(%s)", + runMode.identity) + } else { + klog.Info("Found OVNKube Master Pod running on this node, registering OVN Northd and DB Metrics") + metricsServer.EnableOVNNorthdMetrics() + metricsServer.EnableOVNDBMetrics() + } + }() } - metrics.RegisterOvnMetrics(ovnClientset.KubeClient, runMode.identity, - ovsClient, metricsScrapeInterval, ctx.Done()) - metrics.StartOVNMetricsServer(config.Metrics.OVNMetricsBindAddress, - config.Metrics.NodeServerCert, config.Metrics.NodeServerPrivKey, ctx.Done(), wg) } } diff --git a/go-controller/go.mod b/go-controller/go.mod index c0e67ebcdd..0f9c593e2b 100644 --- a/go-controller/go.mod +++ b/go-controller/go.mod @@ -30,7 +30,7 @@ require ( github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 github.com/mdlayher/ndp v1.0.1 github.com/mdlayher/socket v0.2.1 - github.com/metallb/frr-k8s v0.0.15 + github.com/metallb/frr-k8s v0.0.21 github.com/miekg/dns v1.1.31 github.com/mitchellh/copystructure v1.2.0 github.com/moby/sys/userns v0.1.0 @@ -40,7 +40,7 @@ require ( github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 github.com/ovn-kubernetes/libovsdb v0.8.1 github.com/prometheus/client_golang v1.22.0 - github.com/prometheus/client_model v0.6.1 + github.com/prometheus/client_model v0.6.2 github.com/safchain/ethtool v0.3.1-0.20231027162144-83e5e0097c91 github.com/spf13/afero v1.14.0 github.com/stretchr/testify v1.10.0 @@ -50,7 +50,7 @@ require ( golang.org/x/net v0.46.0 golang.org/x/sync v0.17.0 golang.org/x/sys v0.37.0 - golang.org/x/time v0.9.0 + golang.org/x/time v0.11.0 google.golang.org/grpc v1.72.1 google.golang.org/grpc/security/advancedtls v0.0.0-20240425232638-1e8b9b7fc655 google.golang.org/protobuf v1.36.10 @@ -63,6 +63,7 @@ require ( k8s.io/client-go v0.34.1 k8s.io/component-helpers v0.34.1 k8s.io/klog/v2 v2.130.1 + k8s.io/kubelet v0.34.1 k8s.io/kubernetes v1.34.1 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 kubevirt.io/api v1.0.0-alpha.0 @@ -88,9 +89,9 @@ require ( github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonpointer v0.21.1 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect @@ -101,13 +102,14 @@ require ( github.com/google/gnostic-models v0.7.0 // indirect github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v0.0.0-20200330140219-3fe23663418f // indirect github.com/juju/testing v0.0.0-20200706033705-4c23f9c453cd // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect github.com/mdlayher/packet v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -120,10 +122,11 @@ require ( github.com/pborman/uuid v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/stretchr/objx v0.5.2 // indirect @@ -137,11 +140,11 @@ require ( go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.43.0 // indirect - golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/oauth2 v0.29.0 // indirect golang.org/x/term v0.36.0 // indirect golang.org/x/text v0.30.0 // indirect golang.org/x/tools v0.38.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -152,7 +155,6 @@ require ( k8s.io/component-base v0.34.1 // indirect k8s.io/controller-manager v0.34.1 // indirect k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect - k8s.io/kubelet v0.34.1 // indirect kubevirt.io/containerized-data-importer-api v1.55.0 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect diff --git a/go-controller/go.sum b/go-controller/go.sum index 7413340ad7..3f42055724 100644 --- a/go-controller/go.sum +++ b/go-controller/go.sum @@ -297,8 +297,8 @@ github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= +github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= @@ -310,8 +310,8 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -445,6 +445,8 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -514,8 +516,9 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -533,8 +536,8 @@ github.com/mdlayher/packet v1.0.0 h1:InhZJbdShQYt6XV2GPj5XHxChzOfhJJOMbvnGAmOfQ8 github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU= github.com/mdlayher/socket v0.2.1 h1:F2aaOwb53VsBE+ebRS9bLd7yPOfYUMC8lOODdCBDY6w= github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= -github.com/metallb/frr-k8s v0.0.15 h1:6M3UGhovX1EFoaSGjrRD7djUAx3w2I+g81FH8OVtHkM= -github.com/metallb/frr-k8s v0.0.15/go.mod h1:TjrGoAf+v00hYGlI8jUdyDxY5udMAOs2GWwrvLWnA4E= +github.com/metallb/frr-k8s v0.0.21 h1:JLlCeXVlW5BLVdPy2u5sS9UCVlnK9x2vzWbIkxb8Atk= +github.com/metallb/frr-k8s v0.0.21/go.mod h1:VMnCZUVXYy7k0Fsa2L3XKwISFs3Thv0Uord7rSZPQZw= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= @@ -654,16 +657,16 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -674,8 +677,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -711,6 +714,8 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -928,8 +933,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= +golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1043,8 +1048,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1093,8 +1098,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= -gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= diff --git a/go-controller/hack/update-codegen.sh b/go-controller/hack/update-codegen.sh index 5f919c4036..ba96791522 100755 --- a/go-controller/hack/update-codegen.sh +++ b/go-controller/hack/update-codegen.sh @@ -4,6 +4,10 @@ set -o errexit set -o nounset set -o pipefail +# Add GOBIN to PATH so installed tools can be found +GOPATH=${GOPATH:-$(go env GOPATH)} +export PATH="${GOPATH}/bin:${PATH}" + crds=$(ls pkg/crd 2> /dev/null) if [ -z "${crds}" ]; then exit @@ -53,6 +57,10 @@ for crd in ${crds}; do api_version=$(get_crd_version "${crd}") + # Clean up previously generated files to avoid stale copies + echo "Cleaning up existing generated files for $crd ($api_version)" + rm -rf "${SCRIPT_ROOT}"/pkg/crd/$crd/${api_version}/apis + echo "Generating deepcopy funcs for $crd ($api_version)" deepcopy-gen \ --go-header-file hack/boilerplate.go.txt \ @@ -139,3 +147,5 @@ echo "Copying routeAdvertisements CRD" cp _output/crds/k8s.ovn.org_routeadvertisements.yaml ../dist/templates/k8s.ovn.org_routeadvertisements.yaml.j2 echo "Copying clusterNetworkConnect CRD" cp _output/crds/k8s.ovn.org_clusternetworkconnects.yaml ../dist/templates/k8s.ovn.org_clusternetworkconnects.yaml.j2 +echo "Copying vtep CRD" +cp _output/crds/k8s.ovn.org_vteps.yaml ../dist/templates/k8s.ovn.org_vteps.yaml.j2 diff --git a/go-controller/pkg/allocator/id/allocator.go b/go-controller/pkg/allocator/id/allocator.go index ef4900c7ce..20ed9798b4 100644 --- a/go-controller/pkg/allocator/id/allocator.go +++ b/go-controller/pkg/allocator/id/allocator.go @@ -16,7 +16,7 @@ const ( type Allocator interface { AllocateID(name string) (int, error) ReserveID(name string, id int) error - ReleaseID(name string) + ReleaseID(name string) int ForName(name string) NamedAllocator GetID(name string) int } @@ -25,7 +25,7 @@ type Allocator interface { type NamedAllocator interface { AllocateID() (int, error) ReserveID(int) error - ReleaseID() + ReleaseID() int } // idAllocator is used to allocate id for a resource and store the resource - id in a map @@ -90,15 +90,18 @@ func (idAllocator *idAllocator) ReserveID(name string, id int) error { return nil } -// ReleaseID releases the id allocated for the resource 'name' -func (idAllocator *idAllocator) ReleaseID(name string) { +// ReleaseID releases the id allocated for the resource 'name'. +// Returns the released id, or -1 if no id was allocated for that name. +func (idAllocator *idAllocator) ReleaseID(name string) int { idAllocator.nameIdMap.LockKey(name) defer idAllocator.nameIdMap.UnlockKey(name) v, ok := idAllocator.nameIdMap.Load(name) if ok { idAllocator.idBitmap.Release(v) idAllocator.nameIdMap.Delete(name) + return v } + return invalidID } func (idAllocator *idAllocator) ForName(name string) NamedAllocator { @@ -129,8 +132,8 @@ func (allocator *namedAllocator) ReserveID(id int) error { return allocator.allocator.ReserveID(allocator.name, id) } -func (allocator *namedAllocator) ReleaseID() { - allocator.allocator.ReleaseID(allocator.name) +func (allocator *namedAllocator) ReleaseID() int { + return allocator.allocator.ReleaseID(allocator.name) } // idsAllocator is used to allocate multiple ids for a resource and store the resource - ids in a map diff --git a/go-controller/pkg/allocator/id/allocator_test.go b/go-controller/pkg/allocator/id/allocator_test.go index 79b783fbf8..2803d6ea81 100644 --- a/go-controller/pkg/allocator/id/allocator_test.go +++ b/go-controller/pkg/allocator/id/allocator_test.go @@ -5,6 +5,43 @@ import ( "testing" ) +func TestIDAllocator_ReleaseID(t *testing.T) { + t.Run("returns allocated ID when releasing", func(t *testing.T) { + allocator := NewIDAllocator("test", 10) + id, err := allocator.AllocateID("resource1") + if err != nil { + t.Fatalf("AllocateID() unexpected error: %v", err) + } + + got := allocator.ReleaseID("resource1") + if got != id { + t.Errorf("ReleaseID() = %d, want %d", got, id) + } + if allocator.GetID("resource1") != -1 { + t.Error("GetID() should return -1 after release") + } + }) + + t.Run("returns -1 when releasing already released resource", func(t *testing.T) { + allocator := NewIDAllocator("test", 10) + if _, err := allocator.AllocateID("resource1"); err != nil { + t.Fatalf("AllocateID() unexpected error: %v", err) + } + allocator.ReleaseID("resource1") + + if got := allocator.ReleaseID("resource1"); got != -1 { + t.Errorf("ReleaseID() = %d, want -1", got) + } + }) + + t.Run("returns -1 when releasing non-existent resource", func(t *testing.T) { + allocator := NewIDAllocator("test", 10) + if got := allocator.ReleaseID("nonexistent"); got != -1 { + t.Errorf("ReleaseID() = %d, want -1", got) + } + }) +} + func TestIDsAllocator(t *testing.T) { // create allocator with range [3, 8] allocator := newIDsAllocator("test", 6, 3) diff --git a/go-controller/pkg/allocator/pod/pod_annotation.go b/go-controller/pkg/allocator/pod/pod_annotation.go index 5e687e1a00..116a2a52ae 100644 --- a/go-controller/pkg/allocator/pod/pod_annotation.go +++ b/go-controller/pkg/allocator/pod/pod_annotation.go @@ -75,6 +75,7 @@ func (allocator *PodAnnotationAllocator) AllocatePodAnnotation( ipAllocator subnet.NamedAllocator, node *corev1.Node, pod *corev1.Pod, + nadKey string, network *nadapi.NetworkSelectionElement, reallocateIP bool, networkRole string) ( @@ -89,6 +90,7 @@ func (allocator *PodAnnotationAllocator) AllocatePodAnnotation( allocator.netInfo, node, pod, + nadKey, network, allocator.ipamClaimsReconciler, allocator.macRegistry, @@ -104,6 +106,7 @@ func allocatePodAnnotation( netInfo util.NetInfo, node *corev1.Node, pod *corev1.Pod, + nadKey string, network *nadapi.NetworkSelectionElement, claimsReconciler persistentips.PersistentAllocations, macRegistry mac.Register, @@ -124,6 +127,7 @@ func allocatePodAnnotation( netInfo, node, pod, + nadKey, network, claimsReconciler, macRegistry, @@ -161,6 +165,7 @@ func (allocator *PodAnnotationAllocator) AllocatePodAnnotationWithTunnelID( idAllocator id.NamedAllocator, node *corev1.Node, pod *corev1.Pod, + nadKey string, network *nadapi.NetworkSelectionElement, reallocateIP bool, networkRole string) ( @@ -176,6 +181,7 @@ func (allocator *PodAnnotationAllocator) AllocatePodAnnotationWithTunnelID( allocator.netInfo, node, pod, + nadKey, network, allocator.ipamClaimsReconciler, allocator.macRegistry, @@ -192,6 +198,7 @@ func allocatePodAnnotationWithTunnelID( netInfo util.NetInfo, node *corev1.Node, pod *corev1.Pod, + nadKey string, network *nadapi.NetworkSelectionElement, claimsReconciler persistentips.PersistentAllocations, macRegistry mac.Register, @@ -209,6 +216,7 @@ func allocatePodAnnotationWithTunnelID( netInfo, node, pod, + nadKey, network, claimsReconciler, macRegistry, @@ -329,6 +337,7 @@ func allocatePodAnnotationWithRollback( netInfo util.NetInfo, node *corev1.Node, pod *corev1.Pod, + nadKey string, network *nadapi.NetworkSelectionElement, claimsReconciler persistentips.PersistentAllocations, macRegistry mac.Register, @@ -339,11 +348,10 @@ func allocatePodAnnotationWithRollback( rollback func(), err error) { - nadName := types.DefaultNetworkName - if netInfo.IsUserDefinedNetwork() { - nadName = util.GetNADName(network.Namespace, network.Name) + if !netInfo.IsUserDefinedNetwork() { + nadKey = types.DefaultNetworkName } - podDesc := fmt.Sprintf("%s/%s/%s", nadName, pod.Namespace, pod.Name) + podDesc := fmt.Sprintf("%s/%s/%s", nadKey, pod.Namespace, pod.Name) macOwnerID := macOwner(pod) networkName := netInfo.GetNetworkName() @@ -388,7 +396,7 @@ func allocatePodAnnotationWithRollback( } }() - podAnnotation, _ = util.UnmarshalPodAnnotation(pod.Annotations, nadName) + podAnnotation, _ = util.UnmarshalPodAnnotation(pod.Annotations, nadKey) isNetworkAllocated := podAnnotation != nil if podAnnotation == nil { podAnnotation = &util.PodAnnotation{} @@ -532,13 +540,13 @@ func allocatePodAnnotationWithRollback( // repeated requests are no-op because mac already reserved if !errors.Is(rerr, mac.ErrMACReserved) { // avoid leaking the network name because this error may reflect of a pod event, which is visible to non-admins. - err = fmt.Errorf("failed to reserve MAC address %q for owner %q on network attachment %q: %w", - tentative.MAC, macOwnerID, nadName, rerr) + err = fmt.Errorf("failed to reserve MAC address %q for owner %q on NAD key %q: %w", + tentative.MAC, macOwnerID, nadKey, rerr) klog.Errorf("%v, network-name: %q", err, networkName) return } } else { - klog.V(5).Infof("Reserved MAC %q for owner %q on network %q nad %q", tentative.MAC, macOwnerID, networkName, nadName) + klog.V(5).Infof("Reserved MAC %q for owner %q on network %q NAD key %q", tentative.MAC, macOwnerID, networkName, nadKey) releaseMAC = tentative.MAC } } @@ -554,7 +562,7 @@ func allocatePodAnnotationWithRollback( if needsAnnotationUpdate { updatedPod = pod - updatedPod.Annotations, err = util.MarshalPodAnnotation(updatedPod.Annotations, tentative, nadName) + updatedPod.Annotations, err = util.MarshalPodAnnotation(updatedPod.Annotations, tentative, nadKey) podAnnotation = tentative } diff --git a/go-controller/pkg/allocator/pod/pod_annotation_test.go b/go-controller/pkg/allocator/pod/pod_annotation_test.go index da7c1979bc..8159c39da7 100644 --- a/go-controller/pkg/allocator/pod/pod_annotation_test.go +++ b/go-controller/pkg/allocator/pod/pod_annotation_test.go @@ -64,8 +64,9 @@ func (a *idAllocatorStub) ReserveID(int) error { return a.reserveIDError } -func (a *idAllocatorStub) ReleaseID() { +func (a *idAllocatorStub) ReleaseID() int { a.releasedID = true + return a.nextID } type persistentIPsStub struct { @@ -978,21 +979,6 @@ func Test_allocatePodAnnotationWithRollback(t *testing.T) { }, wantErr: true, }, - { - // ID allocation error - name: "expect ID allocation error", - idAllocation: true, - args: args{ - idAllocator: &idAllocatorStub{ - reserveIDError: errors.New("ID allocation error"), - }, - }, - podAnnotation: &util.PodAnnotation{ - MAC: randomMac, - TunnelID: 200, - }, - wantErr: true, - }, { // expect ID release on error name: "expect error, release ID", @@ -1262,6 +1248,7 @@ func Test_allocatePodAnnotationWithRollback(t *testing.T) { tt.netInfo, node, pod, + fmt.Sprintf("%s/%s", network.Namespace, network.Name), network, claimsReconciler, macRegistry, diff --git a/go-controller/pkg/clustermanager/clustermanager.go b/go-controller/pkg/clustermanager/clustermanager.go index de03ddea0b..12ce158aa0 100644 --- a/go-controller/pkg/clustermanager/clustermanager.go +++ b/go-controller/pkg/clustermanager/clustermanager.go @@ -23,6 +23,7 @@ import ( udntemplate "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork/template" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" networkconnectclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1/apis/clientset/versioned" + vtepinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" @@ -160,6 +161,10 @@ func NewClusterManager( } if util.IsNetworkSegmentationSupportEnabled() { + var vtepInformer vtepinformer.VTEPInformer + if util.IsEVPNEnabled() { + vtepInformer = wf.VTEPInformer() + } udnController := udncontroller.New( ovnClient.NetworkAttchDefClient, wf.NADInformer(), ovnClient.UserDefinedNetworkClient, @@ -168,6 +173,7 @@ func NewClusterManager( cm.networkManager.Interface(), wf.PodCoreInformer(), wf.NamespaceInformer(), + vtepInformer, cm.recorder, ) cm.userDefinedNetworkController = udnController @@ -254,7 +260,6 @@ func (cm *ClusterManager) Start(ctx context.Context) error { return err } } - return nil } diff --git a/go-controller/pkg/clustermanager/egressip_controller.go b/go-controller/pkg/clustermanager/egressip_controller.go index 7b0f2a2e39..ded7851375 100644 --- a/go-controller/pkg/clustermanager/egressip_controller.go +++ b/go-controller/pkg/clustermanager/egressip_controller.go @@ -531,7 +531,26 @@ func (eIPC *egressIPClusterController) getSortedEgressData() ([]*egressNode, map return assignableNodes, allAllocations } -func (eIPC *egressIPClusterController) initEgressNodeReachability(_ []interface{}) error { +func (eIPC *egressIPClusterController) initEgressNodeReachability(objs []interface{}) error { + for _, obj := range objs { + node := obj.(*corev1.Node) + if err := eIPC.initEgressIPAllocator(node); err != nil { + klog.Warningf("Egress node initialization error: %v", err) + } + } + + // Before reconciling unassigned EgressIPs, ensure the allocator cache is populated + // with existing assignments from EgressIP statuses. This prevents duplicate IP + // assignments when two EgressIPs have the same IP in their specs but only one has + // it assigned in status (e.g., after control-plane restart or during initial sync). + egressIPs, err := eIPC.kube.GetEgressIPs() + if err != nil { + return fmt.Errorf("unable to list EgressIPs, err: %v", err) + } + for _, egressIP := range egressIPs { + eIPC.ensureAllocatorEgressIPAssignments(egressIP) + } + go eIPC.checkEgressNodesReachability() return nil } @@ -990,11 +1009,6 @@ func (eIPC *egressIPClusterController) reconcileEgressIP(old, new *egressipv1.Eg statusToRemove = append(statusToRemove, status) ipsToRemove.Insert(status.EgressIP) } - // Adding the mark to annotations is bundled with status update in-order to minimise updates, cover the case where there is no update to status - // and mark annotation has been modified / removed. This should only occur for an update and the mark was previous set. - if ipsToAssign.Len() == 0 && ipsToRemove.Len() == 0 { - eIPC.ensureMark(old, new) - } if ipsToRemove.Len() > 0 { // The following is added as to ensure that we only add after having @@ -1825,10 +1839,21 @@ func generateStatusPatchOp(statusItems []egressipv1.EgressIPStatusItem) jsonPatc } } +// ensureAllocatorEgressIPAssignments adds EgressIP assignments to the allocator cache +// if the EgressIP has status items. This is critical to prevent duplicate IP assignments +// during restart when EgressIPs are processed in arbitrary order. +func (eIPC *egressIPClusterController) ensureAllocatorEgressIPAssignments(egressIP *egressipv1.EgressIP) { + if len(egressIP.Status.Items) > 0 { + eIPC.addAllocatorEgressIPAssignments(egressIP.Name, egressIP.Status.Items) + } +} + // syncEgressIPMarkAllocator iterates over all existing EgressIPs. It builds a mark cache of existing marks stored on each -// EgressIP annotation or allocates and adds a new mark to an EgressIP if it doesn't exist +// EgressIP annotation or allocates and adds a new mark to an EgressIP if it doesn't exist. func (eIPC *egressIPClusterController) syncEgressIPMarkAllocator(egressIPs []interface{}) error { - // reserve previously assigned marks + // Reserve previously assigned marks. Note: the allocator cache is pre-populated with + // existing assignments from EgressIP statuses in initEgressNodeReachability, which runs + // before this sync function. for _, object := range egressIPs { egressIP, ok := object.(*egressipv1.EgressIP) if !ok { @@ -1880,22 +1905,6 @@ func getEgressIPMarkAllocator() id.Allocator { return id.NewIDAllocator("eip_mark", eipMarkMax-eipMarkMin) } -// ensureMark ensures that if a mark was remove or changed value, then restore the mark. -func (eIPC *egressIPClusterController) ensureMark(old, new *egressipv1.EgressIP) { - // Adding the mark to annotations is bundled with status update in-order to minimise updates, cover the case where there is no update to status - // and mark annotation has been modified / removed. This should only occur for an update and the mark was previous set. - if old != nil && new != nil { - if util.IsEgressIPMarkSet(old.Annotations) && util.EgressIPMarkAnnotationChanged(old.Annotations, new.Annotations) { - mark, _, err := eIPC.getOrAllocMark(new.Name) - if err != nil { - klog.Errorf("Failed to restore EgressIP %s mark because unable to retrieve mark: %v", new.Name, err) - } else if err = eIPC.patchEgressIP(new.Name, generateMarkPatchOp(mark)); err != nil { - klog.Errorf("Failed to restore EgressIP %s mark because patching failed: %v", new.Name, err) - } - } - } -} - // getOrAllocMark allocates a new mark integer for name using round-robin strategy if none was already allocated for name otherwise // returns the previously allocated mark. // The mark is bounded by util.EgressIPMarkBase & util.EgressIPMarkMax inclusive. diff --git a/go-controller/pkg/clustermanager/egressip_controller_test.go b/go-controller/pkg/clustermanager/egressip_controller_test.go index c89b8b58cf..593671f7b8 100644 --- a/go-controller/pkg/clustermanager/egressip_controller_test.go +++ b/go-controller/pkg/clustermanager/egressip_controller_test.go @@ -22,7 +22,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/util/retry" utilnet "k8s.io/utils/net" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" @@ -3288,24 +3287,7 @@ var _ = ginkgo.Describe("OVN cluster-manager EgressIP Operations", func() { assignedMark, err := strconv.Atoi(assignedMarkStr) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "failed to convert mark to string") - ginkgo.By("clear mark to cause update and expect restoration of mark") - gomega.Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { - eIP, err := fakeClusterManagerOVN.fakeClient.EgressIPClient.K8sV1().EgressIPs().Get(context.TODO(), eIP.Name, metav1.GetOptions{}) - if err != nil { - return err - } - eIP.Annotations = map[string]string{} - _, err = fakeClusterManagerOVN.fakeClient.EgressIPClient.K8sV1().EgressIPs().Update(context.TODO(), eIP, metav1.UpdateOptions{}) - return err - })).ShouldNot(gomega.HaveOccurred(), "failed to update EgressIP object") - ginkgo.By("confirm the original mark is restored") - gomega.Eventually(getEgressIPAnnotationValue(eIP.Name)).ShouldNot(gomega.BeEmpty()) - assignedMarkStr, err = getEgressIPAnnotationValue(eIP.Name)() - gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "failed to get egress IP mark from annotations") - assignedMarkAfterUpdate, err := strconv.Atoi(assignedMarkStr) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "failed to convert mark to string") - gomega.Expect(assignedMark).Should(gomega.Equal(assignedMarkAfterUpdate), "Mark should be identical if annotation is cleared") - ginkgo.By("confirm cache is unchanged") + ginkgo.By("confirm cache is set correctly") cachedMark, _, err := fakeClusterManagerOVN.eIPC.getOrAllocMark(eIP.Name) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) gomega.Expect(cachedMark).Should(gomega.Equal(assignedMark), "EIP annotation and cache mark integer must be the same") @@ -4218,6 +4200,122 @@ var _ = ginkgo.Describe("OVN cluster-manager EgressIP Operations", func() { err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) + + // This test validates that when two EgressIP CRs have the same IP in their specs, + // and one already has the IP assigned in status (from before restart), the sync + // function properly pre-populates the allocator cache to prevent duplicate assignment. + // This is a regression test for the bug where duplicate IPs were assigned during + // control-plane pod restart because the allocator cache wasn't populated from + // existing EgressIP statuses before processing individual ADD events. + ginkgo.It("should not assign duplicate IP during restart when two EgressIPs have same IP in spec", func() { + app.Action = func(*cli.Context) error { + duplicateIP := "192.168.126.101" + node1IPv4 := "192.168.126.12/24" + node2IPv4 := "192.168.126.51/24" + + node1 := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Annotations: map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\", \"%s\"]}", v4NodeSubnet, v6NodeSubnet), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4), + }, + Labels: map[string]string{ + "k8s.ovn.org/egress-assignable": "", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } + node2 := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Annotations: map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\": [\"%s\",\"%s\"]}", v4NodeSubnet, v6NodeSubnet), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4), + }, + Labels: map[string]string{ + "k8s.ovn.org/egress-assignable": "", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } + + // eIP1 has the IP assigned in status (simulating state from before restart) + eIP1 := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMeta("egressip-1"), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{duplicateIP}, + }, + Status: egressipv1.EgressIPStatus{ + Items: []egressipv1.EgressIPStatusItem{ + { + EgressIP: duplicateIP, + Node: node1Name, + }, + }, + }, + } + + // eIP2 has the same IP in spec but NOT in status (unassigned, but was created + // with duplicate IP - which should have been rejected but wasn't due to a bug + // or manual API manipulation) + eIP2 := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMeta("egressip-2"), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{duplicateIP}, + }, + Status: egressipv1.EgressIPStatus{ + Items: []egressipv1.EgressIPStatusItem{}, + }, + } + + fakeClusterManagerOVN.start( + &corev1.NodeList{Items: []corev1.Node{node1, node2}}, + // Both EgressIPs exist at startup - simulating restart scenario + &egressipv1.EgressIPList{Items: []egressipv1.EgressIP{eIP1, eIP2}}, + ) + + // Use WatchEgressNodes to properly initialize the allocator cache + // (simulating real startup behavior rather than manually setting up cache) + _, err := fakeClusterManagerOVN.eIPC.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + _, err = fakeClusterManagerOVN.eIPC.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // eIP1 should keep its assignment (the IP was already assigned) + gomega.Eventually(getEgressIPStatusLen("egressip-1")).Should(gomega.Equal(1)) + egressIPs1, nodes1 := getEgressIPStatus("egressip-1") + gomega.Expect(nodes1[0]).To(gomega.Equal(node1Name)) + gomega.Expect(egressIPs1[0]).To(gomega.Equal(duplicateIP)) + + // eIP2 should NOT get the duplicate IP assigned (not even to node2) - + // it should remain unassigned because initEgressNodeReachability pre-populated the + // cache with eIP1's assignment + gomega.Eventually(getEgressIPStatusLen("egressip-2")).Should(gomega.Equal(0)) + + return nil + } + + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) }) ginkgo.Context("AddEgressIP for IPv4", func() { diff --git a/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller.go b/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller.go index 191695d58d..66f95f1e83 100644 --- a/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller.go +++ b/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller.go @@ -258,6 +258,17 @@ func (c *Controller) syncDefaultEndpointSlice(ctx context.Context, key string) e klog.Infof("Processing %s/%s EndpointSlice in %q primary network", namespace, name, namespacePrimaryNetwork.GetNetworkName()) + nadKey, err := c.networkManager.GetPrimaryNADForNamespace(namespace) + if err != nil { + return err + } + if nadKey == types.DefaultNetworkName { + return fmt.Errorf("no primary NAD found for namespace %s", namespace) + } + if networkName := c.networkManager.GetNetworkNameForNADKey(nadKey); networkName == "" || networkName != namespacePrimaryNetwork.GetNetworkName() { + return fmt.Errorf("primary NAD %s does not match network %s", nadKey, namespacePrimaryNetwork.GetNetworkName()) + } + defaultEndpointSlice, err := c.endpointSliceLister.EndpointSlices(namespace).Get(name) if err != nil && !apierrors.IsNotFound(err) { return err @@ -316,7 +327,7 @@ func (c *Controller) syncDefaultEndpointSlice(ctx context.Context, key string) e } } - currentMirror, err := c.mirrorEndpointSlice(mirroredEndpointSlice, defaultEndpointSlice, namespacePrimaryNetwork) + currentMirror, err := c.mirrorEndpointSlice(mirroredEndpointSlice, defaultEndpointSlice, namespacePrimaryNetwork, nadKey) if err != nil { return err } @@ -349,7 +360,7 @@ func isManagedByDefault(endpointSlice *v1.EndpointSlice) bool { // getPodIP retrieves the IP address of a specified Pod within a given namespace and network. // If the pod is host networked it returns default pod IP from the status ignoring the network. // Otherwise, it unmarshals the Pod's network annotation, and matches the IP from the provided network. -func (c *Controller) getPodIP(name, namespace, network string, isIPv6 bool) (string, error) { +func (c *Controller) getPodIP(name, namespace, nadKey string, isIPv6 bool) (string, error) { var podIP string pod, err := c.podLister.Pods(namespace).Get(name) if err != nil { @@ -370,7 +381,7 @@ func (c *Controller) getPodIP(name, namespace, network string, isIPv6 bool) (str podIP = ipAddr.String() } else { - net, err := util.UnmarshalPodAnnotation(pod.Annotations, network) + net, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { return "", err } @@ -388,7 +399,7 @@ func (c *Controller) getPodIP(name, namespace, network string, isIPv6 bool) (str // mirrorEndpointSlice creates or updates a mirrored EndpointSlice based on the provided defaultEndpointSlice. // The mirrored EndpointSlice will have custom labels set and will be managed by the current controller. -func (c *Controller) mirrorEndpointSlice(mirroredEndpointSlice, defaultEndpointSlice *v1.EndpointSlice, network util.NetInfo) (*v1.EndpointSlice, error) { +func (c *Controller) mirrorEndpointSlice(mirroredEndpointSlice, defaultEndpointSlice *v1.EndpointSlice, network util.NetInfo, nadKey string) (*v1.EndpointSlice, error) { var currentMirror *v1.EndpointSlice if mirroredEndpointSlice != nil { currentMirror = mirroredEndpointSlice.DeepCopy() @@ -428,13 +439,9 @@ func (c *Controller) mirrorEndpointSlice(mirroredEndpointSlice, defaultEndpointS currentMirror.Endpoints = make([]v1.Endpoint, len(defaultEndpointSlice.Endpoints)) isIPv6 := defaultEndpointSlice.AddressType == v1.AddressTypeIPv6 - nadList := network.GetNADs() - if len(nadList) != 1 { - return nil, fmt.Errorf("expected one NAD in %s network, got: %d", network.GetNetworkName(), len(nadList)) - } for i, endpoint := range defaultEndpointSlice.Endpoints { if endpoint.TargetRef != nil && endpoint.TargetRef.Kind == "Pod" { - podIP, err := c.getPodIP(endpoint.TargetRef.Name, endpoint.TargetRef.Namespace, nadList[0], isIPv6) + podIP, err := c.getPodIP(endpoint.TargetRef.Name, endpoint.TargetRef.Namespace, nadKey, isIPv6) if err != nil { return nil, fmt.Errorf("failed to determine the Pod IP of: %s/%s: %v", endpoint.TargetRef.Namespace, endpoint.TargetRef.Name, err) } diff --git a/go-controller/pkg/clustermanager/network_cluster_controller.go b/go-controller/pkg/clustermanager/network_cluster_controller.go index 870345a6f4..63dfacc54a 100644 --- a/go-controller/pkg/clustermanager/network_cluster_controller.go +++ b/go-controller/pkg/clustermanager/network_cluster_controller.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" cache "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" @@ -28,6 +29,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/persistentips" objretry "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" @@ -81,9 +83,90 @@ type networkClusterController struct { // To avoid changing that error report with every update, we store reported error node. reportedErrorNode string + // dynamicUDNNodeRefs tracks active nodes for dynamic UDN allocation. + dynamicUDNNodeRefsLock sync.Mutex + dynamicUDNNodeRefs map[string]bool + dynamicUDNNodeCount int + + nadKeysLock sync.Mutex + lastNADKeys sets.Set[string] + util.ReconcilableNetInfo } +// HandleNetworkRefChange satisfies the NetworkController interface; it updates dynamic UDN metrics and status. +func (ncc *networkClusterController) HandleNetworkRefChange(nodeName string, active bool) { + if !config.OVNKubernetesFeature.EnableDynamicUDNAllocation || !ncc.IsUserDefinedNetwork() { + return + } + + nodeCount, changed := ncc.updateDynamicUDNNodeRefs(nodeName, active) + if !changed { + return + } + networkName := ncc.GetNetworkName() + metrics.SetDynamicUDNNodeCount(networkName, float64(nodeCount)) + klog.V(5).Infof("Updated metric: network=%s nodes=%d", networkName, nodeCount) + + var cond *metav1.Condition + if nodeCount == 0 { + msg := "no nodes currently rendered with network" + cond = &metav1.Condition{ + Type: "NodesSelected", + Status: metav1.ConditionFalse, + Reason: "DynamicAllocation", + Message: msg, + LastTransitionTime: metav1.Now(), + } + } else { + msg := fmt.Sprintf("%d node(s) rendered with network", nodeCount) + cond = &metav1.Condition{ + Type: "NodesSelected", + Status: metav1.ConditionTrue, + Reason: "DynamicAllocation", + Message: msg, + LastTransitionTime: metav1.Now(), + } + } + if ncc.statusReporter != nil { + if err := ncc.statusReporter( + networkName, + "ClusterManager", // FieldManager - must be unique per subsystem + cond, + ); err != nil { + klog.Errorf("Failed to update NodesSelected condition for %s: %v", networkName, err) + } else { + klog.V(4).Infof("Updated Dynamic Allocation NodesSelected condition for %s: %s", networkName, cond.Message) + } + } +} + +func (ncc *networkClusterController) updateDynamicUDNNodeRefs(nodeName string, active bool) (int, bool) { + ncc.dynamicUDNNodeRefsLock.Lock() + defer ncc.dynamicUDNNodeRefsLock.Unlock() + + if ncc.dynamicUDNNodeRefs == nil { + ncc.dynamicUDNNodeRefs = map[string]bool{} + } + + current := ncc.dynamicUDNNodeRefs[nodeName] + if active == current { + return ncc.dynamicUDNNodeCount, false + } + + if active { + ncc.dynamicUDNNodeRefs[nodeName] = true + ncc.dynamicUDNNodeCount++ + return ncc.dynamicUDNNodeCount, true + } + + delete(ncc.dynamicUDNNodeRefs, nodeName) + if ncc.dynamicUDNNodeCount > 0 { + ncc.dynamicUDNNodeCount-- + } + return ncc.dynamicUDNNodeCount, true +} + func newNetworkClusterController( netInfo util.NetInfo, ovnClient *util.OVNClusterManagerClientset, @@ -479,7 +562,8 @@ func (ncc *networkClusterController) Cleanup() error { } func (ncc *networkClusterController) Reconcile(netInfo util.NetInfo) error { - reconcilePendingPods := !ncc.ReconcilableNetInfo.EqualNADs(netInfo.GetNADs()...) + nadKeys := ncc.networkManager.GetNADKeysForNetwork(netInfo.GetNetworkName()) + reconcilePendingPods := ncc.updateNADKeysChanged(nadKeys) // update network information, point of no return err := util.ReconcileNetInfo(ncc.ReconcilableNetInfo, netInfo) if err != nil { @@ -493,6 +577,16 @@ func (ncc *networkClusterController) Reconcile(netInfo util.NetInfo) error { return nil } +func (ncc *networkClusterController) updateNADKeysChanged(nadKeys []string) bool { + ncc.nadKeysLock.Lock() + defer ncc.nadKeysLock.Unlock() + + next := sets.New(nadKeys...) + changed := ncc.lastNADKeys == nil || !next.Equal(ncc.lastNADKeys) + ncc.lastNADKeys = next + return changed +} + // networkClusterControllerEventHandler object handles the events // from retry framework. type networkClusterControllerEventHandler struct { diff --git a/go-controller/pkg/clustermanager/network_cluster_controller_test.go b/go-controller/pkg/clustermanager/network_cluster_controller_test.go index f4c5976c4b..db4f95d569 100644 --- a/go-controller/pkg/clustermanager/network_cluster_controller_test.go +++ b/go-controller/pkg/clustermanager/network_cluster_controller_test.go @@ -1,209 +1,129 @@ package clustermanager import ( - "context" - "net" - "sync" + "fmt" + "testing" - "github.com/onsi/ginkgo/v2" + cnitypes "github.com/containernetworking/cni/pkg/types" "github.com/onsi/gomega" - "github.com/urfave/cli/v2" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/record" + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" - ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) -var _ = ginkgo.Describe("Network Cluster Controller", func() { - var ( - app *cli.App - f *factory.WatchFactory - recorder record.EventRecorder - stopChan chan struct{} - wg *sync.WaitGroup +func TestHandleNetworkRefChangeUpdatesStatusAndMetrics(t *testing.T) { + g := gomega.NewWithT(t) + + err := config.PrepareTestConfig() + g.Expect(err).ToNot(gomega.HaveOccurred()) + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + + metrics.RegisterClusterManagerFunctional() + + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: "ns1_udn1_refchange_test", + Type: "ovn-k8s-cni-overlay", + }, + Topology: types.Layer3Topology, + Role: types.NetworkRolePrimary, + Subnets: "10.1.0.0/16", + } + netInfo, err := util.NewNetInfo(netConf) + g.Expect(err).ToNot(gomega.HaveOccurred()) + networkName := netInfo.GetNetworkName() + defer metrics.DeleteDynamicUDNNodeCount(networkName) + + var gotNetwork string + var gotCondStatus string + var gotCondMsg string + ncc := &networkClusterController{ + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), + statusReporter: func(networkName, _ string, condition *metav1.Condition, _ ...*util.EventDetails) error { + gotNetwork = networkName + if condition != nil { + gotCondStatus = string(condition.Status) + gotCondMsg = condition.Message + } + return nil + }, + } + + ncc.HandleNetworkRefChange("node1", true) + g.Expect(gotNetwork).To(gomega.Equal(networkName)) + g.Expect(gotCondStatus).To(gomega.Equal(string(metav1.ConditionTrue))) + g.Expect(gotCondMsg).To(gomega.Equal("1 node(s) rendered with network")) + g.Expect(getUDNNodesRenderedMetric(t, networkName)).To(gomega.Equal(1.0)) + + ncc.HandleNetworkRefChange("node1", true) + g.Expect(gotNetwork).To(gomega.Equal(networkName)) + g.Expect(gotCondStatus).To(gomega.Equal(string(metav1.ConditionTrue))) + g.Expect(gotCondMsg).To(gomega.Equal("1 node(s) rendered with network")) + g.Expect(getUDNNodesRenderedMetric(t, networkName)).To(gomega.Equal(1.0)) + + ncc.HandleNetworkRefChange("node2", true) + g.Expect(gotNetwork).To(gomega.Equal(networkName)) + g.Expect(gotCondStatus).To(gomega.Equal(string(metav1.ConditionTrue))) + g.Expect(gotCondMsg).To(gomega.Equal("2 node(s) rendered with network")) + g.Expect(getUDNNodesRenderedMetric(t, networkName)).To(gomega.Equal(2.0)) + + ncc.HandleNetworkRefChange("node2", false) + g.Expect(gotNetwork).To(gomega.Equal(networkName)) + g.Expect(gotCondStatus).To(gomega.Equal(string(metav1.ConditionTrue))) + g.Expect(gotCondMsg).To(gomega.Equal("1 node(s) rendered with network")) + g.Expect(getUDNNodesRenderedMetric(t, networkName)).To(gomega.Equal(1.0)) + + ncc.HandleNetworkRefChange("node1", false) + ncc.HandleNetworkRefChange("node1", false) + g.Expect(gotCondStatus).To(gomega.Equal(string(metav1.ConditionFalse))) + g.Expect(gotCondMsg).To(gomega.Equal("no nodes currently rendered with network")) + g.Expect(getUDNNodesRenderedMetric(t, networkName)).To(gomega.Equal(0.0)) +} + +func getUDNNodesRenderedMetric(t *testing.T, networkName string) float64 { + t.Helper() + + metricName := fmt.Sprintf("%s_%s_%s", + types.MetricOvnkubeNamespace, + types.MetricOvnkubeSubsystemClusterManager, + "udn_nodes_rendered", ) - - ginkgo.BeforeEach(func() { - // Restore global default values before each testcase - gomega.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) - - app = cli.NewApp() - app.Name = "test" - app.Flags = config.Flags - stopChan = make(chan struct{}) - wg = &sync.WaitGroup{} - recorder = record.NewFakeRecorder(10) - }) - - ginkgo.AfterEach(func() { - close(stopChan) - if f != nil { - f.Shutdown() + mfs, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatalf("failed to gather metrics: %v", err) + } + for _, mf := range mfs { + if mf.GetName() != metricName { + continue } - wg.Wait() - }) - - ginkgo.Context("Host Subnets", func() { - ginkgo.It("removes an unused dual-stack allocation from single-stack cluster", func() { - app.Action = func(ctx *cli.Context) error { - nodes := []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - Annotations: map[string]string{ - ovnNodeIDAnnotaton: "3", - "k8s.ovn.org/node-subnets": "{\"default\":[\"10.128.0.0/24\", \"fd02:0:0:2::2895/64\"]}", - }, - }, - }, - } - kubeFakeClient := fake.NewSimpleClientset(&corev1.NodeList{ - Items: nodes, - }) - fakeClient := &util.OVNClusterManagerClientset{ - KubeClient: kubeFakeClient, - } - - _, err := config.InitConfig(ctx, nil, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - config.Kubernetes.HostNetworkNamespace = "" - - f, err = factory.NewClusterManagerWatchFactory(fakeClient) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = f.Start() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ncc := newDefaultNetworkClusterController(&util.DefaultNetInfo{}, fakeClient, f, recorder) - gomega.Expect(ncc.Start(ctx.Context)).To(gomega.Succeed()) - defer ncc.Stop() - - // Check that the default network controller removes the unused v6 node subnet annotation - gomega.Eventually(func() ([]*net.IPNet, error) { - updatedNode, err := fakeClient.KubeClient.CoreV1().Nodes().Get(context.TODO(), nodes[0].Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - return util.ParseNodeHostSubnetAnnotation(updatedNode, ovntypes.DefaultNetworkName) - }, 2).Should(gomega.Equal(ovntest.MustParseIPNets("10.128.0.0/24"))) - - return nil - } - - err := app.Run([]string{ - app.Name, - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.It("allocates a subnet for a new node", func() { - app.Action = func(ctx *cli.Context) error { - nodes := []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - Annotations: map[string]string{ - ovnNodeIDAnnotaton: "3", - }, - }, - }, - } - kubeFakeClient := fake.NewSimpleClientset(&corev1.NodeList{ - Items: nodes, - }) - fakeClient := &util.OVNClusterManagerClientset{ - KubeClient: kubeFakeClient, - } - - _, err := config.InitConfig(ctx, nil, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - config.Kubernetes.HostNetworkNamespace = "" - - f, err = factory.NewClusterManagerWatchFactory(fakeClient) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = f.Start() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ncc := newDefaultNetworkClusterController(&util.DefaultNetInfo{}, fakeClient, f, recorder) - gomega.Expect(ncc.Start(ctx.Context)).To(gomega.Succeed()) - defer ncc.Stop() - - // Check that the default network controller adds a subnet for the new node - gomega.Eventually(func() ([]*net.IPNet, error) { - updatedNode, err := fakeClient.KubeClient.CoreV1().Nodes().Get(context.TODO(), nodes[0].Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - return util.ParseNodeHostSubnetAnnotation(updatedNode, ovntypes.DefaultNetworkName) - }, 2).Should(gomega.Equal(ovntest.MustParseIPNets("10.128.0.0/23"))) - - return nil + for _, metric := range mf.GetMetric() { + if labelValue(metric.GetLabel(), "network_name") != networkName { + continue } - - err := app.Run([]string{ - app.Name, - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.It("removes an invalid single-stack annotation", func() { - app.Action = func(ctx *cli.Context) error { - nodes := []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - Annotations: map[string]string{ - "k8s.ovn.org/node-subnets": "{\"default\":[\"10.128.0.0/24\", \"1.2.3.0/24\"]}", - ovnNodeIDAnnotaton: "3", - }, - }, - }, - } - kubeFakeClient := fake.NewSimpleClientset(&corev1.NodeList{ - Items: nodes, - }) - fakeClient := &util.OVNClusterManagerClientset{ - KubeClient: kubeFakeClient, - } - - _, err := config.InitConfig(ctx, nil, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - config.Kubernetes.HostNetworkNamespace = "" - - f, err = factory.NewClusterManagerWatchFactory(fakeClient) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = f.Start() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ncc := newDefaultNetworkClusterController(&util.DefaultNetInfo{}, fakeClient, f, recorder) - gomega.Expect(ncc.Start(ctx.Context)).To(gomega.Succeed()) - defer ncc.Stop() - - // Check that the default network controller removes the unused v6 node subnet annotation - gomega.Eventually(func() ([]*net.IPNet, error) { - updatedNode, err := fakeClient.KubeClient.CoreV1().Nodes().Get(context.TODO(), nodes[0].Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - return util.ParseNodeHostSubnetAnnotation(updatedNode, ovntypes.DefaultNetworkName) - }, 2).Should(gomega.Equal(ovntest.MustParseIPNets("10.128.0.0/24"))) - - return nil + if metric.GetGauge() == nil { + t.Fatalf("metric %s for %s is not a gauge", metricName, networkName) } - - err := app.Run([]string{ - app.Name, - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - }) -}) + return metric.GetGauge().GetValue() + } + } + t.Fatalf("metric %s with network_name=%s not found", metricName, networkName) + return 0 +} + +func labelValue(labels []*dto.LabelPair, name string) string { + for _, label := range labels { + if label.GetName() == name { + return label.GetValue() + } + } + return "" +} diff --git a/go-controller/pkg/clustermanager/networkconnect/cluster_network_connect.go b/go-controller/pkg/clustermanager/networkconnect/cluster_network_connect.go index 3b94bd72b6..89e0eb2b8b 100644 --- a/go-controller/pkg/clustermanager/networkconnect/cluster_network_connect.go +++ b/go-controller/pkg/clustermanager/networkconnect/cluster_network_connect.go @@ -51,37 +51,34 @@ func getPrimaryNADForNamespace(networkMgr networkmanager.Interface, namespaceNam // No primary UDN in this namespace return "", nil, nil } - // Get the NAD key for the primary network in this namespace. - // Since this is for namespace-scoped UDNs, we expect exactly one NAD per network. - // Today we don't support multiple primary NADs for a namespace, so this is safe. - // Also note if the user misconfigures and ends up with CUDN and UDN for the same namespace, - // and if the CUDN was created first - which means the UDN won't be created successfully, - // then the user uses the P-UDN selector, the CUDN's NAD will be chosen here for this selector - // but that's a design flaw in the user's configuration, and expectation is for users to use - // the selectors correctly. - primaryNADs := namespacePrimaryNetwork.GetNADs() - if len(primaryNADs) != 1 { - return "", nil, fmt.Errorf("expected exactly one primary NAD for namespace %s, got %d", namespaceName, len(primaryNADs)) + primaryNADKey, err := networkMgr.GetPrimaryNADForNamespace(namespaceName) + if err != nil { + if util.IsInvalidPrimaryNetworkError(err) || util.IsUnprocessedActiveNetworkError(err) { + return "", nil, nil + } + return "", nil, err + } + if primaryNADKey == ovntypes.DefaultNetworkName { + return "", nil, nil } // There is a race condition where NAD is already deleted from kapi // but network manager is too slow to update the network manager cache. - // In this case, GetNADs() will return the NADs even though they are deleted. + // In this case, the primary NAD key may still be cached even though it is deleted. // So let's fetch the NAD again from the kapi to double confirm it exists // before returning it. - nadNamespace, nadName, err := cache.SplitMetaNamespaceKey(primaryNADs[0]) + nadNamespace, nadName, err := cache.SplitMetaNamespaceKey(primaryNADKey) if err != nil { - return "", nil, fmt.Errorf("failed to split NAD key %s: %w", primaryNADs[0], err) + return "", nil, fmt.Errorf("failed to split NAD key %s: %w", primaryNADKey, err) } _, err = nadLister.NetworkAttachmentDefinitions(nadNamespace).Get(nadName) if err != nil { if apierrors.IsNotFound(err) { - klog.Warningf("NAD %s not found in kapi, returning empty network info even if network manager cache says it exists", primaryNADs[0]) + klog.Warningf("NAD %s not found in kapi, returning empty network info even if network manager cache says it exists", primaryNADKey) return "", nil, nil } return "", nil, err } - // GetNADs() returns NADs in "namespace/name" format, so use directly - return primaryNADs[0], namespacePrimaryNetwork, nil + return primaryNADKey, namespacePrimaryNetwork, nil } func (c *Controller) reconcileClusterNetworkConnect(key string) error { @@ -269,22 +266,6 @@ func (c *Controller) discoverSelectedNetworks(cnc *networkconnectv1.ClusterNetwo return discoveredNetworks, allMatchingNADKeys, kerrors.NewAggregate(errs) } -func computeNetworkOwner(networkType string, networkID int) string { - return fmt.Sprintf("%s_%d", networkType, networkID) -} - -// parseNetworkOwnerTopology extracts the topology type from an owner key. -// Owner keys are formatted as "{topology}_{networkID}" (e.g., "layer3_1", "layer2_2"). -func parseNetworkOwnerTopology(owner string) (topologyType string, ok bool) { - if len(owner) > len(ovntypes.Layer3Topology)+1 && owner[:len(ovntypes.Layer3Topology)] == ovntypes.Layer3Topology { - return ovntypes.Layer3Topology, true - } - if len(owner) > len(ovntypes.Layer2Topology)+1 && owner[:len(ovntypes.Layer2Topology)] == ovntypes.Layer2Topology { - return ovntypes.Layer2Topology, true - } - return "", false -} - // allocateSubnets allocates subnets for the given discovered networks // It returns a map of owner to subnets // NOTE: If owner already had its subnets allocated, it will simply return those existing subnets @@ -302,14 +283,14 @@ func (c *Controller) allocateSubnets(discoveredNetworks []util.NetInfo, allocato } var err error if network.TopologyType() == ovntypes.Layer3Topology { - owner = computeNetworkOwner(ovntypes.Layer3Topology, networkID) + owner = util.ComputeNetworkOwner(ovntypes.Layer3Topology, networkID) subnets, err = allocator.AllocateLayer3Subnet(owner) if err != nil { errs = append(errs, fmt.Errorf("failed to allocate Layer3 subnet for network %s: %w", network.GetNetworkName(), err)) continue } } else if network.TopologyType() == ovntypes.Layer2Topology { - owner = computeNetworkOwner(ovntypes.Layer2Topology, networkID) + owner = util.ComputeNetworkOwner(ovntypes.Layer2Topology, networkID) subnets, err = allocator.AllocateLayer2Subnet(owner) if err != nil { errs = append(errs, fmt.Errorf("failed to allocate Layer2 subnet for network %s: %w", network.GetNetworkName(), err)) @@ -333,8 +314,8 @@ func (c *Controller) releaseSubnets(networksNeedingRelease sets.Set[string], allocator HybridConnectSubnetAllocator) error { var errs []error for networkKey := range networksNeedingRelease { - topologyType, ok := parseNetworkOwnerTopology(networkKey) - if !ok { + topologyType, _, err := util.ParseNetworkOwner(networkKey) + if err != nil { errs = append(errs, fmt.Errorf("invalid network key format: %s", networkKey)) continue } diff --git a/go-controller/pkg/clustermanager/networkconnect/controller.go b/go-controller/pkg/clustermanager/networkconnect/controller.go index df97a9f3a9..8bec1787ee 100644 --- a/go-controller/pkg/clustermanager/networkconnect/controller.go +++ b/go-controller/pkg/clustermanager/networkconnect/controller.go @@ -415,7 +415,8 @@ func (c *Controller) mustProcessCNCForNAD(nad *nadv1.NetworkAttachmentDefinition klog.Errorf("Failed to get active network for namespace %s: %v", namespace.Name, err) continue } - if primaryNAD.HasNAD(nadKey) { + networkName := c.networkManager.GetNetworkNameForNADKey(nadKey) + if networkName != "" && networkName == primaryNAD.GetNetworkName() { isSelected = true break selectorLoop } diff --git a/go-controller/pkg/clustermanager/networkconnect/controller_components_test.go b/go-controller/pkg/clustermanager/networkconnect/controller_components_test.go index 6e5d051466..461eda3c40 100644 --- a/go-controller/pkg/clustermanager/networkconnect/controller_components_test.go +++ b/go-controller/pkg/clustermanager/networkconnect/controller_components_test.go @@ -745,6 +745,7 @@ func TestController_reconcileClusterNetworkConnect(t *testing.T) { // Create fake network manager and auto-configure from nads and namespaces fakeNM := &networkmanager.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), + NADNetworks: make(map[string]util.NetInfo), } // Auto-populate PrimaryNetworks from NADs with IsUDN=true and IsPrimary=true @@ -770,6 +771,14 @@ func TestController_reconcileClusterNetworkConnect(t *testing.T) { } fakeNM.PrimaryNetworks[namespace] = mutableNetInfo } + for _, nad := range tt.nads { + nadKey := fmt.Sprintf("%s/%s", nad.Namespace, nad.Name) + nadObj, err := wf.NADInformer().Lister().NetworkAttachmentDefinitions(nad.Namespace).Get(nad.Name) + g.Expect(err).ToNot(gomega.HaveOccurred(), "NAD %s should exist", nadKey) + netInfo, err := util.ParseNADInfo(nadObj) + g.Expect(err).ToNot(gomega.HaveOccurred(), "ParseNADInfo for %s failed", nadKey) + fakeNM.NADNetworks[nadKey] = netInfo + } // Auto-configure UDN namespaces from namespaces with RequiresUDN=true for _, ns := range tt.namespaces { @@ -1162,6 +1171,16 @@ func TestController_reconcileNAD(t *testing.T) { fakeNM := &networkmanager.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), + NADNetworks: make(map[string]util.NetInfo), + } + + for _, nad := range tt.nads { + nadKey := fmt.Sprintf("%s/%s", nad.Namespace, nad.Name) + nadObj, err := wf.NADInformer().Lister().NetworkAttachmentDefinitions(nad.Namespace).Get(nad.Name) + g.Expect(err).ToNot(gomega.HaveOccurred(), "NAD %s should exist", nadKey) + netInfo, err := util.ParseNADInfo(nadObj) + g.Expect(err).ToNot(gomega.HaveOccurred(), "ParseNADInfo for %s failed", nadKey) + fakeNM.NADNetworks[nadKey] = netInfo } tunnelKeysAllocator := id.NewTunnelKeyAllocator("TunnelKeys") @@ -1757,6 +1776,7 @@ func TestMustProcessCNCForNAD(t *testing.T) { fakeNM := &networkmanager.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), + NADNetworks: make(map[string]util.NetInfo), } // Auto-configure primary network from NAD when IsUDN && IsPrimary @@ -1771,6 +1791,13 @@ func TestMustProcessCNCForNAD(t *testing.T) { mutableNetInfo.SetNADs(tt.nad.Namespace + "/" + tt.nad.Name) fakeNM.PrimaryNetworks[tt.nad.Namespace] = mutableNetInfo } + if tt.nad != nil { + nadKey := fmt.Sprintf("%s/%s", tt.nad.Namespace, tt.nad.Name) + nadObj := tt.nad.NAD() + netInfo, err := util.ParseNADInfo(nadObj) + g.Expect(err).ToNot(gomega.HaveOccurred(), "ParseNADInfo for %s failed", nadKey) + fakeNM.NADNetworks[nadKey] = netInfo + } tunnelKeysAllocator := id.NewTunnelKeyAllocator("TunnelKeys") c := NewController(wf, fakeClientset, fakeNM.Interface(), tunnelKeysAllocator) @@ -2451,6 +2478,7 @@ func TestController_reconcileNamespace(t *testing.T) { // Create fake network manager and auto-configure from nads fakeNM := &networkmanager.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), + NADNetworks: make(map[string]util.NetInfo), } // Auto-populate PrimaryNetworks from NADs with IsUDN=true and IsPrimary=true @@ -2473,6 +2501,14 @@ func TestController_reconcileNamespace(t *testing.T) { } fakeNM.PrimaryNetworks[namespace] = mutableNetInfo } + for _, nad := range tt.nads { + nadKey := fmt.Sprintf("%s/%s", nad.Namespace, nad.Name) + nadObj, err := wf.NADInformer().Lister().NetworkAttachmentDefinitions(nad.Namespace).Get(nad.Name) + g.Expect(err).ToNot(gomega.HaveOccurred(), "NAD %s should exist", nadKey) + netInfo, err := util.ParseNADInfo(nadObj) + g.Expect(err).ToNot(gomega.HaveOccurred(), "ParseNADInfo for %s failed", nadKey) + fakeNM.NADNetworks[nadKey] = netInfo + } tunnelKeysAllocator := id.NewTunnelKeyAllocator("TunnelKeys") c := NewController(wf, fakeClientset, fakeNM.Interface(), tunnelKeysAllocator) diff --git a/go-controller/pkg/clustermanager/networkconnect/controller_test.go b/go-controller/pkg/clustermanager/networkconnect/controller_test.go index 28f531731d..a712abfe34 100644 --- a/go-controller/pkg/clustermanager/networkconnect/controller_test.go +++ b/go-controller/pkg/clustermanager/networkconnect/controller_test.go @@ -121,6 +121,7 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller Integration Te fakeNM = &networkmanager.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), + NADNetworks: make(map[string]util.NetInfo), } tunnelKeysAllocator := id.NewTunnelKeyAllocator("TunnelKeys") @@ -675,6 +676,19 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller Integration Te mutableNetInfo := util.NewMutableNetInfo(netInfo) mutableNetInfo.AddNADs(nsName + "/" + nadName) fakeNM.PrimaryNetworks[nsName] = mutableNetInfo + fakeNM.NADNetworks[nsName+"/"+nadName] = netInfo + + // Ensure namespace and NAD are visible in informer caches before CNC creation. + gomega.Eventually(func() bool { + _, err := controller.namespaceLister.Get(nsName) + return err == nil + }).WithTimeout(2 * time.Second).Should(gomega.BeTrue()) + gomega.Eventually(func() bool { + _, err := controller.nadLister.NetworkAttachmentDefinitions(nsName).Get(nadName) + return err == nil + }).WithTimeout(2 * time.Second).Should(gomega.BeTrue()) + fakeNM.NADNetworks[nsName+"/"+nadName] = netInfo + fakeNM.NADNetworks[nsName+"/"+nadName] = netInfo // Create CNC with Primary UDN selector cnc := testCNC(cncName, []apitypes.NetworkSelector{ @@ -736,6 +750,7 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller Integration Te mutableNetInfo1 := util.NewMutableNetInfo(netInfo1) mutableNetInfo1.AddNADs("frontend-a/primary-udn") fakeNM.PrimaryNetworks["frontend-a"] = mutableNetInfo1 + fakeNM.NADNetworks["frontend-a/primary-udn"] = netInfo1 // Create second namespace and UDN ns2 := &corev1.Namespace{ @@ -762,6 +777,7 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller Integration Te mutableNetInfo2 := util.NewMutableNetInfo(netInfo2) mutableNetInfo2.AddNADs("frontend-b/primary-udn") fakeNM.PrimaryNetworks["frontend-b"] = mutableNetInfo2 + fakeNM.NADNetworks["frontend-b/primary-udn"] = netInfo2 // Create a non-matching namespace ns3 := &corev1.Namespace{ @@ -856,6 +872,17 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller Integration Te mutableNetInfo := util.NewMutableNetInfo(netInfo) mutableNetInfo.AddNADs(nsName + "/" + nadName) fakeNM.PrimaryNetworks[nsName] = mutableNetInfo + fakeNM.NADNetworks[nsName+"/"+nadName] = netInfo + + // Ensure namespace and NAD are visible in informer caches before CNC creation. + gomega.Eventually(func() bool { + _, err := controller.namespaceLister.Get(nsName) + return err == nil + }).WithTimeout(2 * time.Second).Should(gomega.BeTrue()) + gomega.Eventually(func() bool { + _, err := controller.nadLister.NetworkAttachmentDefinitions(nsName).Get(nadName) + return err == nil + }).WithTimeout(2 * time.Second).Should(gomega.BeTrue()) // Still no subnet annotation gomega.Consistently(func() bool { @@ -920,6 +947,17 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller Integration Te mutableNetInfo := util.NewMutableNetInfo(netInfo) mutableNetInfo.AddNADs(nsName + "/" + nadName) fakeNM.PrimaryNetworks[nsName] = mutableNetInfo + fakeNM.NADNetworks[nsName+"/"+nadName] = netInfo + + // Ensure namespace and NAD are visible in informer caches before CNC creation. + gomega.Eventually(func() bool { + _, err := controller.namespaceLister.Get(nsName) + return err == nil + }).WithTimeout(2 * time.Second).Should(gomega.BeTrue()) + gomega.Eventually(func() bool { + _, err := controller.nadLister.NetworkAttachmentDefinitions(nsName).Get(nadName) + return err == nil + }).WithTimeout(2 * time.Second).Should(gomega.BeTrue()) // Create CNC with Primary UDN selector cnc := testCNC(cncName, []apitypes.NetworkSelector{ @@ -1257,6 +1295,7 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller InitialSync Te fakeNM := &networkmanager.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), + NADNetworks: make(map[string]util.NetInfo), } tunnelKeysAllocator := id.NewTunnelKeyAllocator("TunnelKeys") @@ -1303,6 +1342,7 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller InitialSync Te mutableNetInfo1 := util.NewMutableNetInfo(netInfo1) mutableNetInfo1.AddNADs("pudn1-ns/primary-udn") fakeNM.PrimaryNetworks["pudn1-ns"] = mutableNetInfo1 + fakeNM.NADNetworks["pudn1-ns/primary-udn"] = netInfo1 ns2 := newTestNamespace("pudn2-ns", map[string]string{ "cnc1-pudn": "true", @@ -1323,6 +1363,7 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller InitialSync Te mutableNetInfo2 := util.NewMutableNetInfo(netInfo2) mutableNetInfo2.AddNADs("pudn2-ns/primary-udn") fakeNM.PrimaryNetworks["pudn2-ns"] = mutableNetInfo2 + fakeNM.NADNetworks["pudn2-ns/primary-udn"] = netInfo2 // ============================================================ // Create NADs for CNC2 (1 CUDN + 1 P-UDN) @@ -1351,6 +1392,7 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller InitialSync Te mutableNetInfo3 := util.NewMutableNetInfo(netInfo3) mutableNetInfo3.AddNADs("pudn3-ns/primary-udn") fakeNM.PrimaryNetworks["pudn3-ns"] = mutableNetInfo3 + fakeNM.NADNetworks["pudn3-ns/primary-udn"] = netInfo3 // ============================================================ // Create CNCs @@ -1478,11 +1520,15 @@ var _ = ginkgo.Describe("NetworkConnect ClusterManager Controller InitialSync Te // Create new FakeNetworkManager with same primary networks config fakeNM2 := &networkmanager.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), + NADNetworks: make(map[string]util.NetInfo), } // Re-setup primary networks (in real deployment this comes from network manager cache) fakeNM2.PrimaryNetworks["pudn1-ns"] = mutableNetInfo1 + fakeNM2.NADNetworks["pudn1-ns/primary-udn"] = netInfo1 fakeNM2.PrimaryNetworks["pudn2-ns"] = mutableNetInfo2 + fakeNM2.NADNetworks["pudn2-ns/primary-udn"] = netInfo2 fakeNM2.PrimaryNetworks["pudn3-ns"] = mutableNetInfo3 + fakeNM2.NADNetworks["pudn3-ns/primary-udn"] = netInfo3 tunnelKeysAllocator2 := id.NewTunnelKeyAllocator("TunnelKeys") controller2 := NewController(wf2, fakeClientset, fakeNM2.Interface(), tunnelKeysAllocator2) diff --git a/go-controller/pkg/clustermanager/networkconnect/hybrid_connect_subnet_allocator.go b/go-controller/pkg/clustermanager/networkconnect/hybrid_connect_subnet_allocator.go index 42ed8a2b78..92786ec8e5 100644 --- a/go-controller/pkg/clustermanager/networkconnect/hybrid_connect_subnet_allocator.go +++ b/go-controller/pkg/clustermanager/networkconnect/hybrid_connect_subnet_allocator.go @@ -14,6 +14,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) var ( @@ -323,8 +324,8 @@ func (hca *hybridConnectSubnetAllocator) MarkAllocatedSubnets(allocatedSubnets m defer hca.mu.Unlock() for owner, subnets := range allocatedSubnets { - topologyType, ok := parseNetworkOwnerTopology(owner) - if !ok { + topologyType, _, err := util.ParseNetworkOwner(owner) + if err != nil { continue } diff --git a/go-controller/pkg/clustermanager/pod/allocator.go b/go-controller/pkg/clustermanager/pod/allocator.go index 9943fa2faf..5e5e65f25d 100644 --- a/go-controller/pkg/clustermanager/pod/allocator.go +++ b/go-controller/pkg/clustermanager/pod/allocator.go @@ -124,7 +124,12 @@ func (a *PodAllocator) getActiveNetworkForPod(pod *corev1.Pod) (util.NetInfo, er // GetNetworkRole returns the role of this controller's network for the given pod func (a *PodAllocator) GetNetworkRole(pod *corev1.Pod) (string, error) { - role, err := util.GetNetworkRole(a.netInfo, a.networkManager.GetActiveNetworkForNamespace, pod) + role, err := util.GetNetworkRole( + a.netInfo, + a.networkManager.GetPrimaryNADForNamespace, + a.networkManager.GetNetworkNameForNADKey, + pod, + ) if err != nil { if util.IsUnprocessedActiveNetworkError(err) { a.recordPodErrorEvent(pod, err) @@ -200,8 +205,9 @@ func (a *PodAllocator) reconcile(old, new *corev1.Pod, releaseFromAllocator bool if err != nil { return err } - for nadName := range podNetworks { - if a.netInfo.HasNAD(nadName) { + for nadKey := range podNetworks { + networkName := a.networkManager.GetNetworkNameForNADKey(nadKey) + if networkName != "" && networkName == a.netInfo.GetNetworkName() { activeNetwork = a.netInfo break } @@ -212,7 +218,13 @@ func (a *PodAllocator) reconcile(old, new *corev1.Pod, releaseFromAllocator bool } } - onNetwork, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, a.netInfo, activeNetwork) + onNetwork, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork( + pod, + a.netInfo, + activeNetwork, + a.networkManager.GetNetworkNameForNADKey, + a.networkManager.GetPrimaryNADForNamespace, + ) if err != nil { a.recordPodErrorEvent(pod, err) return fmt.Errorf("failed to get NAD to network mapping: %w", err) @@ -226,8 +238,8 @@ func (a *PodAllocator) reconcile(old, new *corev1.Pod, releaseFromAllocator bool } // reconcile for each NAD - for nadName, network := range networkMap { - err = a.reconcileForNAD(old, new, nadName, network, releaseFromAllocator) + for nadKey, network := range networkMap { + err = a.reconcileForNAD(old, new, nadKey, network, releaseFromAllocator) if err != nil { return err } @@ -236,7 +248,7 @@ func (a *PodAllocator) reconcile(old, new *corev1.Pod, releaseFromAllocator bool return nil } -func (a *PodAllocator) reconcileForNAD(old, new *corev1.Pod, nad string, network *nettypes.NetworkSelectionElement, releaseIPsFromAllocator bool) error { +func (a *PodAllocator) reconcileForNAD(old, new *corev1.Pod, nadKey string, network *nettypes.NetworkSelectionElement, releaseIPsFromAllocator bool) error { var pod *corev1.Pod if old != nil { pod = old @@ -248,15 +260,15 @@ func (a *PodAllocator) reconcileForNAD(old, new *corev1.Pod, nad string, network podCompleted := util.PodCompleted(pod) if podCompleted || podDeleted { - return a.releasePodOnNAD(pod, nad, network, podDeleted, releaseIPsFromAllocator) + return a.releasePodOnNAD(pod, nadKey, network, podDeleted, releaseIPsFromAllocator) } - return a.allocatePodOnNAD(pod, nad, network) + return a.allocatePodOnNAD(pod, nadKey, network) } -func (a *PodAllocator) releasePodOnNAD(pod *corev1.Pod, nad string, network *nettypes.NetworkSelectionElement, +func (a *PodAllocator) releasePodOnNAD(pod *corev1.Pod, nadKey string, network *nettypes.NetworkSelectionElement, podDeleted, releaseFromAllocator bool) error { - podAnnotation, _ := util.UnmarshalPodAnnotation(pod.Annotations, nad) + podAnnotation, _ := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if podAnnotation == nil { // track release pods even if they have no annotation in case a user // might have removed it manually @@ -298,12 +310,12 @@ func (a *PodAllocator) releasePodOnNAD(pod *corev1.Pod, nad string, network *net // do not release from the allocators if not flaged to do so or if they // were already previosuly released - doRelease := releaseFromAllocator && !a.isPodReleased(nad, uid) + doRelease := releaseFromAllocator && !a.isPodReleased(nadKey, uid) doReleaseIDs := doRelease && hasIDAllocation doReleaseIPs := doRelease && hasIPAM && !hasIPAMClaim if doReleaseIDs { - name := podIdAllocationName(nad, uid) + name := podIdAllocationName(nadKey, uid) a.idAllocator.ReleaseID(name) klog.V(5).Infof("Released ID %d", podAnnotation.TunnelID) } @@ -311,11 +323,11 @@ func (a *PodAllocator) releasePodOnNAD(pod *corev1.Pod, nad string, network *net if doReleaseIPs { err := a.ipAllocator.ReleaseIPs(a.netInfo.GetNetworkName(), podAnnotation.IPs) if err != nil { - return fmt.Errorf("failed to release ips %v for pod %s/%s and nad %s: %w", + return fmt.Errorf("failed to release ips %v for pod %s/%s and NAD key %s: %w", util.StringSlice(podAnnotation.IPs), pod.Name, pod.Namespace, - nad, + nadKey, err, ) } @@ -330,15 +342,15 @@ func (a *PodAllocator) releasePodOnNAD(pod *corev1.Pod, nad string, network *net } if podDeleted { - a.deleteReleasedPod(nad, string(pod.UID)) + a.deleteReleasedPod(nadKey, string(pod.UID)) } else { - a.addReleasedPod(nad, string(pod.UID)) + a.addReleasedPod(nadKey, string(pod.UID)) } return nil } -func (a *PodAllocator) allocatePodOnNAD(pod *corev1.Pod, nad string, network *nettypes.NetworkSelectionElement) error { +func (a *PodAllocator) allocatePodOnNAD(pod *corev1.Pod, nadKey string, network *nettypes.NetworkSelectionElement) error { var ipAllocator subnet.NamedAllocator if util.DoesNetworkRequireIPAM(a.netInfo) { ipAllocator = a.ipAllocator.ForSubnet(a.netInfo.GetNetworkName()) @@ -346,7 +358,7 @@ func (a *PodAllocator) allocatePodOnNAD(pod *corev1.Pod, nad string, network *ne var idAllocator id.NamedAllocator if util.DoesNetworkRequireTunnelIDs(a.netInfo) { - name := podIdAllocationName(nad, string(pod.UID)) + name := podIdAllocationName(nadKey, string(pod.UID)) idAllocator = a.idAllocator.ForName(name) } @@ -372,6 +384,7 @@ func (a *PodAllocator) allocatePodOnNAD(pod *corev1.Pod, nad string, network *ne idAllocator, node, pod, + nadKey, network, reallocate, networkRole, @@ -388,46 +401,46 @@ func (a *PodAllocator) allocatePodOnNAD(pod *corev1.Pod, nad string, network *ne } if updatedPod != nil { - klog.V(5).Infof("Allocated IP addresses %v, mac address %s, gateways %v, routes %s and tunnel id %d for pod %s/%s on nad %s", + klog.V(5).Infof("Allocated IP addresses %v, mac address %s, gateways %v, routes %s and tunnel id %d for pod %s/%s on NAD key %s", util.StringSlice(podAnnotation.IPs), podAnnotation.MAC, util.StringSlice(podAnnotation.Gateways), util.StringSlice(podAnnotation.Routes), podAnnotation.TunnelID, - pod.Namespace, pod.Name, nad, + pod.Namespace, pod.Name, nadKey, ) } return err } -func (a *PodAllocator) addReleasedPod(nad, uid string) { +func (a *PodAllocator) addReleasedPod(nadKey, uid string) { a.releasedPodsMutex.Lock() defer a.releasedPodsMutex.Unlock() - releasedPods := a.releasedPods[nad] + releasedPods := a.releasedPods[nadKey] if releasedPods == nil { - a.releasedPods[nad] = sets.New(uid) + a.releasedPods[nadKey] = sets.New(uid) return } releasedPods.Insert(uid) } -func (a *PodAllocator) deleteReleasedPod(nad, uid string) { +func (a *PodAllocator) deleteReleasedPod(nadKey, uid string) { a.releasedPodsMutex.Lock() defer a.releasedPodsMutex.Unlock() - releasedPods := a.releasedPods[nad] + releasedPods := a.releasedPods[nadKey] if releasedPods != nil { releasedPods.Delete(uid) if releasedPods.Len() == 0 { - delete(a.releasedPods, nad) + delete(a.releasedPods, nadKey) } } } -func (a *PodAllocator) isPodReleased(nad, uid string) bool { +func (a *PodAllocator) isPodReleased(nadKey, uid string) bool { a.releasedPodsMutex.Lock() defer a.releasedPodsMutex.Unlock() - releasedPods := a.releasedPods[nad] + releasedPods := a.releasedPods[nadKey] if releasedPods != nil { return releasedPods.Has(uid) } @@ -445,6 +458,6 @@ func (a *PodAllocator) recordPodErrorEvent(pod *corev1.Pod, podErr error) { } } -func podIdAllocationName(nad, uid string) string { - return fmt.Sprintf("%s/%s", nad, uid) +func podIdAllocationName(nadKey, uid string) string { + return fmt.Sprintf("%s/%s", nadKey, uid) } diff --git a/go-controller/pkg/clustermanager/pod/allocator_test.go b/go-controller/pkg/clustermanager/pod/allocator_test.go index 0ba54e5504..e11f287d8f 100644 --- a/go-controller/pkg/clustermanager/pod/allocator_test.go +++ b/go-controller/pkg/clustermanager/pod/allocator_test.go @@ -9,6 +9,7 @@ import ( "sync" "testing" + cnitypes "github.com/containernetworking/cni/pkg/types" ipamclaimsapi "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1" fakeipamclaimclient "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1/apis/clientset/versioned/fake" ipamclaimsfactory "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1/apis/informers/externalversions" @@ -146,8 +147,9 @@ func (a *idAllocatorStub) ReserveID(string, int) error { panic("not implemented") // TODO: Implement } -func (a *idAllocatorStub) ReleaseID(string) { +func (a *idAllocatorStub) ReleaseID(string) int { a.released = true + return 0 } func (a *idAllocatorStub) ForName(string) id.NamedAllocator { @@ -544,13 +546,13 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { }, }, nads: []*nadapi.NetworkAttachmentDefinition{ - ovntest.GenerateNAD("surya", "nad", "namespace", + ovntest.GenerateNAD("nad", "nad", "namespace", types.Layer3Topology, "100.128.0.0/16", types.NetworkRolePrimary), }, }, role: types.NetworkRolePrimary, - expectError: "failed to get NAD to network mapping: unexpected primary network \"\" specified with a NetworkSelectionElement &{Name:nad Namespace:namespace IPRequest:[] MacRequest: InfinibandGUIDRequest: InterfaceRequest: PortMappingsRequest:[] BandwidthRequest: CNIArgs: GatewayRequest:[] IPAMClaimReference:}", - expectEvents: []string{"Warning ErrorAllocatingPod unexpected primary network \"\" specified with a NetworkSelectionElement &{Name:nad Namespace:namespace IPRequest:[] MacRequest: InfinibandGUIDRequest: InterfaceRequest: PortMappingsRequest:[] BandwidthRequest: CNIArgs: GatewayRequest:[] IPAMClaimReference:}"}, + expectError: "failed to get NAD to network mapping: unexpected primary network \"nad\" specified with a NetworkSelectionElement &{Name:nad Namespace:namespace IPRequest:[] MacRequest: InfinibandGUIDRequest: InterfaceRequest: PortMappingsRequest:[] BandwidthRequest: CNIArgs: GatewayRequest:[] IPAMClaimReference:}", + expectEvents: []string{"Warning ErrorAllocatingPod unexpected primary network \"nad\" specified with a NetworkSelectionElement &{Name:nad Namespace:namespace IPRequest:[] MacRequest: InfinibandGUIDRequest: InterfaceRequest: PortMappingsRequest:[] BandwidthRequest: CNIArgs: GatewayRequest:[] IPAMClaimReference:}"}, }, { name: "Pod on network with exhausted ip pool, expect event and error", @@ -597,7 +599,7 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { network: &nadapi.NetworkSelectionElement{Namespace: "namespace", Name: "nad", MacRequest: "0a:0a:0a:0a:0a:0a"}, }, }, - expectError: `failed to update pod namespace/pod: failed to reserve MAC address "0a:0a:0a:0a:0a:0a" for owner "namespace/pod" on network attachment "namespace/nad": test reserve failure`, + expectError: `failed to update pod namespace/pod: failed to reserve MAC address "0a:0a:0a:0a:0a:0a" for owner "namespace/pod" on NAD key "namespace/nad": test reserve failure`, }, { name: "should emit pod event when macRegistry fail to reserve pod's MAC due to MAC conflict", @@ -609,8 +611,8 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { network: &nadapi.NetworkSelectionElement{Namespace: "namespace", Name: "nad", MacRequest: "0a:0a:0a:0a:0a:0a"}, }, }, - expectError: `failed to update pod namespace/pod: failed to reserve MAC address "0a:0a:0a:0a:0a:0a" for owner "namespace/pod" on network attachment "namespace/nad": MAC address already in use`, - expectEvents: []string{`Warning ErrorAllocatingPod failed to update pod namespace/pod: failed to reserve MAC address "0a:0a:0a:0a:0a:0a" for owner "namespace/pod" on network attachment "namespace/nad": MAC address already in use`}, + expectError: `failed to update pod namespace/pod: failed to reserve MAC address "0a:0a:0a:0a:0a:0a" for owner "namespace/pod" on NAD key "namespace/nad": MAC address already in use`, + expectEvents: []string{`Warning ErrorAllocatingPod failed to update pod namespace/pod: failed to reserve MAC address "0a:0a:0a:0a:0a:0a" for owner "namespace/pod" on NAD key "namespace/nad": MAC address already in use`}, }, { name: "should NOT fail when macRegistry gets repeated reserve requests (same mac and owner)", @@ -673,7 +675,7 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { network: &nadapi.NetworkSelectionElement{Namespace: "namespace", Name: "nad", MacRequest: "0a:0a:0a:0a:0a:0a"}, }, }, - expectError: `failed to release pod "namespace/pod" mac "0a:0a:0a:0a:0a:0a": failed to release MAC address "0a:0a:0a:0a:0a:0a" for owner "namespace/pod" on network "": test release failure`, + expectError: `failed to release pod "namespace/pod" mac "0a:0a:0a:0a:0a:0a": failed to release MAC address "0a:0a:0a:0a:0a:0a" for owner "namespace/pod" on network "nad": test release failure`, expectIPRelease: true, }, { @@ -827,6 +829,7 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { nodeListerMock.On("Get", mock.AnythingOfType("string")).Return(&corev1.Node{}, nil) netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "nad"}, Topology: types.Layer2Topology, AllowPersistentIPs: tt.ipam && tt.args.ipamClaim != nil, } @@ -877,9 +880,21 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { testNs := "namespace" nadNetworks := map[string]util.NetInfo{} + nadKeyToNetInfo := map[string]util.NetInfo{} for _, nad := range tt.args.nads { if nad.Namespace == testNs { - nadNetwork, _ := util.ParseNADInfo(nad) + nadNetwork, err := util.ParseNADInfo(nad) + if err != nil { + t.Fatalf("ParseNADInfo failed for %s: %v", util.GetNADName(nad.Namespace, nad.Name), err) + } + if nadNetwork == nil { + t.Fatalf("ParseNADInfo returned nil for %s", util.GetNADName(nad.Namespace, nad.Name)) + } + mutableNADNetInfo := util.NewMutableNetInfo(nadNetwork) + nadKey := util.GetNADName(nad.Namespace, nad.Name) + mutableNADNetInfo.AddNADs(nadKey) + nadNetwork = mutableNADNetInfo + nadKeyToNetInfo[nadKey] = nadNetwork if nadNetwork.IsPrimaryNetwork() { if _, ok := nadNetworks[testNs]; !ok { nadNetworks[testNs] = nadNetwork @@ -887,8 +902,17 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { } } } - - fakeNetworkManager := &networkmanager.FakeNetworkManager{PrimaryNetworks: nadNetworks} + fakeNetworkManager := &networkmanager.FakeNetworkManager{ + PrimaryNetworks: nadNetworks, + NADNetworks: nadKeyToNetInfo, + } + // Ensure resolver can map the test NAD key used by pod annotations. + if _, ok := fakeNetworkManager.NADNetworks["namespace/nad"]; !ok { + fakeNetworkManager.NADNetworks["namespace/nad"] = netInfo + } + if netInfo.IsPrimaryNetwork() && fakeNetworkManager.PrimaryNetworks["namespace"] == nil { + fakeNetworkManager.PrimaryNetworks["namespace"] = netInfo + } fakeRecorder := record.NewFakeRecorder(10) diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller.go b/go-controller/pkg/clustermanager/routeadvertisements/controller.go index cffbb3425e..75ce469089 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/controller.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller.go @@ -50,6 +50,8 @@ import ( const ( generateName = "ovnk-generated-" fieldManager = "clustermanager-routeadvertisements-controller" + // evpnRawConfigPriority is set to an arbitrary value that still allows users to override EVPN config if needed. + evpnRawConfigPriority = 10 ) var ( @@ -222,7 +224,7 @@ func (c *Controller) ReconcileNetwork(_ string, old, new util.NetInfo) { } if new != nil && !newNamespaces.Equal(oldNamespaces) { // we use one of the NADs of the network to reconcile it - nads := new.GetNADs() + nads := c.nm.GetNADKeysForNetwork(new.GetNetworkName()) if len(nads) > 0 { c.nadController.Reconcile(nads[0]) } @@ -324,6 +326,33 @@ type selectedNetworks struct { prefixLength map[string]uint32 // networkType is a map of selected network to their topology networkTopology map[string]string + // macVRFConfigs is an ordered list of MAC-VRF EVPN configurations for selected networks + macVRFConfigs []*vrfConfig + // ipVRFConfigs is an ordered list of IP-VRF EVPN configurations for selected networks + ipVRFConfigs []*ipVRFConfig + // networkTransport is a map of selected network to their transport mode + networkTransport map[string]string +} + +// vrfConfig holds base VRF EVPN configuration for a network +type vrfConfig struct { + // VNI is the VXLAN Network Identifier + VNI int32 + // RouteTarget is the BGP route target, empty means use FRR defaults + RouteTarget string +} + +// ipVRFConfig holds IP-VRF EVPN configuration for a network +type ipVRFConfig struct { + vrfConfig + // NetworkName is the name of the network this config belongs to + NetworkName string + // VRFName is the Linux VRF name + VRFName string + // HasIPv4 indicates if the network has IPv4 subnets + HasIPv4 bool + // HasIPv6 indicates if the network has IPv6 subnets + HasIPv6 bool } // generateFRRConfigurations generates FRRConfigurations for the route @@ -351,10 +380,11 @@ func (c *Controller) generateFRRConfigurations(ra *ratypes.RouteAdvertisements) // validate and gather information about the networks networkSet := sets.New[string]() selectedNetworks := &selectedNetworks{ - networkVRFs: map[string]string{}, - networkSubnets: map[string][]string{}, - prefixLength: map[string]uint32{}, - networkTopology: map[string]string{}, + networkVRFs: map[string]string{}, + networkSubnets: map[string][]string{}, + prefixLength: map[string]uint32{}, + networkTopology: map[string]string{}, + networkTransport: map[string]string{}, } for _, nad := range nads { networkName := util.GetAnnotatedNetworkName(nad) @@ -385,6 +415,43 @@ func (c *Controller) generateFRRConfigurations(ra *ratypes.RouteAdvertisements) selectedNetworks.vrfs = append(selectedNetworks.vrfs, vrf) selectedNetworks.networkVRFs[vrf] = networkName selectedNetworks.networkTopology[networkName] = network.TopologyType() + selectedNetworks.networkTransport[networkName] = network.Transport() + + // MAC-VRF configuration + if macVNI := network.EVPNMACVRFVNI(); macVNI > 0 { + selectedNetworks.macVRFConfigs = append(selectedNetworks.macVRFConfigs, &vrfConfig{ + VNI: macVNI, + RouteTarget: network.EVPNMACVRFRouteTarget(), + }) + } + + // IP-VRF configuration + if ipVNI := network.EVPNIPVRFVNI(); ipVNI > 0 { + // Compute IP families from network subnets + hasIPv4, hasIPv6 := false, false + for _, subnet := range network.Subnets() { + if subnet.CIDR.IP.To4() == nil { + hasIPv6 = true + } else { + hasIPv4 = true + } + } + selectedNetworks.ipVRFConfigs = append(selectedNetworks.ipVRFConfigs, &ipVRFConfig{ + vrfConfig: vrfConfig{ + VNI: ipVNI, + RouteTarget: network.EVPNIPVRFRouteTarget(), + }, + NetworkName: networkName, + VRFName: vrf, + HasIPv4: hasIPv4, + HasIPv6: hasIPv6, + }) + } + hasEVPNConfig := network.EVPNMACVRFVNI() > 0 || network.EVPNIPVRFVNI() > 0 + if hasEVPNConfig && ra.Spec.TargetVRF != "auto" && ra.Spec.TargetVRF != vrf { + return nil, nil, fmt.Errorf("%w: EVPN network %q with VRF %q requires TargetVRF to be 'auto' or %q, got %q", + errConfig, networkName, vrf, vrf, ra.Spec.TargetVRF) + } // TODO check overlaps? for _, cidr := range network.Subnets() { subnet := cidr.CIDR.String() @@ -399,6 +466,8 @@ func (c *Controller) generateFRRConfigurations(ra *ratypes.RouteAdvertisements) // ordered slices.Sort(selectedNetworks.vrfs) slices.Sort(selectedNetworks.subnets) + slices.SortFunc(selectedNetworks.macVRFConfigs, func(a, b *vrfConfig) int { return int(a.VNI - b.VNI) }) + slices.SortFunc(selectedNetworks.ipVRFConfigs, func(a, b *ipVRFConfig) int { return int(a.VNI - b.VNI) }) selectedNetworks.networks = sets.List(networkSet) // gather selected nodes @@ -435,6 +504,8 @@ func (c *Controller) generateFRRConfigurations(ra *ratypes.RouteAdvertisements) if len(frrConfigs) == 0 { return nil, nil, fmt.Errorf("%w: no FRRConfigurations selected", errPending) } + + frrRouterVRFs := sets.New[string]() for _, frrConfig := range frrConfigs { if strings.HasPrefix(frrConfig.Name, generateName) { klog.V(4).Infof("Skipping FRRConfiguration %q selected by RouteAdvertisements %q as it was generated by ovn-kubernetes", frrConfig.Name, ra.Name) @@ -455,6 +526,27 @@ func (c *Controller) generateFRRConfigurations(ra *ratypes.RouteAdvertisements) } nodeToFRRConfig[node.Name] = append(nodeToFRRConfig[node.Name], frrConfig) } + for _, router := range frrConfig.Spec.BGP.Routers { + frrRouterVRFs.Insert(router.VRF) + } + } + + // Validate EVPN configuration requirements + hasEVPNConfig := len(selectedNetworks.macVRFConfigs) > 0 || len(selectedNetworks.ipVRFConfigs) > 0 + if hasEVPNConfig && !util.IsEVPNEnabled() { + return nil, nil, fmt.Errorf("%w: EVPN networks selected but EVPN feature is not enabled", errConfig) + } + // Require a router with default VRF for any EVPN configuration, since the + // global EVPN section with advertise-all-vni is required for EVPN to work properly. + if hasEVPNConfig && !frrRouterVRFs.Has("") { + return nil, nil, fmt.Errorf("%w: EVPN requires a router with default VRF but none were found in selected FRRConfigurations", errConfig) + } + // Validate IP-VRF networks: each needs either an existing VRF router or + // the default VRF router to create one from. + for _, cfg := range selectedNetworks.ipVRFConfigs { + if !frrRouterVRFs.Has(cfg.VRFName) && !frrRouterVRFs.Has("") { + return nil, nil, fmt.Errorf("%w: IP-VRF EVPN network %q requires a router with VRF %q or a router with default VRF, but none were found in selected FRRConfigurations", errConfig, cfg.NetworkName, cfg.VRFName) + } } // helper to gather host subnets and cache during reconcile @@ -562,6 +654,7 @@ func (c *Controller) generateFRRConfigurations(ra *ratypes.RouteAdvertisements) nodeName, selectedNetworks, matchedNetworks, + frrRouterVRFs, ) if err != nil { return nil, nil, err @@ -591,8 +684,9 @@ func (c *Controller) generateFRRConfiguration( nodeName string, selectedNetworks *selectedNetworks, matchedNetworks sets.Set[string], + frrRouterVRFs sets.Set[string], ) (*frrtypes.FRRConfiguration, error) { - routers := []frrtypes.Router{} + var routers []frrtypes.Router // go over the source routers for i, router := range source.Spec.BGP.Routers { @@ -670,6 +764,32 @@ func (c *Controller) generateFRRConfiguration( Prefixes: advertisePrefixes, }, } + + // For no-overlay networks, add routes to pod subnets to the accepted routes list + // frr-k8s will merge the prefixes from both the generated and the base FRRConfiguration + if selectedNetworks.networkTransport[matchedNetwork] == types.NetworkTransportNoOverlay { + // Get the pod subnets for this network (the network subnets, not host subnets) + podSubnets := selectedNetworks.networkSubnets[matchedNetwork] + if len(podSubnets) > 0 { + // Filter pod subnets by IP family to match the neighbor + filteredPodSubnets := util.MatchAllIPNetsStringFamily(isIPV6, podSubnets) + if len(filteredPodSubnets) > 0 { + neighbor.ToReceive = frrtypes.Receive{ + Allowed: frrtypes.AllowedInPrefixes{ + Mode: frrtypes.AllowRestricted, + }, + } + for _, subnet := range filteredPodSubnets { + neighbor.ToReceive.Allowed.Prefixes = append(neighbor.ToReceive.Allowed.Prefixes, frrtypes.PrefixSelector{ + Prefix: subnet, + LE: selectedNetworks.prefixLength[subnet], + GE: selectedNetworks.prefixLength[subnet], + }) + } + } + } + } + targetRouter.Neighbors = append(targetRouter.Neighbors, neighbor) } if len(targetRouter.Neighbors) == 0 { @@ -720,11 +840,65 @@ func (c *Controller) generateFRRConfiguration( routers = append(routers, importRouter) } } - if len(routers) == 0 { - // we ended up with no routers, bail out - return nil, nil + var globalRouterASN uint32 + var neighbors []string + vrfASNs := map[string]uint32{} + + if len(selectedNetworks.macVRFConfigs) > 0 || len(selectedNetworks.ipVRFConfigs) > 0 { + // Look for global router in the source FRRConfiguration, not in the filtered routers + for _, router := range source.Spec.BGP.Routers { + if router.VRF == "" { // default VRF + globalRouterASN = router.ASN + for _, neighbor := range router.Neighbors { + neighbors = append(neighbors, neighbor.Address) + } + break + } + } + } + + // For IP-VRF: Find or create routers for each EVPN network's VRF. + // IP-VRF routers don't need neighbors for EVPN (they use the global router's neighbors). + for _, cfg := range selectedNetworks.ipVRFConfigs { + if frrRouterVRFs.Has(cfg.VRFName) { + // VRF router exists somewhere - check if it's in the current source + for _, router := range source.Spec.BGP.Routers { + if router.VRF == cfg.VRFName { + vrfASNs[cfg.VRFName] = router.ASN + if !slices.ContainsFunc(routers, func(r frrtypes.Router) bool { return r.VRF == cfg.VRFName }) { + routers = append(routers, frrtypes.Router{ + ASN: router.ASN, + VRF: cfg.VRFName, + Prefixes: selectedNetworks.hostNetworkSubnets[cfg.NetworkName], + }) + } + break + } + } + // If not in current source, another source will handle it + } else if globalRouterASN > 0 { + // VRF router doesn't exist anywhere - create with global ASN + klog.Infof("Creating router for EVPN network %q VRF %q with ASN=%d, prefixes=%v", + cfg.NetworkName, cfg.VRFName, globalRouterASN, selectedNetworks.hostNetworkSubnets[cfg.NetworkName]) + matchedNetworks.Insert(cfg.NetworkName) + vrfASNs[cfg.VRFName] = globalRouterASN + routers = append(routers, frrtypes.Router{ + ASN: globalRouterASN, + VRF: cfg.VRFName, + Prefixes: selectedNetworks.hostNetworkSubnets[cfg.NetworkName], + }) + } } + // Check if we have anything to generate: routers or EVPN raw config. + // EVPN raw config is generated when we have: + // - A global router (globalRouterASN > 0 && len(neighbors) > 0) for the global EVPN section + // - IP-VRF configs for VRF VNI and VRF EVPN sections + hasEVPNRawConfig := (globalRouterASN > 0 && len(neighbors) > 0) || len(selectedNetworks.ipVRFConfigs) > 0 + if len(routers) == 0 && !hasEVPNRawConfig { + // we ended up with no routers and no EVPN raw config to generate, bail out + return nil, nil + } new := &frrtypes.FRRConfiguration{} new.GenerateName = generateName new.Namespace = source.Namespace @@ -748,6 +922,18 @@ func (c *Controller) generateFRRConfiguration( }, } + // Generate EVPN raw config for the EVPN-specific parts. + // TODO: once frr-k8s provides a typed EVPN API, we can use that instead of raw config + if len(selectedNetworks.macVRFConfigs) > 0 || len(selectedNetworks.ipVRFConfigs) > 0 { + rawConfig := generateEVPNRawConfig(selectedNetworks, globalRouterASN, neighbors, vrfASNs) + if rawConfig != "" { + new.Spec.Raw = frrtypes.RawConfig{ + Priority: evpnRawConfigPriority, + Config: rawConfig, + } + } + } + return new, nil } diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go index a6b8e8b664..1bad4f1ad5 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go @@ -2,6 +2,7 @@ package routeadvertisements import ( "context" + "encoding/json" "fmt" "strings" "sync" @@ -148,11 +149,18 @@ func (tn testNode) Node() *corev1.Node { } } +type testPrefixSelector struct { + Prefix string + LE uint32 + GE uint32 +} + type testNeighbor struct { ASN uint32 Address string DisableMP *bool Advertise []string + Receive []testPrefixSelector } func (tn testNeighbor) Neighbor() frrapi.Neighbor { @@ -170,6 +178,22 @@ func (tn testNeighbor) Neighbor() frrapi.Neighbor { if tn.DisableMP != nil { n.DisableMP = *tn.DisableMP } + if len(tn.Receive) > 0 { + prefixSelectors := make([]frrapi.PrefixSelector, 0, len(tn.Receive)) + for _, ps := range tn.Receive { + prefixSelectors = append(prefixSelectors, frrapi.PrefixSelector{ + Prefix: ps.Prefix, + LE: ps.LE, + GE: ps.GE, + }) + } + n.ToReceive = frrapi.Receive{ + Allowed: frrapi.AllowedInPrefixes{ + Mode: frrapi.AllowRestricted, + Prefixes: prefixSelectors, + }, + } + } return n } @@ -198,14 +222,16 @@ func (tr testRouter) Router() frrapi.Router { } type testFRRConfig struct { - Name string - Namespace string - Generation int - Labels map[string]string - Annotations map[string]string - Routers []*testRouter - NodeSelector map[string]string - OwnUpdate bool + Name string + Namespace string + Generation int + Labels map[string]string + Annotations map[string]string + Routers []*testRouter + NodeSelector map[string]string + OwnUpdate bool + RawConfig string + RawConfigPriority int } func (tf testFRRConfig) FRRConfiguration() *frrapi.FRRConfiguration { @@ -226,6 +252,10 @@ func (tf testFRRConfig) FRRConfiguration() *frrapi.FRRConfiguration { for _, r := range tf.Routers { f.Spec.BGP.Routers = append(f.Spec.BGP.Routers, r.Router()) } + if tf.RawConfig != "" { + f.Spec.Raw.Config = tf.RawConfig + f.Spec.Raw.Priority = tf.RawConfigPriority + } if tf.OwnUpdate { f.ManagedFields = append(f.ManagedFields, metav1.ManagedFieldsEntry{ Manager: fieldManager, @@ -264,15 +294,19 @@ func (te testEIP) EgressIP() *eiptypes.EgressIP { } type testNAD struct { - Name string - Namespace string - Network string - Subnet string - Labels map[string]string - Annotations map[string]string - IsSecondary bool - Topology string - OwnUpdate bool + Name string + Namespace string + Network string + Subnet string + Labels map[string]string + Annotations map[string]string + IsSecondary bool + Topology string + OwnUpdate bool + EVPNMACVRFVNI int32 + EVPNMACVRFRouteTarget string + EVPNIPVRFVNI int32 + EVPNIPVRFRouteTarget string } func (tn testNAD) NAD() *nadtypes.NetworkAttachmentDefinition { @@ -295,27 +329,52 @@ func (tn testNAD) NAD() *nadtypes.NetworkAttachmentDefinition { ) nad.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ownerRef} } - topology := tn.Topology - switch { - case tn.IsSecondary: - nad.Spec.Config = fmt.Sprintf("{\"cniVersion\": \"0.4.0\", \"name\": \"%s\", \"type\": \"%s\", \"topology\": \"%s\", \"netAttachDefName\": \"%s\", \"subnets\": \"%s\"}", - tn.Network, - config.CNI.Plugin, - topology, - tn.Namespace+"/"+tn.Name, - tn.Subnet, - ) - case tn.Topology != "": - nad.Spec.Config = fmt.Sprintf("{\"cniVersion\": \"0.4.0\", \"name\": \"%s\", \"type\": \"%s\", \"topology\": \"%s\", \"netAttachDefName\": \"%s\", \"role\": \"primary\", \"subnets\": \"%s\"}", - tn.Network, - config.CNI.Plugin, - topology, - tn.Namespace+"/"+tn.Name, - tn.Subnet, - ) - default: - nad.Spec.Config = fmt.Sprintf("{\"cniVersion\": \"0.4.0\", \"name\": \"%s\", \"type\": \"%s\"}", tn.Network, config.CNI.Plugin) + + // Build the config as a map to properly marshal EVPN config + cniConfig := map[string]interface{}{ + "cniVersion": "0.4.0", + "name": tn.Network, + "type": config.CNI.Plugin, + "netAttachDefName": tn.Namespace + "/" + tn.Name, + } + + if tn.Topology != "" { + cniConfig["topology"] = tn.Topology + } + if tn.Subnet != "" { + cniConfig["subnets"] = tn.Subnet + } + if tn.Topology != "" && !tn.IsSecondary { + cniConfig["role"] = "primary" + } + + // Add EVPN configuration if present + if tn.EVPNMACVRFVNI > 0 || tn.EVPNIPVRFVNI > 0 { + evpnConfig := map[string]interface{}{} + if tn.EVPNMACVRFVNI > 0 { + macvrf := map[string]interface{}{ + "vni": tn.EVPNMACVRFVNI, + } + if tn.EVPNMACVRFRouteTarget != "" { + macvrf["routeTarget"] = tn.EVPNMACVRFRouteTarget + } + evpnConfig["macVRF"] = macvrf + } + if tn.EVPNIPVRFVNI > 0 { + ipvrf := map[string]interface{}{ + "vni": tn.EVPNIPVRFVNI, + } + if tn.EVPNIPVRFRouteTarget != "" { + ipvrf["routeTarget"] = tn.EVPNIPVRFRouteTarget + } + evpnConfig["ipVRF"] = ipvrf + } + cniConfig["evpn"] = evpnConfig } + + configBytes, _ := json.Marshal(cniConfig) + nad.Spec.Config = string(configBytes) + if tn.OwnUpdate { nad.ManagedFields = append(nad.ManagedFields, metav1.ManagedFieldsEntry{ Manager: fieldManager, @@ -372,6 +431,7 @@ func TestController_reconcile(t *testing.T) { namespaces []*testNamespace eips []*testEIP reconcile string + transport string wantErr bool expectAcceptedStatus metav1.ConditionStatus expectFRRConfigs []*testFRRConfig @@ -781,6 +841,37 @@ func TestController_reconcile(t *testing.T) { reconcile: "ra", expectAcceptedStatus: metav1.ConditionTrue, }, + { + name: "reconciles pod RouteAdvertisement for default network in no-overlay mode with ToReceive routes", + ra: &testRA{Name: "ra", AdvertisePods: true, SelectsDefault: true}, + transport: types.NetworkTransportNoOverlay, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Receive: []testPrefixSelector{{Prefix: "1.2.0.0/16"}}}, + }}, + }, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.0.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.1.0.0/24"}, Receive: []testPrefixSelector{{Prefix: "1.1.0.0/16", LE: 24, GE: 24}}}, + }}, + }}, + }, + expectNADAnnotations: map[string]map[string]string{"default": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, { name: "fails to reconcile a secondary network", ra: &testRA{Name: "ra", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, @@ -942,6 +1033,338 @@ func TestController_reconcile(t *testing.T) { reconcile: "ra", expectAcceptedStatus: metav1.ConditionFalse, }, + { + name: "fails to reconcile EVPN-enabled network to default VRF", + ra: &testRA{Name: "ra", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nads: []*testNAD{ + {Name: "evpn-net", Namespace: "test", Network: util.GenerateCUDNNetworkName("evpn-net"), + Topology: "layer2", Subnet: "1.2.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNMACVRFVNI: 1000}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "reconciles EVPN MAC-VRF l2 network with a specific target VRF without a VRF router", + ra: &testRA{Name: "ra", TargetVRF: "red", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 65000, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1"}, + }}, + }, + }, + }, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: util.GenerateCUDNNetworkName("red"), + Topology: "layer2", Subnet: "10.1.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNMACVRFVNI: 1000, EVPNMACVRFRouteTarget: "65000:1000"}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + RawConfigPriority: 10, + RawConfig: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + advertise-all-vni + vni 1000 + route-target import 65000:1000 + route-target export 65000:1000 + exit-vni + exit-address-family +exit +! +`, + }, + }, + expectNADAnnotations: map[string]map[string]string{"red": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, + { + name: "reconciles EVPN IP-VRF network with auto target and creates a router", + ra: &testRA{Name: "ra", TargetVRF: "auto", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 65000, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1"}, + }}, + }, + }, + }, + nads: []*testNAD{ + {Name: "blue", Namespace: "blue", Network: util.GenerateCUDNNetworkName("blue"), + Topology: "layer3", Subnet: "10.2.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNIPVRFVNI: 2000, EVPNIPVRFRouteTarget: "65000:2000"}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"cluster_udn_blue\":\"10.2.1.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + RawConfigPriority: 10, + RawConfig: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + advertise-all-vni + exit-address-family +exit +! +vrf blue + vni 2000 +exit-vrf +! +router bgp 65000 vrf blue + address-family l2vpn evpn + advertise ipv4 unicast + route-target import 65000:2000 + route-target export 65000:2000 + exit-address-family +exit +! +`, + Routers: []*testRouter{ + {ASN: 65000, VRF: "blue", Prefixes: []string{"10.2.1.0/24"}}, + }, + }, + }, + expectNADAnnotations: map[string]map[string]string{"blue": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, + { + name: "reconciles EVPN IP-VRF with router ASN from another FRRConfiguration", + ra: &testRA{Name: "ra", TargetVRF: "auto", AdvertisePods: true, SelectsDefault: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfigGlobal", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 65000, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1"}, + }}, + }, + }, + { + Name: "frrConfigVRF", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 65100, VRF: "blue"}, + }, + }, + }, + nads: []*testNAD{ + {Name: "blue", Namespace: "blue", Network: util.GenerateCUDNNetworkName("blue"), + Topology: "layer3", Subnet: "10.2.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNIPVRFVNI: 2000, EVPNIPVRFRouteTarget: "65000:2000"}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\",\"cluster_udn_blue\":\"10.2.1.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigGlobal/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + RawConfigPriority: 10, + RawConfig: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + advertise-all-vni + exit-address-family +exit +! +vrf blue + vni 2000 +exit-vrf +! +`, + Routers: []*testRouter{ + {ASN: 65000, Prefixes: []string{"1.1.0.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1", Advertise: []string{"1.1.0.0/24"}}, + }}, + }, + }, + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigVRF/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + RawConfigPriority: 10, + RawConfig: `vrf blue + vni 2000 +exit-vrf +! +router bgp 65100 vrf blue + address-family l2vpn evpn + advertise ipv4 unicast + route-target import 65000:2000 + route-target export 65000:2000 + exit-address-family +exit +! +`, + Routers: []*testRouter{ + {ASN: 65100, VRF: "blue", Prefixes: []string{"10.2.1.0/24"}}, + }, + }, + }, + expectNADAnnotations: map[string]map[string]string{"blue": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, + { + name: "fails to reconcile MACVRF EVPN without global router", + ra: &testRA{Name: "ra", TargetVRF: "red", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 65000, VRF: "red", Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1"}, + }}, + }, + }, + }, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: util.GenerateCUDNNetworkName("red"), + Topology: "layer2", Subnet: "10.1.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNMACVRFVNI: 1000}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"cluster_udn_red\":\"10.1.1.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile IPVRF EVPN without global router", + ra: &testRA{Name: "ra", TargetVRF: "red", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 65000, VRF: "red", Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1"}, + }}, + }, + }, + }, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: util.GenerateCUDNNetworkName("red"), + Topology: "layer2", Subnet: "10.1.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNIPVRFVNI: 1000}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"cluster_udn_red\":\"10.1.1.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "fails to reconcile EVPN with global router but no neighbors", + ra: &testRA{Name: "ra", TargetVRF: "red", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 65000}, + }, + }, + }, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: util.GenerateCUDNNetworkName("red"), + Topology: "layer2", Subnet: "10.1.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNMACVRFVNI: 1000}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"cluster_udn_red\":\"10.1.1.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, + { + name: "reconciles EVPN when global router is in a different FRRConfiguration than VRF router", + ra: &testRA{Name: "ra", TargetVRF: "red", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfigGlobal", + Namespace: frrNamespace, + Routers: []*testRouter{ + // Global router with neighbors - provides ASN and neighbors for EVPN + {ASN: 65000, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1"}, + }}, + }, + }, + { + Name: "frrConfigVRF", + Namespace: frrNamespace, + Routers: []*testRouter{ + // VRF-specific router - matches the target VRF + {ASN: 65000, VRF: "red", Prefixes: []string{"10.1.0.0/16"}, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1"}, + }}, + }, + }, + }, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: util.GenerateCUDNNetworkName("red"), + Topology: "layer2", Subnet: "10.1.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNMACVRFVNI: 1000, EVPNMACVRFRouteTarget: "65000:1000"}, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}"}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigGlobal/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + RawConfigPriority: 10, + RawConfig: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + advertise-all-vni + vni 1000 + route-target import 65000:1000 + route-target export 65000:1000 + exit-vni + exit-address-family +exit +! +`, + }, + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigVRF/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 65000, VRF: "red", Prefixes: []string{"10.1.0.0/16"}, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1", Advertise: []string{"10.1.0.0/16"}}, + }}, + }, + }, + }, + expectNADAnnotations: map[string]map[string]string{"red": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -960,9 +1383,11 @@ func TestController_reconcile(t *testing.T) { HostSubnetLength: 64, }, } + config.Default.Transport = tt.transport config.OVNKubernetesFeature.EnableMultiNetwork = true config.OVNKubernetesFeature.EnableRouteAdvertisements = true config.OVNKubernetesFeature.EnableEgressIP = true + config.OVNKubernetesFeature.EnableEVPN = true fakeClientset := util.GetOVNClientset().GetClusterManagerClientset() addGenerateNameReactor[*frrfake.Clientset](fakeClientset.FRRClient) @@ -1315,6 +1740,7 @@ func TestUpdates(t *testing.T) { config.OVNKubernetesFeature.EnableMultiNetwork = true config.OVNKubernetesFeature.EnableRouteAdvertisements = true config.OVNKubernetesFeature.EnableEgressIP = true + config.OVNKubernetesFeature.EnableEVPN = true fakeClientset := util.GetOVNClientset().GetClusterManagerClientset() diff --git a/go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig.go b/go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig.go new file mode 100644 index 0000000000..7e3fc9bd47 --- /dev/null +++ b/go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig.go @@ -0,0 +1,140 @@ +package routeadvertisements + +import ( + "fmt" + "strings" +) + +// generateEVPNRawConfig generates raw FRR configuration for EVPN. +// If asn/neighbors aren't provided the related sections are skipped. +// +// Generated config structure: +// +// router bgp <- genGlobalEVPNSection +// address-family l2vpn evpn +// neighbor activate +// advertise-all-vni +// vni <- (one per MAC-VRF with RT, section only added when MAC-VRF RT is set) +// route-target import +// route-target export +// exit-vni +// exit-address-family +// exit +// ! +// vrf <- genVRFVNISection (one per IP-VRF) +// vni +// exit-vrf +// ! +// router bgp vrf <- genVRFEVPNSection (one per IP-VRF) +// address-family l2vpn evpn +// advertise ipv4 unicast +// advertise ipv6 unicast +// route-target import +// route-target export +// exit-address-family +// exit +// ! +func generateEVPNRawConfig(selected *selectedNetworks, asn uint32, neighbors []string, vrfASNs map[string]uint32) string { + var buf strings.Builder + + if asn > 0 && len(neighbors) > 0 { + buf.WriteString(genGlobalEVPNSection(asn, neighbors, selected.macVRFConfigs)) + } + for _, cfg := range selected.ipVRFConfigs { + buf.WriteString(genVRFVNISection(cfg)) + } + // Generate VRF-specific EVPN sections using each config's ASN + for _, cfg := range selected.ipVRFConfigs { + if vrfASN := vrfASNs[cfg.VRFName]; vrfASN > 0 { + buf.WriteString(genVRFEVPNSection(vrfASN, cfg)) + } + } + return buf.String() +} + +// genVRFVNISection generates VRF-to-VNI mapping. +// +// vrf +// vni +// exit-vrf +// ! +func genVRFVNISection(cfg *ipVRFConfig) string { + return fmt.Sprintf(`vrf %s + vni %d +exit-vrf +! +`, cfg.VRFName, cfg.VNI) +} + +// genGlobalEVPNSection generates the global router's EVPN address-family. +// +// router bgp +// address-family l2vpn evpn +// neighbor activate +// advertise-all-vni +// vni <- (Section only added when MAC-VRF RT is set) +// route-target import +// route-target export +// exit-vni +// exit-address-family +// exit +// ! +func genGlobalEVPNSection(asn uint32, neighbors []string, macVRFs []*vrfConfig) string { + var buf strings.Builder + + fmt.Fprintf(&buf, "router bgp %d\n", asn) + buf.WriteString(" address-family l2vpn evpn\n") + + for _, neighbor := range neighbors { + fmt.Fprintf(&buf, " neighbor %s activate\n", neighbor) + } + buf.WriteString(" advertise-all-vni\n") + + for _, cfg := range macVRFs { + if cfg.RouteTarget == "" { + continue + } + fmt.Fprintf(&buf, " vni %d\n", cfg.VNI) + fmt.Fprintf(&buf, " route-target import %s\n", cfg.RouteTarget) + fmt.Fprintf(&buf, " route-target export %s\n", cfg.RouteTarget) + buf.WriteString(" exit-vni\n") + } + + buf.WriteString(" exit-address-family\n") + buf.WriteString("exit\n!\n") + + return buf.String() +} + +// genVRFEVPNSection generates a VRF router's EVPN address-family. +// +// router bgp 65000 vrf red +// address-family l2vpn evpn +// advertise ipv4 unicast +// advertise ipv6 unicast +// route-target import 65000:100 +// route-target export 65000:100 +// exit-address-family +// exit +// ! +func genVRFEVPNSection(asn uint32, cfg *ipVRFConfig) string { + var buf strings.Builder + fmt.Fprintf(&buf, "router bgp %d vrf %s\n", asn, cfg.VRFName) + buf.WriteString(" address-family l2vpn evpn\n") + + if cfg.HasIPv4 { + buf.WriteString(" advertise ipv4 unicast\n") + } + if cfg.HasIPv6 { + buf.WriteString(" advertise ipv6 unicast\n") + } + if cfg.RouteTarget != "" { + fmt.Fprintf(&buf, " route-target import %s\n", cfg.RouteTarget) + fmt.Fprintf(&buf, " route-target export %s\n", cfg.RouteTarget) + } + + buf.WriteString(" exit-address-family\n") + buf.WriteString("exit\n!\n") + + return buf.String() +} diff --git a/go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig_test.go b/go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig_test.go new file mode 100644 index 0000000000..1b1460441c --- /dev/null +++ b/go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig_test.go @@ -0,0 +1,183 @@ +package routeadvertisements + +import ( + "testing" +) + +func TestGenerateEVPNRawConfig(t *testing.T) { + tests := []struct { + name string + selected *selectedNetworks + asn uint32 + neighbors []string + want string + }{ + { + name: "MAC-VRF without route target", + selected: &selectedNetworks{ + macVRFConfigs: []*vrfConfig{ + {VNI: 1000}, + }, + }, + asn: 65000, + neighbors: []string{"192.168.1.1"}, + want: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + advertise-all-vni + exit-address-family +exit +! +`, + }, + { + name: "MAC-VRF with route target", + selected: &selectedNetworks{ + macVRFConfigs: []*vrfConfig{ + {VNI: 1000, RouteTarget: "65000:1000"}, + }, + }, + asn: 65000, + neighbors: []string{"192.168.1.1"}, + want: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + advertise-all-vni + vni 1000 + route-target import 65000:1000 + route-target export 65000:1000 + exit-vni + exit-address-family +exit +! +`, + }, + { + name: "IP-VRF IPv6", + selected: &selectedNetworks{ + ipVRFConfigs: []*ipVRFConfig{ + { + vrfConfig: vrfConfig{VNI: 2000, RouteTarget: "65000:2000"}, + VRFName: "blue", + HasIPv6: true, + }, + }, + }, + asn: 65000, + neighbors: []string{"192.168.1.1"}, + want: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + advertise-all-vni + exit-address-family +exit +! +vrf blue + vni 2000 +exit-vrf +! +router bgp 65000 vrf blue + address-family l2vpn evpn + advertise ipv6 unicast + route-target import 65000:2000 + route-target export 65000:2000 + exit-address-family +exit +! +`, + }, + { + name: "IP-VRF dual stack", + selected: &selectedNetworks{ + ipVRFConfigs: []*ipVRFConfig{ + { + vrfConfig: vrfConfig{VNI: 2000, RouteTarget: "65000:2000"}, + VRFName: "blue", + HasIPv4: true, + HasIPv6: true, + }, + }, + }, + asn: 65000, + neighbors: []string{"192.168.1.1"}, + want: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + advertise-all-vni + exit-address-family +exit +! +vrf blue + vni 2000 +exit-vrf +! +router bgp 65000 vrf blue + address-family l2vpn evpn + advertise ipv4 unicast + advertise ipv6 unicast + route-target import 65000:2000 + route-target export 65000:2000 + exit-address-family +exit +! +`, + }, + { + name: "MAC-VRF and IP-VRF combined", + selected: &selectedNetworks{ + macVRFConfigs: []*vrfConfig{ + {VNI: 1000, RouteTarget: "65000:1000"}, + }, + ipVRFConfigs: []*ipVRFConfig{ + { + vrfConfig: vrfConfig{VNI: 2000, RouteTarget: "65000:2000"}, + VRFName: "blue", + HasIPv4: true, + }, + }, + }, + asn: 65000, + neighbors: []string{"192.168.1.1", "192.168.1.2"}, + want: `router bgp 65000 + address-family l2vpn evpn + neighbor 192.168.1.1 activate + neighbor 192.168.1.2 activate + advertise-all-vni + vni 1000 + route-target import 65000:1000 + route-target export 65000:1000 + exit-vni + exit-address-family +exit +! +vrf blue + vni 2000 +exit-vrf +! +router bgp 65000 vrf blue + address-family l2vpn evpn + advertise ipv4 unicast + route-target import 65000:2000 + route-target export 65000:2000 + exit-address-family +exit +! +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vrfASNs := map[string]uint32{} + for _, cfg := range tt.selected.ipVRFConfigs { + if cfg.VRFName != "" { + vrfASNs[cfg.VRFName] = tt.asn + } + } + got := generateEVPNRawConfig(tt.selected, tt.asn, tt.neighbors, vrfASNs) + if got != tt.want { + t.Errorf("generateEVPNRawConfig() mismatch\nGot:\n%s\nWant:\n%s", got, tt.want) + } + }) + } +} diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller.go index 10a1fee9dd..f1ffd0dcc3 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" metaapplyv1 "k8s.io/client-go/applyconfigurations/meta/v1" corev1informer "k8s.io/client-go/informers/core/v1" @@ -29,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork/notifier" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork/template" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" @@ -38,14 +40,42 @@ import ( userdefinednetworkscheme "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/scheme" userdefinednetworkinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/userdefinednetwork/v1" userdefinednetworklister "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/listers/userdefinednetwork/v1" + vtepinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1" + vteplister "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/listers/vtep/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) -const conditionTypeNetworkCreated = "NetworkCreated" +const ( + conditionTypeNetworkCreated = "NetworkCreated" + + // Condition reasons + reasonNADCreated = "NetworkAttachmentDefinitionCreated" + reasonSyncError = "SyncError" + reasonVTEPNotFound = "VTEPNotFound" + reasonNADDeleted = "NetworkAttachmentDefinitionDeleted" + reasonNADSyncError = "NetworkAttachmentDefinitionSyncError" + + // MaxEVPNVIDs is the maximum number of VIDs available for EVPN networks (0-4094, but 0 and 1 are reserved). + MaxEVPNVIDs = 4095 + // reservedVIDZeroKey is the key used to reserve VID 0 (reserved per IEEE 802.1Q for priority tagging). + reservedVIDZeroKey = "__vid_zero_reserved__" + // reservedVIDOneKey is the key used to reserve VID 1 (default VLAN on many switches, avoided by convention). + reservedVIDOneKey = "__vid_one_reserved__" +) + +// macVRFKey returns the VID allocator key for a network's MAC-VRF. +func macVRFKey(networkName string) string { + return networkName + "/macvrf" +} + +// ipVRFKey returns the VID allocator key for a network's IP-VRF. +func ipVRFKey(networkName string) string { + return networkName + "/ipvrf" +} -type RenderNetAttachDefManifest func(obj client.Object, targetNamespace string) (*netv1.NetworkAttachmentDefinition, error) +type RenderNetAttachDefManifest func(obj client.Object, targetNamespace string, opts ...template.RenderOption) (*netv1.NetworkAttachmentDefinition, error) type networkInUseError struct { err error @@ -55,6 +85,15 @@ func (n *networkInUseError) Error() string { return n.err.Error() } +// vtepNotFoundError indicates that a required VTEP CR does not exist. +type vtepNotFoundError struct { + vtepName string +} + +func (e *vtepNotFoundError) Error() string { + return fmt.Sprintf("VTEP %q does not exist", e.vtepName) +} + type Controller struct { // cudnController manage ClusterUserDefinedNetwork CRs. cudnController controller.Controller @@ -76,6 +115,10 @@ type Controller struct { networkManager networkmanager.Interface + // vidAllocator allocates cluster-wide VLAN IDs for EVPN networks. + // VIDs are allocated per network name and stored in the NAD config JSON. + vidAllocator id.Allocator + udnClient userdefinednetworkclientset.Interface udnLister userdefinednetworklister.UserDefinedNetworkLister cudnLister userdefinednetworklister.ClusterUserDefinedNetworkLister @@ -83,6 +126,10 @@ type Controller struct { nadLister netv1lister.NetworkAttachmentDefinitionLister podInformer corev1informer.PodInformer namespaceInformer corev1informer.NamespaceInformer + // vtepLister provides read access to VTEP CRs for validating EVPN configuration. + vtepLister vteplister.VTEPLister + // vtepNotifier notifies subscribing controllers about VTEP events. + vtepNotifier *notifier.VTEPNotifier networkInUseRequeueInterval time.Duration eventRecorder record.EventRecorder @@ -98,10 +145,15 @@ func New( networkManager networkmanager.Interface, podInformer corev1informer.PodInformer, namespaceInformer corev1informer.NamespaceInformer, + vtepInformer vtepinformer.VTEPInformer, eventRecorder record.EventRecorder, ) *Controller { udnLister := udnInformer.Lister() cudnLister := cudnInformer.Lister() + + // Allocates VIDs in range 1-4094 (0 is reserved per IEEE 802.1Q). + vidAllocator := id.NewIDAllocator("EVPN-VIDs", MaxEVPNVIDs) + c := &Controller{ nadClient: nadClient, nadLister: nadInfomer.Lister(), @@ -113,6 +165,7 @@ func New( namespaceInformer: namespaceInformer, networkManager: networkManager, namespaceTracker: map[string]sets.Set[string]{}, + vidAllocator: vidAllocator, eventRecorder: eventRecorder, } udnCfg := &controller.ControllerConfig[userdefinednetworkv1.UserDefinedNetwork]{ @@ -138,18 +191,30 @@ func New( c.nadNotifier = notifier.NewNetAttachDefNotifier(nadInfomer, c) c.namespaceNotifier = notifier.NewNamespaceNotifier(namespaceInformer, c) + // Setup EVPN components only when EVPN is enabled. + if util.IsEVPNEnabled() && vtepInformer != nil { + // Setup VTEP watching for EVPN support. + c.vtepLister = vtepInformer.Lister() + c.vtepNotifier = notifier.NewVTEPNotifier(vtepInformer, c) + } + return c } func (c *Controller) Run() error { klog.Infof("Starting user-defined network controllers") - if err := controller.StartWithInitialSync( - c.initializeNamespaceTracker, + + controllers := []controller.Reconciler{ c.cudnController, c.udnController, c.nadNotifier.Controller, c.namespaceNotifier.Controller, - ); err != nil { + } + if c.vtepNotifier != nil { + controllers = append(controllers, c.vtepNotifier.Controller) + } + + if err := controller.StartWithInitialSync(c.initializeController, controllers...); err != nil { return fmt.Errorf("unable to start user-defined network controller: %v", err) } @@ -162,57 +227,233 @@ func (c *Controller) Run() error { return nil } -// initializeNamespaceTracker populates the namespace-tracker with NAD namespaces who owned by the controller. -func (c *Controller) initializeNamespaceTracker() error { - cudns, err := c.cudnLister.List(labels.Everything()) +// initializeController performs all startup initialization before controllers begin processing. +func (c *Controller) initializeController() error { + // Reserve VID 0 and VID 1 to ensure they're never allocated to any network. + // VID 0 is reserved per IEEE 802.1Q standard. + // VID 1 is the default VLAN on many switches and avoided by convention. + if err := c.vidAllocator.ReserveID(reservedVIDZeroKey, 0); err != nil { + return fmt.Errorf("failed to reserve VID 0: %w", err) + } + if err := c.vidAllocator.ReserveID(reservedVIDOneKey, 1); err != nil { + return fmt.Errorf("failed to reserve VID 1: %w", err) + } + + cudnNADs, err := c.buildCUDNToNADs() if err != nil { return err } - if len(cudns) == 0 { + if len(cudnNADs) == 0 { return nil } + c.initializeNamespaceTracker(cudnNADs) + if util.IsEVPNEnabled() { + // Recover VID allocations from existing EVPN CUDNs. + // Recovery failures are logged and the affected CUDNs are enqueued for reconciliation, + // but don't block startup - this prevents a DoS where a malicious NAD could + // crash the entire cluster-manager. + c.recoverEVPNVIDs(cudnNADs) + } + + return nil +} + +// cudnWithNADs pairs a CUDN with its owned NADs. +type cudnWithNADs struct { + cudn *userdefinednetworkv1.ClusterUserDefinedNetwork + nads []netv1.NetworkAttachmentDefinition +} + +// cudnToNADs maps CUDN name to its object and owned NADs. +type cudnToNADs map[string]*cudnWithNADs + +// buildCUDNToNADs builds an index of CUDNs to their owned NADs. +// It returns an entry for every existing CUDN, including CUDNs that currently own no NADs +func (c *Controller) buildCUDNToNADs() (cudnToNADs, error) { + cudns, err := c.cudnLister.List(labels.Everything()) + if err != nil { + return nil, err + } + if len(cudns) == 0 { + return nil, nil + } + nads, err := c.nadLister.List(labels.Everything()) if err != nil { - return err + return nil, err } - if len(nads) == 0 { - return nil + + cudnByUID := make(map[types.UID]*userdefinednetworkv1.ClusterUserDefinedNetwork, len(cudns)) + index := make(cudnToNADs, len(cudns)) + for _, cudn := range cudns { + cudnByUID[cudn.UID] = cudn + index[cudn.Name] = &cudnWithNADs{cudn: cudn} } - indexedNADs := map[string]netv1.NetworkAttachmentDefinition{} + for _, nad := range nads { - if nad != nil { - indexedNADs[nad.Namespace+"/"+nad.Name] = *nad.DeepCopy() + if nad == nil { + continue + } + controllerRef := metav1.GetControllerOfNoCopy(nad) + if controllerRef == nil { + continue + } + if cudn, ok := cudnByUID[controllerRef.UID]; ok { + index[cudn.Name].nads = append(index[cudn.Name].nads, *nad.DeepCopy()) } } - for _, cudn := range cudns { - c.namespaceTracker[cudn.Name] = sets.New[string]() + return index, nil +} - for nadKey, nad := range indexedNADs { - if !metav1.IsControlledBy(&nad, cudn) { - continue - } - c.namespaceTracker[cudn.Name].Insert(nad.Namespace) +// initializeNamespaceTracker populates the namespace tracker with NAD namespaces owned by each CUDN. +func (c *Controller) initializeNamespaceTracker(cudnNADs cudnToNADs) { + for cudnName, entry := range cudnNADs { + c.namespaceTracker[cudnName] = sets.New[string]() + for _, nad := range entry.nads { + c.namespaceTracker[cudnName].Insert(nad.Namespace) + } + } +} - // Usually we don't want to mutate an iterated map, in this case - // the processed entry is removed because it shouldn't be processed - // again and not expected to be visited again, i.e.: the NAD should - // be recorded by the namespaceTracker once. - delete(indexedNADs, nadKey) +// recoverEVPNVIDs recovers VID allocations from existing EVPN CUDNs using +// NetworkManager's cached NetInfo. NetworkManager has already processed all NADs +// by the time this function is called (it starts before UDN controller). +// +// CUDNs are processed in order of creation timestamp (oldest first) to ensure +// deterministic VID assignment when conflicts occur. If two CUDNs have NADs +// claiming the same VID, the oldest CUDN wins ("first come, first served"). +// CUDN name is used as tie-breaker when timestamps are equal. +// +// If VID recovery fails for a CUDN (e.g., NetworkManager couldn't parse the NAD), +// this logs an error and enqueues the CUDN for reconciliation. +func (c *Controller) recoverEVPNVIDs(cudnNADs cudnToNADs) { + // Extract EVPN CUDNs with NADs into a slice for deterministic ordering. + evpnCUDNs := make([]*cudnWithNADs, 0, len(cudnNADs)) + for _, entry := range cudnNADs { + if entry.cudn.Spec.Network.Transport != userdefinednetworkv1.TransportOptionEVPN { + continue + } + if len(entry.nads) == 0 { + klog.V(4).Infof("EVPN CUDN %s has no NADs, skipping VID recovery", entry.cudn.Name) + continue } + evpnCUDNs = append(evpnCUDNs, entry) } + // Sort by creation timestamp (oldest first) for deterministic conflict resolution. + // When two CUDNs have conflicting VIDs, the oldest one wins. + // Use name as tie-breaker when timestamps are equal for consistent ordering. + slices.SortFunc(evpnCUDNs, func(a, b *cudnWithNADs) int { + if a.cudn.CreationTimestamp.Before(&b.cudn.CreationTimestamp) { + return -1 + } + if b.cudn.CreationTimestamp.Before(&a.cudn.CreationTimestamp) { + return 1 + } + return strings.Compare(a.cudn.Name, b.cudn.Name) + }) + + for _, entry := range evpnCUDNs { + if err := c.recoverEVPNVIDsForCUDN(entry.cudn.Name); err != nil { + klog.Errorf("VID recovery failed for EVPN CUDN %s: %v. "+ + "The CUDN will be reconciled and existing NAD VIDs will be preserved if possible.", + entry.cudn.Name, err) + c.cudnController.Reconcile(entry.cudn.Name) + } + } +} + +// recoverEVPNVIDsForCUDN attempts to recover VIDs for a single CUDN using NetworkManager's cache. +// Returns nil if VIDs were successfully recovered or if no VIDs are allocated yet. +// Returns error if VID reservation fails (e.g., conflict with another network). +func (c *Controller) recoverEVPNVIDsForCUDN(cudnName string) error { + networkName := util.GenerateCUDNNetworkName(cudnName) + + // Use NetworkManager's cached NetInfo - it has already parsed the NAD + netInfo := c.networkManager.GetNetwork(networkName) + if netInfo == nil { + // NetworkManager doesn't have this network cached. This can happen if: + // - NetworkManager failed to parse the NAD (corrupted) + // - NAD doesn't exist yet + return fmt.Errorf("network %s not found in NetworkManager cache", networkName) + } + + macVRFVID := netInfo.EVPNMACVRFVID() + ipVRFVID := netInfo.EVPNIPVRFVID() + + // Check if this network has EVPN VIDs allocated + if macVRFVID == 0 && ipVRFVID == 0 { + klog.V(4).Infof("EVPN CUDN %s has no VIDs allocated yet, skipping recovery", cudnName) + return nil // No VIDs to recover + } + + if err := c.reserveRecoveredVIDs(cudnName, macVRFVID, ipVRFVID); err != nil { + return fmt.Errorf("failed to reserve VIDs for cudn %s: %w", cudnName, err) + } + + klog.V(4).Infof("Recovered VIDs for CUDN %s (macVRF=%d, ipVRF=%d)", cudnName, macVRFVID, ipVRFVID) return nil } +// reserveRecoveredVIDs reserves the given VIDs in the allocator for a network. +// VIDs of 0 are skipped (not allocated). +// +// Both VIDs are attempted even if one fails - this maximizes recovery and protects +// as many VIDs as possible. We don't release successfully reserved VIDs on partial +// failure because they represent state that already exists in NADs; releasing them +// could allow another network to "steal" the VID, causing route leakage. +func (c *Controller) reserveRecoveredVIDs(networkName string, macVRFVID, ipVRFVID int) error { + var errs []error + + if macVRFVID > 0 { + if err := c.vidAllocator.ReserveID(macVRFKey(networkName), macVRFVID); err != nil { + errs = append(errs, fmt.Errorf("failed to reserve VID %d for MAC-VRF of network %s: %w", macVRFVID, networkName, err)) + } else { + klog.V(4).Infof("Recovered VID %d for MAC-VRF of network %s", macVRFVID, networkName) + } + } + if ipVRFVID > 0 { + if err := c.vidAllocator.ReserveID(ipVRFKey(networkName), ipVRFVID); err != nil { + errs = append(errs, fmt.Errorf("failed to reserve VID %d for IP-VRF of network %s: %w", ipVRFVID, networkName, err)) + } else { + klog.V(4).Infof("Recovered VID %d for IP-VRF of network %s", ipVRFVID, networkName) + } + } + + return errors.Join(errs...) +} + +// releaseVIDForNetwork releases the VIDs allocated for a network's VRFs. +// +// NOTE: VID release is not synchronized with node-side dataplane cleanup. +// In theory, a rapidly created new network could get the same VID while nodes +// are still tearing down the old network's bridge configuration. In practice, +// VID collisions are unlikely because the allocator is monotonic and won't +// reallocate the same VID unless the pool fills up or CUDNs are recycled rapidly. +// The actual mitigation is on the node-side: nodes should check for VID conflicts +// and refuse to configure a VID already in use by a different network, waiting +// until the old network is cleaned up. +func (c *Controller) releaseVIDForNetwork(networkName string) { + macVID := c.vidAllocator.ReleaseID(macVRFKey(networkName)) + ipVID := c.vidAllocator.ReleaseID(ipVRFKey(networkName)) + if macVID >= 0 || ipVID >= 0 { + klog.V(4).Infof("Released VIDs for network %s: MAC-VRF=%d, IP-VRF=%d", networkName, macVID, ipVID) + } +} + func (c *Controller) Shutdown() { - controller.Stop( + controllers := []controller.Reconciler{ c.cudnController, c.udnController, c.nadNotifier.Controller, c.namespaceNotifier.Controller, - ) + } + if c.vtepNotifier != nil { + controllers = append(controllers, c.vtepNotifier.Controller) + } + controller.Stop(controllers...) } // ReconcileNetAttachDef enqueue NAD requests following NAD events. @@ -283,7 +524,7 @@ func (c *Controller) ReconcileNamespace(key string) error { if !affectedNamespace { cudn, err := c.cudnLister.Get(cudnName) if err != nil { - return fmt.Errorf("faild to get CUDN %q from cache: %w", cudnName, err) + return fmt.Errorf("failed to get CUDN %q from cache: %w", cudnName, err) } cudnSelector, err := metav1.LabelSelectorAsSelector(&cudn.Spec.NamespaceSelector) if err != nil { @@ -427,6 +668,7 @@ func (c *Controller) syncUserDefinedNetwork(udn *userdefinednetworkv1.UserDefine } klog.Infof("Finalizer removed from UserDefinedNetworks [%s/%s]", udn.Namespace, udn.Name) metrics.DecrementUDNCount(role, topology) + metrics.DeleteDynamicUDNNodeCount(util.GenerateUDNNetworkName(udn.Namespace, udn.Name)) } return nil, nil @@ -487,19 +729,19 @@ func newNetworkCreatedCondition(nad *netv1.NetworkAttachmentDefinition, syncErro networkCreatedCondition := &metav1.Condition{ Type: conditionTypeNetworkCreated, Status: metav1.ConditionTrue, - Reason: "NetworkAttachmentDefinitionCreated", + Reason: reasonNADCreated, Message: "NetworkAttachmentDefinition has been created", LastTransitionTime: now, } if nad != nil && !nad.DeletionTimestamp.IsZero() { networkCreatedCondition.Status = metav1.ConditionFalse - networkCreatedCondition.Reason = "NetworkAttachmentDefinitionDeleted" + networkCreatedCondition.Reason = reasonNADDeleted networkCreatedCondition.Message = "NetworkAttachmentDefinition is being deleted" } if syncError != nil { networkCreatedCondition.Status = metav1.ConditionFalse - networkCreatedCondition.Reason = "SyncError" + networkCreatedCondition.Reason = reasonSyncError networkCreatedCondition.Message = syncError.Error() } @@ -510,8 +752,8 @@ func (c *Controller) cudnNeedUpdate(_ *userdefinednetworkv1.ClusterUserDefinedNe return true } -// reconcileUDN get ClusterUserDefinedNetwork CR key and reconcile it according to spec. -// It creates NADs according to spec at the spesified selected namespaces. +// reconcileCUDN get ClusterUserDefinedNetwork CR key and reconcile it according to spec. +// It creates NADs according to spec at the specified selected namespaces. // The NAD objects are created with the same key as the request CR, having both kinds have the same key enable // the controller to act on NAD changes as well and reconciles NAD objects (e.g: in case NAD is deleted it will be re-created). func (c *Controller) reconcileCUDN(key string) error { @@ -538,6 +780,14 @@ func (c *Controller) reconcileCUDN(key string) error { return updateStatusErr } + // vtepNotFoundError is non-fatal: the status has been updated to reflect + // the missing VTEP, and the VTEPNotifier will re-queue this CUDN when + // the VTEP is created. No need to return an error that would cause retries. + var vtepNotFound *vtepNotFoundError + if errors.As(syncErr, &vtepNotFound) { + return updateStatusErr + } + return errors.Join(syncErr, updateStatusErr) } @@ -594,6 +844,8 @@ func (c *Controller) syncClusterUDN(cudn *userdefinednetworkv1.ClusterUserDefine klog.Infof("Finalizer removed from ClusterUserDefinedNetwork %q", cudn.Name) delete(c.namespaceTracker, cudnName) metrics.DecrementCUDNCount(role, topology) + metrics.DeleteDynamicUDNNodeCount(util.GenerateCUDNNetworkName(cudn.Name)) + c.releaseVIDForNetwork(cudnName) } return nil, nil @@ -614,6 +866,10 @@ func (c *Controller) syncClusterUDN(cudn *userdefinednetworkv1.ClusterUserDefine metrics.IncrementCUDNCount(role, topology) } + if err := c.validateEVPNVTEP(cudn); err != nil { + return nil, err + } + selectedNamespaces, err := c.getSelectedNamespaces(cudn.Spec.NamespaceSelector) if err != nil { return nil, fmt.Errorf("failed to get selected namespaces: %w", err) @@ -671,7 +927,7 @@ func (c *Controller) updateClusterUDNStatus(cudn *userdefinednetworkv1.ClusterUs return strings.Compare(a.Namespace, b.Namespace) }) - networkCreatedCondition := newClusterNetworCreatedCondition(nads, syncError) + networkCreatedCondition := newClusterNetworkCreatedCondition(nads, syncError) updated := meta.SetStatusCondition(&cudn.Status.Conditions, networkCreatedCondition) if !updated { @@ -705,7 +961,7 @@ func (c *Controller) updateClusterUDNStatus(cudn *userdefinednetworkv1.ClusterUs return nil } -func newClusterNetworCreatedCondition(nads []netv1.NetworkAttachmentDefinition, syncError error) metav1.Condition { +func newClusterNetworkCreatedCondition(nads []netv1.NetworkAttachmentDefinition, syncError error) metav1.Condition { var namespaces []string for _, nad := range nads { namespaces = append(namespaces, nad.Namespace) @@ -716,7 +972,7 @@ func newClusterNetworCreatedCondition(nads []netv1.NetworkAttachmentDefinition, condition := metav1.Condition{ Type: conditionTypeNetworkCreated, Status: metav1.ConditionTrue, - Reason: "NetworkAttachmentDefinitionCreated", + Reason: reasonNADCreated, Message: fmt.Sprintf("NetworkAttachmentDefinition has been created in following namespaces: [%s]", affectedNamespaces), LastTransitionTime: now, } @@ -729,15 +985,80 @@ func newClusterNetworCreatedCondition(nads []netv1.NetworkAttachmentDefinition, } if len(deletedNadKeys) > 0 { condition.Status = metav1.ConditionFalse - condition.Reason = "NetworkAttachmentDefinitionDeleted" + condition.Reason = reasonNADDeleted condition.Message = fmt.Sprintf("NetworkAttachmentDefinition are being deleted: %v", deletedNadKeys) } if syncError != nil { condition.Status = metav1.ConditionFalse - condition.Reason = "NetworkAttachmentDefinitionSyncError" - condition.Message = syncError.Error() + + // Check for specific error types to provide better status reasons + var vtepNotFound *vtepNotFoundError + if errors.As(syncError, &vtepNotFound) { + condition.Reason = reasonVTEPNotFound + condition.Message = fmt.Sprintf("Cannot create network: VTEP '%s' does not exist. "+ + "Create the VTEP CR first or update the CUDN to reference an existing VTEP.", + vtepNotFound.vtepName) + } else { + condition.Reason = reasonNADSyncError + condition.Message = syncError.Error() + } } return condition } + +// validateEVPNVTEP validates EVPN configuration for a CUDN. +// Returns an error if EVPN is requested but disabled, or if the referenced VTEP doesn't exist. +func (c *Controller) validateEVPNVTEP(cudn *userdefinednetworkv1.ClusterUserDefinedNetwork) error { + if cudn.Spec.Network.Transport != userdefinednetworkv1.TransportOptionEVPN { + return nil // Not an EVPN network + } + + if !util.IsEVPNEnabled() { + return fmt.Errorf("EVPN transport requested but EVPN feature is not enabled") + } + + // CEL validation ensures EVPN is set when transport is EVPN. + vtepName := cudn.Spec.Network.EVPN.VTEP + _, err := c.vtepLister.Get(vtepName) + if err != nil { + if apierrors.IsNotFound(err) { + return &vtepNotFoundError{vtepName: vtepName} + } + return fmt.Errorf("failed to get VTEP %q: %w", vtepName, err) + } + + return nil +} + +// ReconcileVTEP handles VTEP events by re-queuing all CUDNs that reference the VTEP. +// +// This uses O(n) iteration over all CUDNs rather than maintaining an index because: +// VTEP create/delete events are expected to be rare; scanning all CUDNs from the +// informer cache keeps the logic simple. If this becomes a hot path at large +// CUDN counts, add an informer indexer keyed by VTEP. +func (c *Controller) ReconcileVTEP(vtepName string) error { + cudns, err := c.cudnLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("failed to list CUDNs: %w", err) + } + + for _, cudn := range cudns { + if cudnReferencesVTEP(cudn, vtepName) { + klog.V(4).InfoS("Re-queueing CUDN following VTEP event", "cudn", cudn.Name, "vtep", vtepName) + c.cudnController.Reconcile(cudn.Name) + } + } + + return nil +} + +// cudnReferencesVTEP returns true if the CUDN is an EVPN network referencing the given VTEP. +// CEL validation ensures EVPN is set when transport is EVPN. +func cudnReferencesVTEP(cudn *userdefinednetworkv1.ClusterUserDefinedNetwork, vtepName string) bool { + if cudn.Spec.Network.Transport != userdefinednetworkv1.TransportOptionEVPN { + return false + } + return cudn.Spec.Network.EVPN.VTEP == vtepName +} diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller_helper.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller_helper.go index 735b0afea2..127552ac93 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller_helper.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller_helper.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork/template" + userdefinednetworkv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" utiludn "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/udn" @@ -35,16 +36,22 @@ func (c *Controller) updateNAD(obj client.Object, namespace string) (*netv1.Netw } } - desiredNAD, err := c.renderNadFn(obj, namespace) + existingNAD, err := c.nadLister.NetworkAttachmentDefinitions(namespace).Get(obj.GetName()) + if err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get NetworkAttachmentDefinition %s/%s from cache: %v", namespace, obj.GetName(), err) + } + + renderOpts, err := c.allocateEVPNVIDsIfNeeded(obj) if err != nil { - return nil, fmt.Errorf("failed to generate NetworkAttachmentDefinition: %w", err) + return nil, fmt.Errorf("failed to allocate EVPN VIDs: %w", err) } - nad, err := c.nadLister.NetworkAttachmentDefinitions(namespace).Get(obj.GetName()) - if err != nil && !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get NetworkAttachmentDefinition %s/%s from cache: %v", namespace, obj.GetName(), err) + desiredNAD, err := c.renderNadFn(obj, namespace, renderOpts...) + if err != nil { + return nil, fmt.Errorf("failed to generate NetworkAttachmentDefinition: %w", err) } - nadCopy := nad.DeepCopy() + + nadCopy := existingNAD.DeepCopy() if nadCopy == nil { // creating NAD in case no primary network exist should be atomic and synchronized with @@ -119,7 +126,7 @@ func (c *Controller) deleteNAD(obj client.Object, namespace string) error { pods, err := c.podInformer.Lister().Pods(nadCopy.Namespace).List(labels.Everything()) if err != nil { - return fmt.Errorf("failed to list pods at target namesapce %q: %w", nadCopy.Namespace, err) + return fmt.Errorf("failed to list pods at target namespace %q: %w", nadCopy.Namespace, err) } // This is best-effort check no pod using the subject NAD, // noting prevent a from being pod creation right after this check. @@ -142,3 +149,55 @@ func (c *Controller) deleteNAD(obj client.Object, namespace string) error { return nil } + +// allocateEVPNVIDsIfNeeded checks if the object is an EVPN network and allocates VIDs if needed. +// Returns render options containing the allocated VIDs, or empty options for non-EVPN networks. +// Returns an error if EVPN transport is requested but the feature flag is disabled. +// +// This function relies on the idempotency of AllocateID: if a VID was already allocated for a key +// (either during recovery or a previous reconciliation), AllocateID returns the same VID. +// This means VIDs are stable across reconciliations without needing to parse the existing NAD. +func (c *Controller) allocateEVPNVIDsIfNeeded(obj client.Object) ([]template.RenderOption, error) { + spec := template.GetSpec(obj) + if spec.GetTransport() != userdefinednetworkv1.TransportOptionEVPN { + return nil, nil + } + + // EVPN transport is requested - ensure the feature is enabled. + if !util.IsEVPNEnabled() { + return nil, fmt.Errorf("EVPN transport requested but EVPN feature is not enabled") + } + + evpnCfg := spec.GetEVPN() + if evpnCfg == nil { + return nil, nil + } + + networkName := obj.GetName() + var macVRFVID, ipVRFVID int + + // Allocate VID for MAC-VRF if present + if evpnCfg.MACVRF != nil { + vid, err := c.vidAllocator.AllocateID(macVRFKey(networkName)) + if err != nil { + return nil, fmt.Errorf("failed to allocate VID for MAC-VRF: %w", err) + } + macVRFVID = vid + klog.V(4).InfoS("Allocated VID for MAC-VRF", "network", networkName, "vid", vid) + } + + // Allocate VID for IP-VRF if present + if evpnCfg.IPVRF != nil { + vid, err := c.vidAllocator.AllocateID(ipVRFKey(networkName)) + if err != nil { + return nil, fmt.Errorf("failed to allocate VID for IP-VRF: %w", err) + } + ipVRFVID = vid + klog.V(4).InfoS("Allocated VID for IP-VRF", "network", networkName, "vid", vid) + } + + // Return render options with allocated VIDs. + // Note: API validation ensures at least one of macVRF or ipVRF is specified, + // so at least one VID will be allocated if we reach here. + return []template.RenderOption{template.WithEVPNVIDs(macVRFVID, ipVRFVID)}, nil +} diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go index 2a07e96dbe..166931625d 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go @@ -2,6 +2,7 @@ package userdefinednetwork import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -22,10 +23,13 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork/template" + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" udnv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1" udnclient "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned" udnfakeclient "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/fake" + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + vtepinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -39,6 +43,7 @@ var _ = Describe("User Defined Network Controller", func() { var ( cs *util.OVNClusterManagerClientset f *factory.WatchFactory + nm networkmanager.Controller ) BeforeEach(func() { @@ -46,9 +51,16 @@ var _ = Describe("User Defined Network Controller", func() { Expect(config.PrepareTestConfig()).To(Succeed()) config.OVNKubernetesFeature.EnableMultiNetwork = true config.OVNKubernetesFeature.EnableNetworkSegmentation = true + // Enable EVPN for EVPN-related tests + config.OVNKubernetesFeature.EnableRouteAdvertisements = true + config.OVNKubernetesFeature.EnableEVPN = true }) AfterEach(func() { + if nm != nil { + nm.Stop() + nm = nil + } if f != nil { f.Shutdown() } @@ -65,7 +77,30 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return New(cs.NetworkAttchDefClient, f.NADInformer(), cs.UserDefinedNetworkClient, f.UserDefinedNetworkInformer(), f.ClusterUserDefinedNetworkInformer(), - renderNADStub, networkManager.Interface(), f.PodCoreInformer(), f.NamespaceInformer(), nil, + renderNADStub, networkManager.Interface(), f.PodCoreInformer(), f.NamespaceInformer(), f.VTEPInformer(), nil, + ) + } + + // newTestControllerWithNetworkManager creates a controller with a started NetworkManager. + newTestControllerWithNetworkManager := func(renderNADStub RenderNetAttachDefManifest, objects ...runtime.Object) *Controller { + cs = util.GetOVNClientset(objects...).GetClusterManagerClientset() + var err error + f, err = factory.NewClusterManagerWatchFactory(cs) + Expect(err).NotTo(HaveOccurred()) + Expect(f.Start()).To(Succeed()) + + nm, err = networkmanager.NewForCluster(&networkmanager.FakeControllerManager{}, f, cs, nil, id.NewTunnelKeyAllocator("TunnelKeys")) + Expect(err).NotTo(HaveOccurred()) + // Start NetworkManager - it will process existing NADs and cache their VIDs + Expect(nm.Start()).To(Succeed()) + + var vtepInformer vtepinformer.VTEPInformer + if util.IsEVPNEnabled() { + vtepInformer = f.VTEPInformer() + } + return New(cs.NetworkAttchDefClient, f.NADInformer(), + cs.UserDefinedNetworkClient, f.UserDefinedNetworkInformer(), f.ClusterUserDefinedNetworkInformer(), + renderNADStub, nm.Interface(), f.PodCoreInformer(), f.NamespaceInformer(), vtepInformer, nil, ) } @@ -445,6 +480,827 @@ var _ = Describe("User Defined Network Controller", func() { } }) + It("should allocate VID for EVPN network NAD", func() { + testNs := testNamespace("evpn-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + Eventually(func() []metav1.Condition { + var err error + cudn, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [evpn-test]", + }})) + + // Verify VID was allocated in the NAD config + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "VID should be allocated for EVPN MAC-VRF (first available after 0,1 reserved)") + }).Should(Succeed()) + }) + + It("should allocate VID for EVPN network NAD with IP-VRF only", func() { + testNs := testNamespace("evpn-ipvrf-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNIPVRFClusterUDN("evpn-ipvrf-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + Eventually(func() []metav1.Condition { + var err error + cudn, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [evpn-ipvrf-test]", + }})) + + // Verify VID was allocated in the NAD config (IP-VRF only, no MAC-VRF) + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, ipVID := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(0), "MAC-VRF should not be present for IP-VRF only config") + g.Expect(ipVID).To(Equal(2), "VID should be allocated for EVPN IP-VRF only (first available after 0,1 reserved)") + }).Should(Succeed()) + }) + + It("should allocate separate VIDs for EVPN network with both MAC-VRF and IP-VRF (symmetric IRB)", func() { + testNs := testNamespace("evpn-irb-test") + vtep := testVTEP("vtep-test") + cudn := testSymmetricIRBClusterUDN("evpn-irb-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + Eventually(func() []metav1.Condition { + var err error + cudn, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [evpn-irb-test]", + }})) + + // Verify both VIDs were allocated with different values + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, ipVID := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "MAC-VRF should get VID 2 (first available)") + g.Expect(ipVID).To(Equal(3), "IP-VRF should get VID 3") + }).Should(Succeed()) + }) + + It("should allocate different VIDs for multiple EVPN networks", func() { + testNs := testNamespace("evpn-multi-test") + vtep := testVTEP("vtep-test") + cudn1 := testEVPNClusterUDN("evpn-cudn-1", vtep.Name, testNs.Name) + cudn2 := testEVPNClusterUDN("evpn-cudn-2", vtep.Name, testNs.Name) + cudn2.UID = "2" // Different UID for second CUDN + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn1, cudn2, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // Wait for both NADs to be created and have VIDs, and verify they are different + Eventually(func(g Gomega) { + nad1, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), "evpn-cudn-1", metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + nad2, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), "evpn-cudn-2", metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + vid1, _ := evpnVIDsFromNAD(nad1) + vid2, _ := evpnVIDsFromNAD(nad2) + g.Expect(vid1).To(BeNumerically(">", 0), "NAD 1 should have VID allocated") + g.Expect(vid2).To(BeNumerically(">", 0), "NAD 2 should have VID allocated") + // VIDs should be different from each other + // Note: Order is non-deterministic due to concurrent CUDN processing + g.Expect(vid1).NotTo(Equal(vid2), "VIDs should be different for different networks") + }).Should(Succeed()) + }) + + It("should release VID when EVPN CUDN is deleted", func() { + testNs := testNamespace("evpn-delete-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-delete-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // Wait for CUDN to be processed and NAD created with VID + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "First CUDN should get VID 2 (first available)") + }).Should(Succeed()) + + // Verify VID is allocated in the controller's allocator + Expect(c.vidAllocator.GetID("evpn-delete-cudn/macvrf")).To(BeNumerically(">=", 0), "VID should be allocated") + + // Trigger deletion by setting DeletionTimestamp and processing + now := metav1.Now() + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + cudn.DeletionTimestamp = &now + _, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Update(context.Background(), cudn, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for finalizer to be removed (indicating deletion was processed) + Eventually(func(g Gomega) { + updatedCUDN, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(updatedCUDN.Finalizers).To(BeEmpty(), "Finalizer should be removed after deletion") + // Verify VID is released from the allocator + g.Expect(c.vidAllocator.GetID("evpn-delete-cudn/macvrf")).To(Equal(-1), "VID should be released after deletion") + }).Should(Succeed()) + }) + + It("should release both MAC-VRF and IP-VRF VIDs when symmetric IRB CUDN is deleted", func() { + testNs := testNamespace("evpn-irb-delete-test") + vtep := testVTEP("vtep-irb-delete") + cudn := testSymmetricIRBClusterUDN("evpn-irb-delete", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // Wait for CUDN to be processed and NAD created with both VIDs + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, ipVID := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "MAC-VRF VID should be allocated (first available)") + g.Expect(ipVID).To(Equal(3), "IP-VRF VID should be allocated") + }).Should(Succeed()) + + // Verify both VIDs are allocated in the controller's allocator + Expect(c.vidAllocator.GetID("evpn-irb-delete/macvrf")).To(Equal(2), "MAC-VRF VID should be allocated (first available)") + Expect(c.vidAllocator.GetID("evpn-irb-delete/ipvrf")).To(Equal(3), "IP-VRF VID should be allocated") + + // Trigger deletion + now := metav1.Now() + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + cudn.DeletionTimestamp = &now + _, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Update(context.Background(), cudn, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for finalizer to be removed and verify both VIDs are released + Eventually(func(g Gomega) { + updatedCUDN, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(updatedCUDN.Finalizers).To(BeEmpty(), "Finalizer should be removed after deletion") + // Verify both VIDs are released from the allocator + g.Expect(c.vidAllocator.GetID("evpn-irb-delete/macvrf")).To(Equal(-1), "MAC-VRF VID should be released after deletion") + g.Expect(c.vidAllocator.GetID("evpn-irb-delete/ipvrf")).To(Equal(-1), "IP-VRF VID should be released after deletion") + }).Should(Succeed()) + }) + + It("should preserve allocated VID when EVPN CUDN is updated", func() { + testNs := testNamespace("evpn-update-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-update-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // Wait for initial VID allocation + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "VID should be allocated (first available)") + }).Should(Succeed()) + + // Update CUDN (trigger reconciliation) + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + cudn.Annotations = map[string]string{"updated": "true"} + _, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Update(context.Background(), cudn, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Ensure VID remains the same after reconciliation + Consistently(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "VID should remain consistent after CUDN update") + }, 500*time.Millisecond, 50*time.Millisecond).Should(Succeed()) + }) + + It("should continue startup and allocate new VID when all NADs are corrupted", func() { + // VID recovery failures no longer block startup to prevent DoS attacks + // via malicious NADs. Instead, the CUDN is enqueued for reconciliation + // and a new VID is allocated. + testNs := testNamespace("evpn-all-corrupted-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-all-corrupted", vtep.Name, testNs.Name) + + // Create a corrupted NAD owned by the CUDN - NetworkManager will fail to parse it + corruptedNAD := testEVPNClusterUdnNADOwnedByCUDN(cudn, testNs.Name, vtep.Name, 0, 0) + corruptedNAD.Spec.Config = `{"transport":"evpn", invalid json - corrupted` + + // Use started NetworkManager - it will fail to parse the corrupted NAD + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep, corruptedNAD) + + // Controller should start successfully (VID recovery failure logged but not fatal) + Expect(c.Run()).To(Succeed()) + + // The CUDN is enqueued for reconciliation and gets a new VID + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "Should allocate new VID since recovery failed (first available)") + }).Should(Succeed()) + }) + + It("should continue startup and allocate new VID when VID recovery encounters a conflict", func() { + // VID conflicts during recovery no longer block startup. + // Instead, the CUDN is enqueued for reconciliation and gets a new VID. + testNs := testNamespace("evpn-vid-conflict-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-conflict", vtep.Name, testNs.Name) + + // Create a NAD with VID 5 for MAC-VRF + existingNAD := testEVPNClusterUdnNADOwnedByCUDN(cudn, testNs.Name, vtep.Name, 5, 0) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep, existingNAD) + + // Pre-reserve VID 5 for a DIFFERENT key to create a conflict during recovery + Expect(c.vidAllocator.ReserveID("conflicting-network/macvrf", 5)).To(Succeed()) + + // Controller should start successfully despite the conflict + Expect(c.Run()).To(Succeed()) + + // Recovery fails due to conflict, CUDN is enqueued for reconciliation and gets a new VID + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "Should allocate new VID since 5 is taken by another network (first available)") + }).Should(Succeed()) + }) + + It("should continue startup and preserve MAC-VRF VID when only IP-VRF VID recovery encounters a conflict", func() { + // When IP-VRF VID conflicts but MAC-VRF VID is available: + // - MAC-VRF recovery succeeds (VID reserved in allocator) + // - IP-VRF recovery fails (conflict) + // - CUDN is enqueued for reconciliation + // - MAC-VRF VID is preserved (already in allocator), IP-VRF gets new VID + testNs := testNamespace("evpn-ipvrf-conflict-test") + vtep := testVTEP("vtep-test") + cudn := testSymmetricIRBClusterUDN("evpn-ipvrf-conflict", vtep.Name, testNs.Name) + + // Create a symmetric IRB NAD with both MAC-VRF (VID 3) and IP-VRF (VID 7) + existingNAD := testEVPNClusterUdnNADOwnedByCUDN(cudn, testNs.Name, vtep.Name, 3, 7) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep, existingNAD) + + // Pre-reserve VID 7 for IP-VRF of a DIFFERENT network to create a conflict + Expect(c.vidAllocator.ReserveID("other-network/ipvrf", 7)).To(Succeed()) + + // Controller should start successfully + Expect(c.Run()).To(Succeed()) + + // MAC-VRF VID 3 was successfully reserved during recovery. + // IP-VRF VID 7 conflicted, so during reconciliation it gets new VID 2 (first available). + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, ipVID := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(3), "MAC-VRF VID should be preserved (recovery succeeded)") + g.Expect(ipVID).To(Equal(2), "IP-VRF gets new VID (first available, 0,1 reserved, 7 is taken)") + }).Should(Succeed()) + }) + + It("should continue startup and preserve IP-VRF VID when only MAC-VRF VID recovery encounters a conflict", func() { + // When MAC-VRF VID conflicts but IP-VRF VID is available: + // - MAC-VRF recovery fails (conflict) + // - IP-VRF recovery succeeds (VID reserved in allocator) + // - CUDN is enqueued for reconciliation + // - MAC-VRF gets new VID, IP-VRF VID is preserved + testNs := testNamespace("evpn-macvrf-conflict-test") + vtep := testVTEP("vtep-test") + cudn := testSymmetricIRBClusterUDN("evpn-macvrf-conflict", vtep.Name, testNs.Name) + + // Create a symmetric IRB NAD with both MAC-VRF (VID 3) and IP-VRF (VID 7) + existingNAD := testEVPNClusterUdnNADOwnedByCUDN(cudn, testNs.Name, vtep.Name, 3, 7) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep, existingNAD) + + // Pre-reserve VID 3 for a DIFFERENT network to create a conflict during recovery + Expect(c.vidAllocator.ReserveID("other-network/macvrf", 3)).To(Succeed()) + + // Controller should start successfully + Expect(c.Run()).To(Succeed()) + + // IP-VRF VID 7 was successfully reserved during recovery. + // MAC-VRF VID 3 conflicted, so during reconciliation it gets new VID 2 (first available). + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, ipVID := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "MAC-VRF gets new VID (first available, 0,1 reserved, 3 is already taken)") + g.Expect(ipVID).To(Equal(7), "IP-VRF VID should be preserved (recovery succeeded)") + }).Should(Succeed()) + }) + + It("should not fail startup when CUDN exists but has no NADs yet", func() { + vtep := testVTEP("vtep-test") + // Create a CUDN without any NADs (namespace doesn't match selector) + cudnWithNoNADs := testEVPNClusterUDN("evpn-no-nads", vtep.Name, "nonexistent-ns") + + c = newTestControllerWithNetworkManager(renderNadStub(nil), cudnWithNoNADs, vtep) + + Expect(c.Run()).To(Succeed(), "Controller should start even when CUDN has no NADs") + + // No VID should be allocated since there are no NADs + Expect(c.vidAllocator.GetID("evpn-no-nads/macvrf")).To(Equal(-1), "No VID should be allocated for CUDN without NADs") + }) + + It("should recover VIDs from NetworkManager cache at startup", func() { + // This tests the production startup recovery path where: + // 1. NetworkManager is started and processes existing NADs + // 2. UDN controller starts and recovers VIDs from NetworkManager's cache + testNs := testNamespace("evpn-nm-recovery-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-nm-recovery", vtep.Name, testNs.Name) + + // Create an existing NAD with VID 42 (simulating a previous controller run) + existingNAD := testEVPNClusterUdnNADOwnedByCUDN(cudn, testNs.Name, vtep.Name, 42, 0) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep, existingNAD) + Expect(c.Run()).To(Succeed()) + + // VID should be recovered from NetworkManager cache at startup + Eventually(func() int { + return c.vidAllocator.GetID("evpn-nm-recovery/macvrf") + }).Should(Equal(42), "VID 42 should be recovered from NetworkManager cache at startup") + }) + + It("should recover VIDs in deterministic order based on CUDN creation timestamp", func() { + // When two CUDNs have NADs claiming the same VID, the older CUDN wins. + // This ensures deterministic behavior across restarts. + testNs1 := testNamespace("evpn-order-test-1") + testNs2 := testNamespace("evpn-order-test-2") + vtep := testVTEP("vtep-test") + + // Create two CUDNs with different creation timestamps and unique UIDs + olderCUDN := testEVPNClusterUDN("aaa-older-cudn", vtep.Name, testNs1.Name) + olderCUDN.UID = "older-uid-1" + olderCUDN.CreationTimestamp = metav1.NewTime(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + + newerCUDN := testEVPNClusterUDN("zzz-newer-cudn", vtep.Name, testNs2.Name) + newerCUDN.UID = "newer-uid-2" + newerCUDN.CreationTimestamp = metav1.NewTime(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC)) + + // Both NADs claim VID 42 - this simulates a conflict scenario + olderNAD := testEVPNClusterUdnNADOwnedByCUDN(olderCUDN, testNs1.Name, vtep.Name, 42, 0) + newerNAD := testEVPNClusterUdnNADOwnedByCUDN(newerCUDN, testNs2.Name, vtep.Name, 42, 0) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, + olderCUDN, newerCUDN, testNs1, testNs2, vtep, olderNAD, newerNAD) + Expect(c.Run()).To(Succeed()) + + // The older CUDN should win the VID 42, regardless of alphabetical name order + // (newerCUDN has name "zzz-newer-cudn" which comes after "aaa-older-cudn" alphabetically, + // but olderCUDN should still win because it was created first) + Eventually(func() int { + return c.vidAllocator.GetID("aaa-older-cudn/macvrf") + }).Should(Equal(42), "Older CUDN should keep VID 42") + + // The newer CUDN loses the conflict and gets a new VID during reconciliation + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs2.Name).Get(context.Background(), newerCUDN.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "Newer CUDN should get new VID (first available) since older CUDN won VID 42") + }).Should(Succeed()) + }) + + It("should return error when VID pool is exhausted", func() { + testNs := testNamespace("evpn-exhaustion-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-exhaust-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + + // Exhaust all available VIDs (2-4094) before starting the controller (0,1 already reserved) + for i := 2; i < MaxEVPNVIDs; i++ { + err := c.vidAllocator.ReserveID(fmt.Sprintf("exhaust-key-%d", i), i) + Expect(err).NotTo(HaveOccurred(), "should allocate VID %d", i) + } + + // Now start the controller - the EVPN CUDN should fail to get a VID + Expect(c.Run()).To(Succeed()) + + // Verify the pool is exhausted + _, err := c.vidAllocator.AllocateID("one-more-key") + Expect(err).To(HaveOccurred(), "VID pool should be exhausted") + + // The CUDN should report a sync error because VID allocation failed + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "False", + Reason: "NetworkAttachmentDefinitionSyncError", + Message: "failed to allocate EVPN VIDs: failed to allocate VID for MAC-VRF: failed to allocate the id for the resource evpn-exhaust-cudn/macvrf", + }}), "should report VID allocation failure in status") + + // Verify NAD was not created + _, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "NAD should not be created when VID allocation fails") + }) + + It("should allocate VID after pool is freed up", func() { + testNs := testNamespace("evpn-free-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-free-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + + // Exhaust all VIDs except one (starting from 2, since 0,1 already reserved) + for i := 2; i < MaxEVPNVIDs-1; i++ { + err := c.vidAllocator.ReserveID(fmt.Sprintf("exhaust-key-%d", i), i) + Expect(err).NotTo(HaveOccurred()) + } + + // Start controller - it should successfully allocate the last available VID + Expect(c.Run()).To(Succeed()) + + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [evpn-free-test]", + }}), "should successfully create network with last available VID") + + // Verify the VID was allocated + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(MaxEVPNVIDs-1), "should get the last available VID") + }).Should(Succeed()) + }) + + It("should fail to start if VID 0 is already reserved by another resource", func() { + // This tests the defensive check that VID 0 (reserved per IEEE 802.1Q) + // must be reservable during controller initialization. + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest) + + // Reserve VID 0 with a DIFFERENT key (simulating corruption/bug) + Expect(c.vidAllocator.ReserveID("some-other-key", 0)).To(Succeed()) + + // Run should fail because initializeController can't reserve VID 0 + err := c.Run() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to reserve VID 0")) + }) + + It("should allocate new VID when namespace and NAD are created at runtime", func() { + // Scenario: Allocator has no VID for this key, namespace/NAD created at runtime + // This can happen when: + // - CUDN exists but had no matching namespaces at startup (no NADs to recover) + // - Admin later creates a namespace + // - Controller reconciles and allocates a new VID + // + // 1. Controller starts with CUDN but NO matching namespaces (no NADs created) + // 2. Allocator has NO VID for this key after startup + // 3. Namespace is created at runtime + // 4. Controller reconciles and allocates VID 2 (first available, 0,1 reserved) + vtep := testVTEP("vtep-test") + + // Namespace that doesn't exist at startup + const runtimeNsName = "runtime-ns-test" + + // CUDN with selector matching a namespace that doesn't exist yet + cudn := testEVPNClusterUDN("evpn-runtime-cudn", vtep.Name, runtimeNsName) + + // Start controller - no NADs to recover, allocator empty for this key + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, vtep) + Expect(c.Run()).To(Succeed()) + + // Create namespace at runtime (NAD will be created by controller) + testNs := testNamespace(runtimeNsName) + _, err := cs.KubeClient.CoreV1().Namespaces().Create(context.Background(), testNs, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Controller reconciles and allocates VID 2 (first available, 0,1 reserved) + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "VID should be allocated (first available, 0,1 reserved)") + }).Should(Succeed()) + }) + + It("should allocate new VID when existing NAD has VID taken by another CUDN", func() { + // Scenario: Allocator has no VID for this key, but NAD's VID is taken by another CUDN + // This can happen when: + // - CUDN-A had no matching namespaces at startup + // - CUDN-B had a NAD with VID 42 that was recovered + // - Someone manually creates NAD for CUDN-A with VID 42 (collision) + // + // 1. Controller starts with CUDN but NO matching namespaces + // 2. VID 42 is already reserved by a different CUDN + // 3. Namespace and NAD with VID 42 are created at runtime + // 4. Controller reconciles + // 5. VID 42 can't be reserved (taken) -> new VID allocated + vtep := testVTEP("vtep-test") + + // Namespace that doesn't exist at startup + const runtimeNsName = "runtime-conflict-test" + + cudn := testEVPNClusterUDN("evpn-runtime-conflict", vtep.Name, runtimeNsName) + + // Start controller - no NADs to recover, allocator empty for this key + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, vtep) + + // VID 42 is already reserved by another CUDN (simulates collision) + Expect(c.vidAllocator.ReserveID("another-cudn/macvrf", 42)).To(Succeed()) + + Expect(c.Run()).To(Succeed()) + + // Create namespace and NAD with VID 42 at runtime (collision with another CUDN) + testNs := testNamespace(runtimeNsName) + runtimeNAD := testEVPNClusterUdnNADOwnedByCUDN(cudn, testNs.Name, vtep.Name, 42, 0) + + _, err := cs.KubeClient.CoreV1().Namespaces().Create(context.Background(), testNs, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Create(context.Background(), runtimeNAD, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Controller reconciles - VID 42 is taken, must allocate new VID + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "VID should be newly allocated since 42 is taken by another CUDN (first available)") + }).Should(Succeed()) + }) + + It("should revert manual NAD VID change when allocator already has VID for this key", func() { + // This tests the case where: + // - Allocator has VID 2 for this key (from initial NAD creation, first available) + // - Someone manually changes NAD to VID 42 + // - Allocator's VID 2 should win, NAD reverted to 2 + // Note: Whether VID 42 is free or taken doesn't matter - the allocator's + // existing VID takes precedence because ReserveID fails when key already has a VID. + testNs := testNamespace("evpn-vid-manual-change-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-manual-change-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // Wait for initial NAD creation (will get VID 2, first available) + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "Initial VID should be 2 (first available)") + }).Should(Succeed()) + + // Now manually update the NAD with VID 42 + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(setNADEVPNVIDs(nad, 42, 0)).To(Succeed()) + _, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Update(context.Background(), nad, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // The NAD update triggers reconciliation. The allocator already has VID 2 + // for this key, so NAD is reverted to 2. + Eventually(func(g Gomega) { + nad, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + macVID, _ := evpnVIDsFromNAD(nad) + g.Expect(macVID).To(Equal(2), "VID should be reverted to allocator's VID") + }).Should(Succeed()) + }) + + It("should report VTEPNotFound when EVPN CUDN references non-existent VTEP", func() { + testNs := testNamespace("evpn-vtep-missing-test") + cudn := testEVPNClusterUDN("evpn-vtep-missing", "default", testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs) + Expect(c.Run()).To(Succeed()) + + // CUDN should report VTEPNotFound status + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "False", + Reason: "VTEPNotFound", + Message: "Cannot create network: VTEP 'default' does not exist. Create the VTEP CR first or update the CUDN to reference an existing VTEP.", + }}), "should report VTEPNotFound in status") + + // NAD should not be created when VTEP is missing + _, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "NAD should not be created when VTEP is missing") + }) + + It("should create NAD when VTEP exists for EVPN CUDN", func() { + testNs := testNamespace("evpn-vtep-exists-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-vtep-exists", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // CUDN should succeed when VTEP exists + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [evpn-vtep-exists-test]", + }}), "should succeed when VTEP exists") + + // NAD should be created + Eventually(func() error { + _, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + return err + }).Should(Succeed(), "NAD should be created when VTEP exists") + }) + + It("should automatically reconcile CUDN when VTEP is created after CUDN", func() { + testNs := testNamespace("evpn-vtep-transition-test") + vtepName := "default" + cudn := testEVPNClusterUDN("evpn-vtep-transition", vtepName, testNs.Name) + + // Start controller WITHOUT the VTEP - CUDN references non-existent VTEP + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs) + Expect(c.Run()).To(Succeed()) + + // Step 1: CUDN should initially report VTEPNotFound + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "False", + Reason: "VTEPNotFound", + Message: "Cannot create network: VTEP '" + vtepName + "' does not exist. Create the VTEP CR first or update the CUDN to reference an existing VTEP.", + }}), "should initially report VTEPNotFound") + + // NAD should NOT exist yet + _, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "NAD should not be created when VTEP is missing") + + // Step 2: Create the VTEP dynamically - this should trigger VTEPNotifier + vtep := testVTEP(vtepName) + _, err = cs.VTEPClient.K8sV1().VTEPs().Create(context.Background(), vtep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Step 3: CUDN should be automatically reconciled and succeed + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [evpn-vtep-transition-test]", + }}), "should succeed after VTEP is created") + + // NAD should now be created + Eventually(func() error { + _, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + return err + }).Should(Succeed(), "NAD should be created after VTEP is created") + }) + + It("should only re-queue EVPN CUDNs when VTEP changes, not non-EVPN CUDNs", func() { + testNs := testNamespace("vtep-filter-test") + vtep := testVTEP("vtep-filter") + + // Create a non-EVPN CUDN (Layer2 without EVPN transport) + nonEvpnCUDN := testClusterUDN("non-evpn-cudn", testNs.Name) + nonEvpnCUDN.UID = "non-evpn-uid" + + // Create an EVPN CUDN that references the VTEP + evpnCUDN := testEVPNClusterUDN("evpn-cudn", vtep.Name, testNs.Name) + evpnCUDN.UID = "evpn-uid" + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, nonEvpnCUDN, evpnCUDN, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // Wait for EVPN NAD to be created + Eventually(func() error { + _, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), evpnCUDN.Name, metav1.GetOptions{}) + return err + }).Should(Succeed()) + + // ReconcileVTEP should iterate over all CUDNs but only match the EVPN one + // This covers the non-EVPN path in cudnReferencesVTEP + err := c.ReconcileVTEP(vtep.Name) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should report VTEPNotFound when VTEP is deleted after CUDN creation", func() { + testNs := testNamespace("evpn-vtep-delete-test") + vtep := testVTEP("vtep-to-delete") + cudn := testEVPNClusterUDN("evpn-vtep-delete", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // Step 1: Verify NAD is created successfully when VTEP exists + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [evpn-vtep-delete-test]", + }}), "should initially succeed when VTEP exists") + + Eventually(func() error { + _, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + return err + }).Should(Succeed(), "NAD should be created when VTEP exists") + + // Step 2: Delete the VTEP - this should trigger VTEPNotifier + err := cs.VTEPClient.K8sV1().VTEPs().Delete(context.Background(), vtep.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Step 3: CUDN should be re-reconciled and report VTEPNotFound + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "False", + Reason: "VTEPNotFound", + Message: "Cannot create network: VTEP '" + vtep.Name + "' does not exist. Create the VTEP CR first or update the CUDN to reference an existing VTEP.", + }}), "should report VTEPNotFound after VTEP is deleted") + }) + + It("should fail when EVPN transport is requested but EVPN feature is disabled", func() { + // Disable EVPN feature flag for this test. + // No defer needed - BeforeEach resets config via PrepareTestConfig(). + config.OVNKubernetesFeature.EnableEVPN = false + + testNs := testNamespace("evpn-disabled-test") + vtep := testVTEP("vtep-test") + cudn := testEVPNClusterUDN("evpn-disabled-cudn", vtep.Name, testNs.Name) + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + // CUDN should report error with message about EVPN flag + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(cudn.Status.Conditions) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "False", + Reason: "NetworkAttachmentDefinitionSyncError", + Message: "EVPN transport requested but EVPN feature is not enabled", + }}), "should report error when EVPN flag is disabled") + + // NAD should not be created when EVPN is disabled + _, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(testNs.Name).Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "NAD should not be created when EVPN is disabled") + }) + It("should update NAD annotations and preserve internal OVNK annotations on UDN update", func() { testNamespaces := []string{"red", "blue"} var objs []runtime.Object @@ -1604,7 +2460,208 @@ func failRenderNadStub(err error) RenderNetAttachDefManifest { } func newRenderNadStub(nad *netv1.NetworkAttachmentDefinition, err error) RenderNetAttachDefManifest { - return func(client.Object, string) (*netv1.NetworkAttachmentDefinition, error) { + return func(client.Object, string, ...template.RenderOption) (*netv1.NetworkAttachmentDefinition, error) { return nad, err } } + +func testEVPNClusterUDN(name string, vtepName string, targetNamespaces ...string) *udnv1.ClusterUserDefinedNetwork { + return &udnv1.ClusterUserDefinedNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"k8s.ovn.org/user-defined-network": ""}, + Finalizers: []string{"k8s.ovn.org/user-defined-network-protection"}, + Name: name, + UID: "1", + }, + Spec: udnv1.ClusterUserDefinedNetworkSpec{ + NamespaceSelector: metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1.LabelMetadataName, + Operator: metav1.LabelSelectorOpIn, + Values: targetNamespaces, + }, + }}, + Network: udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRoleSecondary, + Subnets: udnv1.DualStackCIDRs{"10.10.10.0/24"}, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: vtepName, + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + }, + }, + }, + }, + } +} + +// testEVPNClusterUdnNADWithVIDs creates an EVPN NAD with specific MAC-VRF and IP-VRF VIDs. +// Pass 0 for ipVID to create a MAC-VRF only NAD. +func testEVPNClusterUdnNADWithVIDs(name, namespace, vtepName string, macVID, ipVID int) *netv1.NetworkAttachmentDefinition { + nad := testClusterUdnNAD(name, namespace) + if ipVID > 0 { + // Symmetric IRB (both MAC-VRF and IP-VRF) + nad.Spec.Config = fmt.Sprintf(`{"cniVersion":"1.0.0","name":"cluster_udn_%s","type":"ovn-k8s-cni-overlay","netAttachDefName":"%s/%s","topology":"layer2","role":"primary","subnets":"10.10.0.0/16","transport":"evpn","evpn":{"vtep":"%s","macVRF":{"vni":100,"vid":%d},"ipVRF":{"vni":200,"vid":%d}}}`, name, namespace, name, vtepName, macVID, ipVID) + } else { + // MAC-VRF only + nad.Spec.Config = fmt.Sprintf(`{"cniVersion":"1.0.0","name":"cluster_udn_%s","type":"ovn-k8s-cni-overlay","netAttachDefName":"%s/%s","topology":"layer2","role":"primary","subnets":"10.10.0.0/16","transport":"evpn","evpn":{"vtep":"%s","macVRF":{"vni":100,"vid":%d}}}`, name, namespace, name, vtepName, macVID) + } + return nad +} + +// testEVPNClusterUdnNADOwnedByCUDN creates an EVPN NAD with specific VIDs and sets up +// the OwnerReferences to indicate ownership by the given CUDN. +func testEVPNClusterUdnNADOwnedByCUDN(cudn *udnv1.ClusterUserDefinedNetwork, namespace, vtepName string, macVID, ipVID int) *netv1.NetworkAttachmentDefinition { + nad := testEVPNClusterUdnNADWithVIDs(cudn.Name, namespace, vtepName, macVID, ipVID) + nad.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "k8s.ovn.org/v1", + Kind: "ClusterUserDefinedNetwork", + Name: cudn.Name, + UID: cudn.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + } + return nad +} + +func testSymmetricIRBClusterUDN(name string, vtepName string, targetNamespaces ...string) *udnv1.ClusterUserDefinedNetwork { + return &udnv1.ClusterUserDefinedNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"k8s.ovn.org/user-defined-network": ""}, + Finalizers: []string{"k8s.ovn.org/user-defined-network-protection"}, + Name: name, + UID: "1", + }, + Spec: udnv1.ClusterUserDefinedNetworkSpec{ + NamespaceSelector: metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1.LabelMetadataName, + Operator: metav1.LabelSelectorOpIn, + Values: targetNamespaces, + }, + }}, + Network: udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRoleSecondary, + Subnets: udnv1.DualStackCIDRs{"10.10.10.0/24"}, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: vtepName, + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + }, + IPVRF: &udnv1.VRFConfig{ + VNI: 200, + }, + }, + }, + }, + } +} + +func testEVPNIPVRFClusterUDN(name string, vtepName string, targetNamespaces ...string) *udnv1.ClusterUserDefinedNetwork { + return &udnv1.ClusterUserDefinedNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"k8s.ovn.org/user-defined-network": ""}, + Finalizers: []string{"k8s.ovn.org/user-defined-network-protection"}, + Name: name, + UID: "1", + }, + Spec: udnv1.ClusterUserDefinedNetworkSpec{ + NamespaceSelector: metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1.LabelMetadataName, + Operator: metav1.LabelSelectorOpIn, + Values: targetNamespaces, + }, + }}, + Network: udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer3, + Layer3: &udnv1.Layer3Config{ + Role: udnv1.NetworkRoleSecondary, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: vtepName, + IPVRF: &udnv1.VRFConfig{ + VNI: 200, + }, + }, + }, + }, + } +} + +func testVTEP(name string) *vtepv1.VTEP { + return &vtepv1.VTEP{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID("vtep-" + name), + }, + Spec: vtepv1.VTEPSpec{ + CIDRs: vtepv1.DualStackCIDRs{"100.64.0.0/24"}, + Mode: vtepv1.VTEPModeManaged, + }, + } +} + +// evpnVIDsFromNAD extracts MAC-VRF and IP-VRF VIDs from a NAD config. +// Returns (macVID, ipVID) where 0 indicates the VRF is not present or has no VID. +func evpnVIDsFromNAD(nad *netv1.NetworkAttachmentDefinition) (macVID, ipVID int) { + if nad == nil { + return 0, 0 + } + var netConf ovncnitypes.NetConf + if err := json.Unmarshal([]byte(nad.Spec.Config), &netConf); err != nil { + return 0, 0 + } + if netConf.EVPN == nil { + return 0, 0 + } + if netConf.EVPN.MACVRF != nil { + macVID = netConf.EVPN.MACVRF.VID + } + if netConf.EVPN.IPVRF != nil { + ipVID = netConf.EVPN.IPVRF.VID + } + return macVID, ipVID +} + +// setNADEVPNVIDs modifies the MAC-VRF and/or IP-VRF VIDs in a NAD config. +// Pass 0 to leave a VID unchanged. This is used in tests to set specific VIDs +// without rewriting the entire config. +func setNADEVPNVIDs(nad *netv1.NetworkAttachmentDefinition, macVID, ipVID int) error { + var netConf ovncnitypes.NetConf + if err := json.Unmarshal([]byte(nad.Spec.Config), &netConf); err != nil { + return err + } + if netConf.EVPN == nil { + return fmt.Errorf("NAD has no EVPN config") + } + if macVID > 0 { + if netConf.EVPN.MACVRF == nil { + return fmt.Errorf("NAD has no EVPN MAC-VRF config") + } + netConf.EVPN.MACVRF.VID = macVID + } + if ipVID > 0 { + if netConf.EVPN.IPVRF == nil { + return fmt.Errorf("NAD has no EVPN IP-VRF config") + } + netConf.EVPN.IPVRF.VID = ipVID + } + configBytes, err := json.Marshal(netConf) + if err != nil { + return err + } + nad.Spec.Config = string(configBytes) + return nil +} diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/notifier/namespace_test.go b/go-controller/pkg/clustermanager/userdefinednetwork/notifier/namespace_test.go index afe0d93c03..50ed844a34 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/notifier/namespace_test.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/notifier/namespace_test.go @@ -15,6 +15,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" udnv1fake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/fake" + vtepv1fake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -40,6 +41,7 @@ var _ = Describe("NamespaceNotifier", func() { KubeClient: kubeClient, NetworkAttchDefClient: netv1fake.NewSimpleClientset(), UserDefinedNetworkClient: udnv1fake.NewSimpleClientset(), + VTEPClient: vtepv1fake.NewSimpleClientset(), } var err error wf, err = factory.NewClusterManagerWatchFactory(fakeClient) diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/notifier/vtep.go b/go-controller/pkg/clustermanager/userdefinednetwork/notifier/vtep.go new file mode 100644 index 0000000000..75eb8d7fcb --- /dev/null +++ b/go-controller/pkg/clustermanager/userdefinednetwork/notifier/vtep.go @@ -0,0 +1,70 @@ +package notifier + +import ( + "errors" + + "k8s.io/client-go/util/workqueue" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + vtepinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1" +) + +// VTEPReconciler is the interface for controllers that need to react to VTEP events. +type VTEPReconciler interface { + ReconcileVTEP(key string) error +} + +// VTEPNotifier watches VTEP objects and notifies subscribers upon change. +// It enqueues the reconciled object keys in the subscribing controllers workqueue. +type VTEPNotifier struct { + Controller controller.Controller + + subscribers []VTEPReconciler +} + +// NewVTEPNotifier creates a new VTEPNotifier that watches VTEP CRs and notifies subscribers. +func NewVTEPNotifier(vtepInformer vtepinformer.VTEPInformer, subscribers ...VTEPReconciler) *VTEPNotifier { + c := &VTEPNotifier{ + subscribers: subscribers, + } + + vtepLister := vtepInformer.Lister() + cfg := &controller.ControllerConfig[vtepv1.VTEP]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: c.reconcile, + ObjNeedsUpdate: c.needUpdate, + Threadiness: 1, + Informer: vtepInformer.Informer(), + Lister: vtepLister.List, + } + c.Controller = controller.NewController("udn-vtep-controller", cfg) + + return c +} + +// needUpdate returns true when the VTEP has been created or deleted. +// We notify on create/delete so that CUDNs referencing this VTEP can be re-queued. +// IMPORTANT: Before adding update notifications, verify that all subscribers +// can handle increased event frequency. +func (c *VTEPNotifier) needUpdate(old, new *vtepv1.VTEP) bool { + vtepCreated := old == nil && new != nil + vtepDeleted := old != nil && new == nil + return vtepCreated || vtepDeleted +} + +// reconcile notifies subscribers with the VTEP key following VTEP events. +func (c *VTEPNotifier) reconcile(key string) error { + var errs []error + for _, subscriber := range c.subscribers { + if subscriber != nil { + // enqueue the reconciled VTEP key in the subscribers workqueue to + // enable the subscriber to act on VTEP changes + if err := subscriber.ReconcileVTEP(key); err != nil { + errs = append(errs, err) + } + } + } + + return errors.Join(errs...) +} diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/notifier/vtep_test.go b/go-controller/pkg/clustermanager/userdefinednetwork/notifier/vtep_test.go new file mode 100644 index 0000000000..111106f3e5 --- /dev/null +++ b/go-controller/pkg/clustermanager/userdefinednetwork/notifier/vtep_test.go @@ -0,0 +1,198 @@ +package notifier + +import ( + "context" + "maps" + "strconv" + "sync" + + netv1fake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" + frrfake "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + rafake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/clientset/versioned/fake" + udnv1fake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/fake" + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + vtepv1fake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("VTEPNotifier", func() { + var ( + vtepClient *vtepv1fake.Clientset + wf *factory.WatchFactory + testVTEPNotifier *VTEPNotifier + ) + + BeforeEach(func() { + vtepClient = vtepv1fake.NewSimpleClientset() + + // enable features to make watch-factory start the VTEP informer + Expect(config.PrepareTestConfig()).To(Succeed()) + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableRouteAdvertisements = true + config.OVNKubernetesFeature.EnableEVPN = true + fakeClient := &util.OVNClusterManagerClientset{ + KubeClient: fake.NewSimpleClientset(), + NetworkAttchDefClient: netv1fake.NewSimpleClientset(), + UserDefinedNetworkClient: udnv1fake.NewSimpleClientset(), + RouteAdvertisementsClient: rafake.NewSimpleClientset(), + FRRClient: frrfake.NewSimpleClientset(), + VTEPClient: vtepClient, + } + var err error + wf, err = factory.NewClusterManagerWatchFactory(fakeClient) + Expect(err).NotTo(HaveOccurred()) + Expect(wf.Start()).To(Succeed()) + }) + + AfterEach(func() { + wf.Shutdown() + }) + + var s *testVTEPSubscriber + + BeforeEach(func() { + s = &testVTEPSubscriber{reconciledKeys: map[string]int64{}} + testVTEPNotifier = NewVTEPNotifier(wf.VTEPInformer(), s) + Expect(controller.Start(testVTEPNotifier.Controller)).Should(Succeed()) + + // create test VTEPs + for i := 0; i < 3; i++ { + vtepName := "test-vtep-" + strconv.Itoa(i) + _, err := vtepClient.K8sV1().VTEPs().Create(context.Background(), testVTEP(vtepName), metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + } + }) + + AfterEach(func() { + if testVTEPNotifier != nil { + controller.Stop(testVTEPNotifier.Controller) + } + }) + + It("should notify VTEP create events", func() { + Eventually(func() map[string]int64 { + return s.GetReconciledKeys() + }).Should(Equal(map[string]int64{ + "test-vtep-0": 1, + "test-vtep-1": 1, + "test-vtep-2": 1, + })) + }) + + It("should notify VTEP delete events", func() { + Eventually(func() map[string]int64 { + return s.GetReconciledKeys() + }).Should(Equal(map[string]int64{ + "test-vtep-0": 1, + "test-vtep-1": 1, + "test-vtep-2": 1, + })) + + Expect(vtepClient.K8sV1().VTEPs().Delete(context.Background(), "test-vtep-2", metav1.DeleteOptions{})).To(Succeed()) + Expect(vtepClient.K8sV1().VTEPs().Delete(context.Background(), "test-vtep-0", metav1.DeleteOptions{})).To(Succeed()) + + Eventually(func() map[string]int64 { + return s.GetReconciledKeys() + }).Should(Equal(map[string]int64{ + "test-vtep-0": 2, + "test-vtep-1": 1, + "test-vtep-2": 2, + }), "should record additional two events, following VTEP deletion") + }) + + It("should NOT notify VTEP update events (spec/status changes)", func() { + Eventually(func() map[string]int64 { + return s.GetReconciledKeys() + }).Should(Equal(map[string]int64{ + "test-vtep-0": 1, + "test-vtep-1": 1, + "test-vtep-2": 1, + })) + + // Update VTEP spec (change CIDRs) + vtep, err := vtepClient.K8sV1().VTEPs().Get(context.Background(), "test-vtep-1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + vtep.Spec.CIDRs = vtepv1.DualStackCIDRs{"192.168.0.0/24"} + _, err = vtepClient.K8sV1().VTEPs().Update(context.Background(), vtep, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Updates should NOT trigger notification (needUpdate returns false for updates) + Consistently(func() map[string]int64 { + return s.GetReconciledKeys() + }).Should(Equal(map[string]int64{ + "test-vtep-0": 1, + "test-vtep-1": 1, + "test-vtep-2": 1, + }), "should NOT record additional events following VTEP update") + }) + + It("should notify multiple subscribers", func() { + // Stop the single-subscriber notifier + controller.Stop(testVTEPNotifier.Controller) + + // Create a second subscriber + s2 := &testVTEPSubscriber{reconciledKeys: map[string]int64{}} + + // Create a new notifier with multiple subscribers + testVTEPNotifier = NewVTEPNotifier(wf.VTEPInformer(), s, s2) + Expect(controller.Start(testVTEPNotifier.Controller)).Should(Succeed()) + + // Create a new VTEP + _, err := vtepClient.K8sV1().VTEPs().Create(context.Background(), testVTEP("test-vtep-new"), metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Both subscribers should be notified exactly once + Eventually(func(g Gomega) { + keys1 := s.GetReconciledKeys() + keys2 := s2.GetReconciledKeys() + g.Expect(keys1["test-vtep-new"]).To(BeEquivalentTo(1), "subscriber 1 should be notified exactly once") + g.Expect(keys2["test-vtep-new"]).To(BeEquivalentTo(1), "subscriber 2 should be notified exactly once") + }).Should(Succeed()) + }) +}) + +func testVTEP(name string) *vtepv1.VTEP { + return &vtepv1.VTEP{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: vtepv1.VTEPSpec{ + CIDRs: vtepv1.DualStackCIDRs{"10.10.10.0/24"}, + Mode: vtepv1.VTEPModeManaged, + }, + } +} + +type testVTEPSubscriber struct { + err error + reconciledKeys map[string]int64 + lock sync.RWMutex +} + +func (s *testVTEPSubscriber) ReconcileVTEP(key string) error { + s.lock.Lock() + defer s.lock.Unlock() + + s.reconciledKeys[key]++ + return s.err +} + +func (s *testVTEPSubscriber) GetReconciledKeys() map[string]int64 { + s.lock.RLock() + defer s.lock.RUnlock() + + cp := map[string]int64{} + maps.Copy(cp, s.reconciledKeys) + return cp +} diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/template/net-attach-def-template.go b/go-controller/pkg/clustermanager/userdefinednetwork/template/net-attach-def-template.go index e451ed3923..62850d4a23 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/template/net-attach-def-template.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/template/net-attach-def-template.go @@ -33,15 +33,17 @@ type SpecGetter interface { GetLayer3() *userdefinednetworkv1.Layer3Config GetLayer2() *userdefinednetworkv1.Layer2Config GetLocalnet() *userdefinednetworkv1.LocalnetConfig + GetTransport() userdefinednetworkv1.TransportOption + GetEVPN() *userdefinednetworkv1.EVPNConfig } -func RenderNetAttachDefManifest(obj client.Object, targetNamespace string) (*netv1.NetworkAttachmentDefinition, error) { +func RenderNetAttachDefManifest(obj client.Object, targetNamespace string, opts ...RenderOption) (*netv1.NetworkAttachmentDefinition, error) { if obj == nil { return nil, nil } if targetNamespace == "" { - return nil, fmt.Errorf("namspace should not be empty") + return nil, fmt.Errorf("namespace should not be empty") } var ownerRef metav1.OwnerReference @@ -62,7 +64,7 @@ func RenderNetAttachDefManifest(obj client.Object, targetNamespace string) (*net nadName := util.GetNADName(targetNamespace, obj.GetName()) - nadSpec, err := RenderNADSpec(networkName, nadName, spec) + nadSpec, err := renderNADSpec(networkName, nadName, spec, applyOptions(opts)) if err != nil { return nil, err } @@ -79,12 +81,12 @@ func RenderNetAttachDefManifest(obj client.Object, targetNamespace string) (*net }, nil } -func RenderNADSpec(networkName, nadName string, spec SpecGetter) (*netv1.NetworkAttachmentDefinitionSpec, error) { +func renderNADSpec(networkName, nadName string, spec SpecGetter, opts *RenderOptions) (*netv1.NetworkAttachmentDefinitionSpec, error) { if err := validateTopology(spec); err != nil { return nil, fmt.Errorf("invalid topology specified: %w", err) } - cniNetConf, err := renderCNINetworkConfig(networkName, nadName, spec) + cniNetConf, err := renderCNINetworkConfig(networkName, nadName, spec, opts) if err != nil { return nil, fmt.Errorf("failed to render CNI network config: %w", err) } @@ -98,7 +100,7 @@ func RenderNADSpec(networkName, nadName string, spec SpecGetter) (*netv1.Network }, nil } -// renderNADLabels copies labels from UDN to help RenderNADSpec +// renderNADLabels copies labels from UDN to help renderNADSpec // function add those labels to corresponding NAD func renderNADLabels(obj client.Object) map[string]string { labels := make(map[string]string) @@ -134,15 +136,16 @@ func validateTopology(spec SpecGetter) error { return nil } -func renderCNINetworkConfig(networkName, nadName string, spec SpecGetter) (map[string]interface{}, error) { +func renderCNINetworkConfig(networkName, nadName string, spec SpecGetter, opts *RenderOptions) (map[string]interface{}, error) { netConfSpec := &ovncnitypes.NetConf{ NetConf: cnitypes.NetConf{ CNIVersion: cniVersion, Type: OvnK8sCNIOverlay, Name: networkName, }, - NADName: nadName, - Topology: strings.ToLower(string(spec.GetTopology())), + NADName: nadName, + Topology: strings.ToLower(string(spec.GetTopology())), + Transport: transportFromCRD(string(spec.GetTransport())), } switch spec.GetTopology() { @@ -194,6 +197,14 @@ func renderCNINetworkConfig(networkName, nadName string, spec SpecGetter) (map[s netConfSpec.VLANID = int(cfg.VLAN.Access.ID) } } + + if spec.GetTransport() == userdefinednetworkv1.TransportOptionEVPN { + if !util.IsEVPNEnabled() { + return nil, fmt.Errorf("EVPN transport requested but EVPN feature is not enabled") + } + netConfSpec.EVPN = renderEVPNConfig(spec, opts) + } + if netConfSpec.AllowPersistentIPs && !config.OVNKubernetesFeature.EnablePersistentIPs { return nil, fmt.Errorf("allowPersistentIPs is set but persistentIPs is Disabled") } @@ -256,9 +267,33 @@ func renderCNINetworkConfig(networkName, nadName string, spec SpecGetter) (map[s cniNetConf["defaultGatewayIPs"] = netConfSpec.DefaultGatewayIPs } } + + if netConfSpec.Transport != "" { + cniNetConf["transport"] = netConfSpec.Transport + } + if netConfSpec.EVPN != nil { + cniNetConf["evpn"] = netConfSpec.EVPN + } + return cniNetConf, nil } +// transportFromCRD converts CRD PascalCase format to canonical format. +// CRD format uses PascalCase: "Geneve", "NoOverlay", "EVPN" +// Returns canonical lowercase format: "geneve", "no-overlay", "evpn" +func transportFromCRD(crdTransport string) string { + switch crdTransport { + case "Geneve": + return types.NetworkTransportGeneve + case "NoOverlay": + return types.NetworkTransportNoOverlay + case "EVPN": + return types.NetworkTransportEVPN + default: + return crdTransport // Return as-is for validation to catch + } +} + func localnetMTU(desiredMTU int32) int { // The MTU for localnet topology should be as the default MTU (1500) because the underlay // is not part of the SDN and compensating for the SDN overhead (100) is not required. @@ -332,6 +367,36 @@ func ipString(ips userdefinednetworkv1.DualStackIPs) string { return strings.Join(ipStrings, ",") } +// renderEVPNConfig converts the EVPN configuration from the spec into the CNI EVPNConfig format. +// Note: evpnCfg is guaranteed to be non-nil by CEL validation on the CRD. +func renderEVPNConfig(spec SpecGetter, opts *RenderOptions) *ovncnitypes.EVPNConfig { + evpnCfg := spec.GetEVPN() + evpnConfig := &ovncnitypes.EVPNConfig{ + VTEP: evpnCfg.VTEP, + } + + if evpnCfg.MACVRF != nil { + evpnConfig.MACVRF = &ovncnitypes.VRFConfig{ + VNI: evpnCfg.MACVRF.VNI, + RouteTarget: string(evpnCfg.MACVRF.RouteTarget), + } + if opts != nil && opts.EVPNVIDs != nil && opts.EVPNVIDs.MACVRFVID > 0 { + evpnConfig.MACVRF.VID = opts.EVPNVIDs.MACVRFVID + } + } + if evpnCfg.IPVRF != nil { + evpnConfig.IPVRF = &ovncnitypes.VRFConfig{ + VNI: evpnCfg.IPVRF.VNI, + RouteTarget: string(evpnCfg.IPVRF.RouteTarget), + } + if opts != nil && opts.EVPNVIDs != nil && opts.EVPNVIDs.IPVRFVID > 0 { + evpnConfig.IPVRF.VID = opts.EVPNVIDs.IPVRFVID + } + } + + return evpnConfig +} + func GetSpec(obj client.Object) SpecGetter { switch o := obj.(type) { case *userdefinednetworkv1.UserDefinedNetwork: diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/template/net-attach-def-template_test.go b/go-controller/pkg/clustermanager/userdefinednetwork/template/net-attach-def-template_test.go index e44cee4366..5881617c6b 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/template/net-attach-def-template_test.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/template/net-attach-def-template_test.go @@ -1,6 +1,7 @@ package template import ( + "encoding/json" "strings" netv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" @@ -21,15 +22,20 @@ import ( var _ = Describe("NetAttachDefTemplate", func() { - // before each test, set the IPv4Mode and IPv6Mode to true BeforeEach(func() { + // Restore global default values before each testcase + Expect(config.PrepareTestConfig()).To(Succeed()) config.IPv4Mode = true config.IPv6Mode = true + // Enable EVPN for tests that use EVPN transport + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableRouteAdvertisements = true + config.OVNKubernetesFeature.EnableEVPN = true }) DescribeTable("should fail to render NAD spec given", func(spec *udnv1.UserDefinedNetworkSpec, expectedError string) { - _, err := RenderNADSpec("foo", "bar", spec) + _, err := renderNADSpec("foo", "bar", spec, nil) Expect(err).To(MatchError(ContainSubstring(expectedError))) }, Entry("invalid layer2 subnets", @@ -631,8 +637,342 @@ var _ = Describe("NetAttachDefTemplate", func() { "allowPersistentIPs": true }`, ), + Entry("primary network, layer2 with EVPN transport and MAC-VRF", + udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRolePrimary, + Subnets: udnv1.DualStackCIDRs{"192.168.100.0/24"}, + MTU: 1500, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: "my-vtep", + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + RouteTarget: "65000:100", + }, + }, + }, + `{ + "cniVersion": "1.0.0", + "type": "ovn-k8s-cni-overlay", + "name": "cluster_udn_test-net", + "netAttachDefName": "mynamespace/test-net", + "role": "primary", + "topology": "layer2", + "joinSubnet": "100.65.0.0/16,fd99::/64", + "transitSubnet": "100.88.0.0/16", + "subnets": "192.168.100.0/24", + "mtu": 1500, + "transport": "evpn", + "evpn": { + "vtep": "my-vtep", + "macVRF": { + "vni": 100, + "routeTarget": "65000:100" + } + } + }`, + ), + Entry("primary network, layer3 with EVPN transport and IP-VRF", + udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer3, + Layer3: &udnv1.Layer3Config{ + Role: udnv1.NetworkRolePrimary, + Subnets: []udnv1.Layer3Subnet{ + {CIDR: "192.168.100.0/16"}, + }, + MTU: 1500, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: "my-vtep", + IPVRF: &udnv1.VRFConfig{ + VNI: 200, + RouteTarget: "65000:200", + }, + }, + }, + `{ + "cniVersion": "1.0.0", + "type": "ovn-k8s-cni-overlay", + "name": "cluster_udn_test-net", + "netAttachDefName": "mynamespace/test-net", + "role": "primary", + "topology": "layer3", + "joinSubnet": "100.65.0.0/16,fd99::/64", + "subnets": "192.168.100.0/16", + "mtu": 1500, + "transport": "evpn", + "evpn": { + "vtep": "my-vtep", + "ipVRF": { + "vni": 200, + "routeTarget": "65000:200" + } + } + }`, + ), + Entry("primary network, layer2 with EVPN transport, MAC-VRF and IP-VRF", + udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRolePrimary, + Subnets: udnv1.DualStackCIDRs{"192.168.100.0/24"}, + MTU: 1500, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: "my-vtep", + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + RouteTarget: "100000:100", // 4-byte ASN format + }, + IPVRF: &udnv1.VRFConfig{ + VNI: 200, + RouteTarget: "192.168.1.1:200", // IPv4 format + }, + }, + }, + `{ + "cniVersion": "1.0.0", + "type": "ovn-k8s-cni-overlay", + "name": "cluster_udn_test-net", + "netAttachDefName": "mynamespace/test-net", + "role": "primary", + "topology": "layer2", + "joinSubnet": "100.65.0.0/16,fd99::/64", + "transitSubnet": "100.88.0.0/16", + "subnets": "192.168.100.0/24", + "mtu": 1500, + "transport": "evpn", + "evpn": { + "vtep": "my-vtep", + "macVRF": { + "vni": 100, + "routeTarget": "100000:100" + }, + "ipVRF": { + "vni": 200, + "routeTarget": "192.168.1.1:200" + } + } + }`, + ), + Entry("primary network, layer2 with EVPN transport, MAC-VRF with VNI only (no RouteTarget)", + udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRolePrimary, + Subnets: udnv1.DualStackCIDRs{"192.168.100.0/24"}, + MTU: 1500, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: "my-vtep", + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + // RouteTarget intentionally omitted + }, + }, + }, + `{ + "cniVersion": "1.0.0", + "type": "ovn-k8s-cni-overlay", + "name": "cluster_udn_test-net", + "netAttachDefName": "mynamespace/test-net", + "role": "primary", + "topology": "layer2", + "joinSubnet": "100.65.0.0/16,fd99::/64", + "transitSubnet": "100.88.0.0/16", + "subnets": "192.168.100.0/24", + "mtu": 1500, + "transport": "evpn", + "evpn": { + "vtep": "my-vtep", + "macVRF": { + "vni": 100 + } + } + }`, + ), ) + Context("EVPN VID injection", func() { + It("should inject VIDs into EVPN config when provided via WithEVPNVIDs", func() { + cudn := &udnv1.ClusterUserDefinedNetwork{ + ObjectMeta: metav1.ObjectMeta{Name: "test-evpn", UID: "1"}, + Spec: udnv1.ClusterUserDefinedNetworkSpec{ + Network: udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRoleSecondary, + Subnets: udnv1.DualStackCIDRs{"192.168.0.0/16"}, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: "my-vtep", + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + RouteTarget: "65000:100", + }, + IPVRF: &udnv1.VRFConfig{ + VNI: 200, + RouteTarget: "65000:200", + }, + }, + }, + }, + } + + nad, err := RenderNetAttachDefManifest(cudn, "test-ns", WithEVPNVIDs(12, 13)) + Expect(err).NotTo(HaveOccurred()) + Expect(nad).NotTo(BeNil()) + + var netConf ovncnitypes.NetConf + err = json.Unmarshal([]byte(nad.Spec.Config), &netConf) + Expect(err).NotTo(HaveOccurred()) + + Expect(netConf.EVPN).NotTo(BeNil(), "evpnConfig should be present") + Expect(netConf.EVPN.MACVRF).NotTo(BeNil(), "macVRF should be present") + Expect(netConf.EVPN.MACVRF.VID).To(Equal(12), "macVRF VID should be 12") + Expect(netConf.EVPN.IPVRF).NotTo(BeNil(), "ipVRF should be present") + Expect(netConf.EVPN.IPVRF.VID).To(Equal(13), "ipVRF VID should be 13") + }) + + It("should omit VID when zero (VID=0 not injected)", func() { + cudn := &udnv1.ClusterUserDefinedNetwork{ + ObjectMeta: metav1.ObjectMeta{Name: "test-evpn-no-vid", UID: "1"}, + Spec: udnv1.ClusterUserDefinedNetworkSpec{ + Network: udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRoleSecondary, + Subnets: udnv1.DualStackCIDRs{"192.168.0.0/16"}, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: "my-vtep", + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + RouteTarget: "65000:100", + }, + }, + }, + }, + } + + // Pass VID=0 for both (should be omitted from JSON, unmarshals as zero value) + nad, err := RenderNetAttachDefManifest(cudn, "test-ns", WithEVPNVIDs(0, 0)) + Expect(err).NotTo(HaveOccurred()) + Expect(nad).NotTo(BeNil()) + + var netConf ovncnitypes.NetConf + err = json.Unmarshal([]byte(nad.Spec.Config), &netConf) + Expect(err).NotTo(HaveOccurred()) + + Expect(netConf.EVPN).NotTo(BeNil(), "evpnConfig should be present") + Expect(netConf.EVPN.MACVRF).NotTo(BeNil(), "macVRF should be present") + Expect(netConf.EVPN.MACVRF.VID).To(Equal(0), "VID should be zero when not injected") + + // Also verify the raw JSON doesn't contain "vid" field (omitempty) + Expect(nad.Spec.Config).NotTo(ContainSubstring(`"vid"`), "vid field should be omitted from JSON when zero") + }) + + It("should omit empty RouteTarget in EVPN config", func() { + cudn := &udnv1.ClusterUserDefinedNetwork{ + ObjectMeta: metav1.ObjectMeta{Name: "test-evpn-no-rt", UID: "1"}, + Spec: udnv1.ClusterUserDefinedNetworkSpec{ + Network: udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRoleSecondary, + Subnets: udnv1.DualStackCIDRs{"192.168.0.0/16"}, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: "my-vtep", + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + // RouteTarget intentionally omitted (empty) + }, + }, + }, + }, + } + + nad, err := RenderNetAttachDefManifest(cudn, "test-ns", WithEVPNVIDs(5, 0)) + Expect(err).NotTo(HaveOccurred()) + Expect(nad).NotTo(BeNil()) + + var netConf ovncnitypes.NetConf + err = json.Unmarshal([]byte(nad.Spec.Config), &netConf) + Expect(err).NotTo(HaveOccurred()) + + // RouteTarget should be empty (omitted in JSON, unmarshals as empty string) + Expect(netConf.EVPN.MACVRF.RouteTarget).To(BeEmpty(), "empty routeTarget should unmarshal as empty string") + + // Also verify the raw JSON doesn't contain "routeTarget" field + Expect(nad.Spec.Config).NotTo(ContainSubstring(`"routeTarget"`), "routeTarget should be omitted from JSON when empty") + + // VID should be present + Expect(netConf.EVPN.MACVRF.VID).To(Equal(5), "macVRF VID should be 5") + }) + + It("should handle nil RenderOption without panic", func() { + cudn := &udnv1.ClusterUserDefinedNetwork{ + ObjectMeta: metav1.ObjectMeta{Name: "test-nil-option", UID: "1"}, + Spec: udnv1.ClusterUserDefinedNetworkSpec{ + Network: udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRoleSecondary, + Subnets: udnv1.DualStackCIDRs{"192.168.0.0/16"}, + }, + }, + }, + } + + // Pass nil option - should not panic + var nilOpt RenderOption + Expect(func() { + _, _ = RenderNetAttachDefManifest(cudn, "test-ns", nilOpt, WithEVPNVIDs(1, 2)) + }).NotTo(Panic()) + }) + + It("should fail when EVPN transport is requested but EVPN feature is disabled", func() { + // Disable EVPN feature flag for this test. + // No defer needed - BeforeEach resets config via PrepareTestConfig(). + config.OVNKubernetesFeature.EnableEVPN = false + + cudn := &udnv1.ClusterUserDefinedNetwork{ + ObjectMeta: metav1.ObjectMeta{Name: "test-evpn-disabled", UID: "1"}, + Spec: udnv1.ClusterUserDefinedNetworkSpec{ + Network: udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRolePrimary, + Subnets: udnv1.DualStackCIDRs{"192.168.100.0/24"}, + }, + Transport: udnv1.TransportOptionEVPN, + EVPN: &udnv1.EVPNConfig{ + VTEP: "my-vtep", + MACVRF: &udnv1.VRFConfig{ + VNI: 100, + RouteTarget: "65000:100", + }, + }, + }, + }, + } + + _, err := RenderNetAttachDefManifest(cudn, "test-ns") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("EVPN transport requested but EVPN feature is not enabled")) + }) + }) + It("should correctly assign transit Subnets", func() { // check no overlap, use default values netConf := &ovncnitypes.NetConf{ diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/template/render_options.go b/go-controller/pkg/clustermanager/userdefinednetwork/template/render_options.go new file mode 100644 index 0000000000..7b93ef34fd --- /dev/null +++ b/go-controller/pkg/clustermanager/userdefinednetwork/template/render_options.go @@ -0,0 +1,41 @@ +package template + +// RenderOption is a functional option for configuring NAD rendering. +type RenderOption func(*RenderOptions) + +// RenderOptions contains optional configuration for NAD rendering. +type RenderOptions struct { + EVPNVIDs *EVPNVIDs +} + +// EVPNVIDs contains pre-allocated VLAN IDs for EVPN MAC-VRF and IP-VRF. +type EVPNVIDs struct { + // MACVRFVID is the VLAN ID for the MAC-VRF (Layer 2 EVPN). + // A value of 0 means no VID is allocated for MAC-VRF. + MACVRFVID int + // IPVRFVID is the VLAN ID for the IP-VRF (Layer 3 EVPN). + // A value of 0 means no VID is allocated for IP-VRF. + IPVRFVID int +} + +// WithEVPNVIDs returns a RenderOption that sets the EVPN VIDs for rendering. +func WithEVPNVIDs(macVRFVID, ipVRFVID int) RenderOption { + return func(opts *RenderOptions) { + opts.EVPNVIDs = &EVPNVIDs{ + MACVRFVID: macVRFVID, + IPVRFVID: ipVRFVID, + } + } +} + +// applyOptions applies the given functional options and returns the resulting RenderOptions. +// Nil options in the slice are safely skipped to prevent panics. +func applyOptions(opts []RenderOption) *RenderOptions { + options := &RenderOptions{} + for _, opt := range opts { + if opt != nil { + opt(options) + } + } + return options +} diff --git a/go-controller/pkg/cni/cni.go b/go-controller/pkg/cni/cni.go index a164eacc24..2a7b71b77d 100644 --- a/go-controller/pkg/cni/cni.go +++ b/go-controller/pkg/cni/cni.go @@ -86,7 +86,7 @@ func extractPodBandwidth(podAnnotations map[string]string, dir direction) (int64 } func (pr *PodRequest) String() string { - return fmt.Sprintf("[%s/%s %s network %s NAD %s]", pr.PodNamespace, pr.PodName, pr.SandboxID, pr.netName, pr.nadName) + return fmt.Sprintf("[%s/%s %s network %s NAD %s NAD key %s]", pr.PodNamespace, pr.PodName, pr.SandboxID, pr.netName, pr.nadName, pr.nadKey) } // checkOrUpdatePodUID validates the given pod UID against the request's existing @@ -119,9 +119,9 @@ func (pr *PodRequest) cmdAdd(kubeAuth *KubeAPIAuth, clientset *ClientSet, networ // primaryDPUReady makes sure previous annotation condition is ready, then if primary UDN interface is needed and it is // in the DPU-HOST/DPU setup, checks if DPU connection annotations for primary UDN interface are ready. func (pr *PodRequest) primaryDPUReady(primaryUDN *udn.UserDefinedPrimaryNetwork, k kube.Interface, podLister corev1listers.PodLister, annotCondFn podAnnotWaitCond) podAnnotWaitCond { - return func(pod *corev1.Pod, nadName string) (*util.PodAnnotation, bool, error) { + return func(pod *corev1.Pod, nadKey string) (*util.PodAnnotation, bool, error) { // First, check the original annotation condition - annotation, isReady, err := annotCondFn(pod, nadName) + annotation, isReady, err := annotCondFn(pod, nadKey) if err != nil || !isReady { return annotation, isReady, err } @@ -135,7 +135,7 @@ func (pr *PodRequest) primaryDPUReady(primaryUDN *udn.UserDefinedPrimaryNetwork, return annotation, false, err } // Check if DPU status annotation is ready (passing nil as we've already checked annotation) - return isDPUReady(nil, primaryUDN.NADName())(pod, nadName) + return isDPUReady(nil, primaryUDN.NADName())(pod, nadKey) } // Non-DPU case: proceed normally return annotation, true, nil @@ -156,6 +156,26 @@ func (pr *PodRequest) cmdAddWithGetCNIResultFunc( } kubecli := &kube.Kube{KClient: clientset.kclient} + + pod, _, _, err := GetPodWithAnnotations(pr.ctx, clientset, namespace, podName, "", + func(*corev1.Pod, string) (*util.PodAnnotation, bool, error) { + return nil, true, nil + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to get pod %s/%s: %v", namespace, podName, err) + } + + if pr.netName != types.DefaultNetworkName { + nadKey, err := GetCNINADKey(pod, pr.IfName, pr.nadName) + if err != nil { + return nil, fmt.Errorf("failed to get NAD key for CNI Add request %v: %v", pr, err) + } + pr.nadKey = nadKey + } else { + pr.nadKey = pr.nadName + } + annotCondFn := isOvnReady netdevName := "" if pr.CNIConf.DeviceID != "" { @@ -191,10 +211,10 @@ func (pr *PodRequest) cmdAddWithGetCNIResultFunc( // now checks for default network's DPU connection status if config.OvnKubeNode.Mode == types.NodeModeDPUHost { if pr.CNIConf.DeviceID != "" { - annotCondFn = isDPUReady(annotCondFn, pr.nadName) + annotCondFn = isDPUReady(annotCondFn, pr.nadKey) } } - pod, annotations, podNADAnnotation, err := GetPodWithAnnotations(pr.ctx, clientset, namespace, podName, pr.nadName, annotCondFn) + pod, annotations, podNADAnnotation, err := GetPodWithAnnotations(pr.ctx, clientset, namespace, podName, pr.nadKey, annotCondFn) if err != nil { return nil, fmt.Errorf("failed to get pod annotation: %v", err) } @@ -217,6 +237,11 @@ func (pr *PodRequest) cmdAddWithGetCNIResultFunc( if err != nil { return nil, err } + // get all the Pod interface names of the same nadName. See if this is a pod with multiple secondary UDN of nadName + podIfNamesOfSameNAD, _ := GetPodIfNamesForNAD(pod, pr.nadName) + if len(podIfNamesOfSameNAD) > 1 { + podInterfaceInfo.PodIfNamesOfSameNAD = podIfNamesOfSameNAD + } podInterfaceInfo.SkipIPConfig = kubevirt.IsPodLiveMigratable(pod) @@ -289,26 +314,43 @@ func (pr *PodRequest) cmdDel(clientset *ClientSet) (*Response, error) { return nil, fmt.Errorf("required CNI variable missing") } + pod, err := clientset.getPod(pr.PodNamespace, pr.PodName) + if err != nil { + if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get pod %s/%s: %w", pr.PodNamespace, pr.PodName, err) + } + } + + if pod != nil && pr.netName != types.DefaultNetworkName { + nadKey, err := GetCNINADKey(pod, pr.IfName, pr.nadName) + if err != nil { + return nil, err + } + pr.nadKey = nadKey + } else { + pr.nadKey = pr.nadName + } + netdevName := "" if pr.CNIConf.DeviceID != "" { if config.OvnKubeNode.Mode == types.NodeModeDPUHost { - pod, err := clientset.getPod(pr.PodNamespace, pr.PodName) - if err != nil { + if pod == nil { + // no need to update DPU connection-details annotation if pod is already removed klog.Warningf("Failed to get pod %s/%s: %v", pr.PodNamespace, pr.PodName, err) return response, nil } - dpuCD, err := util.UnmarshalPodDPUConnDetails(pod.Annotations, pr.nadName) + dpuCD, err := util.UnmarshalPodDPUConnDetails(pod.Annotations, pr.nadKey) if err != nil { - klog.Warningf("Failed to get DPU connection details annotation for pod %s/%s NAD %s: %v", pr.PodNamespace, - pr.PodName, pr.nadName, err) + klog.Warningf("Failed to get DPU connection details annotation for pod %s/%s NAD key %s: %v", pr.PodNamespace, + pr.PodName, pr.nadKey, err) return response, nil } // check if this cmdDel is meant for the current sandbox, if not, directly return if dpuCD.SandboxId != pr.SandboxID { klog.Infof("The cmdDel request for sandbox %s is not meant for the currently configured "+ - "pod %s/%s on NAD %s with sandbox %s. Ignoring this request.", - pr.SandboxID, namespace, podName, pr.nadName, dpuCD.SandboxId) + "pod %s/%s on NAD key %s with sandbox %s. Ignoring this request.", + pr.SandboxID, namespace, podName, pr.nadKey, dpuCD.SandboxId) return response, nil } @@ -333,26 +375,33 @@ func (pr *PodRequest) cmdDel(clientset *ClientSet) (*Response, error) { } // not an error if pod has already been deleted if err != nil && !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to cleanup the DPU connection details annotation for NAD %s: %v", pr.nadName, err) + return nil, fmt.Errorf("failed to cleanup the DPU connection details annotation for NAD key %s: %v", pr.nadKey, err) } } else { // Find the hostInterface name condString := []string{"external-ids:sandbox=" + pr.SandboxID} - if pr.netName != types.DefaultNetworkName { - condString = append(condString, fmt.Sprintf("external_ids:%s=%s", types.NADExternalID, pr.nadName)) - } else { - condString = append(condString, fmt.Sprintf("external_ids:%s{=}[]", types.NADExternalID)) - } + condString = append(condString, fmt.Sprintf("external_ids:pod-if-name=%s", pr.IfName)) ovsIfNames, err := ovsFind("Interface", "name", condString...) if err != nil || len(ovsIfNames) != 1 { - klog.Warningf("Couldn't find the OVS interface for pod %s/%s NAD %s: %v", - pr.PodNamespace, pr.PodName, pr.nadName, err) + // the pod was added before "external_ids:pod-if-name" was introduced, fall back to the old way to find + // out the OVS interface associated with this CNIDel request + condString = []string{"external-ids:sandbox=" + pr.SandboxID} + if pr.netName != types.DefaultNetworkName { + condString = append(condString, fmt.Sprintf("external_ids:%s=%s", types.NADExternalID, pr.nadKey)) + } else { + condString = append(condString, fmt.Sprintf("external_ids:%s{=}[]", types.NADExternalID)) + } + ovsIfNames, err = ovsFind("Interface", "name", condString...) + } + + if err != nil || len(ovsIfNames) != 1 { + klog.Warningf("Couldn't find the OVS interface for pod %s/%s NAD key %s: %v", + pr.PodNamespace, pr.PodName, pr.nadKey, err) } else { - ovsIfName := ovsIfNames[0] - out, err := ovsGet("interface", ovsIfName, "external_ids", "vf-netdev-name") + out, err := ovsGet("interface", ovsIfNames[0], "external_ids", "vf-netdev-name") if err != nil { klog.Warningf("Couldn't find the original Netdev name from OVS interface %s for pod %s/%s: %v", - ovsIfName, pr.PodNamespace, pr.PodName, err) + ovsIfNames[0], pr.PodNamespace, pr.PodName, err) } else { netdevName = out } @@ -501,6 +550,7 @@ func (pr *PodRequest) buildPrimaryUDNPodRequest( IsVFIO: isVFIO, netName: primaryUDN.NetworkName(), nadName: primaryUDN.NADName(), + nadKey: primaryUDN.NADName(), deviceInfo: *deviceInfo, } @@ -514,7 +564,7 @@ func (pr *PodRequest) buildPodInterfaceInfo(annotations map[string]string, podAn podAnnotation, pr.PodUID, netDevice, - pr.nadName, + pr.nadKey, pr.netName, pr.CNIConf.MTU, ) diff --git a/go-controller/pkg/cni/cni_dpu.go b/go-controller/pkg/cni/cni_dpu.go index fa57d87a37..613f80a9ed 100644 --- a/go-controller/pkg/cni/cni_dpu.go +++ b/go-controller/pkg/cni/cni_dpu.go @@ -21,7 +21,7 @@ func (pr *PodRequest) updatePodDPUConnDetailsWithRetry(kube kube.Interface, podL kube, pod, dpuConnDetails, - pr.nadName, + pr.nadKey, ) if util.IsAnnotationAlreadySetError(err) { return nil diff --git a/go-controller/pkg/cni/cni_dpu_test.go b/go-controller/pkg/cni/cni_dpu_test.go index 4ff79027eb..1185def539 100644 --- a/go-controller/pkg/cni/cni_dpu_test.go +++ b/go-controller/pkg/cni/cni_dpu_test.go @@ -48,6 +48,7 @@ var _ = Describe("cni_dpu tests", func() { IsVFIO: false, netName: ovntypes.DefaultNetworkName, nadName: ovntypes.DefaultNetworkName, + nadKey: ovntypes.DefaultNetworkName, deviceInfo: nadapi.DeviceInfo{}, } pod = &corev1.Pod{ diff --git a/go-controller/pkg/cni/cni_test.go b/go-controller/pkg/cni/cni_test.go index 2535300190..a8954a0836 100644 --- a/go-controller/pkg/cni/cni_test.go +++ b/go-controller/pkg/cni/cni_test.go @@ -93,6 +93,7 @@ var _ = Describe("Network Segmentation", func() { IsVFIO: false, netName: ovntypes.DefaultNetworkName, nadName: ovntypes.DefaultNetworkName, + nadKey: ovntypes.DefaultNetworkName, } pr.ctx, pr.cancel = context.WithTimeout(context.Background(), 2*time.Minute) @@ -364,7 +365,7 @@ var _ = Describe("Network Segmentation", func() { PodAnnotation: *podNADAnnotation, MTU: 1400, NetName: "tenantred", - NADName: "foo-ns/meganet", + NADKey: "foo-ns/meganet", })) Expect(response.PrimaryUDNPodReq.IfName).To(Equal("ovn-udn1")) Expect(response.PodIFInfo.NetName).To(Equal("default")) diff --git a/go-controller/pkg/cni/cnishim_test.go b/go-controller/pkg/cni/cnishim_test.go index 6c056791ec..4793feeec5 100644 --- a/go-controller/pkg/cni/cnishim_test.go +++ b/go-controller/pkg/cni/cnishim_test.go @@ -120,7 +120,7 @@ func TestCmdAdd_UnprivilegedMode(t *testing.T) { PodAnnotation: *defaultPodNADAnnotation, MTU: 1400, NetName: "default", - NADName: "foo-ns/default", + NADKey: "foo-ns/default", // hack to bypass OVS exec check IsDPUHostMode: true, }, @@ -128,7 +128,7 @@ func TestCmdAdd_UnprivilegedMode(t *testing.T) { PodAnnotation: *udnPodNADAnnotation, MTU: 1400, NetName: "tenantred", - NADName: "foo-ns/meganet", + NADKey: "foo-ns/meganet", // hack to bypass OVS exec check IsDPUHostMode: true, }, @@ -255,12 +255,12 @@ func TestCmdDel_UnprivilegedMode(t *testing.T) { Result: nil, PodIFInfo: &PodInterfaceInfo{ NetName: "default", - NADName: "foo-ns/default", + NADKey: "foo-ns/default", IsDPUHostMode: true, }, PrimaryUDNPodInfo: &PodInterfaceInfo{ NetName: "tenantred", - NADName: "foo-ns/meganet", + NADKey: "foo-ns/meganet", IsDPUHostMode: true, }, PrimaryUDNPodReq: &PodRequest{ diff --git a/go-controller/pkg/cni/helper_linux.go b/go-controller/pkg/cni/helper_linux.go index bcb3687364..6da9008df4 100644 --- a/go-controller/pkg/cni/helper_linux.go +++ b/go-controller/pkg/cni/helper_linux.go @@ -28,7 +28,8 @@ import ( ) type CNIPluginLibOps interface { - AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link, mtu int) error + AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link, mtu, table int) error + ReplaceRouteECMP(ipn *net.IPNet, gw net.IP, devs []netlink.Link, mtu int) error SetupVeth(contVethName string, hostVethName string, mtu int, contVethMac string, hostNS ns.NetNS) (net.Interface, net.Interface, error) } @@ -36,18 +37,36 @@ type defaultCNIPluginLibOps struct{} var cniPluginLibOps CNIPluginLibOps = &defaultCNIPluginLibOps{} -func (defaultCNIPluginLibOps) AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link, mtu int) error { +func (defaultCNIPluginLibOps) AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link, mtu, table int) error { route := &netlink.Route{ LinkIndex: dev.Attrs().Index, Scope: netlink.SCOPE_UNIVERSE, Dst: ipn, Gw: gw, MTU: mtu, + Table: table, } return util.GetNetLinkOps().RouteAdd(route) } +func (defaultCNIPluginLibOps) ReplaceRouteECMP(ipn *net.IPNet, gw net.IP, devs []netlink.Link, mtu int) error { + ecmpRoute := &netlink.Route{ + Dst: ipn, + MTU: mtu, + } + + ecmpRoute.MultiPath = make([]*netlink.NexthopInfo, len(devs)) + for i, dev := range devs { + ecmpRoute.MultiPath[i] = &netlink.NexthopInfo{ + LinkIndex: dev.Attrs().Index, + Gw: gw, + Hops: 0, // Weight (0 means weight of 1) + } + } + return util.GetNetLinkOps().RouteReplace(ecmpRoute) +} + func (defaultCNIPluginLibOps) SetupVeth(contVethName string, hostVethName string, mtu int, contVethMac string, hostNS ns.NetNS) (net.Interface, net.Interface, error) { return ip.SetupVethWithName(contVethName, hostVethName, mtu, contVethMac, hostNS) } @@ -193,13 +212,100 @@ func setupNetwork(link netlink.Link, ifInfo *PodInterfaceInfo) error { } } for _, gw := range ifInfo.Gateways { - if err := cniPluginLibOps.AddRoute(nil, gw, link, ifInfo.RoutableMTU); err != nil { + if err := cniPluginLibOps.AddRoute(nil, gw, link, ifInfo.RoutableMTU, 0); err != nil { return fmt.Errorf("failed to add gateway route to link '%s': %v", link.Attrs().Name, err) } } + + // all pod links of the same secondary UDN up to the current one + var links []netlink.Link + var iptableNum int + + if len(ifInfo.PodIfNamesOfSameNAD) != 0 { + // this is a pod with multiple interfaces of the same non-primary UDN, we need to configure the following sysctl configuration + // stop ARP flux and stop RP filter drop from happening + // sysctl -w net.ipv4.conf..arp_ignore=1 # only reply to ARP requests if its sent to the IP on this interface + // sysctl -w net.ipv4.conf..arp_announce=2 # when sending ARP use the IP that belongs to this interface + // sysctl -w net.ipv4.conf..rp_filter=2 # allow reverse path filter check to pass if there is any route to the source + linkName := link.Attrs().Name + if err := setSysctl(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/arp_ignore", linkName), 1); err != nil { + return fmt.Errorf("failed to set arp_ignore for %s: %v", linkName, err) + } + if err := setSysctl(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/arp_announce", linkName), 2); err != nil { + return fmt.Errorf("failed to set arp_announce for %s: %v", linkName, err) + } + if err := setSysctl(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/rp_filter", linkName), 2); err != nil { + return fmt.Errorf("failed to set rp_filter for %s: %v", linkName, err) + } + + if len(ifInfo.Routes) != 0 { + // needs ip table for each Pod interface of the same secondary UDN, table number start from 100 + iptableNum = 100 + link.Attrs().Index + + // ECMP routes are needed + // get all the Pod interface names of the same nadName + // up to the current link name. This is used for configuring ECMP routes via multiple pod interfaces + links = make([]netlink.Link, 0, len(ifInfo.PodIfNamesOfSameNAD)) + for _, ifName := range ifInfo.PodIfNamesOfSameNAD { + if ifName == link.Attrs().Name { + // loop all pod interfaces of the same secondary UDN until the current pod interface + links = append(links, link) + break + } + ifLink, err := util.GetNetLinkOps().LinkByName(ifName) + if err != nil { + return fmt.Errorf("failed to lookup pod interface %s when setup route for link %s: %v", ifName, link.Attrs().Name, err) + } + links = append(links, ifLink) + } + + // ip rules are needed to force traffic from an ip to egress the correct interface via a route table for that interface: + // ip rule add from table + for _, ip := range ifInfo.IPs { + rule := netlink.NewRule() + rule.Src = util.GetIPNetFullMaskFromIP(ip.IP) + rule.Table = iptableNum + if err := util.GetNetLinkOps().RuleAdd(rule); err != nil { + return fmt.Errorf("failed to add IP rule for %s to table %d: %v", ip.IP, iptableNum, err) + } + + // add scope link route to the specific table + route := &netlink.Route{ + LinkIndex: link.Attrs().Index, + Scope: netlink.SCOPE_LINK, + Dst: util.IPsToNetworkIPs(ip)[0], + Table: iptableNum, + } + if err := util.GetNetLinkOps().RouteAdd(route); err != nil { + return fmt.Errorf("failed to add link scope route to %v via %v to table %v: %v", ip, linkName, iptableNum, err) + } + } + } + } + for _, route := range ifInfo.Routes { - if err := cniPluginLibOps.AddRoute(route.Dest, route.NextHop, link, ifInfo.RoutableMTU); err != nil { - return fmt.Errorf("failed to add pod route %v via %v: %v", route.Dest, route.NextHop, err) + if len(ifInfo.PodIfNamesOfSameNAD) == 0 { + // if there is no other interface of the same NAD, just add route directly + if err := cniPluginLibOps.AddRoute(route.Dest, route.NextHop, link, ifInfo.RoutableMTU, 0); err != nil { + return fmt.Errorf("failed to add pod route %v via %v: %v", route.Dest, route.NextHop, err) + } + } else { + if len(links) == 1 { + // if this is the first pod interface of this NAD, just add route directly + if err := cniPluginLibOps.AddRoute(route.Dest, route.NextHop, link, ifInfo.RoutableMTU, 0); err != nil { + return fmt.Errorf("failed to add pod route %v via %v: %v", route.Dest, route.NextHop, err) + } + } else { + // otherwise replace it with ECMP routes + if err := cniPluginLibOps.ReplaceRouteECMP(route.Dest, route.NextHop, links, ifInfo.RoutableMTU); err != nil { + return fmt.Errorf("failed to replace pod route %v via %v through links %v: %v", route.Dest, route.NextHop, links, err) + } + } + + // add ECMP route to specific IP route table + if err := cniPluginLibOps.AddRoute(route.Dest, route.NextHop, link, ifInfo.RoutableMTU, iptableNum); err != nil { + return fmt.Errorf("failed to add pod route %v nexthop %v via %v table %v: %v", route.Dest, route.NextHop, link.Attrs().Name, iptableNum, err) + } } } @@ -429,12 +535,12 @@ func getPfEncapIP(deviceID string) (string, error) { } // ConfigureOVS performs OVS configurations in order to set up Pod networking -func ConfigureOVS(ctx context.Context, namespace, podName, hostIfaceName string, +func ConfigureOVS(ctx context.Context, namespace, podName, podIfName, hostIfaceName string, ifInfo *PodInterfaceInfo, sandboxID, deviceID string, getter PodInfoGetter) error { ifaceID := util.GetIfaceId(namespace, podName) if ifInfo.NetName != types.DefaultNetworkName { - ifaceID = util.GetUDNIfaceId(namespace, podName, ifInfo.NADName) + ifaceID = util.GetUDNIfaceId(namespace, podName, ifInfo.NADKey) } initialPodUID := ifInfo.PodUID ipStrs := make([]string, len(ifInfo.IPs)) @@ -448,7 +554,7 @@ func ConfigureOVS(ctx context.Context, namespace, podName, hostIfaceName string, } klog.Infof("ConfigureOVS: namespace: %s, podName: %s, hostIfaceName: %s, network: %s, NAD %s, SandboxID: %q, PCI device ID: %s, UID: %q, MAC: %s, IPs: %v", - namespace, podName, hostIfaceName, ifInfo.NetName, ifInfo.NADName, sandboxID, deviceID, initialPodUID, ifInfo.MAC, ipStrs) + namespace, podName, hostIfaceName, ifInfo.NetName, ifInfo.NADKey, sandboxID, deviceID, initialPodUID, ifInfo.MAC, ipStrs) // Find and remove any existing OVS port with this iface-id. Pods can // have multiple sandboxes if some are waiting for garbage collection, @@ -471,16 +577,16 @@ func ConfigureOVS(ctx context.Context, namespace, podName, hostIfaceName string, if err == nil && len(extIds) == 1 { extId := extIds[0] ifaceIDStr := util.GetExternalIDValByKey(extId, "iface-id") - nadNameString := util.GetExternalIDValByKey(extId, types.NADExternalID) + nadKeyString := util.GetExternalIDValByKey(extId, types.NADExternalID) // if NADExternalID does not exists, it is default network - if nadNameString == "" { - nadNameString = types.DefaultNetworkName + if nadKeyString == "" { + nadKeyString = types.DefaultNetworkName } if ifaceIDStr != ifaceID { return fmt.Errorf("OVS port %s was added for iface-id (%s), now readding it for (%s)", hostIfaceName, ifaceIDStr, ifaceID) } - if nadNameString != ifInfo.NADName { - return fmt.Errorf("OVS port %s was added for NAD (%s), expect (%s)", hostIfaceName, nadNameString, ifInfo.NADName) + if nadKeyString != ifInfo.NADKey { + return fmt.Errorf("OVS port %s was added for NAD (%s), expect (%s)", hostIfaceName, nadKeyString, ifInfo.NADKey) } } @@ -495,6 +601,11 @@ func ConfigureOVS(ctx context.Context, namespace, podName, hostIfaceName string, fmt.Sprintf("external_ids:sandbox=%s", sandboxID), } + // pod interface name, used to identify CNI request with the same NAD + if podIfName != "" { + ovsArgs = append(ovsArgs, fmt.Sprintf("external_ids:pod-if-name=%s", podIfName)) + } + // In case of multi-vtep, host has multipe NICs and each NIC has a VTEP interface, the mapping // of VTEP IP to NIC is stored in Open_vSwitch table's `external_ids:ovn-pf-encap-ip-mapping`, // the value's format is: @@ -539,7 +650,7 @@ func ConfigureOVS(ctx context.Context, namespace, podName, hostIfaceName string, if ifInfo.NetName != types.DefaultNetworkName { ovsArgs = append(ovsArgs, fmt.Sprintf("external_ids:%s=%s", types.NetworkExternalID, ifInfo.NetName)) - ovsArgs = append(ovsArgs, fmt.Sprintf("external_ids:%s=%s", types.NADExternalID, ifInfo.NADName)) + ovsArgs = append(ovsArgs, fmt.Sprintf("external_ids:%s=%s", types.NADExternalID, ifInfo.NADKey)) } else { ovsArgs = append(ovsArgs, []string{"--", "--if-exists", "remove", "interface", hostIfaceName, "external_ids", types.NetworkExternalID}...) ovsArgs = append(ovsArgs, []string{"--", "--if-exists", "remove", "interface", hostIfaceName, "external_ids", types.NADExternalID}...) @@ -625,7 +736,7 @@ func (*defaultPodRequestInterfaceOps) ConfigureInterface(pr *PodRequest, getter // END OCP HACK if !ifInfo.IsDPUHostMode { - err = ConfigureOVS(pr.ctx, pr.PodNamespace, pr.PodName, hostIface.Name, ifInfo, pr.SandboxID, pr.CNIConf.DeviceID, getter) + err = ConfigureOVS(pr.ctx, pr.PodNamespace, pr.PodName, pr.IfName, hostIface.Name, ifInfo, pr.SandboxID, pr.CNIConf.DeviceID, getter) if err != nil { pr.deletePort(hostIface.Name, pr.PodNamespace, pr.PodName) return nil, err diff --git a/go-controller/pkg/cni/helper_linux_test.go b/go-controller/pkg/cni/helper_linux_test.go index fb744c3018..0b99996770 100644 --- a/go-controller/pkg/cni/helper_linux_test.go +++ b/go-controller/pkg/cni/helper_linux_test.go @@ -320,7 +320,7 @@ func TestSetupNetwork(t *testing.T) { {OnCallMethodName: "AddrAdd", OnCallMethodArgType: []string{"*mocks.Link", "*netlink.Addr"}, RetArgList: []interface{}{nil}}, }, cniPluginMockHelper: []ovntest.TestifyMockHelper{ - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, }, linkMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Name: "testIfaceName"}}, CallTimes: 2}, @@ -348,8 +348,8 @@ func TestSetupNetwork(t *testing.T) { {OnCallMethodName: "AddrAdd", OnCallMethodArgType: []string{"*mocks.Link", "*netlink.Addr"}, RetArgList: []interface{}{nil}}, }, cniPluginMockHelper: []ovntest.TestifyMockHelper{ - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{nil}}, - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{nil}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, }, linkMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Name: "testIfaceName"}}}, @@ -376,8 +376,8 @@ func TestSetupNetwork(t *testing.T) { {OnCallMethodName: "AddrAdd", OnCallMethodArgType: []string{"*mocks.Link", "*netlink.Addr"}, RetArgList: []interface{}{nil}}, }, cniPluginMockHelper: []ovntest.TestifyMockHelper{ - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{nil}}, - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{nil}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{nil}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{nil}}, }, linkMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Name: "testIfaceName"}}}, @@ -403,8 +403,8 @@ func TestSetupNetwork(t *testing.T) { {OnCallMethodName: "AddrAdd", OnCallMethodArgType: []string{"*mocks.Link", "*netlink.Addr"}, RetArgList: []interface{}{nil}}, }, cniPluginMockHelper: []ovntest.TestifyMockHelper{ - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{nil}}, - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{nil}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{nil}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{nil}}, }, linkMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Name: "testIfaceName", Flags: net.FlagUp}}}, @@ -1105,7 +1105,7 @@ func TestSetupSriovInterface(t *testing.T) { {OnCallMethodName: "AddrAdd", OnCallMethodArgType: []string{"*mocks.Link", "*netlink.Addr"}, RetArgList: []interface{}{nil}}, }, cniPluginMockHelper: []ovntest.TestifyMockHelper{ - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, }, linkMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Flags: net.FlagUp}}, CallTimes: 2}, @@ -1150,8 +1150,8 @@ func TestSetupSriovInterface(t *testing.T) { {OnCallMethodName: "AddrAdd", OnCallMethodArgType: []string{"*mocks.Link", "*netlink.Addr"}, RetArgList: []interface{}{nil}}, }, cniPluginMockHelper: []ovntest.TestifyMockHelper{ - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{nil}}, - {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{nil}}, + {OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, }, linkMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Flags: net.FlagUp}}}, @@ -1503,7 +1503,7 @@ func TestConfigureOVS(t *testing.T) { podLister.On("Pods", mock.AnythingOfType("string")).Return(&podNamespaceLister) fakeClient := fake.NewSimpleClientset(&corev1.PodList{Items: []corev1.Pod{pod}}) clientset := NewClientSet(fakeClient, &podLister) - err = ConfigureOVS(ctx, tc.podNs, tc.podName, tc.vfRep, + err = ConfigureOVS(ctx, tc.podNs, tc.podName, "", tc.vfRep, tc.ifInfo, sandboxID, vfPciAddress, clientset) if tc.errMatch != nil { assert.Contains(t, err.Error(), tc.errMatch.Error()) @@ -1638,7 +1638,7 @@ func TestConfigureOVS_getPfEncapIpWithError(t *testing.T) { var podLister v1mocks.PodLister podLister.On("Pods", mock.AnythingOfType("string")).Return(&podNamespaceLister) - err = ConfigureOVS(ctx, tc.podNs, tc.podName, tc.vfRep, + err = ConfigureOVS(ctx, tc.podNs, tc.podName, "", tc.vfRep, tc.ifInfo, sandboxID, vfPciAddress, nil) if tc.errMatch != nil { assert.Contains(t, err.Error(), tc.errMatch.Error()) diff --git a/go-controller/pkg/cni/mocks/CNIPluginLibOps.go b/go-controller/pkg/cni/mocks/CNIPluginLibOps.go index 43e437a01d..5d6cad7d9a 100644 --- a/go-controller/pkg/cni/mocks/CNIPluginLibOps.go +++ b/go-controller/pkg/cni/mocks/CNIPluginLibOps.go @@ -17,17 +17,35 @@ type CNIPluginLibOps struct { mock.Mock } -// AddRoute provides a mock function with given fields: ipn, gw, dev, mtu -func (_m *CNIPluginLibOps) AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link, mtu int) error { - ret := _m.Called(ipn, gw, dev, mtu) +// AddRoute provides a mock function with given fields: ipn, gw, dev, mtu, table +func (_m *CNIPluginLibOps) AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link, mtu, table int) error { + ret := _m.Called(ipn, gw, dev, mtu, table) if len(ret) == 0 { panic("no return value specified for AddRoute") } var r0 error - if rf, ok := ret.Get(0).(func(*net.IPNet, net.IP, netlink.Link, int) error); ok { - r0 = rf(ipn, gw, dev, mtu) + if rf, ok := ret.Get(0).(func(*net.IPNet, net.IP, netlink.Link, int, int) error); ok { + r0 = rf(ipn, gw, dev, mtu, table) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ReplaceRouteECMP provides a mock function with given fields: ipn, gw, devs, mtu +func (_m *CNIPluginLibOps) ReplaceRouteECMP(ipn *net.IPNet, gw net.IP, devs []netlink.Link, mtu int) error { + ret := _m.Called(ipn, gw, devs, mtu) + + if len(ret) == 0 { + panic("no return value specified for ReplaceRouteECMP") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*net.IPNet, net.IP, []netlink.Link, int) error); ok { + r0 = rf(ipn, gw, devs, mtu) } else { r0 = ret.Error(0) } diff --git a/go-controller/pkg/cni/ovs.go b/go-controller/pkg/cni/ovs.go index f287f4ab9b..e5f5cad86e 100644 --- a/go-controller/pkg/cni/ovs.go +++ b/go-controller/pkg/cni/ovs.go @@ -234,7 +234,7 @@ func doPodFlowsExist(mac string, ifAddrs []*net.IPNet, ofPort int) bool { // have a 1:1 relationship determined by pod UID. If we detect that the pod // has changed either UID or MAC terminate this sandbox request early instead // of waiting for OVN to set up flows that will never exist. -func checkCancelSandbox(mac string, getter PodInfoGetter, namespace, name, nadName, initialPodUID string) error { +func checkCancelSandbox(mac string, getter PodInfoGetter, namespace, name, nadKey, initialPodUID string) error { // Not all node CNI modes may have access to kube api, those will pass nil as getter. if getter == nil { return nil @@ -254,7 +254,7 @@ func checkCancelSandbox(mac string, getter PodInfoGetter, namespace, name, nadNa return fmt.Errorf("canceled old pod sandbox") } - ovnAnnot, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + ovnAnnot, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { return fmt.Errorf("pod OVN annotations deleted or invalid") } @@ -324,7 +324,7 @@ func waitForPodInterface(ctx context.Context, ifInfo *PodInterfaceInfo, } } - if err := checkCancelSandbox(mac, getter, namespace, name, ifInfo.NADName, initialPodUID); err != nil { + if err := checkCancelSandbox(mac, getter, namespace, name, ifInfo.NADKey, initialPodUID); err != nil { return fmt.Errorf("%v waiting for OVS port binding for %s %v", err, mac, ifAddrs) } diff --git a/go-controller/pkg/cni/types.go b/go-controller/pkg/cni/types.go index f405954721..866d5e2749 100644 --- a/go-controller/pkg/cni/types.go +++ b/go-controller/pkg/cni/types.go @@ -59,8 +59,11 @@ type PodInterfaceInfo struct { // network name, for default network, it is "default", otherwise it is net-attach-def's netconf spec name NetName string `json:"netName"` - // NADName, for default network, it is "default", otherwise, in the form of net-attach-def's / - NADName string `json:"nadName"` + // NADKey, for default network, it is "default", otherwise, in the form of net-attach-def's /{/index} + NADKey string `json:"nadKey"` + // pod interface names of the same NAD, in plumbing order. + // Only set for when there are more than one pod interface with the same UDN + PodIfNamesOfSameNAD []string `json:"pod-if-names"` } // Explicit type for CNI commands the server handles @@ -168,6 +171,10 @@ type PodRequest struct { // also, need to find the pod annotation, dpu pod connection/status annotations of the given NAD ("default" // for default network). nadName string + // for default/primary UDN network, nadKey is the same as nadName, for secondary UDN, if a Pod requests + // network attachment of multiple same secondary UDN, nadKey would be nadName for its first interface CNI request, + // and / (index starting from 1) for the subsequent interface CNI request + nadKey string // the DeviceInfo struct deviceInfo nadapi.DeviceInfo diff --git a/go-controller/pkg/cni/types/types.go b/go-controller/pkg/cni/types/types.go index 90fcb47ef0..8f8007e1fb 100644 --- a/go-controller/pkg/cni/types/types.go +++ b/go-controller/pkg/cni/types/types.go @@ -80,6 +80,15 @@ type NetConf struct { // network mapping in the hosts. PhysicalNetworkName string `json:"physicalNetworkName,omitempty"` + // Transport describes the transport protocol for east-west traffic. + // Valid values are "no-overlay", "geneve", and "evpn". + // Defaults to "geneve". + Transport string `json:"transport,omitempty"` + + // EVPNConfig contains configuration for EVPN mode. + // Only valid when Transport is "evpn". + EVPN *EVPNConfig `json:"evpn,omitempty"` + // PciAddrs in case of using sriov or Auxiliry device name in case of SF DeviceID string `json:"deviceID,omitempty"` // LogFile to log all the messages from cni shim binary to @@ -102,6 +111,27 @@ type NetConf struct { } `json:"runtimeConfig,omitempty"` } +// EVPNConfig contains EVPN-specific configuration for the network. +type EVPNConfig struct { + // VTEP is the name of the VTEP CR that defines VTEP IPs for EVPN. + VTEP string `json:"vtep"` + // MACVRF contains the MAC-VRF configuration for Layer 2 EVPN. + MACVRF *VRFConfig `json:"macVRF,omitempty"` + // IPVRF contains the IP-VRF configuration for Layer 3 EVPN. + IPVRF *VRFConfig `json:"ipVRF,omitempty"` +} + +// VRFConfig contains configuration for a VRF in EVPN. +type VRFConfig struct { + // VNI is the VXLAN Network Identifier for this VRF. + VNI int32 `json:"vni"` + // RouteTarget is the BGP route target for this VRF. + RouteTarget string `json:"routeTarget,omitempty"` + // VID is the VLAN ID used for local traffic segmentation on each node. + // Allocated cluster-wide by the UDN controller, one per VRF. + VID int `json:"vid,omitempty"` +} + // NetworkSelectionElement represents one element of the JSON format // Network Attachment Selection Annotation as described in section 4.1.2 // of the CRD specification. diff --git a/go-controller/pkg/cni/utils.go b/go-controller/pkg/cni/utils.go index 542f1813d3..76fcde449e 100644 --- a/go-controller/pkg/cni/utils.go +++ b/go-controller/pkg/cni/utils.go @@ -20,8 +20,8 @@ import ( type podAnnotWaitCond = func(*corev1.Pod, string) (*util.PodAnnotation, bool, error) // isOvnReady is a wait condition for OVN master to set pod-networks annotation -func isOvnReady(pod *corev1.Pod, nadName string) (*util.PodAnnotation, bool, error) { - podNADAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) +func isOvnReady(pod *corev1.Pod, nadKey string) (*util.PodAnnotation, bool, error) { + podNADAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { if util.IsAnnotationNotSetError(err) { return nil, false, nil @@ -33,7 +33,7 @@ func isOvnReady(pod *corev1.Pod, nadName string) (*util.PodAnnotation, bool, err // isDPUReady is a wait condition which waits for OVN master to set pod-networks annotation and // ovnkube running on DPU to set connection-status pod annotation and its status is Ready -func isDPUReady(annotCondFn podAnnotWaitCond, nadName string) podAnnotWaitCond { +func isDPUReady(annotCondFn podAnnotWaitCond, nadKey string) podAnnotWaitCond { return func(pod *corev1.Pod, nad string) (annotation *util.PodAnnotation, ready bool, err error) { if annotCondFn != nil { annotation, ready, err = annotCondFn(pod, nad) @@ -42,7 +42,7 @@ func isDPUReady(annotCondFn podAnnotWaitCond, nadName string) podAnnotWaitCond { } } // check DPU connection status of the given nad name - status, err := util.UnmarshalPodDPUConnStatus(pod.Annotations, nadName) + status, err := util.UnmarshalPodDPUConnStatus(pod.Annotations, nadKey) if err != nil { if util.IsAnnotationNotSetError(err) { return annotation, false, nil @@ -118,11 +118,11 @@ func GetPodWithAnnotations(ctx context.Context, getter PodInfoGetter, // PodAnnotation2PodInfo creates PodInterfaceInfo from Pod annotations and additional attributes func PodAnnotation2PodInfo(podAnnotation map[string]string, podNADAnnotation *util.PodAnnotation, podUID, - netdevname, nadName, netName string, mtu int) (*PodInterfaceInfo, error) { + netdevname, nadKey, netName string, mtu int) (*PodInterfaceInfo, error) { var err error // get pod's annotation of the given NAD if it is not available if podNADAnnotation == nil { - podNADAnnotation, err = util.UnmarshalPodAnnotation(podAnnotation, nadName) + podNADAnnotation, err = util.UnmarshalPodAnnotation(podAnnotation, nadKey) if err != nil { return nil, err } @@ -146,12 +146,55 @@ func PodAnnotation2PodInfo(podAnnotation map[string]string, podNADAnnotation *ut PodUID: podUID, NetdevName: netdevname, NetName: netName, - NADName: nadName, + NADKey: nadKey, EnableUDPAggregation: config.Default.EnableUDPAggregation, } return podInterfaceInfo, nil } +// GetPodIfNamesForNAD gets the pod's all interface names of the given secondary NAD name +// +// Note that the names of secondary UDN pod interfaces are determined by multus: it is either specified by +// network selection itself, or it is net ( is determined by order of the pod interface, +// started from 1 for the 1st non-default interface). +func GetPodIfNamesForNAD(pod *corev1.Pod, nadName string) ([]string, error) { + networks, err := util.GetK8sPodAllNetworkSelections(pod) + if err != nil { + return nil, fmt.Errorf("failed to getting network selection elements for pod %s/%s: %v", + pod.Namespace, pod.Name, err) + } + ifNames := make([]string, 0, len(networks)) + for idx, network := range networks { + nad := util.GetNADName(network.Namespace, network.Name) + if nad != nadName { + continue + } + if network.InterfaceRequest != "" { + ifNames = append(ifNames, network.InterfaceRequest) + } else { + ifNames = append(ifNames, fmt.Sprintf("net%d", idx+1)) + } + } + return ifNames, nil +} + +// GetCNINADKey gets the pod's nadKey (nadName with index in case there are multiple same NADs in the pod) +// Based on the given ifName, find out which number of this CNI request is for the given nadName, then +// determine its associated NAD key. +func GetCNINADKey(pod *corev1.Pod, ifName, nadName string) (string, error) { + ifNames, err := GetPodIfNamesForNAD(pod, nadName) + if err != nil { + return "", err + } + for idx, name := range ifNames { + if ifName == name { + return util.GetIndexedNADKey(nadName, idx), nil + } + } + return "", fmt.Errorf("failed to find NAD key associated with CNI request for pod %s/%s with ifName %s", + pod.Namespace, pod.Name, ifName) +} + // START taken from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/types/pod_update.go const ( ConfigSourceAnnotationKey = "kubernetes.io/config.source" diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index 42d8764cb2..55c985a187 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -1,6 +1,7 @@ package config import ( + "encoding/base64" "flag" "fmt" "net" @@ -13,7 +14,7 @@ import ( "strings" "time" - "github.com/openshift/api/config/v1" + v1 "github.com/openshift/api/config/v1" "github.com/urfave/cli/v2" gcfg "gopkg.in/gcfg.v1" lumberjack "gopkg.in/natefinch/lumberjack.v2" @@ -76,7 +77,7 @@ var ( // ovn-kubernetes build date BuildDate = "" // ovn-kubernetes version, to be changed with every release - Version = "1.1.0" + Version = "1.2.0" // version of the go runtime used to compile ovn-kubernetes GoVersion = runtime.Version() // os and architecture used to build ovn-kubernetes @@ -101,6 +102,7 @@ var ( RawClusterSubnets: "10.128.0.0/14/23", Zone: types.OvnDefaultZone, RawUDNAllowedDefaultServices: "default/kubernetes,kube-system/kube-dns", + Transport: types.NetworkTransportGeneve, } // Logging holds logging-related parsed config file parameters and command-line overrides @@ -154,10 +156,11 @@ var ( // Metrics holds Prometheus metrics-related parameters. Metrics MetricsConfig - // OVNKubernetesFeatureConfig holds OVN-Kubernetes feature enhancement config file parameters and command-line overrides + // OVNKubernetesFeature config holds OVN-Kubernetes feature enhancement config file parameters and command-line overrides OVNKubernetesFeature = OVNKubernetesFeatureConfig{ EgressIPReachabiltyTotalTimeout: 1, AdvertisedUDNIsolationMode: AdvertisedUDNIsolationModeStrict, + UDNDeletionGracePeriod: 120 * time.Second, } // OvnNorth holds northbound OVN database client and server authentication and location details @@ -241,6 +244,14 @@ var ( V6TransitSubnet: "fd97::/64", } + // NoOverlay holds no-overlay mode configuration + NoOverlay = NoOverlayConfig{} + + // ManagedBGP holds managed BGP configuration + ManagedBGP = ManagedBGPConfig{ + ASNumber: 64512, // Default AS number + } + // Layer2UsesTransitRouter indicated whether the layer2 primary networks will use transit router. // It is a per-node setting and is also reflected in the node annotations. Layer2UsesTransitRouter bool @@ -252,6 +263,22 @@ const ( kubeServiceAccountFileCACert string = "ca.crt" ) +// No-overlay mode configuration option constants +const ( + // NoOverlayRoutingManaged indicates OVN-Kubernetes manages the routing + NoOverlayRoutingManaged string = "managed" + // NoOverlayRoutingUnmanaged indicates users manage the routing themselves + NoOverlayRoutingUnmanaged string = "unmanaged" + + // ManagedBGPTopologyFullMesh represents a full-mesh BGP topology + ManagedBGPTopologyFullMesh string = "full-mesh" + + // NoOverlaySNATEnabled enables SNAT for outbound traffic + NoOverlaySNATEnabled string = "enabled" + // NoOverlaySNATDisabled disables SNAT for outbound traffic + NoOverlaySNATDisabled string = "disabled" +) + // DefaultConfig holds parsed config file parameters and command-line overrides type DefaultConfig struct { // MTU value used for the overlay networks. @@ -335,6 +362,11 @@ type DefaultConfig struct { // UDNAllowedDefaultServices holds a list of namespaced names of // default cluster network services accessible from primary user-defined networks UDNAllowedDefaultServices []string + + // Transport specifies the transport technology used for the default network. + // Accepts: "geneve" or "no-overlay". + // Defaults to "geneve". + Transport string `gcfg:"transport"` } // LoggingConfig holds logging-related parsed config file parameters and command-line overrides @@ -406,6 +438,7 @@ type KubernetesConfig struct { CertDuration time.Duration `gcfg:"cert-duration"` Kubeconfig string `gcfg:"kubeconfig"` CACert string `gcfg:"cacert"` + CACertData string `gcfg:"cacert-data"` CAData []byte APIServer string `gcfg:"apiserver"` Token string `gcfg:"token"` @@ -464,6 +497,7 @@ type OVNKubernetesFeatureConfig struct { EnableNetworkConnect bool `gcfg:"enable-network-connect"` EnablePreconfiguredUDNAddresses bool `gcfg:"enable-preconfigured-udn-addresses"` EnableRouteAdvertisements bool `gcfg:"enable-route-advertisements"` + EnableEVPN bool `gcfg:"enable-evpn"` EnableMultiNetworkPolicy bool `gcfg:"enable-multi-networkpolicy"` EnableStatelessNetPol bool `gcfg:"enable-stateless-netpol"` EnableInterconnect bool `gcfg:"enable-interconnect"` @@ -476,6 +510,10 @@ type OVNKubernetesFeatureConfig struct { // This feature requires a kernel fix https://github.com/torvalds/linux/commit/7f3287db654395f9c5ddd246325ff7889f550286 // to work on a kind cluster. Flag allows to disable it for current CI, will be turned on when github runners have this fix. AdvertisedUDNIsolationMode string `gcfg:"advertised-udn-isolation-mode"` + EnableDynamicUDNAllocation bool `gcfg:"enable-dynamic-udn-allocation"` + // UDNDeletionGracePeriod specified in number of seconds to wait before garbage collecting a UDN. Applies + // only when Dynamic UDN Allocation is enabled. + UDNDeletionGracePeriod time.Duration `gcfg:"udn-deletion-grace-period"` } // GatewayMode holds the node gateway mode @@ -614,6 +652,32 @@ type ClusterManagerConfig struct { V6TransitSubnet string `gcfg:"v6-transit-subnet"` } +// NoOverlayConfig holds configuration for no-overlay mode +type NoOverlayConfig struct { + // OutboundSNAT configures SNAT behavior for outbound traffic from pods on the default network. + // Supported values: "enabled" or "disabled". + // Required when transport=no-overlay. + OutboundSNAT string `gcfg:"outbound-snat"` + // Routing configures whether the pod network routing configuration is managed by + // OVN-Kubernetes or users. Supported values: "managed" or "unmanaged". + // Required when transport=no-overlay. + Routing string `gcfg:"routing"` +} + +// ManagedBGPConfig holds configuration for managed BGP +type ManagedBGPConfig struct { + // ASNumber specifies the AS number to be used by the BGP speakers on each node for its + // default VRF when no-overlay networks are configured with managed routing. + // It is shared by both the cluster default network and CUDNs. + // Supports both 16-bit (1-65535) and 32-bit (1-4294967295) AS numbers. + // Optional. Defaults to 64512 if not specified. + ASNumber uint32 `gcfg:"as-number"` + // Topology configures the BGP peering topology when routing is managed. + // Supported values: "full-mesh". + // Required when transport=no-overlay and routing=managed. + Topology string `gcfg:"topology"` +} + // OvnDBScheme describes the OVN database connection transport method type OvnDBScheme string @@ -645,6 +709,8 @@ type config struct { OvnKubeNode OvnKubeNodeConfig ClusterManager ClusterManagerConfig OvsPaths OvsPathConfig + NoOverlay NoOverlayConfig `gcfg:"no-overlay"` + ManagedBGP ManagedBGPConfig `gcfg:"bgp-managed"` } var ( @@ -665,6 +731,8 @@ var ( savedOvnKubeNode OvnKubeNodeConfig savedClusterManager ClusterManagerConfig savedOvsPaths OvsPathConfig + savedNoOverlay NoOverlayConfig + savedManagedBGP ManagedBGPConfig // legacy service-cluster-ip-range CLI option serviceClusterIPRange string @@ -695,6 +763,8 @@ func init() { savedOvnKubeNode = OvnKubeNode savedClusterManager = ClusterManager savedOvsPaths = OvsPaths + savedNoOverlay = NoOverlay + savedManagedBGP = ManagedBGP cli.VersionPrinter = func(_ *cli.Context) { fmt.Printf("Version: %s\n", Version) fmt.Printf("Git commit: %s\n", Commit) @@ -726,6 +796,8 @@ func PrepareTestConfig() error { OvnKubeNode = savedOvnKubeNode ClusterManager = savedClusterManager OvsPaths = savedOvsPaths + NoOverlay = savedNoOverlay + ManagedBGP = savedManagedBGP Kubernetes.DisableRequestedChassis = false EnableMulticast = false UnprivilegedMode = false @@ -748,6 +820,7 @@ func PrepareTestConfig() error { // Don't pick up defaults from the environment os.Unsetenv("KUBECONFIG") os.Unsetenv("K8S_CACERT") + os.Unsetenv("K8S_CACERT_DATA") os.Unsetenv("K8S_APISERVER") os.Unsetenv("K8S_TOKEN") os.Unsetenv("K8S_TOKEN_FILE") @@ -864,7 +937,7 @@ var CommonFlags = []cli.Flag{ }, &cli.StringFlag{ Name: "encap-type", - Usage: "The encapsulation protocol to use to transmit packets between hypervisors", + Usage: "The encapsulation protocol to use to transmit packets between hypervisors by OVN in overlay mode (geneve, vxlan, gre)", Destination: &cliConfig.Default.EncapType, Value: Default.EncapType, }, @@ -959,6 +1032,12 @@ var CommonFlags = []cli.Flag{ "it defaults to 24 if unspecified.", Destination: &cliConfig.Default.RawClusterSubnets, }, + &cli.StringFlag{ + Name: "transport", + Value: Default.Transport, + Usage: "Transport technology used for the default network, default to geneve if unspecified. (geneve, no-overlay)", + Destination: &cliConfig.Default.Transport, + }, &cli.BoolFlag{ Name: "unprivileged-mode", Usage: "Run ovnkube-node container in unprivileged mode. Valid only with --init-node option.", @@ -1170,6 +1249,12 @@ var OVNK8sFeatureFlags = []cli.Flag{ Destination: &cliConfig.OVNKubernetesFeature.EnableRouteAdvertisements, Value: OVNKubernetesFeature.EnableRouteAdvertisements, }, + &cli.BoolFlag{ + Name: "enable-evpn", + Usage: "Use EVPN feature with ovn-kubernetes. Requires route advertisements.", + Destination: &cliConfig.OVNKubernetesFeature.EnableEVPN, + Value: OVNKubernetesFeature.EnableEVPN, + }, &cli.StringFlag{ Name: "advertised-udn-isolation-mode", Usage: "Use pod isolation for BGP advertised UDN networks. Valid values are 'strict' or 'loose'.", @@ -1230,6 +1315,19 @@ var OVNK8sFeatureFlags = []cli.Flag{ Destination: &cliConfig.OVNKubernetesFeature.EnableNetworkQoS, Value: OVNKubernetesFeature.EnableNetworkQoS, }, + &cli.BoolFlag{ + Name: "enable-dynamic-udn-allocation", + Usage: "Configure to use the dynamic UDN allocation feature with ovn-kubernetes.", + Destination: &cliConfig.OVNKubernetesFeature.EnableDynamicUDNAllocation, + Value: OVNKubernetesFeature.EnableDynamicUDNAllocation, + }, + &cli.DurationFlag{ + Name: "udn-deletion-grace-period", + Usage: "Delay time in seconds that a node will wait before removing a UDN when the dynamic UDN allocation " + + "feature is used.", + Destination: &cliConfig.OVNKubernetesFeature.UDNDeletionGracePeriod, + Value: OVNKubernetesFeature.UDNDeletionGracePeriod, + }, } // K8sFlags capture Kubernetes-related options @@ -1285,6 +1383,11 @@ var K8sFlags = []cli.Flag{ Usage: "the absolute path to the Kubernetes API CA certificate (not required if --k8s-kubeconfig is given)", Destination: &cliConfig.Kubernetes.CACert, }, + &cli.StringFlag{ + Name: "k8s-cacert-data", + Usage: "the Base64 encoded Kubernetes API CA certificate data (not required if --k8s-kubeconfig is given)", + Destination: &cliConfig.Kubernetes.CACertData, + }, &cli.StringFlag{ Name: "k8s-token", Usage: "the Kubernetes API authentication token (not required if --k8s-kubeconfig is given)", @@ -1851,8 +1954,46 @@ func setOVSExternalID(exec kexec.Interface, key, value string) error { return nil } +// reconcileKubernetesAuthFields ensures that if a config stage provides Token/TokenFile +// or CACert/CACertData, stale value for any of these set by previous stage is cleared. +// This is required since any combination of these fields could be set by any stage +// and might get overwritten only partially. +func reconcileKubernetesAuthFields(k *KubernetesConfig, override *KubernetesConfig) { + // If this stage provided either Token or TokenFile, clear the other field + // not provided by this stage. + overrideHasToken := override.Token != "" + overrideHasTokenFile := override.TokenFile != "" + + if overrideHasToken || overrideHasTokenFile { + if !overrideHasToken { + k.Token = "" + } + if !overrideHasTokenFile { + k.TokenFile = "" + } + } + + // If this stage provided either CACert or CACertData, clear the other field + // not provided by this stage. + overrideHasCACert := override.CACert != "" + overrideHasCACertData := override.CACertData != "" + + if overrideHasCACert || overrideHasCACertData { + if !overrideHasCACert { + k.CACert = "" + } + if !overrideHasCACertData { + k.CACertData = "" + } + } +} + func buildKubernetesConfig(exec kexec.Interface, cli, file *config, saPath string, defaults *Defaults) error { - // token adn ca.crt may be from files mounted in container. + // values for token, cacert, kubeconfig, api-server may be found in several places. + // Priority order (highest first): OVS config, command line options, config file, + // environment variables, service account files + + // token and ca.crt may be from files mounted in container. saConfig := savedKubernetes if data, err := os.ReadFile(filepath.Join(saPath, kubeServiceAccountFileToken)); err == nil { saConfig.Token = string(data) @@ -1866,16 +2007,13 @@ func buildKubernetesConfig(exec kexec.Interface, cli, file *config, saPath strin return err } - // values for token, cacert, kubeconfig, api-server may be found in several places. - // Priority order (highest first): OVS config, command line options, config file, - // environment variables, service account files - envConfig := savedKubernetes envVarsMap := map[string]string{ "Kubeconfig": "KUBECONFIG", "BootstrapKubeconfig": "BOOTSTRAP_KUBECONFIG", "CertDir": "CERT_DIR", "CACert": "K8S_CACERT", + "CACertData": "K8S_CACERT_DATA", "APIServer": "K8S_APISERVER", "Token": "K8S_TOKEN", "TokenFile": "K8S_TOKEN_FILE", @@ -1890,16 +2028,19 @@ func buildKubernetesConfig(exec kexec.Interface, cli, file *config, saPath strin if err := overrideFields(&Kubernetes, &envConfig, &savedKubernetes); err != nil { return err } + reconcileKubernetesAuthFields(&Kubernetes, &envConfig) // Copy config file values over default values if err := overrideFields(&Kubernetes, &file.Kubernetes, &savedKubernetes); err != nil { return err } + reconcileKubernetesAuthFields(&Kubernetes, &file.Kubernetes) // And CLI overrides over config file and default values if err := overrideFields(&Kubernetes, &cli.Kubernetes, &savedKubernetes); err != nil { return err } + reconcileKubernetesAuthFields(&Kubernetes, &cli.Kubernetes) // Grab default values from OVS external IDs if defaults.K8sAPIServer { @@ -1920,8 +2061,15 @@ func buildKubernetesConfig(exec kexec.Interface, cli, file *config, saPath strin return fmt.Errorf("kubernetes kubeconfig file %q not found", Kubernetes.Kubeconfig) } - if Kubernetes.CACert != "" { - bytes, err := os.ReadFile(Kubernetes.CACert) + if Kubernetes.CACert != "" || Kubernetes.CACertData != "" { + var bytes []byte + var err error + if Kubernetes.CACert != "" { + bytes, err = os.ReadFile(Kubernetes.CACert) + } else { + bytes, err = base64.StdEncoding.DecodeString(Kubernetes.CACertData) + } + if err != nil { return err } @@ -2122,6 +2270,12 @@ func buildOVNKubernetesFeatureConfig(cli, file *config) error { return fmt.Errorf("invalid advertised-udn-isolation-mode %q: expect one of %s or %s", OVNKubernetesFeature.AdvertisedUDNIsolationMode, AdvertisedUDNIsolationModeStrict, AdvertisedUDNIsolationModeLoose) } + if OVNKubernetesFeature.EnableEVPN && !OVNKubernetesFeature.EnableRouteAdvertisements { + return fmt.Errorf("invalid feature configuration: EVPN requires route advertisements but route advertisements are disabled") + } + if OVNKubernetesFeature.EnableDynamicUDNAllocation && !OVNKubernetesFeature.EnableNetworkSegmentation { + return fmt.Errorf("the Dynamic UDN Allocation feature cannot be enabled without also enabling Network Segmentation") + } return nil } @@ -2269,6 +2423,115 @@ func buildClusterManagerConfig(cli, file *config) error { return nil } +// buildNoOverlayConfig updates NoOverlay config from config file only +// NoOverlay configuration is only available in config file, not via CLI flags +func buildNoOverlayConfig(file *config) error { + // Copy config file values over default values + if err := overrideFields(&NoOverlay, &file.NoOverlay, &savedNoOverlay); err != nil { + return err + } + + return nil +} + +// validateNoOverlayConfig validates the no-overlay configuration +func validateNoOverlayConfig() error { + // Validate transport option + if Default.Transport != types.NetworkTransportGeneve && Default.Transport != types.NetworkTransportNoOverlay { + return fmt.Errorf("invalid transport %q: must be %q or %q", Default.Transport, types.NetworkTransportGeneve, types.NetworkTransportNoOverlay) + } + + // If transport is no-overlay, validate required no-overlay options + if Default.Transport == types.NetworkTransportNoOverlay { + if !OVNKubernetesFeature.EnableRouteAdvertisements { + return fmt.Errorf("enable-route-advertisements must be true when transport=%q", types.NetworkTransportNoOverlay) + } + if NoOverlay.OutboundSNAT == "" { + return fmt.Errorf("outbound-snat is required when transport=no-overlay") + } + if NoOverlay.OutboundSNAT != NoOverlaySNATEnabled && NoOverlay.OutboundSNAT != NoOverlaySNATDisabled { + return fmt.Errorf("invalid outbound-snat %q: must be %q or %q", NoOverlay.OutboundSNAT, NoOverlaySNATEnabled, NoOverlaySNATDisabled) + } + + if NoOverlay.Routing == "" { + return fmt.Errorf("routing is required when transport=no-overlay") + } + if NoOverlay.Routing != NoOverlayRoutingManaged && NoOverlay.Routing != NoOverlayRoutingUnmanaged { + return fmt.Errorf("invalid routing %q: must be %q or %q", NoOverlay.Routing, NoOverlayRoutingManaged, NoOverlayRoutingUnmanaged) + } + + // If routing is managed, topology is required + if NoOverlay.Routing == NoOverlayRoutingManaged { + if ManagedBGP.Topology == "" { + return fmt.Errorf("topology is required when routing=managed") + } + if ManagedBGP.Topology != ManagedBGPTopologyFullMesh { + return fmt.Errorf("invalid topology %q: must be %q", ManagedBGP.Topology, ManagedBGPTopologyFullMesh) + } + } + } else { + // Warn if no-overlay or BGP config is specified but transport is not no-overlay + if NoOverlay.OutboundSNAT != "" || NoOverlay.Routing != "" { + klog.Warningf("[no-overlay] configuration specified but transport is %q; configuration will be ignored", Default.Transport) + } + } + + return nil +} + +// validateConfig performs all configuration validations after configs are built and completed. +// This is the centralized place called after completeConfig() that orchestrates all validations. +func validateConfig() error { + // Validate managed BGP configuration + if err := validateManagedBGPConfig(); err != nil { + return err + } + + // Validate no-overlay/transport configuration + if err := validateNoOverlayConfig(); err != nil { + return err + } + + return nil +} + +// buildManagedBGPConfig updates managed BGP config from config file only +// ManagedBGP configuration is only available in config file, not via CLI flags +func buildManagedBGPConfig(file *config) error { + // Copy config file values over default values + if err := overrideFields(&ManagedBGP, &file.ManagedBGP, &savedManagedBGP); err != nil { + return err + } + + return nil +} + +// validateManagedBGPConfig validates the managed BGP configuration +func validateManagedBGPConfig() error { + // Validate AS number is in valid range + // Valid AS numbers: 1-4294967295 (32-bit) + // Reserved ranges: + // 0 - Reserved (RFC 7607) + // 23456 - AS_TRANS (RFC 6793) + // 65535 - Reserved (RFC 7300) + // 4294967295 - Reserved (RFC 7300) + + if ManagedBGP.ASNumber == 0 { + return fmt.Errorf("invalid as-number: 0 is reserved") + } + if ManagedBGP.ASNumber == 23456 { + return fmt.Errorf("invalid as-number: 23456 is reserved (AS_TRANS for 16-bit to 32-bit AS translation)") + } + if ManagedBGP.ASNumber == 65535 { + return fmt.Errorf("invalid as-number: 65535 is reserved") + } + if ManagedBGP.ASNumber == 4294967295 { + return fmt.Errorf("invalid as-number: 4294967295 is reserved") + } + + return nil +} + // completeClusterManagerConfig completes the ClusterManager config by parsing raw values // into their final form. func completeClusterManagerConfig(allSubnets *ConfigSubnets) error { @@ -2321,6 +2584,7 @@ func buildDefaultConfig(cli, file *config) error { if Default.Zone == "" { Default.Zone = types.OvnDefaultZone } + return nil } @@ -2401,6 +2665,7 @@ func stripTokenFromK8sConfig() KubernetesConfig { // Token and CAData are sensitive fields so stripping // them while logging. k8sConf.Token = "" + k8sConf.CACertData = "" k8sConf.CAData = []byte{} return k8sConf } @@ -2431,6 +2696,8 @@ func initConfigWithPath(ctx *cli.Context, exec kexec.Interface, saPath string, d OvnKubeNode: savedOvnKubeNode, ClusterManager: savedClusterManager, OvsPaths: savedOvsPaths, + NoOverlay: savedNoOverlay, + ManagedBGP: savedManagedBGP, } configFile, configFileIsDefault = getConfigFilePath(ctx) @@ -2556,6 +2823,14 @@ func initConfigWithPath(ctx *cli.Context, exec kexec.Interface, saPath string, d return "", err } + if err = buildNoOverlayConfig(&cfg); err != nil { + return "", err + } + + if err = buildManagedBGPConfig(&cfg); err != nil { + return "", err + } + tmpAuth, err := buildOvnAuth(exec, true, &cliConfig.OvnNorth, &cfg.OvnNorth, defaults.OvnNorthAddress) if err != nil { return "", err @@ -2572,6 +2847,12 @@ func initConfigWithPath(ctx *cli.Context, exec kexec.Interface, saPath string, d return "", err } + // Perform cross-configuration validations + if err := validateConfig(); err != nil { + return "", err + } + + klog.V(5).Infof("Features config: %+v", OVNKubernetesFeature) klog.V(5).Infof("Default config: %+v", Default) klog.V(5).Infof("Logging config: %+v", Logging) klog.V(5).Infof("Monitoring config: %+v", Monitoring) @@ -2585,6 +2866,8 @@ func initConfigWithPath(ctx *cli.Context, exec kexec.Interface, saPath string, d klog.V(5).Infof("Ovnkube Node config: %+v", OvnKubeNode) klog.V(5).Infof("Ovnkube Cluster Manager config: %+v", ClusterManager) klog.V(5).Infof("OVS Paths config: %+v", OvsPaths) + klog.V(5).Infof("No Overlay config: %+v", NoOverlay) + klog.V(5).Infof("Managed BGP config: %+v", ManagedBGP) return retConfigFile, nil } diff --git a/go-controller/pkg/config/config_test.go b/go-controller/pkg/config/config_test.go index 6127dff90e..2a108e39d6 100644 --- a/go-controller/pkg/config/config_test.go +++ b/go-controller/pkg/config/config_test.go @@ -2121,4 +2121,255 @@ udn-allowed-default-services= ns/svc, ns1/svc1 gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) + + Describe("No-Overlay Configuration", func() { + BeforeEach(func() { + err := PrepareTestConfig() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Enable route advertisements - required for no-overlay transport + OVNKubernetesFeature.EnableRouteAdvertisements = true + }) + + It("validates transport option correctly", func() { + // Test valid geneve transport + Default.Transport = types.NetworkTransportGeneve + err := validateNoOverlayConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test valid no-overlay transport with required options + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.OutboundSNAT = NoOverlaySNATEnabled + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.Topology = ManagedBGPTopologyFullMesh + err = validateNoOverlayConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test invalid transport + Default.Transport = "invalid-transport" + err = validateNoOverlayConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("invalid transport")) + }) + + It("requires outbound-snat when transport is no-overlay", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.OutboundSNAT = "" + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.Topology = ManagedBGPTopologyFullMesh + err := validateNoOverlayConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("outbound-snat is required")) + }) + + It("validates outbound-snat values", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.Topology = ManagedBGPTopologyFullMesh + + // Test valid enable + NoOverlay.OutboundSNAT = NoOverlaySNATEnabled + err := validateNoOverlayConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test valid disable + NoOverlay.OutboundSNAT = NoOverlaySNATDisabled + err = validateNoOverlayConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test invalid value + NoOverlay.OutboundSNAT = "maybe" + err = validateNoOverlayConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("invalid outbound-snat")) + }) + + It("requires routing when transport is no-overlay", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.OutboundSNAT = NoOverlaySNATEnabled + NoOverlay.Routing = "" + err := validateNoOverlayConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("routing is required")) + }) + + It("validates routing values", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.OutboundSNAT = NoOverlaySNATEnabled + + // Test valid managed (requires topology) + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.Topology = ManagedBGPTopologyFullMesh + err := validateNoOverlayConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test valid unmanaged (topology not required) + NoOverlay.Routing = NoOverlayRoutingUnmanaged + ManagedBGP.Topology = "" + err = validateNoOverlayConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test invalid value + NoOverlay.Routing = "automatic" + err = validateNoOverlayConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("invalid routing")) + }) + + It("builds no-overlay config from file only", func() { + fileConfig := config{ + NoOverlay: NoOverlayConfig{ + OutboundSNAT: NoOverlaySNATEnabled, + Routing: NoOverlayRoutingManaged, + }, + ManagedBGP: ManagedBGPConfig{ + Topology: ManagedBGPTopologyFullMesh, + }, + } + err := buildNoOverlayConfig(&fileConfig) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = buildManagedBGPConfig(&fileConfig) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + // Config file values should be applied + gomega.Expect(NoOverlay.OutboundSNAT).To(gomega.Equal(NoOverlaySNATEnabled)) + gomega.Expect(NoOverlay.Routing).To(gomega.Equal(NoOverlayRoutingManaged)) + gomega.Expect(ManagedBGP.Topology).To(gomega.Equal(ManagedBGPTopologyFullMesh)) + }) + + It("requires topology when routing is managed", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.OutboundSNAT = NoOverlaySNATEnabled + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.Topology = "" + err := validateNoOverlayConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("topology is required when routing=managed")) + }) + + It("validates topology values", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.OutboundSNAT = NoOverlaySNATEnabled + NoOverlay.Routing = NoOverlayRoutingManaged + + // Test valid full-mesh + ManagedBGP.Topology = ManagedBGPTopologyFullMesh + err := validateNoOverlayConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test invalid value + ManagedBGP.Topology = "route-reflector" + err = validateNoOverlayConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("invalid topology")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring(`must be "full-mesh"`)) + }) + + It("does not require topology when routing is unmanaged", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.OutboundSNAT = NoOverlaySNATEnabled + NoOverlay.Routing = NoOverlayRoutingUnmanaged + ManagedBGP.Topology = "" + err := validateNoOverlayConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + }) + + Describe("BGP Configuration", func() { + BeforeEach(func() { + err := PrepareTestConfig() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + It("parses BGP config from file with all fields set", func() { + fileConfig := config{ + ManagedBGP: ManagedBGPConfig{ + Topology: ManagedBGPTopologyFullMesh, + ASNumber: 64500, + }, + } + err := buildManagedBGPConfig(&fileConfig) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(ManagedBGP.Topology).To(gomega.Equal(ManagedBGPTopologyFullMesh)) + gomega.Expect(ManagedBGP.ASNumber).To(gomega.Equal(uint32(64500))) + }) + + It("handles partial BGP config in file", func() { + fileConfig := config{ + ManagedBGP: savedManagedBGP, + } + fileConfig.ManagedBGP.Topology = ManagedBGPTopologyFullMesh + + err := buildManagedBGPConfig(&fileConfig) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(ManagedBGP.Topology).To(gomega.Equal(ManagedBGPTopologyFullMesh)) + // ASNumber should retain default value from init + gomega.Expect(ManagedBGP.ASNumber).To(gomega.Equal(uint32(64512))) + }) + + It("handles empty BGP config in file", func() { + fileConfig := config{ + ManagedBGP: savedManagedBGP, + } + err := buildManagedBGPConfig(&fileConfig) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + // Should retain default values without panicking + gomega.Expect(ManagedBGP.ASNumber).To(gomega.Equal(uint32(64512))) // default value + }) + + It("validates reserved AS number 0", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.ASNumber = 0 + err := validateManagedBGPConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("0 is reserved")) + }) + + It("validates reserved AS number 23456 (AS_TRANS)", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.ASNumber = 23456 + err := validateManagedBGPConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("23456 is reserved")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("AS_TRANS")) + }) + + It("validates reserved AS number 65535", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.ASNumber = 65535 + err := validateManagedBGPConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("65535 is reserved")) + }) + + It("validates reserved AS number 4294967295", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.Routing = NoOverlayRoutingManaged + ManagedBGP.ASNumber = 4294967295 + err := validateManagedBGPConfig() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("4294967295 is reserved")) + }) + + It("accepts valid AS numbers", func() { + Default.Transport = types.NetworkTransportNoOverlay + NoOverlay.Routing = NoOverlayRoutingManaged + + // Test valid 16-bit AS number + ManagedBGP.ASNumber = 64500 + err := validateManagedBGPConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test default AS number + ManagedBGP.ASNumber = 64512 + err = validateManagedBGPConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Test valid 32-bit AS number + ManagedBGP.ASNumber = 100000 + err = validateManagedBGPConfig() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + }) }) diff --git a/go-controller/pkg/controller/fake.go b/go-controller/pkg/controller/fake.go new file mode 100644 index 0000000000..5d4bff4f47 --- /dev/null +++ b/go-controller/pkg/controller/fake.go @@ -0,0 +1,36 @@ +package controller + +import ( + "fmt" + "sync" + "time" +) + +type FakeController struct { + sync.Mutex + Reconciles []string +} + +func (f *FakeController) Reconcile(key string) { + f.Lock() + defer f.Unlock() + f.Reconciles = append(f.Reconciles, fmt.Sprintf("Reconcile:%s", key)) +} + +func (f *FakeController) ReconcileRateLimited(key string) { + f.Lock() + defer f.Unlock() + f.Reconciles = append(f.Reconciles, fmt.Sprintf("RateLimited:%s", key)) +} + +func (f *FakeController) ReconcileAfter(key string, _ time.Duration) { + f.Lock() + defer f.Unlock() + f.Reconciles = append(f.Reconciles, fmt.Sprintf("After:%s", key)) +} + +func (f *FakeController) addHandler() error { return nil } +func (f *FakeController) startWorkers() error { return nil } +func (f *FakeController) stop() {} + +func (f *FakeController) ReconcileAll() {} diff --git a/go-controller/pkg/controllermanager/controller_manager.go b/go-controller/pkg/controllermanager/controller_manager.go index 1da184c0af..61a342f77a 100644 --- a/go-controller/pkg/controllermanager/controller_manager.go +++ b/go-controller/pkg/controllermanager/controller_manager.go @@ -81,9 +81,11 @@ func (cm *ControllerManager) NewNetworkController(nInfo util.NetInfo) (networkma topoType := nInfo.TopologyType() switch topoType { case ovntypes.Layer3Topology: - return ovn.NewLayer3UserDefinedNetworkController(cnci, nInfo, cm.networkManager.Interface(), cm.routeImportManager, cm.eIPController, cm.portCache) + return ovn.NewLayer3UserDefinedNetworkController(cnci, nInfo, cm.networkManager.Interface(), cm.routeImportManager, + cm.eIPController, cm.portCache) case ovntypes.Layer2Topology: - return ovn.NewLayer2UserDefinedNetworkController(cnci, nInfo, cm.networkManager.Interface(), cm.routeImportManager, cm.portCache, cm.eIPController) + return ovn.NewLayer2UserDefinedNetworkController(cnci, nInfo, cm.networkManager.Interface(), cm.routeImportManager, + cm.portCache, cm.eIPController) case ovntypes.LocalnetTopology: return ovn.NewLocalnetUserDefinedNetworkController(cnci, nInfo, cm.networkManager.Interface()), nil } @@ -494,6 +496,7 @@ func (cm *ControllerManager) Start(ctx context.Context) error { return fmt.Errorf("failed to initialize advertised network isolation: %w", err) } } + if cm.networkManager != nil { if err = cm.networkManager.Start(); err != nil { return fmt.Errorf("failed to start NAD Controller :%v", err) diff --git a/go-controller/pkg/controllermanager/node_controller_manager_test.go b/go-controller/pkg/controllermanager/node_controller_manager_test.go index 00243aa8f1..d9f862fbbf 100644 --- a/go-controller/pkg/controllermanager/node_controller_manager_test.go +++ b/go-controller/pkg/controllermanager/node_controller_manager_test.go @@ -62,6 +62,7 @@ var _ = Describe("Healthcheck tests", func() { var err error BeforeEach(func() { + Expect(config.PrepareTestConfig()).To(Succeed()) execMock = ovntest.NewFakeExec() Expect(util.SetExec(execMock)).To(Succeed()) factoryMock = factoryMocks.NodeWatchFactory{} @@ -138,10 +139,19 @@ var _ = Describe("Healthcheck tests", func() { BeforeEach(func() { // setup kube output - factoryMock.On("NADInformer").Return(nil) + factoryMock.On("GetPods", "").Return(podList, nil) + nadListerMock := &nadlistermocks.NetworkAttachmentDefinitionLister{} + nadInformerMock := &nadinformermocks.NetworkAttachmentDefinitionInformer{} + nadInformerMock.On("Lister").Return(nadListerMock) + nadInformerMock.On("Informer").Return(nil) + factoryMock.On("NADInformer").Return(nadInformerMock) + nodeInformerMock := &coreinformermocks.NodeInformer{} + nodeListerMock := &corelistermocks.NodeLister{} + nodeListerMock.On("List", mock.Anything).Return(nil, nil) + nodeInformerMock.On("Lister").Return(nodeListerMock) + factoryMock.On("NodeCoreInformer").Return(nodeInformerMock) ncm, err = NewNodeControllerManager(fakeClient, &factoryMock, nodeName, &sync.WaitGroup{}, nil, routeManager, nil) Expect(err).NotTo(HaveOccurred()) - factoryMock.On("GetPods", "").Return(podList, nil) }) Context("bridge has stale representor ports", func() { diff --git a/go-controller/pkg/crd/clusternetworkconnect/v1/apis/applyconfiguration/clusternetworkconnect/v1/connectednetworkstatus.go b/go-controller/pkg/crd/clusternetworkconnect/v1/apis/applyconfiguration/clusternetworkconnect/v1/connectednetworkstatus.go deleted file mode 100644 index 17f6642fa3..0000000000 --- a/go-controller/pkg/crd/clusternetworkconnect/v1/apis/applyconfiguration/clusternetworkconnect/v1/connectednetworkstatus.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1 - -import ( - types "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/types" -) - -// ConnectedNetworkStatusApplyConfiguration represents a declarative configuration of the ConnectedNetworkStatus type for use -// with apply. -type ConnectedNetworkStatusApplyConfiguration struct { - NetworkSelectionType *types.NetworkSelectionType `json:"networkSelectionType,omitempty"` - NetworkName *string `json:"networkName,omitempty"` - Namespace *string `json:"namespace,omitempty"` - Ready *bool `json:"ready,omitempty"` - Message *string `json:"message,omitempty"` -} - -// ConnectedNetworkStatusApplyConfiguration constructs a declarative configuration of the ConnectedNetworkStatus type for use with -// apply. -func ConnectedNetworkStatus() *ConnectedNetworkStatusApplyConfiguration { - return &ConnectedNetworkStatusApplyConfiguration{} -} - -// WithNetworkSelectionType sets the NetworkSelectionType field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the NetworkSelectionType field is set to the value of the last call. -func (b *ConnectedNetworkStatusApplyConfiguration) WithNetworkSelectionType(value types.NetworkSelectionType) *ConnectedNetworkStatusApplyConfiguration { - b.NetworkSelectionType = &value - return b -} - -// WithNetworkName sets the NetworkName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the NetworkName field is set to the value of the last call. -func (b *ConnectedNetworkStatusApplyConfiguration) WithNetworkName(value string) *ConnectedNetworkStatusApplyConfiguration { - b.NetworkName = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *ConnectedNetworkStatusApplyConfiguration) WithNamespace(value string) *ConnectedNetworkStatusApplyConfiguration { - b.Namespace = &value - return b -} - -// WithReady sets the Ready field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Ready field is set to the value of the last call. -func (b *ConnectedNetworkStatusApplyConfiguration) WithReady(value bool) *ConnectedNetworkStatusApplyConfiguration { - b.Ready = &value - return b -} - -// WithMessage sets the Message field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Message field is set to the value of the last call. -func (b *ConnectedNetworkStatusApplyConfiguration) WithMessage(value string) *ConnectedNetworkStatusApplyConfiguration { - b.Message = &value - return b -} diff --git a/go-controller/pkg/crd/routeadvertisements/v1/apis/applyconfiguration/routeadvertisements/v1/advertisements.go b/go-controller/pkg/crd/routeadvertisements/v1/apis/applyconfiguration/routeadvertisements/v1/advertisements.go deleted file mode 100644 index 49300610c3..0000000000 --- a/go-controller/pkg/crd/routeadvertisements/v1/apis/applyconfiguration/routeadvertisements/v1/advertisements.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1 - -// AdvertisementsApplyConfiguration represents a declarative configuration of the Advertisements type for use -// with apply. -type AdvertisementsApplyConfiguration struct { - PodNetwork *bool `json:"podNetwork,omitempty"` - EgressIP *bool `json:"egressIP,omitempty"` -} - -// AdvertisementsApplyConfiguration constructs a declarative configuration of the Advertisements type for use with -// apply. -func Advertisements() *AdvertisementsApplyConfiguration { - return &AdvertisementsApplyConfiguration{} -} - -// WithPodNetwork sets the PodNetwork field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the PodNetwork field is set to the value of the last call. -func (b *AdvertisementsApplyConfiguration) WithPodNetwork(value bool) *AdvertisementsApplyConfiguration { - b.PodNetwork = &value - return b -} - -// WithEgressIP sets the EgressIP field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the EgressIP field is set to the value of the last call. -func (b *AdvertisementsApplyConfiguration) WithEgressIP(value bool) *AdvertisementsApplyConfiguration { - b.EgressIP = &value - return b -} diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/evpnconfig.go b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/evpnconfig.go new file mode 100644 index 0000000000..a01cc75607 --- /dev/null +++ b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/evpnconfig.go @@ -0,0 +1,56 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1 + +// EVPNConfigApplyConfiguration represents a declarative configuration of the EVPNConfig type for use +// with apply. +type EVPNConfigApplyConfiguration struct { + VTEP *string `json:"vtep,omitempty"` + MACVRF *VRFConfigApplyConfiguration `json:"macVRF,omitempty"` + IPVRF *VRFConfigApplyConfiguration `json:"ipVRF,omitempty"` +} + +// EVPNConfigApplyConfiguration constructs a declarative configuration of the EVPNConfig type for use with +// apply. +func EVPNConfig() *EVPNConfigApplyConfiguration { + return &EVPNConfigApplyConfiguration{} +} + +// WithVTEP sets the VTEP field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the VTEP field is set to the value of the last call. +func (b *EVPNConfigApplyConfiguration) WithVTEP(value string) *EVPNConfigApplyConfiguration { + b.VTEP = &value + return b +} + +// WithMACVRF sets the MACVRF field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the MACVRF field is set to the value of the last call. +func (b *EVPNConfigApplyConfiguration) WithMACVRF(value *VRFConfigApplyConfiguration) *EVPNConfigApplyConfiguration { + b.MACVRF = value + return b +} + +// WithIPVRF sets the IPVRF field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the IPVRF field is set to the value of the last call. +func (b *EVPNConfigApplyConfiguration) WithIPVRF(value *VRFConfigApplyConfiguration) *EVPNConfigApplyConfiguration { + b.IPVRF = value + return b +} diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/networkspec.go b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/networkspec.go index fa3320e318..cb74580abe 100644 --- a/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/networkspec.go +++ b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/networkspec.go @@ -24,12 +24,13 @@ import ( // NetworkSpecApplyConfiguration represents a declarative configuration of the NetworkSpec type for use // with apply. type NetworkSpecApplyConfiguration struct { - Topology *userdefinednetworkv1.NetworkTopology `json:"topology,omitempty"` - Layer3 *Layer3ConfigApplyConfiguration `json:"layer3,omitempty"` - Layer2 *Layer2ConfigApplyConfiguration `json:"layer2,omitempty"` - Localnet *LocalnetConfigApplyConfiguration `json:"localnet,omitempty"` - Transport *userdefinednetworkv1.TransportOption `json:"transport,omitempty"` - NoOverlayOptions *NoOverlayOptionsApplyConfiguration `json:"noOverlayOptions,omitempty"` + Topology *userdefinednetworkv1.NetworkTopology `json:"topology,omitempty"` + Layer3 *Layer3ConfigApplyConfiguration `json:"layer3,omitempty"` + Layer2 *Layer2ConfigApplyConfiguration `json:"layer2,omitempty"` + Localnet *LocalnetConfigApplyConfiguration `json:"localnet,omitempty"` + Transport *userdefinednetworkv1.TransportOption `json:"transport,omitempty"` + NoOverlay *NoOverlayConfigApplyConfiguration `json:"noOverlay,omitempty"` + EVPN *EVPNConfigApplyConfiguration `json:"evpn,omitempty"` } // NetworkSpecApplyConfiguration constructs a declarative configuration of the NetworkSpec type for use with @@ -78,10 +79,18 @@ func (b *NetworkSpecApplyConfiguration) WithTransport(value userdefinednetworkv1 return b } -// WithNoOverlayOptions sets the NoOverlayOptions field in the declarative configuration to the given value +// WithNoOverlay sets the NoOverlay field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the NoOverlayOptions field is set to the value of the last call. -func (b *NetworkSpecApplyConfiguration) WithNoOverlayOptions(value *NoOverlayOptionsApplyConfiguration) *NetworkSpecApplyConfiguration { - b.NoOverlayOptions = value +// If called multiple times, the NoOverlay field is set to the value of the last call. +func (b *NetworkSpecApplyConfiguration) WithNoOverlay(value *NoOverlayConfigApplyConfiguration) *NetworkSpecApplyConfiguration { + b.NoOverlay = value + return b +} + +// WithEVPN sets the EVPN field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the EVPN field is set to the value of the last call. +func (b *NetworkSpecApplyConfiguration) WithEVPN(value *EVPNConfigApplyConfiguration) *NetworkSpecApplyConfiguration { + b.EVPN = value return b } diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/nooverlayoptions.go b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/nooverlayconfig.go similarity index 70% rename from go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/nooverlayoptions.go rename to go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/nooverlayconfig.go index eb91057663..048c6004df 100644 --- a/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/nooverlayoptions.go +++ b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/nooverlayconfig.go @@ -21,23 +21,23 @@ import ( userdefinednetworkv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1" ) -// NoOverlayOptionsApplyConfiguration represents a declarative configuration of the NoOverlayOptions type for use +// NoOverlayConfigApplyConfiguration represents a declarative configuration of the NoOverlayConfig type for use // with apply. -type NoOverlayOptionsApplyConfiguration struct { +type NoOverlayConfigApplyConfiguration struct { OutboundSNAT *userdefinednetworkv1.SNATOption `json:"outboundSNAT,omitempty"` Routing *userdefinednetworkv1.RoutingOption `json:"routing,omitempty"` } -// NoOverlayOptionsApplyConfiguration constructs a declarative configuration of the NoOverlayOptions type for use with +// NoOverlayConfigApplyConfiguration constructs a declarative configuration of the NoOverlayConfig type for use with // apply. -func NoOverlayOptions() *NoOverlayOptionsApplyConfiguration { - return &NoOverlayOptionsApplyConfiguration{} +func NoOverlayConfig() *NoOverlayConfigApplyConfiguration { + return &NoOverlayConfigApplyConfiguration{} } // WithOutboundSNAT sets the OutboundSNAT field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the OutboundSNAT field is set to the value of the last call. -func (b *NoOverlayOptionsApplyConfiguration) WithOutboundSNAT(value userdefinednetworkv1.SNATOption) *NoOverlayOptionsApplyConfiguration { +func (b *NoOverlayConfigApplyConfiguration) WithOutboundSNAT(value userdefinednetworkv1.SNATOption) *NoOverlayConfigApplyConfiguration { b.OutboundSNAT = &value return b } @@ -45,7 +45,7 @@ func (b *NoOverlayOptionsApplyConfiguration) WithOutboundSNAT(value userdefinedn // WithRouting sets the Routing field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the Routing field is set to the value of the last call. -func (b *NoOverlayOptionsApplyConfiguration) WithRouting(value userdefinednetworkv1.RoutingOption) *NoOverlayOptionsApplyConfiguration { +func (b *NoOverlayConfigApplyConfiguration) WithRouting(value userdefinednetworkv1.RoutingOption) *NoOverlayConfigApplyConfiguration { b.Routing = &value return b } diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/vrfconfig.go b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/vrfconfig.go new file mode 100644 index 0000000000..da519e3136 --- /dev/null +++ b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/userdefinednetwork/v1/vrfconfig.go @@ -0,0 +1,51 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1 + +import ( + userdefinednetworkv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1" +) + +// VRFConfigApplyConfiguration represents a declarative configuration of the VRFConfig type for use +// with apply. +type VRFConfigApplyConfiguration struct { + VNI *int32 `json:"vni,omitempty"` + RouteTarget *userdefinednetworkv1.RouteTargetString `json:"routeTarget,omitempty"` +} + +// VRFConfigApplyConfiguration constructs a declarative configuration of the VRFConfig type for use with +// apply. +func VRFConfig() *VRFConfigApplyConfiguration { + return &VRFConfigApplyConfiguration{} +} + +// WithVNI sets the VNI field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the VNI field is set to the value of the last call. +func (b *VRFConfigApplyConfiguration) WithVNI(value int32) *VRFConfigApplyConfiguration { + b.VNI = &value + return b +} + +// WithRouteTarget sets the RouteTarget field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the RouteTarget field is set to the value of the last call. +func (b *VRFConfigApplyConfiguration) WithRouteTarget(value userdefinednetworkv1.RouteTargetString) *VRFConfigApplyConfiguration { + b.RouteTarget = &value + return b +} diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/utils.go b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/utils.go index c28c93a108..8257867595 100644 --- a/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/utils.go +++ b/go-controller/pkg/crd/userdefinednetwork/v1/apis/applyconfiguration/utils.go @@ -39,6 +39,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &userdefinednetworkv1.ClusterUserDefinedNetworkSpecApplyConfiguration{} case v1.SchemeGroupVersion.WithKind("ClusterUserDefinedNetworkStatus"): return &userdefinednetworkv1.ClusterUserDefinedNetworkStatusApplyConfiguration{} + case v1.SchemeGroupVersion.WithKind("EVPNConfig"): + return &userdefinednetworkv1.EVPNConfigApplyConfiguration{} case v1.SchemeGroupVersion.WithKind("IPAMConfig"): return &userdefinednetworkv1.IPAMConfigApplyConfiguration{} case v1.SchemeGroupVersion.WithKind("Layer2Config"): @@ -51,8 +53,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &userdefinednetworkv1.LocalnetConfigApplyConfiguration{} case v1.SchemeGroupVersion.WithKind("NetworkSpec"): return &userdefinednetworkv1.NetworkSpecApplyConfiguration{} - case v1.SchemeGroupVersion.WithKind("NoOverlayOptions"): - return &userdefinednetworkv1.NoOverlayOptionsApplyConfiguration{} + case v1.SchemeGroupVersion.WithKind("NoOverlayConfig"): + return &userdefinednetworkv1.NoOverlayConfigApplyConfiguration{} case v1.SchemeGroupVersion.WithKind("UserDefinedNetwork"): return &userdefinednetworkv1.UserDefinedNetworkApplyConfiguration{} case v1.SchemeGroupVersion.WithKind("UserDefinedNetworkSpec"): @@ -61,6 +63,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &userdefinednetworkv1.UserDefinedNetworkStatusApplyConfiguration{} case v1.SchemeGroupVersion.WithKind("VLANConfig"): return &userdefinednetworkv1.VLANConfigApplyConfiguration{} + case v1.SchemeGroupVersion.WithKind("VRFConfig"): + return &userdefinednetworkv1.VRFConfigApplyConfiguration{} } return nil diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/cudn.go b/go-controller/pkg/crd/userdefinednetwork/v1/cudn.go index 6c3510a979..8440bef105 100644 --- a/go-controller/pkg/crd/userdefinednetwork/v1/cudn.go +++ b/go-controller/pkg/crd/userdefinednetwork/v1/cudn.go @@ -34,8 +34,14 @@ type ClusterUserDefinedNetworkSpec struct { // +kubebuilder:validation:XValidation:rule="has(self.topology) && self.topology == 'Layer2' ? has(self.layer2): !has(self.layer2)", message="spec.layer2 is required when topology is Layer2 and forbidden otherwise" // +kubebuilder:validation:XValidation:rule="has(self.topology) && self.topology == 'Localnet' ? has(self.localnet): !has(self.localnet)", message="spec.localnet is required when topology is Localnet and forbidden otherwise" // +kubebuilder:validation:XValidation:rule="!has(self.transport) || self.transport != 'NoOverlay' || (self.topology == 'Layer3' && has(self.layer3) && self.layer3.role == 'Primary')", message="transport 'NoOverlay' is only supported for Layer3 primary networks" - // +kubebuilder:validation:XValidation:rule="!has(self.transport) || self.transport != 'NoOverlay' || has(self.noOverlayOptions)", message="noOverlayOptions is required when transport is 'NoOverlay'" - // +kubebuilder:validation:XValidation:rule="self.transport == 'NoOverlay' || !has(self.noOverlayOptions)", message="noOverlayOptions is forbidden when transport is not 'NoOverlay'" + // +kubebuilder:validation:XValidation:rule="!has(self.transport) || self.transport != 'NoOverlay' || has(self.noOverlay)", message="spec.noOverlay is required when type transport is 'NoOverlay'" + // +kubebuilder:validation:XValidation:rule="self.transport == 'NoOverlay' || !has(self.noOverlay)", message="spec.noOverlay is forbidden when transport type is not 'NoOverlay'" + // +kubebuilder:validation:XValidation:rule="!has(self.transport) || self.transport != 'EVPN' || ((self.topology == 'Layer2' && has(self.layer2) && self.layer2.role == 'Primary') || (self.topology == 'Layer3' && has(self.layer3) && self.layer3.role == 'Primary'))", message="transport 'EVPN' is only supported for Layer2 or Layer3 primary networks" + // +kubebuilder:validation:XValidation:rule="!has(self.transport) || self.transport != 'EVPN' || has(self.evpn)", message="spec.evpn field is required when transport is 'EVPN'" + // +kubebuilder:validation:XValidation:rule="self.transport == 'EVPN' || !has(self.evpn)", message="spec.evpn field is forbidden when transport is not 'EVPN'" + // +kubebuilder:validation:XValidation:rule="!has(self.transport) || self.transport != 'EVPN' || self.topology != 'Layer2' || (has(self.evpn) && has(self.evpn.macVRF))", message="spec.evpn.macVRF field is required for Layer2 topology when transport is 'EVPN'" + // +kubebuilder:validation:XValidation:rule="!has(self.transport) || self.transport != 'EVPN' || self.topology != 'Layer3' || (has(self.evpn) && has(self.evpn.ipVRF))", message="spec.evpn.ipVRF field is required for Layer3 topology when transport is 'EVPN'" + // +kubebuilder:validation:XValidation:rule="!has(self.transport) || self.transport != 'EVPN' || self.topology != 'Layer3' || !has(self.evpn) || !has(self.evpn.macVRF)", message="spec.evpn.macVRF field is forbidden for Layer3 topology when transport is 'EVPN'" // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Network spec is immutable" // +required Network NetworkSpec `json:"network"` @@ -70,17 +76,22 @@ type NetworkSpec struct { Localnet *LocalnetConfig `json:"localnet,omitempty"` // Transport describes the transport technology for pod-to-pod traffic. - // Allowed values are "NoOverlay" and "Geneve". + // Allowed values are "NoOverlay", "Geneve", and "EVPN". // - "NoOverlay": The network operates in no-overlay mode. // - "Geneve": The network uses Geneve overlay. + // - "EVPN": The network uses EVPN transport. // When omitted, the default behaviour is Geneve. - // +kubebuilder:validation:Enum=NoOverlay;Geneve + // +kubebuilder:validation:Enum=NoOverlay;Geneve;EVPN // +optional Transport TransportOption `json:"transport,omitempty"` - // NoOverlayOptions contains configuration for no-overlay mode. + // NoOverlay contains configuration for no-overlay mode. // This is only allowed when Transport is "NoOverlay". // +optional - NoOverlayOptions *NoOverlayOptions `json:"noOverlayOptions,omitempty"` + NoOverlay *NoOverlayConfig `json:"noOverlay,omitempty"` + // EVPN contains configuration for EVPN mode. + // This is only allowed when Transport is "EVPN". + // +optional + EVPN *EVPNConfig `json:"evpn,omitempty"` } // ClusterUserDefinedNetworkStatus contains the observed status of the ClusterUserDefinedNetwork. @@ -242,6 +253,7 @@ type RoutingOption string const ( TransportOptionNoOverlay TransportOption = "NoOverlay" TransportOptionGeneve TransportOption = "Geneve" + TransportOptionEVPN TransportOption = "EVPN" SNATEnabled SNATOption = "Enabled" SNATDisabled SNATOption = "Disabled" @@ -250,8 +262,8 @@ const ( RoutingUnmanaged RoutingOption = "Unmanaged" ) -// NoOverlayOptions contains configuration options for networks operating in no-overlay mode. -type NoOverlayOptions struct { +// NoOverlayConfig contains configuration options for networks operating in no-overlay mode. +type NoOverlayConfig struct { // OutboundSNAT defines the SNAT behavior for outbound traffic from pods. // +kubebuilder:validation:Enum=Enabled;Disabled // +required diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/evpn.go b/go-controller/pkg/crd/userdefinednetwork/v1/evpn.go new file mode 100644 index 0000000000..5cfb599b42 --- /dev/null +++ b/go-controller/pkg/crd/userdefinednetwork/v1/evpn.go @@ -0,0 +1,95 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// EVPNConfig contains configuration options for networks operating in EVPN mode. +// +kubebuilder:validation:XValidation:rule="has(self.macVRF) || has(self.ipVRF)", message="at least one of macVRF or ipVRF must be specified" +type EVPNConfig struct { + // VTEP is the name of the VTEP CR that defines VTEP IPs for EVPN. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +required + VTEP string `json:"vtep"` + + // MACVRF contains the MAC-VRF configuration for Layer 2 EVPN. + // This field is required for Layer2 topology and forbidden for Layer3 topology. + // +optional + MACVRF *VRFConfig `json:"macVRF,omitempty"` + + // IPVRF contains the IP-VRF configuration for Layer 3 EVPN. + // This field is required for Layer3 topology and optional for Layer2 topology. + // +optional + IPVRF *VRFConfig `json:"ipVRF,omitempty"` +} + +// RouteTargetString represents the 6-byte value of a BGP extended community route target (RFC 4360). +// BGP Extended Communities are 8 bytes total: 2-byte type field + 6-byte value field. +// This string encodes the 6-byte value, split between a global administrator (Autonomous System or IPv4) and a local administrator. +// +// When auto-generated, the local administrator is set to the VNI, creating a natural mapping +// between Route Targets and VXLAN network segments (e.g., "65000:100" for AS 65000 and VNI 100). +// When explicitly specified, the local administrator can be any value within the type constraints. +// +// FRR EVPN L3 Route-Targets use format (A.B.C.D:MN|EF:OPQR|GHJK:MN|*:OPQR|*:MN) where: +// - EF:OPQR = 2-byte AS (1-65535) : local administrator (4 bytes, 0-4294967295) +// - GHJK:MN = 4-byte AS (65536-4294967295) : local administrator (2 bytes, 0-65535) +// - A.B.C.D:MN = IPv4 address (4 bytes) : local administrator (2 bytes, 0-65535) +// - *:OPQR = wildcard AS : local administrator (4 bytes, 0-4294967295) - for import matching +// - *:MN = wildcard AS : local administrator (2 bytes, 0-65535) - for import matching +// +// The 6-byte constraint means: if AS is 4 bytes, local admin can only be 2 bytes, and vice versa. +// Wildcard (*) matches any AS and is useful for import rules in Downstream VNI scenarios. +// Note: VNI is 24-bit (max 16777215), so auto-generation with 4-byte AS or IPv4 only works if VNI <= 65535. +// See: https://docs.frrouting.org/en/stable-8.5/bgp.html#evpn-l3-route-targets +// +// +kubebuilder:validation:MaxLength=21 +// +kubebuilder:validation:XValidation:rule="self.split(':').size() == 2",message="RT must contain exactly one colon" +// +kubebuilder:validation:XValidation:rule="self.split(':').size() != 2 || (self.startsWith('*:') || isIP(self.split(':')[0]) || self.split(':')[0].matches('[0-9]+'))",message="RT global administrator must be either '*', an IPv4 address, or a number" +// +kubebuilder:validation:XValidation:rule="self.split(':').size() != 2 || self.split(':')[1].matches('[0-9]+')",message="RT local administrator must be a number" +// +kubebuilder:validation:XValidation:rule="self.split(':').size() != 2 || !self.startsWith('*:') || (self.split(':')[1].matches('[0-9]+') && uint(self.split(':')[1]) <= 4294967295u)",message="RT with wildcard global administrator must have format *:OPQR where OPQR <= 4294967295" +// +kubebuilder:validation:XValidation:rule="self.split(':').size() != 2 || !self.split(':')[0].contains('.') || (self.split(':')[1].matches('[0-9]+') && uint(self.split(':')[1]) <= 65535u)",message="RT with IPv4 global administrator must have format A.B.C.D:MN where MN <= 65535" +// +kubebuilder:validation:XValidation:rule="self.split(':').size() != 2 || self.startsWith('*:') || self.split(':')[0].contains('.') || !self.split(':')[0].matches('[0-9]+') || !self.split(':')[1].matches('[0-9]+') || uint(self.split(':')[0]) <= 65535u || uint(self.split(':')[1]) <= 65535u",message="RT with 4-byte ASN global administrator must have format GHJK:MN where GHJK <= 4294967295 and MN <= 65535" +// +kubebuilder:validation:XValidation:rule="self.split(':').size() != 2 || self.startsWith('*:') || self.split(':')[0].contains('.') || !self.split(':')[0].matches('[0-9]+') || !self.split(':')[1].matches('[0-9]+') || uint(self.split(':')[0]) > 65535u || uint(self.split(':')[1]) <= 4294967295u",message="RT with 2-byte ASN global administrator must have format EF:OPQR where EF <= 65535 and OPQR <= 4294967295" +type RouteTargetString string + +// VRFConfig contains configuration for a VRF in EVPN. +type VRFConfig struct { + // VNI is the Virtual Network Identifier for this VRF. + // VNI is a 24-bit field in the VXLAN header (RFC 7348), allowing values from 1 to 16777215. + // but in the future this could be having different limit for other dataplane implementations. + // Must be unique across all EVPN configurations in the cluster. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=16777215 + // +required + VNI int32 `json:"vni"` + + // RouteTarget is the import/export route target for this VRF. + // If not specified, it will be auto-generated as ":". + // Auto-generation will use 2-byte AS if VNI > 65535, since 4-byte AS/IPv4 only allows 2-byte local admin. + // + // Follows FRR EVPN L3 Route-Target format (A.B.C.D:MN|EF:OPQR|GHJK:MN|*:OPQR|*:MN): + // - EF:OPQR = 2-byte AS (1-65535) : local admin (4 bytes, 1-4294967295) + // - GHJK:MN = 4-byte AS (65536-4294967295) : local admin (2 bytes, 1-65535) + // - A.B.C.D:MN = IPv4 address : local admin (2 bytes, 1-65535) + // - *:OPQR = wildcard AS : local admin (4 bytes, 1-4294967295) - for import matching + // - *:MN = wildcard AS : local admin (2 bytes, 1-65535) - for import matching + // + // The 6-byte value constraint (RFC 4360) means AS size + local admin size = 6 bytes. + // +optional + RouteTarget RouteTargetString `json:"routeTarget,omitempty"` +} diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/spec.go b/go-controller/pkg/crd/userdefinednetwork/v1/spec.go index cd65f08223..b4fc651575 100644 --- a/go-controller/pkg/crd/userdefinednetwork/v1/spec.go +++ b/go-controller/pkg/crd/userdefinednetwork/v1/spec.go @@ -17,6 +17,16 @@ func (s *UserDefinedNetworkSpec) GetLocalnet() *LocalnetConfig { return nil } +func (s *UserDefinedNetworkSpec) GetTransport() TransportOption { + // UDN (namespace-scoped) does not support EVPN transport + return "" +} + +func (s *UserDefinedNetworkSpec) GetEVPN() *EVPNConfig { + // UDN (namespace-scoped) does not support EVPN + return nil +} + func (s *NetworkSpec) GetTopology() NetworkTopology { return s.Topology } @@ -32,3 +42,11 @@ func (s *NetworkSpec) GetLayer2() *Layer2Config { func (s *NetworkSpec) GetLocalnet() *LocalnetConfig { return s.Localnet } + +func (s *NetworkSpec) GetTransport() TransportOption { + return s.Transport +} + +func (s *NetworkSpec) GetEVPN() *EVPNConfig { + return s.EVPN +} diff --git a/go-controller/pkg/crd/userdefinednetwork/v1/zz_generated.deepcopy.go b/go-controller/pkg/crd/userdefinednetwork/v1/zz_generated.deepcopy.go index ee6487a6c4..5899b65c23 100644 --- a/go-controller/pkg/crd/userdefinednetwork/v1/zz_generated.deepcopy.go +++ b/go-controller/pkg/crd/userdefinednetwork/v1/zz_generated.deepcopy.go @@ -183,6 +183,32 @@ func (in DualStackIPs) DeepCopy() DualStackIPs { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EVPNConfig) DeepCopyInto(out *EVPNConfig) { + *out = *in + if in.MACVRF != nil { + in, out := &in.MACVRF, &out.MACVRF + *out = new(VRFConfig) + **out = **in + } + if in.IPVRF != nil { + in, out := &in.IPVRF, &out.IPVRF + *out = new(VRFConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EVPNConfig. +func (in *EVPNConfig) DeepCopy() *EVPNConfig { + if in == nil { + return nil + } + out := new(EVPNConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IPAMConfig) DeepCopyInto(out *IPAMConfig) { *out = *in @@ -341,11 +367,16 @@ func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { *out = new(LocalnetConfig) (*in).DeepCopyInto(*out) } - if in.NoOverlayOptions != nil { - in, out := &in.NoOverlayOptions, &out.NoOverlayOptions - *out = new(NoOverlayOptions) + if in.NoOverlay != nil { + in, out := &in.NoOverlay, &out.NoOverlay + *out = new(NoOverlayConfig) **out = **in } + if in.EVPN != nil { + in, out := &in.EVPN, &out.EVPN + *out = new(EVPNConfig) + (*in).DeepCopyInto(*out) + } return } @@ -360,17 +391,17 @@ func (in *NetworkSpec) DeepCopy() *NetworkSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NoOverlayOptions) DeepCopyInto(out *NoOverlayOptions) { +func (in *NoOverlayConfig) DeepCopyInto(out *NoOverlayConfig) { *out = *in return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NoOverlayOptions. -func (in *NoOverlayOptions) DeepCopy() *NoOverlayOptions { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NoOverlayConfig. +func (in *NoOverlayConfig) DeepCopy() *NoOverlayConfig { if in == nil { return nil } - out := new(NoOverlayOptions) + out := new(NoOverlayConfig) in.DeepCopyInto(out) return out } @@ -505,3 +536,19 @@ func (in *VLANConfig) DeepCopy() *VLANConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VRFConfig) DeepCopyInto(out *VRFConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VRFConfig. +func (in *VRFConfig) DeepCopy() *VRFConfig { + if in == nil { + return nil + } + out := new(VRFConfig) + in.DeepCopyInto(out) + return out +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/internal/internal.go b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/internal/internal.go new file mode 100644 index 0000000000..14f5f8b5c2 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/internal/internal.go @@ -0,0 +1,61 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package internal + +import ( + fmt "fmt" + sync "sync" + + typed "sigs.k8s.io/structured-merge-diff/v6/typed" +) + +func Parser() *typed.Parser { + parserOnce.Do(func() { + var err error + parser, err = typed.NewParser(schemaYAML) + if err != nil { + panic(fmt.Sprintf("Failed to parse schema: %v", err)) + } + }) + return parser +} + +var parserOnce sync.Once +var parser *typed.Parser +var schemaYAML = typed.YAMLObject(`types: +- name: __untyped_atomic_ + scalar: untyped + list: + elementType: + namedType: __untyped_atomic_ + elementRelationship: atomic + map: + elementType: + namedType: __untyped_atomic_ + elementRelationship: atomic +- name: __untyped_deduced_ + scalar: untyped + list: + elementType: + namedType: __untyped_atomic_ + elementRelationship: atomic + map: + elementType: + namedType: __untyped_deduced_ + elementRelationship: separable +`) diff --git a/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/utils.go b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/utils.go new file mode 100644 index 0000000000..e7a97fb946 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/utils.go @@ -0,0 +1,47 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package applyconfiguration + +import ( + v1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + internal "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/internal" + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + managedfields "k8s.io/apimachinery/pkg/util/managedfields" +) + +// ForKind returns an apply configuration type for the given GroupVersionKind, or nil if no +// apply configuration type exists for the given GroupVersionKind. +func ForKind(kind schema.GroupVersionKind) interface{} { + switch kind { + // Group=k8s.ovn.org, Version=v1 + case v1.SchemeGroupVersion.WithKind("VTEP"): + return &vtepv1.VTEPApplyConfiguration{} + case v1.SchemeGroupVersion.WithKind("VTEPSpec"): + return &vtepv1.VTEPSpecApplyConfiguration{} + case v1.SchemeGroupVersion.WithKind("VTEPStatus"): + return &vtepv1.VTEPStatusApplyConfiguration{} + + } + return nil +} + +func NewTypeConverter(scheme *runtime.Scheme) managedfields.TypeConverter { + return managedfields.NewSchemeTypeConverter(scheme, internal.Parser()) +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtep.go b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtep.go new file mode 100644 index 0000000000..d40b22acb6 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtep.go @@ -0,0 +1,240 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1 + +import ( + apismetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + metav1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// VTEPApplyConfiguration represents a declarative configuration of the VTEP type for use +// with apply. +type VTEPApplyConfiguration struct { + metav1.TypeMetaApplyConfiguration `json:",inline"` + *metav1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Spec *VTEPSpecApplyConfiguration `json:"spec,omitempty"` + Status *VTEPStatusApplyConfiguration `json:"status,omitempty"` +} + +// VTEP constructs a declarative configuration of the VTEP type for use with +// apply. +func VTEP(name string) *VTEPApplyConfiguration { + b := &VTEPApplyConfiguration{} + b.WithName(name) + b.WithKind("VTEP") + b.WithAPIVersion("k8s.ovn.org/v1") + return b +} +func (b VTEPApplyConfiguration) IsApplyConfiguration() {} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithKind(value string) *VTEPApplyConfiguration { + b.TypeMetaApplyConfiguration.Kind = &value + return b +} + +// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the APIVersion field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithAPIVersion(value string) *VTEPApplyConfiguration { + b.TypeMetaApplyConfiguration.APIVersion = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithName(value string) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithGenerateName(value string) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithNamespace(value string) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithUID(value types.UID) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithResourceVersion(value string) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithGeneration(value int64) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithCreationTimestamp(value apismetav1.Time) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithDeletionTimestamp(value apismetav1.Time) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *VTEPApplyConfiguration) WithLabels(entries map[string]string) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Labels == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *VTEPApplyConfiguration) WithAnnotations(entries map[string]string) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Annotations == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *VTEPApplyConfiguration) WithOwnerReferences(values ...*metav1.OwnerReferenceApplyConfiguration) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.ObjectMetaApplyConfiguration.OwnerReferences = append(b.ObjectMetaApplyConfiguration.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *VTEPApplyConfiguration) WithFinalizers(values ...string) *VTEPApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.ObjectMetaApplyConfiguration.Finalizers = append(b.ObjectMetaApplyConfiguration.Finalizers, values[i]) + } + return b +} + +func (b *VTEPApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &metav1.ObjectMetaApplyConfiguration{} + } +} + +// WithSpec sets the Spec field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Spec field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithSpec(value *VTEPSpecApplyConfiguration) *VTEPApplyConfiguration { + b.Spec = value + return b +} + +// WithStatus sets the Status field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Status field is set to the value of the last call. +func (b *VTEPApplyConfiguration) WithStatus(value *VTEPStatusApplyConfiguration) *VTEPApplyConfiguration { + b.Status = value + return b +} + +// GetKind retrieves the value of the Kind field in the declarative configuration. +func (b *VTEPApplyConfiguration) GetKind() *string { + return b.TypeMetaApplyConfiguration.Kind +} + +// GetAPIVersion retrieves the value of the APIVersion field in the declarative configuration. +func (b *VTEPApplyConfiguration) GetAPIVersion() *string { + return b.TypeMetaApplyConfiguration.APIVersion +} + +// GetName retrieves the value of the Name field in the declarative configuration. +func (b *VTEPApplyConfiguration) GetName() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Name +} + +// GetNamespace retrieves the value of the Namespace field in the declarative configuration. +func (b *VTEPApplyConfiguration) GetNamespace() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Namespace +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtepspec.go b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtepspec.go new file mode 100644 index 0000000000..057ea5bc84 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtepspec.go @@ -0,0 +1,51 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1 + +import ( + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" +) + +// VTEPSpecApplyConfiguration represents a declarative configuration of the VTEPSpec type for use +// with apply. +type VTEPSpecApplyConfiguration struct { + CIDRs *vtepv1.DualStackCIDRs `json:"cidrs,omitempty"` + Mode *vtepv1.VTEPMode `json:"mode,omitempty"` +} + +// VTEPSpecApplyConfiguration constructs a declarative configuration of the VTEPSpec type for use with +// apply. +func VTEPSpec() *VTEPSpecApplyConfiguration { + return &VTEPSpecApplyConfiguration{} +} + +// WithCIDRs sets the CIDRs field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CIDRs field is set to the value of the last call. +func (b *VTEPSpecApplyConfiguration) WithCIDRs(value vtepv1.DualStackCIDRs) *VTEPSpecApplyConfiguration { + b.CIDRs = &value + return b +} + +// WithMode sets the Mode field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Mode field is set to the value of the last call. +func (b *VTEPSpecApplyConfiguration) WithMode(value vtepv1.VTEPMode) *VTEPSpecApplyConfiguration { + b.Mode = &value + return b +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtepstatus.go b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtepstatus.go new file mode 100644 index 0000000000..c105191558 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1/vtepstatus.go @@ -0,0 +1,47 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1 + +import ( + metav1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// VTEPStatusApplyConfiguration represents a declarative configuration of the VTEPStatus type for use +// with apply. +type VTEPStatusApplyConfiguration struct { + Conditions []metav1.ConditionApplyConfiguration `json:"conditions,omitempty"` +} + +// VTEPStatusApplyConfiguration constructs a declarative configuration of the VTEPStatus type for use with +// apply. +func VTEPStatus() *VTEPStatusApplyConfiguration { + return &VTEPStatusApplyConfiguration{} +} + +// WithConditions adds the given value to the Conditions field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Conditions field. +func (b *VTEPStatusApplyConfiguration) WithConditions(values ...*metav1.ConditionApplyConfiguration) *VTEPStatusApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithConditions") + } + b.Conditions = append(b.Conditions, *values[i]) + } + return b +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/clientset.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/clientset.go new file mode 100644 index 0000000000..ff4bc15c22 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/clientset.go @@ -0,0 +1,119 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + fmt "fmt" + http "net/http" + + k8sv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1" + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + K8sV1() k8sv1.K8sV1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + *discovery.DiscoveryClient + k8sV1 *k8sv1.K8sV1Client +} + +// K8sV1 retrieves the K8sV1Client +func (c *Clientset) K8sV1() k8sv1.K8sV1Interface { + return c.k8sV1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.k8sV1, err = k8sv1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.k8sV1 = k8sv1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/clientset_generated.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 0000000000..d7d8ec5ce3 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,130 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + applyconfiguration "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration" + clientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned" + k8sv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1" + fakek8sv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +// +// DEPRECATED: NewClientset replaces this with support for field management, which significantly improves +// server side apply testing. NewClientset is only available when apply configurations are generated (e.g. +// via --with-applyconfig). +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchActcion, ok := action.(testing.WatchActionImpl); ok { + opts = watchActcion.ListOptions + } + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns, opts) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +// NewClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewClientset(objects ...runtime.Object) *Clientset { + o := testing.NewFieldManagedObjectTracker( + scheme, + codecs.UniversalDecoder(), + applyconfiguration.NewTypeConverter(scheme), + ) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchAction, ok := action.(testing.WatchActionImpl); ok { + opts = watchAction.ListOptions + } + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns, opts) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// K8sV1 retrieves the K8sV1Client +func (c *Clientset) K8sV1() k8sv1.K8sV1Interface { + return &fakek8sv1.FakeK8sV1{Fake: &c.Fake} +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/doc.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/doc.go new file mode 100644 index 0000000000..19e0028ffb --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/doc.go @@ -0,0 +1,19 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/register.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/register.go new file mode 100644 index 0000000000..892bc7c866 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake/register.go @@ -0,0 +1,55 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + k8sv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + k8sv1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/scheme/doc.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/scheme/doc.go new file mode 100644 index 0000000000..1aec4021fc --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/scheme/doc.go @@ -0,0 +1,19 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/scheme/register.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/scheme/register.go new file mode 100644 index 0000000000..80420d42f1 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/scheme/register.go @@ -0,0 +1,55 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + k8sv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + k8sv1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/doc.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/doc.go new file mode 100644 index 0000000000..b22b05acdb --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/doc.go @@ -0,0 +1,19 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1 diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/doc.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/doc.go new file mode 100644 index 0000000000..422564f2d5 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/doc.go @@ -0,0 +1,19 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/fake_vtep.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/fake_vtep.go new file mode 100644 index 0000000000..757b7ca4d8 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/fake_vtep.go @@ -0,0 +1,48 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1" + typedvtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1" + gentype "k8s.io/client-go/gentype" +) + +// fakeVTEPs implements VTEPInterface +type fakeVTEPs struct { + *gentype.FakeClientWithListAndApply[*v1.VTEP, *v1.VTEPList, *vtepv1.VTEPApplyConfiguration] + Fake *FakeK8sV1 +} + +func newFakeVTEPs(fake *FakeK8sV1) typedvtepv1.VTEPInterface { + return &fakeVTEPs{ + gentype.NewFakeClientWithListAndApply[*v1.VTEP, *v1.VTEPList, *vtepv1.VTEPApplyConfiguration]( + fake.Fake, + "", + v1.SchemeGroupVersion.WithResource("vteps"), + v1.SchemeGroupVersion.WithKind("VTEP"), + func() *v1.VTEP { return &v1.VTEP{} }, + func() *v1.VTEPList { return &v1.VTEPList{} }, + func(dst, src *v1.VTEPList) { dst.ListMeta = src.ListMeta }, + func(list *v1.VTEPList) []*v1.VTEP { return gentype.ToPointerSlice(list.Items) }, + func(list *v1.VTEPList, items []*v1.VTEP) { list.Items = gentype.FromPointerSlice(items) }, + ), + fake, + } +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/fake_vtep_client.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/fake_vtep_client.go new file mode 100644 index 0000000000..dc4c87a818 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/fake/fake_vtep_client.go @@ -0,0 +1,39 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeK8sV1 struct { + *testing.Fake +} + +func (c *FakeK8sV1) VTEPs() v1.VTEPInterface { + return newFakeVTEPs(c) +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeK8sV1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/generated_expansion.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/generated_expansion.go new file mode 100644 index 0000000000..2055782379 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/generated_expansion.go @@ -0,0 +1,20 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +type VTEPExpansion interface{} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/vtep.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/vtep.go new file mode 100644 index 0000000000..6e57b9755f --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/vtep.go @@ -0,0 +1,73 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + applyconfigurationvtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/applyconfiguration/vtep/v1" + scheme "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// VTEPsGetter has a method to return a VTEPInterface. +// A group's client should implement this interface. +type VTEPsGetter interface { + VTEPs() VTEPInterface +} + +// VTEPInterface has methods to work with VTEP resources. +type VTEPInterface interface { + Create(ctx context.Context, vTEP *vtepv1.VTEP, opts metav1.CreateOptions) (*vtepv1.VTEP, error) + Update(ctx context.Context, vTEP *vtepv1.VTEP, opts metav1.UpdateOptions) (*vtepv1.VTEP, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, vTEP *vtepv1.VTEP, opts metav1.UpdateOptions) (*vtepv1.VTEP, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*vtepv1.VTEP, error) + List(ctx context.Context, opts metav1.ListOptions) (*vtepv1.VTEPList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *vtepv1.VTEP, err error) + Apply(ctx context.Context, vTEP *applyconfigurationvtepv1.VTEPApplyConfiguration, opts metav1.ApplyOptions) (result *vtepv1.VTEP, err error) + // Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). + ApplyStatus(ctx context.Context, vTEP *applyconfigurationvtepv1.VTEPApplyConfiguration, opts metav1.ApplyOptions) (result *vtepv1.VTEP, err error) + VTEPExpansion +} + +// vTEPs implements VTEPInterface +type vTEPs struct { + *gentype.ClientWithListAndApply[*vtepv1.VTEP, *vtepv1.VTEPList, *applyconfigurationvtepv1.VTEPApplyConfiguration] +} + +// newVTEPs returns a VTEPs +func newVTEPs(c *K8sV1Client) *vTEPs { + return &vTEPs{ + gentype.NewClientWithListAndApply[*vtepv1.VTEP, *vtepv1.VTEPList, *applyconfigurationvtepv1.VTEPApplyConfiguration]( + "vteps", + c.RESTClient(), + scheme.ParameterCodec, + "", + func() *vtepv1.VTEP { return &vtepv1.VTEP{} }, + func() *vtepv1.VTEPList { return &vtepv1.VTEPList{} }, + ), + } +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/vtep_client.go b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/vtep_client.go new file mode 100644 index 0000000000..262601161b --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/typed/vtep/v1/vtep_client.go @@ -0,0 +1,100 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + http "net/http" + + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + scheme "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type K8sV1Interface interface { + RESTClient() rest.Interface + VTEPsGetter +} + +// K8sV1Client is used to interact with features provided by the k8s.ovn.org group. +type K8sV1Client struct { + restClient rest.Interface +} + +func (c *K8sV1Client) VTEPs() VTEPInterface { + return newVTEPs(c) +} + +// NewForConfig creates a new K8sV1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*K8sV1Client, error) { + config := *c + setConfigDefaults(&config) + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new K8sV1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*K8sV1Client, error) { + config := *c + setConfigDefaults(&config) + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &K8sV1Client{client}, nil +} + +// NewForConfigOrDie creates a new K8sV1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *K8sV1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new K8sV1Client for the given RESTClient. +func New(c rest.Interface) *K8sV1Client { + return &K8sV1Client{c} +} + +func setConfigDefaults(config *rest.Config) { + gv := vtepv1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = rest.CodecFactoryForGeneratedClient(scheme.Scheme, scheme.Codecs).WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *K8sV1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/factory.go b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/factory.go new file mode 100644 index 0000000000..37852fa5d0 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/factory.go @@ -0,0 +1,261 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + reflect "reflect" + sync "sync" + time "time" + + versioned "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned" + internalinterfaces "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/internalinterfaces" + vtep "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + transform cache.TransformFunc + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// WithTransform sets a transform on all informers. +func WithTransform(transform cache.TransformFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.transform = transform + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.shuttingDown { + return + } + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + f.wg.Add(1) + // We need a new variable in each loop iteration, + // otherwise the goroutine would use the loop variable + // and that keeps changing. + informer := informer + go func() { + defer f.wg.Done() + informer.Run(stopCh) + }() + f.startedInformers[informerType] = true + } + } +} + +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() +} + +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + informer.SetTransform(f.transform) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.Background() +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + // Warning: Start does not block. When run in a go-routine, it will race with a later WaitForCacheSync. + Start(stopCh <-chan struct{}) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + + K8s() vtep.Interface +} + +func (f *sharedInformerFactory) K8s() vtep.Interface { + return vtep.New(f, f.namespace, f.tweakListOptions) +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/generic.go b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/generic.go new file mode 100644 index 0000000000..1d1e148d57 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/generic.go @@ -0,0 +1,61 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + fmt "fmt" + + v1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=k8s.ovn.org, Version=v1 + case v1.SchemeGroupVersion.WithResource("vteps"): + return &genericInformer{resource: resource.GroupResource(), informer: f.K8s().V1().VTEPs().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/internalinterfaces/factory_interfaces.go b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 0000000000..5a28e34e3a --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,39 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + versioned "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/interface.go b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/interface.go new file mode 100644 index 0000000000..4e298c44b9 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/interface.go @@ -0,0 +1,45 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package vtep + +import ( + internalinterfaces "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/internalinterfaces" + v1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1 provides access to shared informers for resources in V1. + V1() v1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1 returns a new v1.Interface. +func (g *group) V1() v1.Interface { + return v1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1/interface.go b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1/interface.go new file mode 100644 index 0000000000..023788d0f7 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1/interface.go @@ -0,0 +1,44 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + internalinterfaces "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // VTEPs returns a VTEPInformer. + VTEPs() VTEPInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// VTEPs returns a VTEPInformer. +func (v *version) VTEPs() VTEPInformer { + return &vTEPInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1/vtep.go b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1/vtep.go new file mode 100644 index 0000000000..d6f854a7db --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1/vtep.go @@ -0,0 +1,100 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + crdvtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + versioned "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned" + internalinterfaces "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/internalinterfaces" + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/listers/vtep/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// VTEPInformer provides access to a shared informer and lister for +// VTEPs. +type VTEPInformer interface { + Informer() cache.SharedIndexInformer + Lister() vtepv1.VTEPLister +} + +type vTEPInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewVTEPInformer constructs a new informer for VTEP type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewVTEPInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredVTEPInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredVTEPInformer constructs a new informer for VTEP type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredVTEPInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.K8sV1().VTEPs().List(context.Background(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.K8sV1().VTEPs().Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.K8sV1().VTEPs().List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.K8sV1().VTEPs().Watch(ctx, options) + }, + }, + &crdvtepv1.VTEP{}, + resyncPeriod, + indexers, + ) +} + +func (f *vTEPInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredVTEPInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *vTEPInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&crdvtepv1.VTEP{}, f.defaultInformer) +} + +func (f *vTEPInformer) Lister() vtepv1.VTEPLister { + return vtepv1.NewVTEPLister(f.Informer().GetIndexer()) +} diff --git a/go-controller/pkg/crd/vtep/v1/apis/listers/vtep/v1/expansion_generated.go b/go-controller/pkg/crd/vtep/v1/apis/listers/vtep/v1/expansion_generated.go new file mode 100644 index 0000000000..9addf2b8b1 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/listers/vtep/v1/expansion_generated.go @@ -0,0 +1,22 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +// VTEPListerExpansion allows custom methods to be added to +// VTEPLister. +type VTEPListerExpansion interface{} diff --git a/go-controller/pkg/crd/vtep/v1/apis/listers/vtep/v1/vtep.go b/go-controller/pkg/crd/vtep/v1/apis/listers/vtep/v1/vtep.go new file mode 100644 index 0000000000..e983f500f2 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/apis/listers/vtep/v1/vtep.go @@ -0,0 +1,47 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// VTEPLister helps list VTEPs. +// All objects returned here must be treated as read-only. +type VTEPLister interface { + // List lists all VTEPs in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*vtepv1.VTEP, err error) + // Get retrieves the VTEP from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*vtepv1.VTEP, error) + VTEPListerExpansion +} + +// vTEPLister implements the VTEPLister interface. +type vTEPLister struct { + listers.ResourceIndexer[*vtepv1.VTEP] +} + +// NewVTEPLister returns a new VTEPLister. +func NewVTEPLister(indexer cache.Indexer) VTEPLister { + return &vTEPLister{listers.New[*vtepv1.VTEP](indexer, vtepv1.Resource("vtep"))} +} diff --git a/go-controller/pkg/crd/vtep/v1/doc.go b/go-controller/pkg/crd/vtep/v1/doc.go new file mode 100644 index 0000000000..7b121f971b --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/doc.go @@ -0,0 +1,4 @@ +// Package v1 contains API Schema definitions for the network v1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=k8s.ovn.org +package v1 diff --git a/go-controller/pkg/crd/vtep/v1/register.go b/go-controller/pkg/crd/vtep/v1/register.go new file mode 100644 index 0000000000..230f0c4f7a --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/register.go @@ -0,0 +1,45 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + GroupName = "k8s.ovn.org" + SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"} + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource takes an unqualified resource and returns a Group qualified GroupResource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +// Adds the list of known types to api.Scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &VTEP{}, + &VTEPList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/go-controller/pkg/crd/vtep/v1/types.go b/go-controller/pkg/crd/vtep/v1/types.go new file mode 100644 index 0000000000..819feda7e9 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/types.go @@ -0,0 +1,104 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VTEP defines VTEP (VXLAN Tunnel Endpoint) IP configuration for EVPN. +// +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:path=vteps,scope=Cluster +// +kubebuilder:singular=vtep +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +type VTEP struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec defines the desired VTEP configuration. + // +kubebuilder:validation:Required + // +required + Spec VTEPSpec `json:"spec"` + + // Status contains the observed state of the VTEP. + // +optional + Status VTEPStatus `json:"status,omitempty"` +} + +// VTEPSpec defines the desired state of VTEP. +type VTEPSpec struct { + // CIDRs is the list of IP ranges from which VTEP IPs are allocated. + // Dual-stack clusters may set 2 CIDRs (one for each IP family), otherwise only 1 CIDR is allowed. + // The format should match standard CIDR notation (for example, "100.64.0.0/24" or "fd00::/64"). + // +kubebuilder:validation:Required + // +required + CIDRs DualStackCIDRs `json:"cidrs"` + + // Mode specifies how VTEP IPs are managed. + // "Managed" means OVN-Kubernetes allocates and assigns VTEP IPs per node automatically. + // "Unmanaged" means an external provider handles IP assignment; OVN-Kubernetes discovers existing IPs on nodes. + // Defaults to "Managed". + // +kubebuilder:validation:Enum=Managed;Unmanaged + // +kubebuilder:default=Managed + // +optional + Mode VTEPMode `json:"mode,omitempty"` +} + +// CIDR represents a CIDR notation IP range. +// +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self) == cidr(self).masked()", message="CIDR must be a valid network address" +// +kubebuilder:validation:MaxLength=43 +type CIDR string + +// DualStackCIDRs is a list of CIDRs that supports dual-stack (IPv4 and IPv6). +// +kubebuilder:validation:MinItems=1 +// +kubebuilder:validation:MaxItems=2 +// +kubebuilder:validation:XValidation:rule="size(self) != 2 || !isCIDR(self[0]) || !isCIDR(self[1]) || cidr(self[0]).ip().family() != cidr(self[1]).ip().family()", message="When 2 CIDRs are set, they must be from different IP families" +type DualStackCIDRs []CIDR + +// VTEPMode defines the mode of VTEP IP allocation. +// +kubebuilder:validation:Enum=Managed;Unmanaged +type VTEPMode string + +const ( + // VTEPModeManaged means OVN-Kubernetes allocates and assigns VTEP IPs per node automatically. + VTEPModeManaged VTEPMode = "Managed" + // VTEPModeUnmanaged means an external provider handles IP assignment; + // OVN-Kubernetes discovers existing IPs on nodes. + VTEPModeUnmanaged VTEPMode = "Unmanaged" +) + +// VTEPStatus contains the observed state of the VTEP. +type VTEPStatus struct { + // Conditions slice of condition objects indicating details about VTEP status. + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// VTEPList contains a list of VTEP. +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type VTEPList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []VTEP `json:"items"` +} diff --git a/go-controller/pkg/crd/vtep/v1/zz_generated.deepcopy.go b/go-controller/pkg/crd/vtep/v1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..1fd9b149a3 --- /dev/null +++ b/go-controller/pkg/crd/vtep/v1/zz_generated.deepcopy.go @@ -0,0 +1,151 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in DualStackCIDRs) DeepCopyInto(out *DualStackCIDRs) { + { + in := &in + *out = make(DualStackCIDRs, len(*in)) + copy(*out, *in) + return + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DualStackCIDRs. +func (in DualStackCIDRs) DeepCopy() DualStackCIDRs { + if in == nil { + return nil + } + out := new(DualStackCIDRs) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VTEP) DeepCopyInto(out *VTEP) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VTEP. +func (in *VTEP) DeepCopy() *VTEP { + if in == nil { + return nil + } + out := new(VTEP) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VTEP) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VTEPList) DeepCopyInto(out *VTEPList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VTEP, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VTEPList. +func (in *VTEPList) DeepCopy() *VTEPList { + if in == nil { + return nil + } + out := new(VTEPList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VTEPList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VTEPSpec) DeepCopyInto(out *VTEPSpec) { + *out = *in + if in.CIDRs != nil { + in, out := &in.CIDRs, &out.CIDRs + *out = make(DualStackCIDRs, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VTEPSpec. +func (in *VTEPSpec) DeepCopy() *VTEPSpec { + if in == nil { + return nil + } + out := new(VTEPSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VTEPStatus) DeepCopyInto(out *VTEPStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VTEPStatus. +func (in *VTEPStatus) DeepCopy() *VTEPStatus { + if in == nil { + return nil + } + out := new(VTEPStatus) + in.DeepCopyInto(out) + return out +} diff --git a/go-controller/pkg/factory/factory.go b/go-controller/pkg/factory/factory.go index f14d0eafc6..ff43fd8476 100644 --- a/go-controller/pkg/factory/factory.go +++ b/go-controller/pkg/factory/factory.go @@ -98,6 +98,8 @@ import ( userdefinednetworkscheme "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/scheme" userdefinednetworkapiinformerfactory "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions" userdefinednetworkinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/userdefinednetwork/v1" + vtepinformerfactory "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions" + vtepinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -129,6 +131,7 @@ type WatchFactory struct { raFactory routeadvertisementsinformerfactory.SharedInformerFactory frrFactory frrinformerfactory.SharedInformerFactory networkQoSFactory networkqosinformerfactory.SharedInformerFactory + vtepFactory vtepinformerfactory.SharedInformerFactory informers map[reflect.Type]*informer stopChan chan struct{} @@ -158,6 +161,7 @@ func (wf *WatchFactory) ShallowClone() *WatchFactory { raFactory: wf.raFactory, frrFactory: wf.frrFactory, networkQoSFactory: wf.networkQoSFactory, + vtepFactory: wf.vtepFactory, informers: wf.informers, stopChan: wf.stopChan, @@ -281,6 +285,13 @@ func NewMasterWatchFactory(ovnClientset *util.OVNMasterClientset) (*WatchFactory } } + // Initialize VTEP factory for EVPN support in combined mode (cluster-manager + ovnkube-controller). + if util.IsEVPNEnabled() { + wf.vtepFactory = vtepinformerfactory.NewSharedInformerFactory(ovnClientset.VTEPClient, resyncInterval) + // make sure shared informer is created for a factory, so on wf.vtepFactory.Start() it is initialized and caches are synced. + wf.vtepFactory.K8s().V1().VTEPs().Informer() + } + return wf, nil } @@ -370,6 +381,10 @@ func NewOVNKubeControllerWatchFactory(ovnClientset *util.OVNKubeControllerClient return nil, err } + if err := networkconnectapi.AddToScheme(networkconnectscheme.Scheme); err != nil { + return nil, err + } + // For Services and Endpoints, pre-populate the shared Informer with one that // has a label selector excluding headless services. wf.iFactory.InformerFor(&corev1.Service{}, func(c kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { @@ -527,6 +542,15 @@ func NewOVNKubeControllerWatchFactory(ovnClientset *util.OVNKubeControllerClient return nil, err } } + if util.IsNetworkConnectEnabled() { + wf.cncFactory = networkconnectinformerfactory.NewSharedInformerFactory(ovnClientset.NetworkConnectClient, resyncInterval) + wf.informers[ClusterNetworkConnectType], err = newQueuedInformer(eventQueueSize, + ClusterNetworkConnectType, + wf.cncFactory.K8s().V1().ClusterNetworkConnects().Informer(), wf.stopChan, minNumEventQueues) + if err != nil { + return nil, err + } + } return wf, nil } @@ -633,6 +657,13 @@ func (wf *WatchFactory) Start() error { } } + if wf.vtepFactory != nil { + wf.vtepFactory.Start(wf.stopChan) + if err := waitForCacheSyncWithTimeout(wf.vtepFactory, wf.stopChan); err != nil { + return err + } + } + if wf.raFactory != nil { wf.raFactory.Start(wf.stopChan) if err := waitForCacheSyncWithTimeout(wf.raFactory, wf.stopChan); err != nil { @@ -693,6 +724,10 @@ func (wf *WatchFactory) Stop() { wf.cncFactory.Shutdown() } + if wf.vtepFactory != nil { + wf.vtepFactory.Shutdown() + } + if wf.raFactory != nil { wf.raFactory.Shutdown() } @@ -1068,6 +1103,13 @@ func NewClusterManagerWatchFactory(ovnClientset *util.OVNClusterManagerClientset wf.iFactory.Core().V1().Pods().Informer() } + // Initialize VTEP factory for EVPN support. + if util.IsEVPNEnabled() { + wf.vtepFactory = vtepinformerfactory.NewSharedInformerFactory(ovnClientset.VTEPClient, resyncInterval) + // make sure shared informer is created for a factory, so on wf.vtepFactory.Start() it is initialized and caches are synced. + wf.vtepFactory.K8s().V1().VTEPs().Informer() + } + if util.IsNetworkConnectEnabled() { wf.cncFactory = networkconnectinformerfactory.NewSharedInformerFactory(ovnClientset.NetworkConnectClient, resyncInterval) wf.informers[ClusterNetworkConnectType], err = newQueuedInformer(eventQueueSize, @@ -1808,6 +1850,10 @@ func (wf *WatchFactory) ClusterNetworkConnectInformer() networkconnectinformer.C return wf.cncFactory.K8s().V1().ClusterNetworkConnects() } +func (wf *WatchFactory) VTEPInformer() vtepinformer.VTEPInformer { + return wf.vtepFactory.K8s().V1().VTEPs() +} + func (wf *WatchFactory) DNSNameResolverInformer() ocpnetworkinformerv1alpha1.DNSNameResolverInformer { return wf.dnsFactory.Network().V1alpha1().DNSNameResolvers() } diff --git a/go-controller/pkg/kubevirt/pod.go b/go-controller/pkg/kubevirt/pod.go index 3e423744fa..4ec93ebb1f 100644 --- a/go-controller/pkg/kubevirt/pod.go +++ b/go-controller/pkg/kubevirt/pod.go @@ -35,16 +35,19 @@ type DefaultGatewayReconciler struct { watchFactory *factory.WatchFactory netInfo util.NetInfo interfaceName string + // getNetworkNameForNADKey resolves NAD keys to network names for UDNs. + getNetworkNameForNADKey func(nadKey string) string } // NewDefaultGatewayReconciler creates a new instance of DefaultGatewayReconciler. // It takes a WatchFactory for managing resource watches, a NetInfo object for network information, // and the name of the network interface to send ARPs or RAs as parameters. -func NewDefaultGatewayReconciler(watchFactory *factory.WatchFactory, netInfo util.NetInfo, interfaceName string) *DefaultGatewayReconciler { +func NewDefaultGatewayReconciler(watchFactory *factory.WatchFactory, netInfo util.NetInfo, interfaceName string, getNetworkNameForNADKey func(nadKey string) string) *DefaultGatewayReconciler { return &DefaultGatewayReconciler{ - watchFactory: watchFactory, - netInfo: netInfo, - interfaceName: interfaceName, + watchFactory: watchFactory, + netInfo: netInfo, + interfaceName: interfaceName, + getNetworkNameForNADKey: getNetworkNameForNADKey, } } @@ -93,7 +96,7 @@ func findVMRelatedPodsWithListerFn(listPodsFn listPodsFn, pod *corev1.Pod) ([]*c // findPodAnnotation will return the the OVN pod // annotation from any other pod annotated with the same VM as pod -func findPodAnnotation(client *factory.WatchFactory, pod *corev1.Pod, nadName string) (*util.PodAnnotation, error) { +func findPodAnnotation(client *factory.WatchFactory, pod *corev1.Pod, nadKey string) (*util.PodAnnotation, error) { vmPods, err := findVMRelatedPods(client, pod) if err != nil { return nil, fmt.Errorf("failed finding related pods for pod %s/%s when looking for network info: %v", pod.Namespace, pod.Name, err) @@ -105,7 +108,7 @@ func findPodAnnotation(client *factory.WatchFactory, pod *corev1.Pod, nadName st } for _, vmPod := range vmPods { - podAnnotation, err := util.UnmarshalPodAnnotation(vmPod.Annotations, nadName) + podAnnotation, err := util.UnmarshalPodAnnotation(vmPod.Annotations, nadKey) if err == nil { return podAnnotation, nil } @@ -118,16 +121,16 @@ func findPodAnnotation(client *factory.WatchFactory, pod *corev1.Pod, nadName st // to the target vm pod so ip address follow vm during migration. This has to // done before creating the LSP to be sure that Address field get configured // correctly at the target VM pod LSP. -func EnsurePodAnnotationForVM(watchFactory *factory.WatchFactory, kube *kube.KubeOVN, pod *corev1.Pod, nadName string) (*util.PodAnnotation, error) { +func EnsurePodAnnotationForVM(watchFactory *factory.WatchFactory, kube *kube.KubeOVN, pod *corev1.Pod, nadKey string) (*util.PodAnnotation, error) { if !IsPodLiveMigratable(pod) { return nil, nil } - if podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName); err == nil { + if podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey); err == nil { return podAnnotation, nil } - podAnnotation, err := findPodAnnotation(watchFactory, pod, nadName) + podAnnotation, err := findPodAnnotation(watchFactory, pod, nadKey) if err != nil { return nil, err } @@ -145,7 +148,7 @@ func EnsurePodAnnotationForVM(watchFactory *factory.WatchFactory, kube *kube.Kub // Informer cache should not be mutated, so get a copy of the object modifiedPod = pod.DeepCopy() if podAnnotation != nil { - modifiedPod.Annotations, err = util.MarshalPodAnnotation(modifiedPod.Annotations, podAnnotation, nadName) + modifiedPod.Annotations, err = util.MarshalPodAnnotation(modifiedPod.Annotations, podAnnotation, nadKey) if err != nil { return err } @@ -218,12 +221,12 @@ func ZoneContainsPodSubnet(lsManager *logicalswitchmanager.LogicalSwitchManager, // nodeContainsPodSubnet will return true if the node subnet annotation // contains the subnets from the argument -func nodeContainsPodSubnet(watchFactory *factory.WatchFactory, nodeName string, podAnnotation *util.PodAnnotation, nadName string) (bool, error) { +func nodeContainsPodSubnet(watchFactory *factory.WatchFactory, nodeName string, podAnnotation *util.PodAnnotation, netName string) (bool, error) { node, err := watchFactory.GetNode(nodeName) if err != nil { return false, err } - nodeHostSubNets, err := util.ParseNodeHostSubnetAnnotation(node, nadName) + nodeHostSubNets, err := util.ParseNodeHostSubnetAnnotation(node, netName) if err != nil { return false, err } @@ -318,7 +321,7 @@ func FindLiveMigratablePods(watchFactory *factory.WatchFactory) ([]*corev1.Pod, // allocateSyncMigratablePodIPs will refill ip pool in // case the node has take over the vm subnet for live migrated vms -func allocateSyncMigratablePodIPs(watchFactory *factory.WatchFactory, lsManager *logicalswitchmanager.LogicalSwitchManager, nodeName, nadName string, pod *corev1.Pod, allocatePodIPsOnSwitch func(*corev1.Pod, *util.PodAnnotation, string, string) (string, error)) (*ktypes.NamespacedName, string, *util.PodAnnotation, error) { +func allocateSyncMigratablePodIPs(watchFactory *factory.WatchFactory, lsManager *logicalswitchmanager.LogicalSwitchManager, nodeName, nadKey string, pod *corev1.Pod, allocatePodIPsOnSwitch func(*corev1.Pod, *util.PodAnnotation, string, string) (string, error)) (*ktypes.NamespacedName, string, *util.PodAnnotation, error) { isStale, err := IsMigratedSourcePodStale(watchFactory, pod) if err != nil { return nil, "", nil, err @@ -331,7 +334,7 @@ func allocateSyncMigratablePodIPs(watchFactory *factory.WatchFactory, lsManager vmKey := ExtractVMNameFromPod(pod) - annotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + annotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { return nil, "", nil, nil } @@ -341,7 +344,7 @@ func allocateSyncMigratablePodIPs(watchFactory *factory.WatchFactory, lsManager if !zoneContainsPodSubnet || (nodeName != "" && switchName != nodeName) { return vmKey, "", annotation, nil } - expectedLogicalPortName, err := allocatePodIPsOnSwitch(pod, annotation, nadName, switchName) + expectedLogicalPortName, err := allocatePodIPsOnSwitch(pod, annotation, nadKey, switchName) if err != nil { return vmKey, "", nil, err } @@ -350,9 +353,9 @@ func allocateSyncMigratablePodIPs(watchFactory *factory.WatchFactory, lsManager // AllocateSyncMigratablePodIPsOnZone will refill ip pool in // with pod's IPs if those IPs belong to the zone -func AllocateSyncMigratablePodIPsOnZone(watchFactory *factory.WatchFactory, lsManager *logicalswitchmanager.LogicalSwitchManager, nadName string, pod *corev1.Pod, allocatePodIPsOnSwitch func(*corev1.Pod, *util.PodAnnotation, string, string) (string, error)) (*ktypes.NamespacedName, string, *util.PodAnnotation, error) { +func AllocateSyncMigratablePodIPsOnZone(watchFactory *factory.WatchFactory, lsManager *logicalswitchmanager.LogicalSwitchManager, nadKey string, pod *corev1.Pod, allocatePodIPsOnSwitch func(*corev1.Pod, *util.PodAnnotation, string, string) (string, error)) (*ktypes.NamespacedName, string, *util.PodAnnotation, error) { // We care about the whole zone so we pass the nodeName empty - return allocateSyncMigratablePodIPs(watchFactory, lsManager, nadName, "", pod, allocatePodIPsOnSwitch) + return allocateSyncMigratablePodIPs(watchFactory, lsManager, "", nadKey, pod, allocatePodIPsOnSwitch) } // ZoneContainsPodSubnetOrUntracked returns whether a pod with its corresponding @@ -577,11 +580,15 @@ func (r *DefaultGatewayReconciler) ReconcileIPv6AfterLiveMigration(liveMigration } targetPod := liveMigration.TargetPod - if len(r.netInfo.GetNADs()) != 1 { - return fmt.Errorf("expected only one nad for network %q, got %d", r.netInfo.GetNetworkName(), len(r.netInfo.GetNADs())) + nadKeys, err := util.PodNADKeys(targetPod, r.netInfo, r.getNetworkNameForNADKey) + if err != nil { + return err + } + if len(nadKeys) != 1 { + return fmt.Errorf("expected only one NAD key for network %q, got %d", r.netInfo.GetNetworkName(), len(nadKeys)) } - targetPodAnnotation, err := util.UnmarshalPodAnnotation(targetPod.Annotations, r.netInfo.GetNADs()[0]) + targetPodAnnotation, err := util.UnmarshalPodAnnotation(targetPod.Annotations, nadKeys[0]) if err != nil { return ovntypes.NewSuppressedError(fmt.Errorf("failed parsing ovn pod annotation for pod '%s/%s' and network %q: %w", targetPod.Namespace, targetPod.Name, r.netInfo.GetNetworkName(), err)) } diff --git a/go-controller/pkg/kubevirt/router.go b/go-controller/pkg/kubevirt/router.go index 779bac1ead..fb8e92fab1 100644 --- a/go-controller/pkg/kubevirt/router.go +++ b/go-controller/pkg/kubevirt/router.go @@ -67,7 +67,7 @@ func DeleteRoutingForMigratedPod(nbClient libovsdbclient.Client, pod *corev1.Pod // Both: // - static route with VM ip as dst-ip prefix and output port the LRP pointing to the VM's node switch func EnsureLocalZonePodAddressesToNodeRoute(watchFactory *factory.WatchFactory, nbClient libovsdbclient.Client, - lsManager *logicalswitchmanager.LogicalSwitchManager, pod *corev1.Pod, nadName string, clusterSubnets []config.CIDRNetworkEntry) error { + lsManager *logicalswitchmanager.LogicalSwitchManager, pod *corev1.Pod, nadKey string, clusterSubnets []config.CIDRNetworkEntry) error { vmReady, err := virtualMachineReady(watchFactory, pod) if err != nil { return err @@ -75,7 +75,7 @@ func EnsureLocalZonePodAddressesToNodeRoute(watchFactory *factory.WatchFactory, if !vmReady { return nil } - podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { return fmt.Errorf("failed reading local pod annotation: %v", err) } @@ -175,7 +175,7 @@ func EnsureLocalZonePodAddressesToNodeRoute(watchFactory *factory.WatchFactory, // port of the node where the pod is running: // - A dst-ip with live migrated pod ip as prefix and nexthop the pod's // current node transit switch port. -func EnsureRemoteZonePodAddressesToNodeRoute(watchFactory *factory.WatchFactory, nbClient libovsdbclient.Client, pod *corev1.Pod, nadName string) error { +func EnsureRemoteZonePodAddressesToNodeRoute(watchFactory *factory.WatchFactory, nbClient libovsdbclient.Client, pod *corev1.Pod) error { vmReady, err := virtualMachineReady(watchFactory, pod) if err != nil { return err @@ -189,12 +189,12 @@ func EnsureRemoteZonePodAddressesToNodeRoute(watchFactory *factory.WatchFactory, return err } - podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, types.DefaultNetworkName) if err != nil { return fmt.Errorf("failed reading remote pod annotation: %v", err) } - vmRunningAtNodeOwningSubnet, err := nodeContainsPodSubnet(watchFactory, pod.Spec.NodeName, podAnnotation, nadName) + vmRunningAtNodeOwningSubnet, err := nodeContainsPodSubnet(watchFactory, pod.Spec.NodeName, podAnnotation, types.DefaultNetworkName) if err != nil { return err } diff --git a/go-controller/pkg/libovsdb/ops/chassis.go b/go-controller/pkg/libovsdb/ops/chassis.go index 83a2d6a3c2..4af8bc58b1 100644 --- a/go-controller/pkg/libovsdb/ops/chassis.go +++ b/go-controller/pkg/libovsdb/ops/chassis.go @@ -2,6 +2,9 @@ package ops import ( "context" + "fmt" + + "github.com/google/uuid" "k8s.io/apimachinery/pkg/util/sets" @@ -171,3 +174,19 @@ func CreateOrUpdateChassis(sbClient libovsdbclient.Client, chassis *sbdb.Chassis return nil } + +// validateRequestedChassisOption is a guard to ensure a caller is using the chassis-id (uuid format) +// for the requested chassis option. +func validateRequestedChassisOption(options map[string]string) error { + if len(options) == 0 { + return nil + } + chassisID, ok := options[RequestedChassis] + if !ok || chassisID == "" { + return nil + } + if _, err := uuid.Parse(chassisID); err != nil { + return fmt.Errorf("requested-chassis must be a valid UUID, got %q", chassisID) + } + return nil +} diff --git a/go-controller/pkg/libovsdb/ops/copp.go b/go-controller/pkg/libovsdb/ops/copp.go index a0f8697b1b..30666758e2 100644 --- a/go-controller/pkg/libovsdb/ops/copp.go +++ b/go-controller/pkg/libovsdb/ops/copp.go @@ -9,6 +9,25 @@ import ( type coppPredicate func(*nbdb.Copp) bool +// GetCOPP looks up a COPP from the cache +func GetCOPP(nbClient libovsdbclient.Client, copp *nbdb.Copp) (*nbdb.Copp, error) { + found := []*nbdb.Copp{} + opModel := operationModel{ + Model: copp, + ExistingResult: &found, + ErrNotFound: true, + BulkOp: false, + } + + m := newModelClient(nbClient) + err := m.Lookup(opModel) + if err != nil { + return nil, err + } + + return found[0], nil +} + // CreateOrUpdateCOPPsOps creates or updates the provided COPP returning the // corresponding ops func CreateOrUpdateCOPPsOps(nbClient libovsdbclient.Client, ops []ovsdb.Operation, copps ...*nbdb.Copp) ([]ovsdb.Operation, error) { diff --git a/go-controller/pkg/libovsdb/ops/db_object_types.go b/go-controller/pkg/libovsdb/ops/db_object_types.go index 87102451c7..375f845ef8 100644 --- a/go-controller/pkg/libovsdb/ops/db_object_types.go +++ b/go-controller/pkg/libovsdb/ops/db_object_types.go @@ -10,6 +10,8 @@ const ( logicalRouterPolicy qos nat + logicalRouterPort + logicalRouterStaticRoute ) const ( @@ -42,21 +44,27 @@ const ( // ClusterOwnerType means the object is cluster-scoped and doesn't belong to any k8s objects ClusterOwnerType ownerType = "Cluster" // UDNIsolationOwnerType means the object is needed to implement UserDefinedNetwork isolation - UDNIsolationOwnerType ownerType = "UDNIsolation" + UDNIsolationOwnerType ownerType = "UDNIsolation" + ClusterNetworkConnectOwnerType ownerType = "ClusterNetworkConnect" // owner extra IDs, make sure to define only 1 ExternalIDKey for every string value - PriorityKey ExternalIDKey = "priority" - PolicyDirectionKey ExternalIDKey = "direction" - GressIdxKey ExternalIDKey = "gress-index" - IPFamilyKey ExternalIDKey = "ip-family" - NetworkKey ExternalIDKey = "network" - TypeKey ExternalIDKey = "type" - IpKey ExternalIDKey = "ip" - PortPolicyIndexKey ExternalIDKey = "port-policy-index" - IpBlockIndexKey ExternalIDKey = "ip-block-index" - RuleIndex ExternalIDKey = "rule-index" - CIDRKey ExternalIDKey = types.OvnK8sPrefix + "/cidr" - PortPolicyProtocolKey ExternalIDKey = "port-policy-protocol" + PriorityKey ExternalIDKey = "priority" + PolicyDirectionKey ExternalIDKey = "direction" + GressIdxKey ExternalIDKey = "gress-index" + IPFamilyKey ExternalIDKey = "ip-family" + NetworkKey ExternalIDKey = "network" + NetworkIDKey ExternalIDKey = "network-id" + SourceNetworkIDKey ExternalIDKey = "source-network-id" + DestinationNetworkIDKey ExternalIDKey = "destination-network-id" + NodeIDKey ExternalIDKey = "node-id" + TypeKey ExternalIDKey = "type" + IpKey ExternalIDKey = "ip" + PortPolicyIndexKey ExternalIDKey = "port-policy-index" + IpBlockIndexKey ExternalIDKey = "ip-block-index" + RuleIndex ExternalIDKey = "rule-index" + CIDRKey ExternalIDKey = types.OvnK8sPrefix + "/cidr" + PortPolicyProtocolKey ExternalIDKey = "port-policy-protocol" + RouterNameKey ExternalIDKey = "router-name" ) // ObjectIDsTypes should only be created here @@ -375,3 +383,64 @@ var NetworkQoS = newObjectIDsType(qos, NetworkQoSOwnerType, []ExternalIDKey{ // rule index RuleIndex, }) + +var LogicalRouterPortClusterNetworkConnect = newObjectIDsType(logicalRouterPort, ClusterNetworkConnectOwnerType, []ExternalIDKey{ + // CNC name + ObjectNameKey, + // connected network's network ID + // value in k8s.ovn.org/network-id annotation set on the NAD + NetworkIDKey, + // node ID + // for layer2 network type ports, the node ID is 0 since there is only one port per network. + // for layer3 network type ports, the node ID is the node ID of the node that the port is connected to. + NodeIDKey, + // router name - stores the name of the router this port belongs to + // This is required for uniqueness since there are two ports per network - + // one on the network router and one on the connect router. + // This also allows cleanup without maintaining a cache of router names + // This is used as a back reference to map the port to the router during + // network deletion, CNC cleanup, etc. It's because our database doesn't + // know the relationship between the port and the router and we always need + // to provide the router name when deleting the port. + RouterNameKey, +}) + +var LogicalRouterPolicyClusterNetworkConnect = newObjectIDsType(logicalRouterPolicy, ClusterNetworkConnectOwnerType, []ExternalIDKey{ + // CNC name + ObjectNameKey, + // source network ID + // value in k8s.ovn.org/network-id annotation set on the NAD + // of the source network whose router contains this policy. + SourceNetworkIDKey, + // destination network ID + // value in k8s.ovn.org/network-id annotation set on the NAD + // of the destination network that this policy routes to. + DestinationNetworkIDKey, + // the IP Family for this policy, ip4 or ip6 or ip(dualstack) + // In future when we support more than one pod subnet from same + // family for the same destination network, we should update the + // the matches of the policies, so in the end its just total of two + // policies, 1 per family each having a match of all subnets belonging + // to that network. + IPFamilyKey, + // router name - stores the name of the router this policy belongs to + // This allows cleanup without maintaining a cache of router names + // This is used as a back reference to map the policy to the router during + // network deletion, CNC cleanup, etc. It's because our database doesn't + // know the relationship between the policy and the router and we always need + // to provide the router name when deleting the policy. + RouterNameKey, +}) + +var LogicalRouterStaticRouteClusterNetworkConnect = newObjectIDsType(logicalRouterStaticRoute, ClusterNetworkConnectOwnerType, []ExternalIDKey{ + // CNC name + ObjectNameKey, + // connected network's network ID + // value in k8s.ovn.org/network-id annotation set on the NAD + // of the connected network that this static route routes to. + NetworkIDKey, + // destination node ID + NodeIDKey, + // the IP Family for this static route, ip4 or ip6 or ip(dualstack) + IPFamilyKey, +}) diff --git a/go-controller/pkg/libovsdb/ops/router.go b/go-controller/pkg/libovsdb/ops/router.go index eb57c3dcc6..3dc443bef3 100644 --- a/go-controller/pkg/libovsdb/ops/router.go +++ b/go-controller/pkg/libovsdb/ops/router.go @@ -174,6 +174,22 @@ func GetLogicalRouterPort(nbClient libovsdbclient.Client, lrp *nbdb.LogicalRoute // router port together with the gateway chassis (if not nil), and adds it to the provided logical router func CreateOrUpdateLogicalRouterPort(nbClient libovsdbclient.Client, router *nbdb.LogicalRouter, lrp *nbdb.LogicalRouterPort, chassis *nbdb.GatewayChassis, fields ...interface{}) error { + ops, err := CreateOrUpdateLogicalRouterPortOps(nbClient, nil, router, lrp, chassis, fields...) + if err != nil { + return err + } + _, err = TransactAndCheck(nbClient, ops) + return err +} + +// CreateOrUpdateLogicalRouterPortOps creates or updates the provided logical +// router port together with the gateway chassis (if not nil), adds it to the provided logical router, +// and returns the corresponding ops +func CreateOrUpdateLogicalRouterPortOps(nbClient libovsdbclient.Client, ops []ovsdb.Operation, router *nbdb.LogicalRouter, + lrp *nbdb.LogicalRouterPort, chassis *nbdb.GatewayChassis, fields ...interface{}) ([]ovsdb.Operation, error) { + if err := validateRequestedChassisOption(lrp.Options); err != nil { + return nil, err + } opModels := []operationModel{} if chassis != nil { opModels = append(opModels, operationModel{ @@ -205,9 +221,9 @@ func CreateOrUpdateLogicalRouterPort(nbClient libovsdbclient.Client, router *nbd BulkOp: false, }) m := newModelClient(nbClient) - _, err := m.CreateOrUpdate(opModels...) + ops, err := m.CreateOrUpdateOps(ops, opModels...) router.Ports = originalPorts - return err + return ops, err } // DeleteLogicalRouterPorts deletes the provided logical router ports and @@ -244,6 +260,29 @@ func DeleteLogicalRouterPorts(nbClient libovsdbclient.Client, router *nbdb.Logic return err } +func DeleteLogicalRouterPortWithPredicateOps(nbClient libovsdbclient.Client, ops []ovsdb.Operation, routerName string, p logicalRouterPortPredicate) ([]ovsdb.Operation, error) { + router := &nbdb.LogicalRouter{Name: routerName} + deleted := []*nbdb.LogicalRouterPort{} + opModels := []operationModel{ + { + ModelPredicate: p, + ExistingResult: &deleted, + DoAfter: func() { router.Ports = extractUUIDsFromModels(&deleted) }, + ErrNotFound: false, + BulkOp: true, + }, + { + Model: router, + OnModelMutations: []interface{}{&router.Ports}, + ErrNotFound: false, + BulkOp: false, + }, + } + + m := newModelClient(nbClient) + return m.DeleteOps(ops, opModels...) +} + // LOGICAL ROUTER POLICY OPs type logicalRouterPolicyPredicate func(*nbdb.LogicalRouterPolicy) bool diff --git a/go-controller/pkg/libovsdb/ops/switch.go b/go-controller/pkg/libovsdb/ops/switch.go index 4136f96bba..ff0216bd03 100644 --- a/go-controller/pkg/libovsdb/ops/switch.go +++ b/go-controller/pkg/libovsdb/ops/switch.go @@ -323,6 +323,9 @@ func createOrUpdateLogicalSwitchPortsOps(nbClient libovsdbclient.Client, ops []o opModels := make([]operationModel, 0, len(lsps)+1) for _, lsp := range lsps { + if err := validateRequestedChassisOption(lsp.Options); err != nil { + return nil, err + } opModel := createOrUpdateLogicalSwitchPortOpModelWithCustomFields(sw, lsp, createLSP, customFields) opModels = append(opModels, opModel) } @@ -480,38 +483,3 @@ func DeleteLogicalSwitchPortsWithPredicateOps(nbClient libovsdbclient.Client, op m := newModelClient(nbClient) return m.DeleteOps(ops, opModels...) } - -// UpdateLogicalSwitchPortSetOptions sets options on the provided logical switch -// port adding any missing, removing the ones set to an empty value and updating -// existing -func UpdateLogicalSwitchPortSetOptions(nbClient libovsdbclient.Client, lsp *nbdb.LogicalSwitchPort) error { - options := lsp.Options - lsp, err := GetLogicalSwitchPort(nbClient, lsp) - if err != nil { - return err - } - - if lsp.Options == nil { - lsp.Options = map[string]string{} - } - - for k, v := range options { - if v == "" { - delete(lsp.Options, k) - } else { - lsp.Options[k] = v - } - } - - opModel := operationModel{ - // For LSP's Name is a valid index, so no predicate is needed - Model: lsp, - OnModelUpdates: []interface{}{&lsp.Options}, - ErrNotFound: true, - BulkOp: false, - } - - m := newModelClient(nbClient) - _, err = m.CreateOrUpdate(opModel) - return err -} diff --git a/go-controller/pkg/metrics/cluster_manager.go b/go-controller/pkg/metrics/cluster_manager.go index 711d4dc026..69c311aa7b 100644 --- a/go-controller/pkg/metrics/cluster_manager.go +++ b/go-controller/pkg/metrics/cluster_manager.go @@ -1,6 +1,7 @@ package metrics import ( + "errors" "runtime" "sync" @@ -11,6 +12,7 @@ import ( ) var registerClusterManagerBaseMetrics sync.Once +var registerClusterManagerFunctionalMetrics sync.Once // MetricClusterManagerLeader identifies whether this instance of ovnkube-cluster-manager is a leader or not var MetricClusterManagerLeader = prometheus.NewGauge(prometheus.GaugeOpts{ @@ -113,6 +115,18 @@ var metricCUDNCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ }, ) +var metricUDNNodesRendered = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, + Name: "udn_nodes_rendered", + Help: "Number of nodes on which a UserDefinedNetwork (UDN) or ClusterUserDefinedNetwork (CUDN) is currently rendered.", + }, + []string{ + "network_name", + }, +) + // RegisterClusterManagerBase registers ovnkube cluster manager base metrics with the Prometheus registry. // This function should only be called once. func RegisterClusterManagerBase() { @@ -143,22 +157,28 @@ func RegisterClusterManagerBase() { // RegisterClusterManagerFunctional is a collection of metrics that help us understand ovnkube-cluster-manager functions. Call once after // LE is won. func RegisterClusterManagerFunctional() { - prometheus.MustRegister(metricV4HostSubnetCount) - prometheus.MustRegister(metricV6HostSubnetCount) - prometheus.MustRegister(metricV4AllocatedHostSubnetCount) - prometheus.MustRegister(metricV6AllocatedHostSubnetCount) - if config.OVNKubernetesFeature.EnableEgressIP { - prometheus.MustRegister(metricEgressIPNodeUnreacheableCount) - prometheus.MustRegister(metricEgressIPRebalanceCount) - prometheus.MustRegister(metricEgressIPCount) - } - prometheus.MustRegister(metricUDNCount) - prometheus.MustRegister(metricCUDNCount) - if err := prometheus.Register(MetricResourceRetryFailuresCount); err != nil { - if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { - panic(err) + registerClusterManagerFunctionalMetrics.Do(func() { + prometheus.MustRegister(metricV4HostSubnetCount) + prometheus.MustRegister(metricV6HostSubnetCount) + prometheus.MustRegister(metricV4AllocatedHostSubnetCount) + prometheus.MustRegister(metricV6AllocatedHostSubnetCount) + if config.OVNKubernetesFeature.EnableEgressIP { + prometheus.MustRegister(metricEgressIPNodeUnreacheableCount) + prometheus.MustRegister(metricEgressIPRebalanceCount) + prometheus.MustRegister(metricEgressIPCount) + } + prometheus.MustRegister(metricUDNCount) + prometheus.MustRegister(metricCUDNCount) + if config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + prometheus.MustRegister(metricUDNNodesRendered) } - } + if err := prometheus.Register(MetricResourceRetryFailuresCount); err != nil { + var alreadyRegistered prometheus.AlreadyRegisteredError + if !errors.As(err, &alreadyRegistered) { + panic(err) + } + } + }) } // RecordSubnetUsage records the number of subnets allocated for nodes @@ -209,3 +229,13 @@ func IncrementCUDNCount(role, topology string) { func DecrementCUDNCount(role, topology string) { metricCUDNCount.WithLabelValues(role, topology).Dec() } + +// SetDynamicUDNNodeCount sets the number of nodes currently active with a CUDN/UDN. +func SetDynamicUDNNodeCount(networkName string, nodeCount float64) { + metricUDNNodesRendered.WithLabelValues(networkName).Set(nodeCount) +} + +// DeleteDynamicUDNNodeCount when CUDN/UDN is deleted. +func DeleteDynamicUDNNodeCount(networkName string) { + metricUDNNodesRendered.DeleteLabelValues(networkName) +} diff --git a/go-controller/pkg/metrics/metrics.go b/go-controller/pkg/metrics/metrics.go index 835569adb9..89b0fa896f 100644 --- a/go-controller/pkg/metrics/metrics.go +++ b/go-controller/pkg/metrics/metrics.go @@ -100,7 +100,7 @@ func parseMetricToFloat(componentName, metricName, value string) float64 { // registerCoverageShowMetrics registers coverage/show metricss for // various components(ovn-northd, ovn-controller, and ovs-vswitchd) with prometheus -func registerCoverageShowMetrics(target string, metricNamespace string, metricSubsystem string) { +func registerCoverageShowMetrics(ovnRegistry prometheus.Registerer, target string, metricNamespace string, metricSubsystem string) { coverageShowMetricsMap := componentCoverageShowMetricsMap[target] for metricName, metricInfo := range coverageShowMetricsMap { metricInfo.metric = prometheus.NewGauge(prometheus.GaugeOpts{ @@ -191,50 +191,41 @@ func ovnKubeLogFileSizeMetricsUpdater(ovnKubeLogFileMetric *prometheus.GaugeVec, } } -// coverageShowMetricsUpdater updates the metric by obtaining values from +// coverageShowMetricsUpdate updates the metric by obtaining values from // getCoverageShowOutputMap for specified component. The counters displayed // by coverage/show output are called events. It could be that the event never // happened, and therefore there will be no counter for it in the output. In such // cases the default value of the counter will be 0. -func coverageShowMetricsUpdater(component string, stopChan <-chan struct{}) { - ticker := time.NewTicker(metricsUpdateInterval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - coverageShowOutputMap, err := getCoverageShowOutputMap(component) - if err != nil { - klog.Errorf("Getting coverage/show metrics for %s failed: %s", component, err.Error()) - continue - } - coverageShowMetricsMap := componentCoverageShowMetricsMap[component] - for metricName, metricInfo := range coverageShowMetricsMap { - var metricValue float64 - if metricInfo.srcName != "" { - metricName = metricInfo.srcName - } - if metricInfo.aggregateFrom != nil { - for _, aggregateMetricName := range metricInfo.aggregateFrom { - if value, ok := coverageShowOutputMap[aggregateMetricName]; ok { - metricValue += parseMetricToFloat(component, aggregateMetricName, value) - } - } - } else { - if value, ok := coverageShowOutputMap[metricName]; ok { - metricValue = parseMetricToFloat(component, metricName, value) - } +func coverageShowMetricsUpdate(component string) { + coverageShowOutputMap, err := getCoverageShowOutputMap(component) + if err != nil { + klog.Errorf("Getting coverage/show metrics for %s failed: %s", component, err.Error()) + return + } + coverageShowMetricsMap := componentCoverageShowMetricsMap[component] + for metricName, metricInfo := range coverageShowMetricsMap { + var metricValue float64 + if metricInfo.srcName != "" { + metricName = metricInfo.srcName + } + if metricInfo.aggregateFrom != nil { + for _, aggregateMetricName := range metricInfo.aggregateFrom { + if value, ok := coverageShowOutputMap[aggregateMetricName]; ok { + metricValue += parseMetricToFloat(component, aggregateMetricName, value) } - metricInfo.metric.Set(metricValue) } - case <-stopChan: - return + } else { + if value, ok := coverageShowOutputMap[metricName]; ok { + metricValue = parseMetricToFloat(component, metricName, value) + } } + metricInfo.metric.Set(metricValue) } } // registerStopwatchShowMetrics registers stopwatch/show metrics for // various components(ovn-northd, ovn-controller) with prometheus -func registerStopwatchShowMetrics(component string, metricNamespace string, metricSubsystem string) { +func registerStopwatchShowMetrics(ovnRegistry prometheus.Registerer, component string, metricNamespace string, metricSubsystem string) { stopwatchShowMetricsMap := componentStopwatchShowMetricsMap[component] for metricName, metricInfo := range stopwatchShowMetricsMap { metricInfo.metrics.totalSamples = prometheus.NewGauge(prometheus.GaugeOpts{ @@ -359,58 +350,50 @@ func parseStopwatchShowOutput(output string) map[string]stopwatchStatistics { return result } -// stopwatchShowMetricsUpdater updates the metric by obtaining the stopwatch/show +// stopwatchShowMetricsUpdate updates the metric by obtaining the stopwatch/show // metrics for the specified component. -func stopwatchShowMetricsUpdater(component string, stopChan <-chan struct{}) { - ticker := time.NewTicker(metricsUpdateInterval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - stopwatchShowOutputMap, err := getStopwatchShowOutputMap(component) - if err != nil { - klog.Errorf("Getting stopwatch/show metrics for %s failed: %s", component, err.Error()) - continue - } - - if len(stopwatchShowOutputMap) == 0 { - klog.Warningf("No stopwatch/show metrics for component %s", component) - continue - } +func stopwatchShowMetricsUpdate(component string) { + stopwatchShowOutputMap, err := getStopwatchShowOutputMap(component) + if err != nil { + klog.Errorf("Getting stopwatch/show metrics for %s failed: %s", component, err.Error()) + return + } - stopwatchShowInterestingMetrics := componentStopwatchShowMetricsMap[component] - for metricName, metricInfo := range stopwatchShowInterestingMetrics { - var totalSamplesMetricValue, maxMetricValue, minMetricValue, percentile95thMetricValue, shortTermAvgMetricValue, longTermAvgMetricValue float64 + if len(stopwatchShowOutputMap) == 0 { + klog.Warningf("No stopwatch/show metrics for component %s", component) + return + } - if metricInfo.srcName != "" { - metricName = metricInfo.srcName - } + stopwatchShowInterestingMetrics := componentStopwatchShowMetricsMap[component] + for metricName, metricInfo := range stopwatchShowInterestingMetrics { + var totalSamplesMetricValue, maxMetricValue, minMetricValue, percentile95thMetricValue, shortTermAvgMetricValue, longTermAvgMetricValue float64 - if value, ok := stopwatchShowOutputMap[metricName]; ok { - totalSamplesMetricValue = parseMetricToFloat(component, metricName, value.totalSamples) - minMetricValue = parseMetricToFloat(component, metricName, value.min) - maxMetricValue = parseMetricToFloat(component, metricName, value.max) - percentile95thMetricValue = parseMetricToFloat(component, metricName, value.percentile95th) - shortTermAvgMetricValue = parseMetricToFloat(component, metricName, value.shortTermAvg) - longTermAvgMetricValue = parseMetricToFloat(component, metricName, value.longTermAvg) - } + if metricInfo.srcName != "" { + metricName = metricInfo.srcName + } - metricInfo.metrics.totalSamples.Set(totalSamplesMetricValue) - metricInfo.metrics.min.Set(minMetricValue / 1000) - metricInfo.metrics.max.Set(maxMetricValue / 1000) - metricInfo.metrics.percentile95th.Set(percentile95thMetricValue / 1000) - metricInfo.metrics.shortTermAvg.Set(shortTermAvgMetricValue / 1000) - metricInfo.metrics.longTermAvg.Set(longTermAvgMetricValue / 1000) - } - case <-stopChan: - return + if value, ok := stopwatchShowOutputMap[metricName]; ok { + totalSamplesMetricValue = parseMetricToFloat(component, metricName, value.totalSamples) + minMetricValue = parseMetricToFloat(component, metricName, value.min) + maxMetricValue = parseMetricToFloat(component, metricName, value.max) + percentile95thMetricValue = parseMetricToFloat(component, metricName, value.percentile95th) + shortTermAvgMetricValue = parseMetricToFloat(component, metricName, value.shortTermAvg) + longTermAvgMetricValue = parseMetricToFloat(component, metricName, value.longTermAvg) } + + metricInfo.metrics.totalSamples.Set(totalSamplesMetricValue) + metricInfo.metrics.min.Set(minMetricValue / 1000) + metricInfo.metrics.max.Set(maxMetricValue / 1000) + metricInfo.metrics.percentile95th.Set(percentile95thMetricValue / 1000) + metricInfo.metrics.shortTermAvg.Set(shortTermAvgMetricValue / 1000) + metricInfo.metrics.longTermAvg.Set(longTermAvgMetricValue / 1000) } + } // The `keepTrying` boolean when set to true will not return an error if we can't find pods with one of the given labels. // This is so that the caller can re-try again to see if the pods have appeared in the k8s cluster. -func checkPodRunsOnGivenNode(clientset kubernetes.Interface, labels []string, k8sNodeName string, +func CheckPodRunsOnGivenNode(clientset kubernetes.Interface, labels []string, k8sNodeName string, keepTrying bool) (bool, error) { for _, label := range labels { pods, err := clientset.CoreV1().Pods(config.Kubernetes.OVNConfigNamespace).List(context.TODO(), metav1.ListOptions{ @@ -520,16 +503,24 @@ func StartMetricsServer(bindAddress string, enablePprof bool, certFile string, k startMetricsServer(bindAddress, certFile, keyFile, mux, stopChan, wg) } -var ovnRegistry = prometheus.NewRegistry() - // StartOVNMetricsServer runs the prometheus listener so that OVN metrics can be collected -func StartOVNMetricsServer(bindAddress, certFile, keyFile string, stopChan <-chan struct{}, wg *sync.WaitGroup) { - handler := promhttp.InstrumentMetricHandler(ovnRegistry, - promhttp.HandlerFor(ovnRegistry, promhttp.HandlerOpts{})) - mux := http.NewServeMux() - mux.Handle("/metrics", handler) +func StartOVNMetricsServer(opts MetricServerOptions, + ovsClient libovsdbclient.Client, + kubeClient kubernetes.Interface, + stopChan <-chan struct{}, wg *sync.WaitGroup) *MetricServer { - startMetricsServer(bindAddress, certFile, keyFile, mux, stopChan, wg) + klog.Infof("Create OVN Metrics Server on address: %s", opts.BindAddress) + metricsServer := NewMetricServer(opts, ovsClient, kubeClient) + metricsServer.registerMetrics() + + wg.Add(1) + go func() { + defer wg.Done() + klog.Infof("OVN Metrics Server starts to run ...") + metricsServer.Run(stopChan) + }() + + return metricsServer } func startMetricsServer(bindAddress, certFile, keyFile string, handler http.Handler, stopChan <-chan struct{}, wg *sync.WaitGroup) { @@ -568,40 +559,3 @@ func startMetricsServer(bindAddress, certFile, keyFile string, handler http.Hand }, 5*time.Second, stopChan) }() } - -func RegisterOvnMetrics(clientset kubernetes.Interface, k8sNodeName string, ovsDBClient libovsdbclient.Client, - metricsScrapeInterval int, stopChan <-chan struct{}) { - go RegisterOvnDBMetrics( - func() bool { - err := utilwait.PollUntilContextTimeout(context.Background(), 1*time.Second, 300*time.Second, true, func(_ context.Context) (bool, error) { - return checkPodRunsOnGivenNode(clientset, []string{"ovn-db-pod=true"}, k8sNodeName, false) - }) - if err != nil { - if utilwait.Interrupted(err) { - klog.Errorf("Timed out while checking if OVN DB Pod runs on this %q K8s Node: %v. "+ - "Not registering OVN DB Metrics on this Node.", k8sNodeName, err) - } else { - klog.Infof("Not registering OVN DB Metrics on this Node since OVN DBs are not running on this node.") - } - return false - } - return true - }, - stopChan, - ) - go RegisterOvnControllerMetrics(ovsDBClient, metricsScrapeInterval, stopChan) - go RegisterOvnNorthdMetrics( - func() bool { - err := utilwait.PollUntilContextTimeout(context.Background(), 1*time.Second, 300*time.Second, true, func(_ context.Context) (bool, error) { - return checkPodRunsOnGivenNode(clientset, []string{"ovn-db-pod=true"}, k8sNodeName, true) - }) - if err != nil { - klog.Infof("Not registering OVN Northd Metrics because OVN DB Pod was not found running on this "+ - "node (%s)", k8sNodeName) - return false - } - return true - }, - stopChan, - ) -} diff --git a/go-controller/pkg/metrics/ovn.go b/go-controller/pkg/metrics/ovn.go index bdc992e2ea..a243aec0ee 100644 --- a/go-controller/pkg/metrics/ovn.go +++ b/go-controller/pkg/metrics/ovn.go @@ -3,7 +3,6 @@ package metrics import ( "fmt" "strings" - "time" "github.com/prometheus/client_golang/prometheus" @@ -310,22 +309,6 @@ func setOvnControllerConfigurationMetrics(ovsDBClient libovsdbclient.Client) (er return nil } -func ovnControllerConfigurationMetricsUpdater(ovsDBClient libovsdbclient.Client, metricsScrapeInterval int, - stopChan <-chan struct{}) { - ticker := time.NewTicker(time.Duration(metricsScrapeInterval) * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := setOvnControllerConfigurationMetrics(ovsDBClient); err != nil { - klog.Errorf("Setting ovn controller config metrics failed: %s", err.Error()) - } - case <-stopChan: - return - } - } -} - func getPortCount(ovsDBClient libovsdbclient.Client, portType string) float64 { var portCount float64 p := func(item *vswitchd.Interface) bool { @@ -351,55 +334,26 @@ func getPortCount(ovsDBClient libovsdbclient.Client, portType string) float64 { return portCount } -// ovnControllerSBDBConnectionCheckUpdater blocks until stopCh closed but before then polls ovn-controllers connection status with -// southbound database periodically. -func ovnControllerSBDBConnectionCheckUpdater(stopCh <-chan struct{}, ovsAppctl ovsClient, period time.Duration) { - // There maybe transient connection issues to SB DB. We want to minimise the risk of reporting this as the current state between - // long poll intervals. - retry := 5 - retrySleep := 5 * time.Second - retryTotal := retrySleep * time.Duration(retry) - - if retryTotal >= period { - panic("period must be greater than retry total time") - } - // update metric to a good initial state - updateSBDBConnectionMetric(ovsAppctl, retry, retrySleep) - - ticker := time.NewTicker(period) - for { - select { - case <-ticker.C: - updateSBDBConnectionMetric(ovsAppctl, retry, retrySleep) - case <-stopCh: - ticker.Stop() - return - } - } -} - -func updateSBDBConnectionMetric(ovsAppctl ovsClient, retry int, retrySleep time.Duration) { +// updateSBDBConnectionMetric updates the connection status with southbound database +func updateSBDBConnectionMetric(ovsAppctl ovsClient) { + // NOTE: This metric had a retry logic, which is removed because metrics should reflect the reality. + // Instead, alert rules should be configured with appropriate thresholds (e.g., "for: 2m") to handle + // transient connection issues and only fire alerts for sustained problems. var stdOut, stdErr string var err error - var connected bool - connected = false - for i := 0; i < retry && !connected; i++ { - stdOut, stdErr, err = ovsAppctl("connection-status") - if err != nil { - klog.Errorf("Failed to get OVN controller southbound database connection status before utilizing "+ - "client ovs-appctl: %v", err) - } else if stdErr != "" { - klog.Errorf("Failed to get OVN controller southbound database connection status because "+ - "ovs-appctl command returned an error: %s", stdErr) - } else if stdOut == "" { - klog.Errorf("Unexpected blank output while attempting to retrieve OVN controller southbound " + - "database connection status") - } else if strings.HasPrefix(stdOut, "connected") { - connected = true - } else { - // sleep and retry - time.Sleep(retrySleep) - } + connected := false + stdOut, stdErr, err = ovsAppctl("connection-status") + if err != nil { + klog.Errorf("Failed to get OVN controller southbound database connection status before utilizing "+ + "client ovs-appctl: %v", err) + } else if stdErr != "" { + klog.Errorf("Failed to get OVN controller southbound database connection status because "+ + "ovs-appctl command returned an error: %s", stdErr) + } else if stdOut == "" { + klog.Errorf("Unexpected blank output while attempting to retrieve OVN controller southbound " + + "database connection status") + } else if strings.HasPrefix(stdOut, "connected") { + connected = true } if connected { @@ -409,8 +363,8 @@ func updateSBDBConnectionMetric(ovsAppctl ovsClient, retry int, retrySleep time. } } -func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, - metricsScrapeInterval int, stopChan <-chan struct{}) { +// RegisterOvnControllerMetrics registers the ovn-controller metrics +func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, ovnRegistry *prometheus.Registry) { getOvnControllerVersionInfo() ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ @@ -483,19 +437,9 @@ func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, ovnRegistry.MustRegister(metricBridgeMappings) // Register the ovn-controller coverage/show metrics componentCoverageShowMetricsMap[ovnController] = ovnControllerCoverageShowMetricsMap - registerCoverageShowMetrics(ovnController, types.MetricOvnNamespace, types.MetricOvnSubsystemController) + registerCoverageShowMetrics(ovnRegistry, ovnController, types.MetricOvnNamespace, types.MetricOvnSubsystemController) // Register the ovn-controller coverage/show metrics componentStopwatchShowMetricsMap[ovnController] = ovnControllerStopwatchShowMetricsMap - registerStopwatchShowMetrics(ovnController, types.MetricOvnNamespace, types.MetricOvnSubsystemController) - - // ovn-controller configuration metrics updater - go ovnControllerConfigurationMetricsUpdater(ovsDBClient, - metricsScrapeInterval, stopChan) - // ovn-controller coverage show metrics updater - go coverageShowMetricsUpdater(ovnController, stopChan) - // ovn-controller stopwatch show metrics updater - go stopwatchShowMetricsUpdater(ovnController, stopChan) - // ovn-controller southbound database connection status updater - go ovnControllerSBDBConnectionCheckUpdater(stopChan, util.RunOVNControllerAppCtl, time.Minute*2) + registerStopwatchShowMetrics(ovnRegistry, ovnController, types.MetricOvnNamespace, types.MetricOvnSubsystemController) } diff --git a/go-controller/pkg/metrics/ovn_db.go b/go-controller/pkg/metrics/ovn_db.go index 2324cac828..e42fa1be3f 100644 --- a/go-controller/pkg/metrics/ovn_db.go +++ b/go-controller/pkg/metrics/ovn_db.go @@ -2,10 +2,8 @@ package metrics import ( "fmt" - "os" "strconv" "strings" - "time" "github.com/prometheus/client_golang/prometheus" @@ -243,7 +241,8 @@ var metricDBClusterConnOutErr = prometheus.NewGaugeVec(prometheus.GaugeOpts{ }, ) -func ovnDBSizeMetricsUpdater(dbProps *util.OvsDbProperties) { +// updateOvnDBSizeMetrics collects and updates the OVN DB size metric +func updateOvnDBSizeMetrics(dbProps *util.OvsDbProperties) { if size, err := getOvnDBSizeViaPath(dbProps); err != nil { klog.Errorf("Failed to update OVN DB size metric: %v", err) } else { @@ -268,7 +267,7 @@ func isOvnDBFoundViaPath(dbProperties []*util.OvsDbProperties) bool { } func getOvnDBSizeViaPath(dbProperties *util.OvsDbProperties) (int64, error) { - fileInfo, err := os.Stat(dbProperties.DbAlias) + fileInfo, err := util.AppFs.Stat(dbProperties.DbAlias) if err != nil { return 0, fmt.Errorf("failed to find OVN DB database %s at path %s: %v", dbProperties.DbName, dbProperties.DbAlias, err) @@ -276,7 +275,8 @@ func getOvnDBSizeViaPath(dbProperties *util.OvsDbProperties) (int64, error) { return fileInfo.Size(), nil } -func ovnDBMemoryMetricsUpdater(dbProperties *util.OvsDbProperties) { +// updateOvnDBMemoryMetrics collects and updates the OVN DB memory metric +func updateOvnDBMemoryMetrics(dbProperties *util.OvsDbProperties) { var stdout, stderr string var err error @@ -323,7 +323,7 @@ var ( func getNBDBSockPath() (string, error) { paths := []string{config.OvsPaths.RunDir, config.OvnNorth.RunDir} for _, basePath := range paths { - if _, err := os.Stat(basePath + "ovnnb_db.sock"); err == nil { + if _, err := util.AppFs.Stat(basePath + "ovnnb_db.sock"); err == nil { klog.Infof("ovnnb_db.sock found at %s", basePath) return basePath, nil } else { @@ -359,14 +359,7 @@ func getOvnDbVersionInfo() { } } -func RegisterOvnDBMetrics(waitTimeoutFunc func() bool, stopChan <-chan struct{}) { - if ok := waitTimeoutFunc(); !ok { - klog.Info("OVN DB metrics registration skipped: readiness gate not satisfied") - return - } - - klog.Info("Found OVN DB Pod running on this node. Registering OVN DB Metrics") - +func RegisterOvnDBMetrics(ovnRegistry *prometheus.Registry) ([]*util.OvsDbProperties, bool, bool) { // get the ovsdb server version info getOvnDbVersionInfo() // register metrics that will be served off of /metrics path @@ -392,17 +385,19 @@ func RegisterOvnDBMetrics(waitTimeoutFunc func() bool, stopChan <-chan struct{}) if err != nil { klog.Errorf("Failed to init nbdb properties: %s", err) } else { + klog.Infof("Found OVN NB DB: %v", nbdbProps) dbProperties = append(dbProperties, nbdbProps) } sbdbProps, err := util.GetOvsDbProperties(config.OvnSouth.DbLocation) if err != nil { klog.Errorf("Failed to init sbdb properties: %s", err) } else { + klog.Infof("Found OVN SB DB: %v", sbdbProps) dbProperties = append(dbProperties, sbdbProps) } if len(dbProperties) == 0 { klog.Errorf("Failed to init properties for all databases") - return + return nil, false, false } // check if DB is clustered or not // the usual way would be to call `ovsdb-tool db-is-standalone`, @@ -441,35 +436,7 @@ func RegisterOvnDBMetrics(waitTimeoutFunc func() bool, stopChan <-chan struct{}) klog.Infof("Unable to enable OVN DB size metric because no OVN DBs found") } - // functions responsible for collecting the values and updating the prometheus metrics - go func() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - // To update not only values but also labels for metrics, we use Reset() to delete previous labels+value - if dbIsClustered { - resetOvnDbClusterMetrics() - } - if dbFoundViaPath { - resetOvnDbSizeMetric() - } - resetOvnDbMemoryMetrics() - for _, dbProperty := range dbProperties { - if dbIsClustered { - ovnDBClusterStatusMetricsUpdater(dbProperty) - } - if dbFoundViaPath { - ovnDBSizeMetricsUpdater(dbProperty) - } - ovnDBMemoryMetricsUpdater(dbProperty) - } - case <-stopChan: - return - } - } - }() + return dbProperties, dbIsClustered, dbFoundViaPath } type OVNDBClusterStatus struct { diff --git a/go-controller/pkg/metrics/ovn_northd.go b/go-controller/pkg/metrics/ovn_northd.go index 02bab346c7..0cc729912a 100644 --- a/go-controller/pkg/metrics/ovn_northd.go +++ b/go-controller/pkg/metrics/ovn_northd.go @@ -110,16 +110,8 @@ var ovnNorthdStopwatchShowMetricsMap = map[string]*stopwatchMetricDetails{ "ovnsb_db_run": {}, } -func RegisterOvnNorthdMetrics( - waitTimeoutFunc func() bool, - stopChan <-chan struct{}, -) { - if ok := waitTimeoutFunc(); !ok { - klog.Info("OVN northd metrics registration skipped: readiness gate not satisfied") - return - } - klog.Info("Registering OVN northd metrics") - +// RegisterOvnNorthdMetrics registers the ovn-northd metrics +func RegisterOvnNorthdMetrics(ovnRegistry prometheus.Registerer) { // ovn-northd metrics getOvnNorthdVersionInfo() ovnRegistry.MustRegister(prometheus.NewGaugeFunc( @@ -186,11 +178,9 @@ func RegisterOvnNorthdMetrics( // Register the ovn-northd coverage/show metrics with prometheus componentCoverageShowMetricsMap[ovnNorthd] = ovnNorthdCoverageShowMetricsMap - registerCoverageShowMetrics(ovnNorthd, types.MetricOvnNamespace, types.MetricOvnSubsystemNorthd) - go coverageShowMetricsUpdater(ovnNorthd, stopChan) + registerCoverageShowMetrics(ovnRegistry, ovnNorthd, types.MetricOvnNamespace, types.MetricOvnSubsystemNorthd) // Register the ovn-northd stopwatch/show metrics with prometheus componentStopwatchShowMetricsMap[ovnNorthd] = ovnNorthdStopwatchShowMetricsMap - registerStopwatchShowMetrics(ovnNorthd, types.MetricOvnNamespace, types.MetricOvnSubsystemNorthd) - go stopwatchShowMetricsUpdater(ovnNorthd, stopChan) + registerStopwatchShowMetrics(ovnRegistry, ovnNorthd, types.MetricOvnNamespace, types.MetricOvnSubsystemNorthd) } diff --git a/go-controller/pkg/metrics/ovs.go b/go-controller/pkg/metrics/ovs.go index 69b3b0a8ee..8cb73610cd 100644 --- a/go-controller/pkg/metrics/ovs.go +++ b/go-controller/pkg/metrics/ovs.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strings" "sync" - "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" @@ -351,7 +350,7 @@ func ovsDatapathMasksMetrics(output, datapath string) { // getOvsDatapaths gives list of datapaths // and updates the corresponding datapath metrics -func getOvsDatapaths(ovsAppctl ovsClient) (datapathsList []string, err error) { +func getOvsDatapaths() (datapathsList []string, err error) { var stdout, stderr string defer func() { @@ -361,7 +360,7 @@ func getOvsDatapaths(ovsAppctl ovsClient) (datapathsList []string, err error) { } }() - stdout, stderr, err = ovsAppctl("dpctl/dump-dps") + stdout, stderr, err = util.RunOvsVswitchdAppCtl("dpctl/dump-dps") if err != nil { return nil, fmt.Errorf("failed to get output of ovs-appctl dpctl/dump-dps "+ "stderr(%s) :(%v)", stderr, err) @@ -382,7 +381,7 @@ func getOvsDatapaths(ovsAppctl ovsClient) (datapathsList []string, err error) { return datapathsList, nil } -func setOvsDatapathMetrics(ovsAppctl ovsClient, datapaths []string) (err error) { +func setOvsDatapathMetrics(datapaths []string) (err error) { var stdout, stderr, datapath string defer func() { @@ -397,7 +396,7 @@ func setOvsDatapathMetrics(ovsAppctl ovsClient, datapaths []string) (err error) // the datapath type and 'ovs-system' the datapath name. To uniquely // identify a datapath, both are required when querying OVS. If type is // omitted, OVS will assume 'system'. - stdout, stderr, err = ovsAppctl("dpctl/show", datapath) + stdout, stderr, err = util.RunOvsVswitchdAppCtl("dpctl/show", datapath) if err != nil { return fmt.Errorf("failed to get datapath stats for %s "+ "stderr(%s) :(%v)", datapath, stderr, err) @@ -430,41 +429,15 @@ func setOvsDatapathMetrics(ovsAppctl ovsClient, datapaths []string) (err error) return nil } -// ovsDatapathMetricsUpdater updates the ovs datapath metrics -func ovsDatapathMetricsUpdater(ovsAppctl ovsClient, metricsScrapeInterval int, stopChan <-chan struct{}) { - ticker := time.NewTicker(time.Duration(metricsScrapeInterval) * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - datapaths, err := getOvsDatapaths(ovsAppctl) - if err != nil { - klog.Errorf("Getting ovs datapath list failed: %s", err.Error()) - continue - } - if err = setOvsDatapathMetrics(ovsAppctl, datapaths); err != nil { - klog.Errorf("Setting ovs datapath metrics failed: %s", err.Error()) - } - case <-stopChan: - return - } +// ovsDatapathMetricsUpdate collects and updates the ovs datapath metrics +func ovsDatapathMetricsUpdate() { + datapaths, err := getOvsDatapaths() + if err != nil { + klog.Errorf("Getting ovs datapath list failed: %s", err.Error()) + return } -} - -// ovsBridgeMetricsUpdater updates bridge related metrics -func ovsBridgeMetricsUpdater(ovsDBClient libovsdbclient.Client, ovsAppctl ovsClient, metricsScrapeInterval int, stopChan <-chan struct{}) { - ticker := time.NewTicker(time.Duration(metricsScrapeInterval) * time.Second) - defer ticker.Stop() - var err error - for { - select { - case <-ticker.C: - if err = updateOvsBridgeMetrics(ovsDBClient, ovsAppctl); err != nil { - klog.Errorf("Getting ovs bridge info failed: %s", err.Error()) - } - case <-stopChan: - return - } + if err = setOvsDatapathMetrics(datapaths); err != nil { + klog.Errorf("Setting ovs datapath metrics failed: %s", err.Error()) } } @@ -513,22 +486,6 @@ func getOvsBridgeOpenFlowsCount(ovsOfctl ovsClient, bridgeName string) (float64, "flow_count field", bridgeName) } -func ovsInterfaceMetricsUpdater(ovsDBClient libovsdbclient.Client, metricsScrapeInterval int, stopChan <-chan struct{}) { - ticker := time.NewTicker(time.Duration(metricsScrapeInterval) * time.Second) - defer ticker.Stop() - var err error - for { - select { - case <-ticker.C: - if err = updateOvsInterfaceMetrics(ovsDBClient); err != nil { - klog.Errorf("Updating OVS interface metrics failed: %s", err.Error()) - } - case <-stopChan: - return - } - } -} - // updateOvsInterfaceMetrics updates the ovs interface metrics obtained from ovsdb func updateOvsInterfaceMetrics(ovsDBClient libovsdbclient.Client) error { interfaceList, err := ovsops.ListInterfaces(ovsDBClient) @@ -608,21 +565,6 @@ func setOvsMemoryMetrics(ovsVswitchdAppctl ovsClient) (err error) { return nil } -func ovsMemoryMetricsUpdater(ovsVswitchdAppctl ovsClient, metricsScrapeInterval int, stopChan <-chan struct{}) { - ticker := time.NewTicker(time.Duration(metricsScrapeInterval) * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := setOvsMemoryMetrics(ovsVswitchdAppctl); err != nil { - klog.Errorf("Setting ovs memory metrics failed: %s", err.Error()) - } - case <-stopChan: - return - } - } -} - // setOvsHwOffloadMetrics updates the hw-offload, tc-policy metrics // obtained from Open_vSwitch table updates func setOvsHwOffloadMetrics(ovsDBClient libovsdbclient.Client) (err error) { @@ -655,21 +597,6 @@ func setOvsHwOffloadMetrics(ovsDBClient libovsdbclient.Client) (err error) { return nil } -func ovsHwOffloadMetricsUpdater(ovsDBClient libovsdbclient.Client, metricsScrapeInterval int, stopChan <-chan struct{}) { - ticker := time.NewTicker(time.Duration(metricsScrapeInterval) * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := setOvsHwOffloadMetrics(ovsDBClient); err != nil { - klog.Errorf("Setting ovs hardware offload metrics failed: %s", err.Error()) - } - case <-stopChan: - return - } - } -} - var ovsVswitchdCoverageShowMetricsMap = map[string]*metricDetails{ "netlink_sent": { help: "Number of netlink message sent to the kernel.", @@ -835,15 +762,8 @@ var ovsVswitchdCoverageShowMetricsMap = map[string]*metricDetails{ } var registerOvsMetricsOnce sync.Once -func RegisterStandaloneOvsMetrics(ovsDBClient libovsdbclient.Client, metricsScrapeInterval int, stopChan <-chan struct{}) { - registerOvsMetrics(ovsDBClient, metricsScrapeInterval, prometheus.DefaultRegisterer, stopChan) -} - -func RegisterOvsMetricsWithOvnMetrics(ovsDBClient libovsdbclient.Client, metricsScrapeInterval int, stopChan <-chan struct{}) { - registerOvsMetrics(ovsDBClient, metricsScrapeInterval, ovnRegistry, stopChan) -} - -func registerOvsMetrics(ovsDBClient libovsdbclient.Client, metricsScrapeInterval int, registry prometheus.Registerer, stopChan <-chan struct{}) { +// registerOvsMetrics registers the ovs metrics +func registerOvsMetrics(ovsDBClient libovsdbclient.Client, registry prometheus.Registerer) { registerOvsMetricsOnce.Do(func() { getOvsVersionInfo(ovsDBClient) registry.MustRegister(prometheus.NewGaugeFunc( @@ -892,7 +812,7 @@ func registerOvsMetrics(ovsDBClient libovsdbclient.Client, metricsScrapeInterval registry.MustRegister(MetricOvsInterfaceUpWait) // Register the OVS coverage/show metrics componentCoverageShowMetricsMap[ovsVswitchd] = ovsVswitchdCoverageShowMetricsMap - registerCoverageShowMetrics(ovsVswitchd, types.MetricOvsNamespace, types.MetricOvsSubsystemVswitchd) + registerCoverageShowMetrics(registry, ovsVswitchd, types.MetricOvsNamespace, types.MetricOvsSubsystemVswitchd) // When ovnkube-node is running in privileged mode, the hostPID will be set to true, // and therefore it can monitor OVS running on the host using PID. @@ -906,18 +826,5 @@ func registerOvsMetrics(ovsDBClient libovsdbclient.Client, metricsScrapeInterval Namespace: fmt.Sprintf("%s_%s", types.MetricOvsNamespace, types.MetricOvsSubsystemDB), })) } - - // OVS datapath metrics updater - go ovsDatapathMetricsUpdater(util.RunOvsVswitchdAppCtl, metricsScrapeInterval, stopChan) - // OVS bridge metrics updater - go ovsBridgeMetricsUpdater(ovsDBClient, util.RunOVSOfctl, metricsScrapeInterval, stopChan) - // OVS interface metrics updater - go ovsInterfaceMetricsUpdater(ovsDBClient, metricsScrapeInterval, stopChan) - // OVS memory metrics updater - go ovsMemoryMetricsUpdater(util.RunOvsVswitchdAppCtl, metricsScrapeInterval, stopChan) - // OVS hw Offload metrics updater - go ovsHwOffloadMetricsUpdater(ovsDBClient, metricsScrapeInterval, stopChan) - // OVS coverage/show metrics updater. - go coverageShowMetricsUpdater(ovsVswitchd, stopChan) }) } diff --git a/go-controller/pkg/metrics/ovs_test.go b/go-controller/pkg/metrics/ovs_test.go index 84d0e929f5..e0580c66b6 100644 --- a/go-controller/pkg/metrics/ovs_test.go +++ b/go-controller/pkg/metrics/ovs_test.go @@ -52,6 +52,44 @@ var _ = ginkgo.Describe("OVS metrics", func() { var rxErrorsTotalMock, txErrorsTotalMock, collisionsTotalMock, bridgeTotalMock *mocks.GaugeMock var hwOffloadMock, tcPolicyMock *mocks.GaugeMock + // These tests overwrite package-level metrics (prometheus Gauges), so save and + // restore them per-spec to avoid leaking state into other tests. + var _metricOvsInterfaceResetsTotal prometheus.Gauge + var _metricOvsInterfaceRxDroppedTotal prometheus.Gauge + var _metricOvsInterfaceTxDroppedTotal prometheus.Gauge + var _metricOvsInterfaceRxErrorsTotal prometheus.Gauge + var _metricOvsInterfaceTxErrorsTotal prometheus.Gauge + var _metricOvsInterfaceCollisionsTotal prometheus.Gauge + var _metricOvsBridgeTotal prometheus.Gauge + var _metricOvsHwOffload prometheus.Gauge + var _metricOvsTcPolicy prometheus.Gauge + + ginkgo.BeforeEach(func() { + // Save all original metrics + _metricOvsInterfaceResetsTotal = metricOvsInterfaceResetsTotal + _metricOvsInterfaceRxDroppedTotal = metricOvsInterfaceRxDroppedTotal + _metricOvsInterfaceTxDroppedTotal = metricOvsInterfaceTxDroppedTotal + _metricOvsInterfaceRxErrorsTotal = metricOvsInterfaceRxErrorsTotal + _metricOvsInterfaceTxErrorsTotal = metricOvsInterfaceTxErrorsTotal + _metricOvsInterfaceCollisionsTotal = metricOvsInterfaceCollisionsTotal + _metricOvsBridgeTotal = metricOvsBridgeTotal + _metricOvsHwOffload = metricOvsHwOffload + _metricOvsTcPolicy = metricOvsTcPolicy + }) + + ginkgo.AfterEach(func() { + // Restore all original metrics + metricOvsInterfaceResetsTotal = _metricOvsInterfaceResetsTotal + metricOvsInterfaceRxDroppedTotal = _metricOvsInterfaceRxDroppedTotal + metricOvsInterfaceTxDroppedTotal = _metricOvsInterfaceTxDroppedTotal + metricOvsInterfaceRxErrorsTotal = _metricOvsInterfaceRxErrorsTotal + metricOvsInterfaceTxErrorsTotal = _metricOvsInterfaceTxErrorsTotal + metricOvsInterfaceCollisionsTotal = _metricOvsInterfaceCollisionsTotal + metricOvsBridgeTotal = _metricOvsBridgeTotal + metricOvsHwOffload = _metricOvsHwOffload + metricOvsTcPolicy = _metricOvsTcPolicy + }) + linkResets := 1 intf1 := vswitchd.Interface{Name: "porta", UUID: buildUUID()} intf2 := vswitchd.Interface{Name: "portb", UUID: buildUUID()} diff --git a/go-controller/pkg/metrics/server.go b/go-controller/pkg/metrics/server.go new file mode 100644 index 0000000000..88d04ce2a4 --- /dev/null +++ b/go-controller/pkg/metrics/server.go @@ -0,0 +1,249 @@ +package metrics + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + utilwait "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + libovsdbclient "github.com/ovn-kubernetes/libovsdb/client" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +// MetricServerOptions defines the configuration options for the new MetricServer +type MetricServerOptions struct { + // Server configuration + BindAddress string + + // TLS configuration + CertFile string + KeyFile string + + // Feature flags + EnableOVSMetrics bool + EnableOVNDBMetrics bool + EnableOVNControllerMetrics bool + EnableOVNNorthdMetrics bool + + // OnFatalError is called when an unrecoverable error occurs (e.g., failed to bind to address). + // If set, it allows the caller to trigger a graceful shutdown. + OnFatalError func() + + // Kubernetes integration + K8sClient kubernetes.Interface + K8sNodeName string + OVSDBClient libovsdbclient.Client + + dbIsClustered bool + dbFoundViaPath bool +} + +// MetricServer represents the new unified metrics server +type MetricServer struct { + // Configuration + opts MetricServerOptions + + ovsDBClient libovsdbclient.Client + kubeClient kubernetes.Interface + + ovsDbProperties []*util.OvsDbProperties + + // HTTP server + server *http.Server + mux *http.ServeMux + + // Prometheus registries + ovnRegistry *prometheus.Registry +} + +// NewMetricServer creates a new MetricServer instance +func NewMetricServer(opts MetricServerOptions, ovsDBClient libovsdbclient.Client, kubeClient kubernetes.Interface) *MetricServer { + // Create server instance + server := &MetricServer{ + opts: opts, + ovsDBClient: ovsDBClient, + ovnRegistry: prometheus.NewRegistry(), + kubeClient: kubeClient, + } + + server.mux = http.NewServeMux() + metricsHandler := promhttp.HandlerForTransactional( + prometheus.ToTransactionalGatherer(server.ovnRegistry), + promhttp.HandlerOpts{}, + ) + server.mux.Handle("/metrics", promhttp.InstrumentMetricHandler( + server.ovnRegistry, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Update metrics in the registry before emitting them. + server.handleMetrics(r) + // Emit the updated metrics using the transactional handler. + metricsHandler.ServeHTTP(w, r) + }), + )) + + return server +} + +// registerMetrics registers the metrics to the OVN registry +func (s *MetricServer) registerMetrics() { + if s.opts.EnableOVSMetrics { + klog.Infof("MetricServer registers OVS metrics") + registerOvsMetrics(s.ovsDBClient, s.ovnRegistry) + } + if s.opts.EnableOVNDBMetrics { + klog.Infof("MetricServer registers OVN DB metrics") + s.ovsDbProperties, s.opts.dbIsClustered, s.opts.dbFoundViaPath = RegisterOvnDBMetrics(s.ovnRegistry) + } + if s.opts.EnableOVNControllerMetrics { + klog.Infof("MetricServer registers OVN Controller metrics") + RegisterOvnControllerMetrics(s.ovsDBClient, s.ovnRegistry) + } + if s.opts.EnableOVNNorthdMetrics { + klog.Infof("MetricServer registers OVN Northd metrics") + RegisterOvnNorthdMetrics(s.ovnRegistry) + } +} + +func (s *MetricServer) EnableOVNNorthdMetrics() { + s.opts.EnableOVNNorthdMetrics = true + klog.Infof("MetricServer registers OVN Northd metrics") + RegisterOvnNorthdMetrics(s.ovnRegistry) +} + +func (s *MetricServer) EnableOVNDBMetrics() { + s.opts.EnableOVNDBMetrics = true + klog.Infof("MetricServer registers OVN DB metrics") + s.ovsDbProperties, s.opts.dbIsClustered, s.opts.dbFoundViaPath = RegisterOvnDBMetrics(s.ovnRegistry) +} + +// updateOvsMetrics updates the OVS metrics +func (s *MetricServer) updateOvsMetrics() { + ovsDatapathMetricsUpdate() + if err := updateOvsBridgeMetrics(s.ovsDBClient, util.RunOVSOfctl); err != nil { + klog.Errorf("Updating ovs bridge metrics failed: %s", err.Error()) + } + if err := updateOvsInterfaceMetrics(s.ovsDBClient); err != nil { + klog.Errorf("Updating ovs interface metrics failed: %s", err.Error()) + } + if err := setOvsMemoryMetrics(util.RunOvsVswitchdAppCtl); err != nil { + klog.Errorf("Updating ovs memory metrics failed: %s", err.Error()) + } + if err := setOvsHwOffloadMetrics(s.ovsDBClient); err != nil { + klog.Errorf("Updating ovs hardware offload metrics failed: %s", err.Error()) + } + coverageShowMetricsUpdate(ovsVswitchd) +} + +// updateOvnControllerMetrics updates the OVN Controller metrics +func (s *MetricServer) updateOvnControllerMetrics() { + if err := setOvnControllerConfigurationMetrics(s.ovsDBClient); err != nil { + klog.Errorf("Setting ovn controller config metrics failed: %s", err.Error()) + } + + coverageShowMetricsUpdate(ovnController) + stopwatchShowMetricsUpdate(ovnController) + updateSBDBConnectionMetric(util.RunOVNControllerAppCtl) + +} + +// updateOvnNorthdMetrics updates the OVN Northd metrics +func (s *MetricServer) updateOvnNorthdMetrics() { + coverageShowMetricsUpdate(ovnNorthd) + stopwatchShowMetricsUpdate(ovnNorthd) +} + +// updateOvnDBMetrics updates the OVN DB metrics +func (s *MetricServer) updateOvnDBMetrics() { + if s.opts.dbIsClustered { + resetOvnDbClusterMetrics() + } + if s.opts.dbFoundViaPath { + resetOvnDbSizeMetric() + } + resetOvnDbMemoryMetrics() + + for _, dbProperty := range s.ovsDbProperties { + if s.opts.dbIsClustered { + ovnDBClusterStatusMetricsUpdater(dbProperty) + } + if s.opts.dbFoundViaPath { + updateOvnDBSizeMetrics(dbProperty) + } + updateOvnDBMemoryMetrics(dbProperty) + } +} + +// handleMetrics handles the /metrics request +func (s *MetricServer) handleMetrics(r *http.Request) { + klog.V(5).Infof("MetricServer starts to handle metrics request from %s", r.RemoteAddr) + + if s.opts.EnableOVSMetrics { + s.updateOvsMetrics() + } + if s.opts.EnableOVNDBMetrics { + s.updateOvnDBMetrics() + } + if s.opts.EnableOVNControllerMetrics { + s.updateOvnControllerMetrics() + } + if s.opts.EnableOVNNorthdMetrics { + s.updateOvnNorthdMetrics() + } +} + +// Run runs the metrics server and blocks until graceful shutdown +func (s *MetricServer) Run(stopChan <-chan struct{}) { + utilwait.Until(func() { + s.server = &http.Server{ + Addr: s.opts.BindAddress, + Handler: s.mux, + } + listenAndServe := func() error { return s.server.ListenAndServe() } + if s.opts.CertFile != "" && s.opts.KeyFile != "" { + s.server.TLSConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(s.opts.CertFile, s.opts.KeyFile) + if err != nil { + return nil, fmt.Errorf("error generating x509 certs for metrics TLS endpoint: %v", err) + } + return &cert, nil + }, + } + listenAndServe = func() error { return s.server.ListenAndServeTLS("", "") } + } + + errCh := make(chan error) + go func() { + errCh <- listenAndServe() + }() + + select { + case err := <-errCh: + if !errors.Is(err, http.ErrServerClosed) { + utilruntime.HandleError(fmt.Errorf("failed while running metrics server at address %q: %w", s.opts.BindAddress, err)) + if s.opts.OnFatalError != nil { + s.opts.OnFatalError() + } + } + case <-stopChan: + klog.Infof("Stopping metrics server at address %q", s.opts.BindAddress) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.server.Shutdown(shutdownCtx); err != nil { + klog.Errorf("Error stopping metrics server at address %q: %v", s.opts.BindAddress, err) + } + } + }, 5*time.Second, stopChan) +} diff --git a/go-controller/pkg/metrics/server_test.go b/go-controller/pkg/metrics/server_test.go new file mode 100644 index 0000000000..32d4144dd8 --- /dev/null +++ b/go-controller/pkg/metrics/server_test.go @@ -0,0 +1,855 @@ +package metrics + +import ( + "bytes" + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/spf13/afero" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + + libovsdbclient "github.com/ovn-kubernetes/libovsdb/client" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + mock_k8s_io_utils_exec "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/utils/exec" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/mocks" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/vswitchd" +) + +func TestNewMetricServerRunAndShutdown(t *testing.T) { + opts := MetricServerOptions{ + BindAddress: "127.0.0.1:0", // Use random port for testing + EnableOVSMetrics: false, + EnableOVNDBMetrics: false, + EnableOVNControllerMetrics: false, + EnableOVNNorthdMetrics: false, + } + + ctx, cancel := context.WithCancel(context.Background()) + + var ovsDBClient libovsdbclient.Client + var kubeClient kubernetes.Interface = fake.NewSimpleClientset() + + server := NewMetricServer(opts, ovsDBClient, kubeClient) + require.NotNil(t, server, "Server should not be nil") + require.NotNil(t, server.mux, "Server mux should not be nil") + require.NotNil(t, server.ovnRegistry, "Server OVN registry should not be nil") + + // Start server in background + serverDone := make(chan struct{}) + go func() { + t.Log("Server starting...") + server.Run(ctx.Done()) + close(serverDone) + }() + + // Give server time to start + time.Sleep(1 * time.Second) + + // Test graceful shutdown + t.Log("Initiating graceful shutdown by cancelling context") + shutdownStart := time.Now() + cancel() + + // Wait for server to stop with timeout + select { + case <-serverDone: + shutdownDuration := time.Since(shutdownStart) + t.Logf("Server stopped gracefully in %v", shutdownDuration) + + // Validate shutdown was reasonably fast (should be under 6 seconds, allowing for 5s grace period) + if shutdownDuration > 6*time.Second { + t.Errorf("Shutdown took too long: %v (expected < 6s)", shutdownDuration) + } + + case <-time.After(10 * time.Second): + t.Fatal("Server did not shut down within timeout period (10s)") + } + + t.Logf("TestNewMetricServer completed successfully") +} + +func TestNewMetricServerRunAndFailOnFatalError(t *testing.T) { + // Occupy the port first so that the metrics server will fail with "address already in use" + addr := "127.0.0.5:9410" + listener, err := net.Listen("tcp", addr) + require.NoError(t, err, "Failed to listen on %s", addr) + defer listener.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + opts := MetricServerOptions{ + BindAddress: addr, + OnFatalError: cancel, + EnableOVSMetrics: false, + EnableOVNDBMetrics: false, + EnableOVNControllerMetrics: false, + EnableOVNNorthdMetrics: false, + } + + var ovsDBClient libovsdbclient.Client + var kubeClient kubernetes.Interface = fake.NewSimpleClientset() + + server := NewMetricServer(opts, ovsDBClient, kubeClient) + require.NotNil(t, server, "Server should not be nil") + require.NotNil(t, server.mux, "Server mux should not be nil") + require.NotNil(t, server.ovnRegistry, "Server OVN registry should not be nil") + + // Start server in background + serverDone := make(chan struct{}) + go func() { + t.Log("Server starting...") + server.Run(ctx.Done()) + close(serverDone) + }() + + // Wait for OnFatalError to be called (context cancelled) or timeout + select { + case <-ctx.Done(): + t.Log("OnFatalError was called as expected (context cancelled)") + case <-time.After(5 * time.Second): + t.Fatal("OnFatalError was not called within timeout - server should have failed to bind") + } + + // Wait for server goroutine to finish + select { + case <-serverDone: + t.Log("Server stopped as expected") + case <-time.After(5 * time.Second): + t.Fatal("Server did not stop within timeout after OnFatalError was called") + } +} + +func setupAppFs(t *testing.T) { + t.Helper() + prevFS := util.AppFs + util.AppFs = afero.NewMemMapFs() + t.Cleanup(func() { + util.AppFs = prevFS + }) + + if err := util.AppFs.MkdirAll("/var/run/openvswitch/", 0o755); err != nil { + t.Fatalf("failed to AppFs.MkdirAlle: %v", err) + } + + files := []ovntest.AferoFileMockHelper{ + {FileName: "/var/run/openvswitch/ovs-vswitchd.pid", Permissions: 0o755, Content: []byte("101")}, + {FileName: "ovn-controller.pid", Permissions: 0o755, Content: []byte("102")}, + {FileName: "ovn-northd.pid", Permissions: 0o755, Content: []byte("103")}, + {FileName: "/var/run/openvswitch/ovnnb_db.sock", Permissions: 0o755, Content: []byte("")}, + {FileName: "/var/run/openvswitch/ovnsb_db.sock", Permissions: 0o755, Content: []byte("")}, + {FileName: "/etc/ovn/ovnnb_db.db", Permissions: 0o755, Content: []byte("abcdefgh")}, + {FileName: "/etc/ovn/ovnsb_db.db", Permissions: 0o755, Content: []byte("xyz")}, + } + + for _, file := range files { + if err := afero.WriteFile(util.AppFs, file.FileName, file.Content, file.Permissions); err != nil { + t.Fatalf("failed to afero.WriteFile: %v", err) + } + } +} + +var ( + dpctlShowOutput = `system@ovs-system: +lookups: hit:123456 missed:11 lost:22 +flows: 77 +masks: hit:23456 total:33 hit/pkt:0.33 +cache: hit:34567 hit-rate:99.99% +caches: + masks-cache: size:256 +port 0: ovs-system (internal) +port 1: breth0 (internal) +port 4: eth0 +` + + ovsMemoryShowOutput = `handlers:8 idl-cells-Open_vSwitch:2892 ofconns:4 ports:48 revalidators:3 rules:1803 udpif keys:86` + + coverageShowOutput = `Event coverage, avg rate over last: 5 seconds, last minute, last hour, hash=f5b8301d: +bridge_reconfigure 0.0/sec 0.000/sec 0.0000/sec total: 194 +ofproto_flush 0.0/sec 0.000/sec 0.0000/sec total: 5 +ofproto_packet_out 0.0/sec 0.000/sec 0.0000/sec total: 26586 +ofproto_recv_openflow 0.0/sec 0.750/sec 0.7525/sec total: 2020864 +ofproto_update_port 0.0/sec 0.000/sec 0.0000/sec total: 174 +rev_reconfigure 0.0/sec 0.000/sec 0.0000/sec total: 50 +rev_port_toggled 0.0/sec 0.000/sec 0.0000/sec total: 20 +rev_flow_table 0.0/sec 0.000/sec 0.0000/sec total: 680 +rev_mac_learning 0.0/sec 0.000/sec 0.0011/sec total: 15447 +dumped_duplicate_flow 0.0/sec 0.000/sec 0.0000/sec total: 1 +handler_duplicate_upcall 0.0/sec 0.517/sec 0.6217/sec total: 1466381 +revalidate_missed_dp_flow 0.0/sec 0.000/sec 0.0000/sec total: 14 +revalidate_missing_dp_flow 0.0/sec 0.000/sec 0.0000/sec total: 479 +ukey_replace_contention 0.0/sec 0.000/sec 0.0003/sec total: 703 +ukey_from_dp_flow 0.0/sec 0.000/sec 0.0000/sec total: 13 +upcall_flow_limit_grew 0.0/sec 0.000/sec 0.0000/sec total: 127 +upcall_ukey_contention 0.0/sec 0.000/sec 0.0000/sec total: 1 +upcall_ukey_replace 0.0/sec 0.000/sec 0.0000/sec total: 77 +upcall_flow_del_rev 0.0/sec 0.000/sec 0.0000/sec total: 139 +upcall_flow_del_idle_or_limit 3.8/sec 5.083/sec 5.0364/sec total: 7281882 +xlate_actions 6.4/sec 9.100/sec 9.0861/sec total: 13299790 +connmgr_async_unsent 0.0/sec 0.000/sec 0.0000/sec total: 2 +ccmap_expand 0.0/sec 0.000/sec 0.0000/sec total: 233 +ccmap_shrink 0.0/sec 0.000/sec 0.0000/sec total: 388 +cmap_expand 0.0/sec 0.000/sec 0.0000/sec total: 3098 +cmap_shrink 0.0/sec 0.000/sec 0.0000/sec total: 1952 +dpif_execute 4.2/sec 6.250/sec 6.1442/sec total: 9164190 +dpif_execute_error 0.0/sec 0.000/sec 0.0000/sec total: 1 +dpif_execute_with_help 0.2/sec 0.200/sec 0.1728/sec total: 215206 +dpif_flow_del 3.8/sec 5.083/sec 5.0364/sec total: 7282525 +dpif_flow_flush 0.0/sec 0.000/sec 0.0000/sec total: 1 +dpif_flow_get 0.0/sec 0.000/sec 0.0000/sec total: 38 +dpif_flow_put 3.8/sec 5.217/sec 5.1142/sec total: 7524160 +dpif_flow_put_error 0.0/sec 0.033/sec 0.0758/sec total: 241346 +dpif_meter_del 0.0/sec 0.000/sec 0.0000/sec total: 55 +dpif_meter_set 0.0/sec 0.000/sec 0.0000/sec total: 58 +dpif_port_add 0.0/sec 0.000/sec 0.0000/sec total: 23 +dpif_port_del 0.0/sec 0.000/sec 0.0000/sec total: 34 +flow_extract 3.8/sec 5.733/sec 5.7367/sec total: 8994358 +miniflow_malloc 0.0/sec 0.467/sec 0.4689/sec total: 1258436 +hindex_pathological 0.0/sec 0.000/sec 0.0000/sec total: 8 +hindex_expand 0.0/sec 0.000/sec 0.0000/sec total: 9 +hmap_pathological 0.0/sec 0.017/sec 0.0269/sec total: 55393 +hmap_expand 28.2/sec 41.100/sec 39.8708/sec total: 85284712 +mac_learning_learned 0.0/sec 0.000/sec 0.0006/sec total: 6175 +mac_learning_expired 0.0/sec 0.000/sec 0.0006/sec total: 6150 +mac_learning_moved 0.0/sec 0.000/sec 0.0000/sec total: 7299 +mac_learning_static_none_move 0.2/sec 0.200/sec 0.1728/sec total: 381251 +netdev_get_stats 9.6/sec 10.400/sec 10.4000/sec total: 20330897 +txn_unchanged 0.0/sec 0.050/sec 0.0500/sec total: 108339 +txn_incomplete 0.2/sec 0.200/sec 0.2111/sec total: 472946 +txn_success 0.2/sec 0.200/sec 0.2000/sec total: 451953 +poll_create_node 186.4/sec 333.033/sec 333.6025/sec total: 719704044 +poll_zero_timeout 3.8/sec 6.017/sec 5.8844/sec total: 8608665 +rconn_queued 0.0/sec 1.900/sec 1.9078/sec total: 4419934 +rconn_sent 0.0/sec 1.900/sec 1.9078/sec total: 4419934 +seq_change 1065.8/sec 1073.883/sec 1077.5122/sec total: 2422874389 +pstream_open 0.0/sec 0.000/sec 0.0000/sec total: 14 +stream_open 0.0/sec 0.000/sec 0.0000/sec total: 1 +unixctl_received 0.0/sec 0.117/sec 0.1167/sec total: 222403 +unixctl_replied 0.0/sec 0.117/sec 0.1167/sec total: 222403 +util_xalloc 1724.0/sec 3564.283/sec 3521.6053/sec total: 6371878350 +vconn_received 0.0/sec 1.067/sec 1.0700/sec total: 2789645 +vconn_sent 0.0/sec 2.217/sec 2.2253/sec total: 5188715 +netdev_set_policing 0.0/sec 0.000/sec 0.0000/sec total: 77 +netdev_get_ifindex 0.0/sec 0.500/sec 0.5000/sec total: 341857 +netdev_set_hwaddr 0.0/sec 0.000/sec 0.0000/sec total: 2 +netdev_get_ethtool 0.0/sec 0.000/sec 0.0000/sec total: 104 +netdev_set_ethtool 0.0/sec 0.000/sec 0.0000/sec total: 20 +netlink_received 25.6/sec 32.067/sec 32.0092/sec total: 60257902 +netlink_recv_jumbo 3.6/sec 4.683/sec 4.6350/sec total: 7520804 +netlink_sent 24.8/sec 32.550/sec 32.2783/sec total: 56841452 +route_table_dump 0.0/sec 0.000/sec 0.0000/sec total: 163 +nln_changed 0.0/sec 0.000/sec 0.0000/sec total: 230 +119 events never hit +` + ovsOfctlDumpAggregateOutput = ` +NXST_AGGREGATE reply (xid=0x4): packet_count=12345 byte_count=67890 flow_count=18000 +` + + ovnDBMemoryShowOutput = `atoms:324341 cells:307671 monitors:2 n-weak-refs:5627 raft-connections:4 raft-log:3403 sessions:12 txn-history:100 txn-history-atoms:52811` + + ovnControllerDumpAggregateOutput = `NXST_AGGREGATE reply (xid=0x4): packet_count=9945601440 byte_count=33370900148508 flow_count=12062` + ovnControllercoverageShowOutput = `Event coverage, avg rate over last: 5 seconds, last minute, last hour, hash=7a3e39bf: +lflow_run 0.0/sec 0.000/sec 0.0000/sec total: 113 +consider_logical_flow 0.0/sec 0.333/sec 0.9019/sec total: 3874864 +lflow_cache_flush 0.0/sec 0.000/sec 0.0000/sec total: 1 +lflow_cache_trim 0.0/sec 0.000/sec 0.0000/sec total: 2 +lflow_conj_alloc 0.0/sec 0.000/sec 0.0097/sec total: 17158 +lflow_conj_free 0.0/sec 0.000/sec 0.0097/sec total: 14792 +pinctrl_notify_main_thread 0.0/sec 0.000/sec 0.0000/sec total: 118 +pinctrl_total_pin_pkts 0.0/sec 0.000/sec 0.0000/sec total: 147 +physical_run 0.0/sec 0.000/sec 0.0000/sec total: 136 +flow_extract 0.0/sec 0.000/sec 0.0000/sec total: 147 +miniflow_malloc 0.0/sec 0.200/sec 3.1769/sec total: 9010830 +hmap_pathological 0.6/sec 1.600/sec 1.4653/sec total: 2410194 +hmap_expand 44.4/sec 116.050/sec 103.9733/sec total: 169419911 +hmap_reserve 3.0/sec 7.667/sec 6.7917/sec total: 10860162 +txn_unchanged 1.2/sec 3.067/sec 2.7164/sec total: 4343791 +txn_incomplete 0.0/sec 0.000/sec 0.0003/sec total: 556 +txn_success 0.0/sec 0.000/sec 0.0003/sec total: 289 +txn_try_again 0.0/sec 0.000/sec 0.0000/sec total: 1 +poll_create_node 14.2/sec 87.233/sec 59.1267/sec total: 93695892 +poll_zero_timeout 0.0/sec 0.017/sec 0.0125/sec total: 15868 +rconn_queued 0.0/sec 1.400/sec 2.0528/sec total: 3211580 +rconn_sent 0.0/sec 1.400/sec 2.0528/sec total: 3211580 +seq_change 5.2/sec 62.550/sec 37.2158/sec total: 58941139 +pstream_open 0.0/sec 0.000/sec 0.0000/sec total: 1 +stream_open 0.0/sec 0.000/sec 0.0000/sec total: 9 +unixctl_received 0.2/sec 0.167/sec 0.1794/sec total: 226274 +unixctl_replied 0.2/sec 0.167/sec 0.1794/sec total: 226274 +util_xalloc 1438.0/sec 4125.033/sec 3639.7153/sec total: 6035801728 +vconn_open 0.0/sec 0.000/sec 0.0000/sec total: 6 +vconn_received 0.0/sec 1.000/sec 0.7628/sec total: 1295583 +vconn_sent 0.0/sec 1.400/sec 2.0528/sec total: 3211584 +nln_changed 0.0/sec 0.000/sec 0.0000/sec total: 6 +netlink_received 0.0/sec 0.000/sec 0.0000/sec total: 582 +netlink_sent 0.0/sec 0.000/sec 0.0000/sec total: 576 +119 events never hit +` + + ovnControllerVersionOutput = `ovn-controller 20.06.0.86f64fc1 +Open vSwitch Library 2.13.0.f945b5c5 +` + ovnNorthdVersionOutput = `ovn-northd 25.03.0.c2144df1.28754012 +Open vSwitch Library 3.5.0 +` +) + +type metricsTestCase struct { + name string + enableOVS bool + enableOVNDB bool + enableOVNController bool + enableOVNNorthd bool + mockRunCommands []ovntest.TestifyMockHelper + expectedMetrics []string +} + +func TestHandleMetrics(t *testing.T) { + // disable Process metrics collector to avoid the test flakiness + savedUnprivilegedMode := config.UnprivilegedMode + config.UnprivilegedMode = true + savedRunner := util.RunCmdExecRunner + defer func() { + config.UnprivilegedMode = savedUnprivilegedMode + util.RunCmdExecRunner = savedRunner + }() + + setupAppFs(t) + + // common OVS DB setup + ovsVersion := "2.17.0" + + intf1 := vswitchd.Interface{Name: "porta", UUID: buildUUID()} + port1 := vswitchd.Port{Name: "porta", UUID: buildUUID()} + br1 := vswitchd.Bridge{Name: "br-int", UUID: buildUUID()} + + testDB := []libovsdbtest.TestData{ + &vswitchd.Interface{ + UUID: intf1.UUID, + Name: intf1.Name, + Statistics: map[string]int{ + "rx_packets": 1000, + "tx_packets": 800, + "rx_bytes": 100000, + "tx_bytes": 80000, + }, + }, + &vswitchd.Port{UUID: port1.UUID, Name: port1.Name, Interfaces: []string{intf1.UUID}}, + &vswitchd.Bridge{UUID: br1.UUID, Name: br1.Name, Ports: []string{port1.UUID}}, + &vswitchd.OpenvSwitch{ + UUID: buildUUID(), + OVSVersion: &ovsVersion, + Bridges: []string{br1.UUID}, + ExternalIDs: map[string]string{ + "ovn-bridge-remote-probe-interval": "100", + "ovn-remote-probe-interval": "200", + "ovn-monitor-all": "false", + "ovn-encap-ip": "192.168.1.1", + "ovn-encap-type": "geneve", + "ovn-remote": "unix:/var/run/ovn/ovnsb_db.sock", + "ovn-k8s-node-port": "false", + "ovn-bridge-mappings": "physnet:breth0", + }, + }, + } + + // Setup OVS test harness + dbSetup := libovsdbtest.TestSetup{ + OVSData: testDB, + } + ovsDBClient, libovsdbCleanup, err := libovsdbtest.NewOVSTestHarness(dbSetup) + if err != nil { + t.Fatalf("Failed to create OVS test harness: %v", err) + } + defer libovsdbCleanup.Cleanup() + + testCases := []metricsTestCase{ + { + name: "OVS metrics", + enableOVS: true, + mockRunCommands: []ovntest.TestifyMockHelper{ + // dpctl/dump-dps + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "dpctl/dump-dps"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("system@ovs-system")), bytes.NewBuffer([]byte("")), nil}, + }, + // dpctl/show + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "dpctl/show", "system@ovs-system"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(dpctlShowOutput)), bytes.NewBuffer([]byte("")), nil}, + }, + // memory/show + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "memory/show"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(ovsMemoryShowOutput)), bytes.NewBuffer([]byte("")), nil}, + }, + // coverage/show + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "coverage/show"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(coverageShowOutput)), bytes.NewBuffer([]byte("")), nil}, + }, + // ovs-ofctl dump-aggregate br-int + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "dump-aggregate", "br-int"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(ovsOfctlDumpAggregateOutput)), bytes.NewBuffer([]byte("")), nil}, + }, + }, + expectedMetrics: []string{ + "ovs_build_info", + "ovs_vswitchd_bridge_flows_total", + "ovs_vswitchd_bridge_ports_total", + "ovs_vswitchd_bridge_reconfigure", + "ovs_vswitchd_bridge_total", + "ovs_vswitchd_bridge", + "ovs_vswitchd_dp_flows_lookup_hit", + "ovs_vswitchd_dp_flows_lookup_lost", + "ovs_vswitchd_dp_flows_lookup_missed", + "ovs_vswitchd_dp_flows_total", + "ovs_vswitchd_dp_if_total", + "ovs_vswitchd_dp_masks_hit_ratio", + "ovs_vswitchd_dp_masks_hit", + "ovs_vswitchd_dp_masks_total", + "ovs_vswitchd_dp_packets_total", + "ovs_vswitchd_dp_total", + "ovs_vswitchd_dp", + "ovs_vswitchd_dpif_execute", + "ovs_vswitchd_dpif_flow_del", + "ovs_vswitchd_dpif_flow_flush", + "ovs_vswitchd_dpif_flow_get", + "ovs_vswitchd_dpif_flow_put", + "ovs_vswitchd_dpif_port_add", + "ovs_vswitchd_dpif_port_del", + "ovs_vswitchd_handlers_total", + "ovs_vswitchd_hw_offload", + "ovs_vswitchd_interface_collisions_total", + "ovs_vswitchd_interface_resets_total", + "ovs_vswitchd_interface_rx_dropped_total", + "ovs_vswitchd_interface_rx_errors_total", + "ovs_vswitchd_interface_tx_dropped_total", + "ovs_vswitchd_interface_tx_errors_total", + "ovs_vswitchd_interface_up_wait_seconds_total", + "ovs_vswitchd_interfaces_total", + "ovs_vswitchd_netlink_overflow", + "ovs_vswitchd_netlink_received", + "ovs_vswitchd_netlink_recv_jumbo", + "ovs_vswitchd_netlink_sent", + "ovs_vswitchd_ofproto_dpif_expired", + "ovs_vswitchd_ofproto_flush", + "ovs_vswitchd_ofproto_packet_out", + "ovs_vswitchd_ofproto_recv_openflow", + "ovs_vswitchd_ofproto_reinit_ports", + "ovs_vswitchd_packet_in_drop", + "ovs_vswitchd_packet_in", + "ovs_vswitchd_pstream_open", + "ovs_vswitchd_rconn_discarded", + "ovs_vswitchd_rconn_overflow", + "ovs_vswitchd_rconn_queued", + "ovs_vswitchd_rconn_sent", + "ovs_vswitchd_revalidators_total", + "ovs_vswitchd_stream_open", + "ovs_vswitchd_tc_policy", + "ovs_vswitchd_txn_aborted", + "ovs_vswitchd_txn_error", + "ovs_vswitchd_txn_incomplete", + "ovs_vswitchd_txn_success", + "ovs_vswitchd_txn_try_again", + "ovs_vswitchd_txn_unchanged", + "ovs_vswitchd_txn_uncommitted", + "ovs_vswitchd_upcall_flow_limit_hit", + "ovs_vswitchd_upcall_flow_limit_kill", + "ovs_vswitchd_vconn_open", + "ovs_vswitchd_vconn_received", + "ovs_vswitchd_vconn_sent", + "ovs_vswitchd_xlate_actions_oversize", + "ovs_vswitchd_xlate_actions_too_many_output", + "ovs_vswitchd_xlate_actions", + "promhttp_metric_handler_requests_in_flight", + "promhttp_metric_handler_requests_total", + }, + }, + { + name: "OVN DB metrics", + enableOVNDB: true, + mockRunCommands: []ovntest.TestifyMockHelper{ + // ovs-appctl version + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "version"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("ovsdb-server (Open vSwitch) 3.5.0")), bytes.NewBuffer([]byte("")), nil}, + }, + // ovsdb-client get-schema-version unix:/var/run/openvswitch/ovnnb_db.sock OVN_Northbound + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "get-schema-version", mock.AnythingOfType("string"), "OVN_Northbound"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("7.11.0")), bytes.NewBuffer([]byte("")), nil}, + }, + // ovsdb-client get-schema-version unix:/var/run/openvswitch/ovnnb_db.sock OVN_Southbound + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "get-schema-version", mock.AnythingOfType("string"), "OVN_Southbound"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("20.41.0")), bytes.NewBuffer([]byte("")), nil}, + }, + // ovs-appctl -t /var/run/openvswitch/ovnnb_db.ctl cluster/status OVN_Northbound + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), mock.AnythingOfType("string"), "cluster/status", "OVN_Northbound"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("")), bytes.NewBuffer([]byte(`"cluster/status" is not a valid command`)), fmt.Errorf("server returned an error")}, + }, + // ovs-appctl -t /var/run/openvswitch/ovnnb_db.ctl memory/show + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), mock.AnythingOfType("string"), "memory/show"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(ovnDBMemoryShowOutput)), bytes.NewBuffer([]byte("")), nil}, + }, + // ovs-appctl -t /var/run/openvswitch/ovnsb_db.ctl memory/show + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), mock.AnythingOfType("string"), "memory/show"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(ovnDBMemoryShowOutput)), bytes.NewBuffer([]byte("")), nil}, + }, + }, + expectedMetrics: []string{ + "ovn_db_build_info", + "ovn_db_db_size_bytes", + "ovn_db_jsonrpc_server_sessions", + "ovn_db_ovsdb_monitors", + "promhttp_metric_handler_requests_in_flight", + "promhttp_metric_handler_requests_total", + }, + }, + { + name: "OVN Controller metrics", + enableOVNController: true, + mockRunCommands: []ovntest.TestifyMockHelper{ + // ovs-ofctl -t 5 dump-aggregate br-int + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "dump-aggregate", "br-int"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(ovnControllerDumpAggregateOutput)), bytes.NewBuffer([]byte("")), nil}, + CallTimes: 2, + }, + // ovs-appctl -t /var/run/openvswitch/ovn-controller.113.ctl coverage/show + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "coverage/show"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(ovnControllercoverageShowOutput)), bytes.NewBuffer([]byte("")), nil}, + }, + // ovs-appctl -t /var/run/openvswitch/ovn-controller.113.ctl version + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "version"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(ovnControllerVersionOutput)), bytes.NewBuffer([]byte("")), nil}, + }, + // ovs-appctl -t /var/run/openvswitch/ovn-controller.113.ctl connection-status + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "connection-status"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("connected")), bytes.NewBuffer([]byte("")), nil}, + }, + }, + expectedMetrics: []string{ + "ovn_controller_bfd_run_95th_percentile", + "ovn_controller_bfd_run_long_term_avg", + "ovn_controller_bfd_run_maximum", + "ovn_controller_bfd_run_minimum", + "ovn_controller_bfd_run_short_term_avg", + "ovn_controller_bfd_run_total_samples", + "ovn_controller_bridge_mappings", + "ovn_controller_build_info", + "ovn_controller_ct_zone_commit_95th_percentile", + "ovn_controller_ct_zone_commit_long_term_avg", + "ovn_controller_ct_zone_commit_maximum", + "ovn_controller_ct_zone_commit_minimum", + "ovn_controller_ct_zone_commit_short_term_avg", + "ovn_controller_ct_zone_commit_total_samples", + "ovn_controller_encap_ip", + "ovn_controller_encap_type", + "ovn_controller_flow_generation_95th_percentile", + "ovn_controller_flow_generation_long_term_avg", + "ovn_controller_flow_generation_maximum", + "ovn_controller_flow_generation_minimum", + "ovn_controller_flow_generation_short_term_avg", + "ovn_controller_flow_generation_total_samples", + "ovn_controller_flow_installation_95th_percentile", + "ovn_controller_flow_installation_long_term_avg", + "ovn_controller_flow_installation_maximum", + "ovn_controller_flow_installation_minimum", + "ovn_controller_flow_installation_short_term_avg", + "ovn_controller_flow_installation_total_samples", + "ovn_controller_if_status_mgr_run_95th_percentile", + "ovn_controller_if_status_mgr_run_long_term_avg", + "ovn_controller_if_status_mgr_run_maximum", + "ovn_controller_if_status_mgr_run_minimum", + "ovn_controller_if_status_mgr_run_short_term_avg", + "ovn_controller_if_status_mgr_run_total_samples", + "ovn_controller_if_status_mgr_update_95th_percentile", + "ovn_controller_if_status_mgr_update_long_term_avg", + "ovn_controller_if_status_mgr_update_maximum", + "ovn_controller_if_status_mgr_update_minimum", + "ovn_controller_if_status_mgr_update_short_term_avg", + "ovn_controller_if_status_mgr_update_total_samples", + "ovn_controller_integration_bridge_geneve_ports", + "ovn_controller_integration_bridge_openflow_total", + "ovn_controller_integration_bridge_patch_ports", + "ovn_controller_lflow_run", + "ovn_controller_monitor_all", + "ovn_controller_netlink_overflow", + "ovn_controller_netlink_received", + "ovn_controller_netlink_recv_jumbo", + "ovn_controller_netlink_sent", + "ovn_controller_ofctrl_seqno_run_95th_percentile", + "ovn_controller_ofctrl_seqno_run_long_term_avg", + "ovn_controller_ofctrl_seqno_run_maximum", + "ovn_controller_ofctrl_seqno_run_minimum", + "ovn_controller_ofctrl_seqno_run_short_term_avg", + "ovn_controller_ofctrl_seqno_run_total_samples", + "ovn_controller_openflow_probe_interval_seconds", + "ovn_controller_packet_in_drop", + "ovn_controller_packet_in", + "ovn_controller_patch_run_95th_percentile", + "ovn_controller_patch_run_long_term_avg", + "ovn_controller_patch_run_maximum", + "ovn_controller_patch_run_minimum", + "ovn_controller_patch_run_short_term_avg", + "ovn_controller_patch_run_total_samples", + "ovn_controller_pinctrl_run_95th_percentile", + "ovn_controller_pinctrl_run_long_term_avg", + "ovn_controller_pinctrl_run_maximum", + "ovn_controller_pinctrl_run_minimum", + "ovn_controller_pinctrl_run_short_term_avg", + "ovn_controller_pinctrl_run_total_samples", + "ovn_controller_rconn_discarded", + "ovn_controller_rconn_overflow", + "ovn_controller_rconn_queued", + "ovn_controller_rconn_sent", + "ovn_controller_remote_probe_interval_seconds", + "ovn_controller_sb_connection_method", + "ovn_controller_southbound_database_connected", + "ovn_controller_stream_open", + "ovn_controller_txn_aborted", + "ovn_controller_txn_error", + "ovn_controller_txn_incomplete", + "ovn_controller_txn_success", + "ovn_controller_txn_try_again", + "ovn_controller_txn_unchanged", + "ovn_controller_txn_uncommitted", + "ovn_controller_vconn_open", + "ovn_controller_vconn_received", + "ovn_controller_vconn_sent", + "promhttp_metric_handler_requests_in_flight", + "promhttp_metric_handler_requests_total", + }, + }, + { + name: "OVN Northd metrics", + enableOVNNorthd: true, + mockRunCommands: []ovntest.TestifyMockHelper{ + // ovs-appctl -t /var/run/openvswitch/ovn-northd.152.ctl version + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "version"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte(ovnNorthdVersionOutput)), bytes.NewBuffer([]byte("")), nil}, + CallTimes: 1, + }, + // ovs-appctl -t /var/run/openvswitch/ovn-northd.152.ctl status + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "status"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("Status: standby")), bytes.NewBuffer([]byte("")), nil}, + CallTimes: 2, + }, + // ovs-appctl -t /var/run/openvswitch/ovn-northd.152.ctl sb-connection-status + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "sb-connection-status"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("connected")), bytes.NewBuffer([]byte("")), nil}, + CallTimes: 2, + }, + // ovs-appctl -t /var/run/openvswitch/ovn-northd.152.ctl nb-connection-status + { + OnCallMethodName: "RunCmd", + OnCallMethodArgs: []interface{}{mock.AnythingOfType("*mocks.Cmd"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string"), "-t", mock.AnythingOfType("string"), "nb-connection-status"}, + RetArgList: []interface{}{bytes.NewBuffer([]byte("connected")), bytes.NewBuffer([]byte("")), nil}, + CallTimes: 2, + }, + }, + expectedMetrics: []string{ + "ovn_northd_build_flows_ctx_95th_percentile", + "ovn_northd_build_flows_ctx_long_term_avg", + "ovn_northd_build_flows_ctx_maximum", + "ovn_northd_build_flows_ctx_minimum", + "ovn_northd_build_flows_ctx_short_term_avg", + "ovn_northd_build_flows_ctx_total_samples", + "ovn_northd_build_info", + "ovn_northd_build_lflows_95th_percentile", + "ovn_northd_build_lflows_long_term_avg", + "ovn_northd_build_lflows_maximum", + "ovn_northd_build_lflows_minimum", + "ovn_northd_build_lflows_short_term_avg", + "ovn_northd_build_lflows_total_samples", + "ovn_northd_clear_lflows_ctx_95th_percentile", + "ovn_northd_clear_lflows_ctx_long_term_avg", + "ovn_northd_clear_lflows_ctx_maximum", + "ovn_northd_clear_lflows_ctx_minimum", + "ovn_northd_clear_lflows_ctx_short_term_avg", + "ovn_northd_clear_lflows_ctx_total_samples", + "ovn_northd_lflows_datapaths_95th_percentile", + "ovn_northd_lflows_datapaths_long_term_avg", + "ovn_northd_lflows_datapaths_maximum", + "ovn_northd_lflows_datapaths_minimum", + "ovn_northd_lflows_datapaths_short_term_avg", + "ovn_northd_lflows_datapaths_total_samples", + "ovn_northd_lflows_dp_groups_95th_percentile", + "ovn_northd_lflows_dp_groups_long_term_avg", + "ovn_northd_lflows_dp_groups_maximum", + "ovn_northd_lflows_dp_groups_minimum", + "ovn_northd_lflows_dp_groups_short_term_avg", + "ovn_northd_lflows_dp_groups_total_samples", + "ovn_northd_lflows_igmp_95th_percentile", + "ovn_northd_lflows_igmp_long_term_avg", + "ovn_northd_lflows_igmp_maximum", + "ovn_northd_lflows_igmp_minimum", + "ovn_northd_lflows_igmp_short_term_avg", + "ovn_northd_lflows_igmp_total_samples", + "ovn_northd_lflows_lbs_95th_percentile", + "ovn_northd_lflows_lbs_long_term_avg", + "ovn_northd_lflows_lbs_maximum", + "ovn_northd_lflows_lbs_minimum", + "ovn_northd_lflows_lbs_short_term_avg", + "ovn_northd_lflows_lbs_total_samples", + "ovn_northd_lflows_ports_95th_percentile", + "ovn_northd_lflows_ports_long_term_avg", + "ovn_northd_lflows_ports_maximum", + "ovn_northd_lflows_ports_minimum", + "ovn_northd_lflows_ports_short_term_avg", + "ovn_northd_lflows_ports_total_samples", + "ovn_northd_nb_connection_status", + "ovn_northd_ovn_northd_loop_95th_percentile", + "ovn_northd_ovn_northd_loop_long_term_avg", + "ovn_northd_ovn_northd_loop_maximum", + "ovn_northd_ovn_northd_loop_minimum", + "ovn_northd_ovn_northd_loop_short_term_avg", + "ovn_northd_ovn_northd_loop_total_samples", + "ovn_northd_ovnnb_db_run_95th_percentile", + "ovn_northd_ovnnb_db_run_long_term_avg", + "ovn_northd_ovnnb_db_run_maximum", + "ovn_northd_ovnnb_db_run_minimum", + "ovn_northd_ovnnb_db_run_short_term_avg", + "ovn_northd_ovnnb_db_run_total_samples", + "ovn_northd_ovnsb_db_run_95th_percentile", + "ovn_northd_ovnsb_db_run_long_term_avg", + "ovn_northd_ovnsb_db_run_maximum", + "ovn_northd_ovnsb_db_run_minimum", + "ovn_northd_ovnsb_db_run_short_term_avg", + "ovn_northd_ovnsb_db_run_total_samples", + "ovn_northd_pstream_open", + "ovn_northd_sb_connection_status", + "ovn_northd_status", + "ovn_northd_stream_open", + "ovn_northd_txn_aborted", + "ovn_northd_txn_error", + "ovn_northd_txn_incomplete", + "ovn_northd_txn_success", + "ovn_northd_txn_try_again", + "ovn_northd_txn_unchanged", + "ovn_northd_txn_uncommitted", + "promhttp_metric_handler_requests_in_flight", + "promhttp_metric_handler_requests_total", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Configure server options + opts := MetricServerOptions{ + BindAddress: "127.0.0.1:0", // Use random port for testing + EnableOVSMetrics: tc.enableOVS, + EnableOVNDBMetrics: tc.enableOVNDB, + EnableOVNControllerMetrics: tc.enableOVNController, + EnableOVNNorthdMetrics: tc.enableOVNNorthd, + } + // Mock the exec runner for RunOvsVswitchdAppCtl calls + mockCmd := new(mock_k8s_io_utils_exec.Cmd) + mockExecRunner := new(mocks.ExecRunner) + util.RunCmdExecRunner = mockExecRunner + ovntest.ProcessMockFnList(&mockExecRunner.Mock, tc.mockRunCommands) + + mockKexecIface := new(mock_k8s_io_utils_exec.Interface) + mockKexecIface.Mock.On("Command", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe().Return(mockCmd) + mockKexecIface.Mock.On("Command", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe().Return(mockCmd) + mockKexecIface.Mock.On("Command", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe().Return(mockCmd) + _ = util.SetSpecificExec(mockKexecIface) + + // Add cleanup for mock expectations + defer func() { + mockExecRunner.AssertExpectations(t) + mockKexecIface.AssertExpectations(t) + }() + + // Create server with OVS client + var kubeClient kubernetes.Interface = fake.NewSimpleClientset() + server := NewMetricServer(opts, ovsDBClient, kubeClient) + server.registerMetrics() + + // iterate s.ovnRegistry to list all registered metrics' names + regMetrics, err := server.ovnRegistry.Gather() + if err != nil { + t.Fatalf("Failed to gather metrics: %v", err) + } + gatherMetrics := []string{} + for _, metric := range regMetrics { + gatherMetrics = append(gatherMetrics, *metric.Name) + } + t.Logf("gatherMetrics: %v", gatherMetrics) + + // Test the /metrics endpoint + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/metrics", nil) + server.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Errorf("Expected status 200, got %d", rec.Code) + } + + got := rec.Body.String() + if len(got) == 0 { + t.Error("Expected non-empty metrics response") + } + + gotMetrics := []string{} + for _, line := range strings.Split(got, "\n") { + if strings.HasPrefix(line, "# TYPE ") { + m := strings.Split(line, " ")[2] + gotMetrics = append(gotMetrics, m) + } + } + + if diff := cmp.Diff(gotMetrics, tc.expectedMetrics, cmpopts.SortSlices(func(x, y string) bool { + return x < y + })); diff != "" { + t.Errorf("mismatch (-got +want):\n%s", diff) + } + }) + } +} diff --git a/go-controller/pkg/metrics/workqueue.go b/go-controller/pkg/metrics/workqueue.go index 98b4b356b1..6da4ed9556 100644 --- a/go-controller/pkg/metrics/workqueue.go +++ b/go-controller/pkg/metrics/workqueue.go @@ -79,7 +79,7 @@ var ( Help: "Total number of retries handled by workqueue", }, []string{"name"}) - metrics = []prometheus.Collector{ + workqueueMetrics = []prometheus.Collector{ depth, adds, latency, workDuration, unfinished, longestRunningProcessor, retries, } ) @@ -124,7 +124,7 @@ func registerWorkqueueMetrics(namespace, subsystem string) { fmt.Sprintf("%s_%s_workqueue_", namespace, subsystem), prometheus.DefaultRegisterer, ) - for _, m := range metrics { + for _, m := range workqueueMetrics { registry.MustRegister(m) } } diff --git a/go-controller/pkg/nbdb/mirror.go b/go-controller/pkg/nbdb/mirror.go index 352cc238af..13e21d00db 100644 --- a/go-controller/pkg/nbdb/mirror.go +++ b/go-controller/pkg/nbdb/mirror.go @@ -19,6 +19,7 @@ var ( MirrorTypeGre MirrorType = "gre" MirrorTypeErspan MirrorType = "erspan" MirrorTypeLocal MirrorType = "local" + MirrorTypeLport MirrorType = "lport" ) // Mirror defines an object in Mirror table @@ -27,6 +28,7 @@ type Mirror struct { ExternalIDs map[string]string `ovsdb:"external_ids"` Filter MirrorFilter `ovsdb:"filter"` Index int `ovsdb:"index"` + MirrorRules []string `ovsdb:"mirror_rules"` Name string `ovsdb:"name"` Sink string `ovsdb:"sink"` Type MirrorType `ovsdb:"type"` @@ -74,6 +76,34 @@ func (a *Mirror) GetIndex() int { return a.Index } +func (a *Mirror) GetMirrorRules() []string { + return a.MirrorRules +} + +func copyMirrorMirrorRules(a []string) []string { + if a == nil { + return nil + } + b := make([]string, len(a)) + copy(b, a) + return b +} + +func equalMirrorMirrorRules(a, b []string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if b[i] != v { + return false + } + } + return true +} + func (a *Mirror) GetName() string { return a.Name } @@ -89,6 +119,7 @@ func (a *Mirror) GetType() MirrorType { func (a *Mirror) DeepCopyInto(b *Mirror) { *b = *a b.ExternalIDs = copyMirrorExternalIDs(a.ExternalIDs) + b.MirrorRules = copyMirrorMirrorRules(a.MirrorRules) } func (a *Mirror) DeepCopy() *Mirror { @@ -111,6 +142,7 @@ func (a *Mirror) Equals(b *Mirror) bool { equalMirrorExternalIDs(a.ExternalIDs, b.ExternalIDs) && a.Filter == b.Filter && a.Index == b.Index && + equalMirrorMirrorRules(a.MirrorRules, b.MirrorRules) && a.Name == b.Name && a.Sink == b.Sink && a.Type == b.Type diff --git a/go-controller/pkg/nbdb/mirror_rule.go b/go-controller/pkg/nbdb/mirror_rule.go new file mode 100644 index 0000000000..2a4f8c1a82 --- /dev/null +++ b/go-controller/pkg/nbdb/mirror_rule.go @@ -0,0 +1,75 @@ +// Code generated by "libovsdb.modelgen" +// DO NOT EDIT. + +package nbdb + +import "github.com/ovn-kubernetes/libovsdb/model" + +const MirrorRuleTable = "Mirror_Rule" + +type ( + MirrorRuleAction = string +) + +var ( + MirrorRuleActionMirror MirrorRuleAction = "mirror" + MirrorRuleActionSkip MirrorRuleAction = "skip" +) + +// MirrorRule defines an object in Mirror_Rule table +type MirrorRule struct { + UUID string `ovsdb:"_uuid"` + Action MirrorRuleAction `ovsdb:"action"` + Match string `ovsdb:"match"` + Priority int `ovsdb:"priority"` +} + +func (a *MirrorRule) GetUUID() string { + return a.UUID +} + +func (a *MirrorRule) GetAction() MirrorRuleAction { + return a.Action +} + +func (a *MirrorRule) GetMatch() string { + return a.Match +} + +func (a *MirrorRule) GetPriority() int { + return a.Priority +} + +func (a *MirrorRule) DeepCopyInto(b *MirrorRule) { + *b = *a +} + +func (a *MirrorRule) DeepCopy() *MirrorRule { + b := new(MirrorRule) + a.DeepCopyInto(b) + return b +} + +func (a *MirrorRule) CloneModelInto(b model.Model) { + c := b.(*MirrorRule) + a.DeepCopyInto(c) +} + +func (a *MirrorRule) CloneModel() model.Model { + return a.DeepCopy() +} + +func (a *MirrorRule) Equals(b *MirrorRule) bool { + return a.UUID == b.UUID && + a.Action == b.Action && + a.Match == b.Match && + a.Priority == b.Priority +} + +func (a *MirrorRule) EqualsModel(b model.Model) bool { + c := b.(*MirrorRule) + return a.Equals(c) +} + +var _ model.CloneableModel = &MirrorRule{} +var _ model.ComparableModel = &MirrorRule{} diff --git a/go-controller/pkg/nbdb/model.go b/go-controller/pkg/nbdb/model.go index 07ca7e0e97..e2ecf1819f 100644 --- a/go-controller/pkg/nbdb/model.go +++ b/go-controller/pkg/nbdb/model.go @@ -38,6 +38,7 @@ func FullDatabaseModel() (model.ClientDBModel, error) { "Meter": &Meter{}, "Meter_Band": &MeterBand{}, "Mirror": &Mirror{}, + "Mirror_Rule": &MirrorRule{}, "NAT": &NAT{}, "NB_Global": &NBGlobal{}, "Port_Group": &PortGroup{}, @@ -52,7 +53,7 @@ func FullDatabaseModel() (model.ClientDBModel, error) { var schema = `{ "name": "OVN_Northbound", - "version": "7.11.0", + "version": "7.12.0", "tables": { "ACL": { "columns": { @@ -1768,6 +1769,17 @@ var schema = `{ "index": { "type": "integer" }, + "mirror_rules": { + "type": { + "key": { + "type": "uuid", + "refTable": "Mirror_Rule", + "refType": "strong" + }, + "min": 0, + "max": "unlimited" + } + }, "name": { "type": "string" }, @@ -1783,7 +1795,8 @@ var schema = `{ [ "gre", "erspan", - "local" + "local", + "lport" ] ] } @@ -1797,6 +1810,36 @@ var schema = `{ ], "isRoot": true }, + "Mirror_Rule": { + "columns": { + "action": { + "type": { + "key": { + "type": "string", + "enum": [ + "set", + [ + "mirror", + "skip" + ] + ] + } + } + }, + "match": { + "type": "string" + }, + "priority": { + "type": { + "key": { + "type": "integer", + "minInteger": 0, + "maxInteger": 32767 + } + } + } + } + }, "NAT": { "columns": { "allowed_ext_ips": { diff --git a/go-controller/pkg/networkmanager/api.go b/go-controller/pkg/networkmanager/api.go index 533d7e6129..7c47997276 100644 --- a/go-controller/pkg/networkmanager/api.go +++ b/go-controller/pkg/networkmanager/api.go @@ -4,9 +4,16 @@ import ( "context" "errors" + nadinformers "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1" + + coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/tools/record" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + egressipinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/egressip/v1" + rainformers "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/informers/externalversions/routeadvertisements/v1" + userdefinednetworkinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/userdefinednetwork/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -18,6 +25,20 @@ const ( MaxNetworks = 4096 ) +// NADReconciler is a level-driven controller notified of NAD key changes. +type NADReconciler controller.Reconciler + +type watchFactory interface { + NADInformer() nadinformers.NetworkAttachmentDefinitionInformer + UserDefinedNetworkInformer() userdefinednetworkinformer.UserDefinedNetworkInformer + ClusterUserDefinedNetworkInformer() userdefinednetworkinformer.ClusterUserDefinedNetworkInformer + NamespaceInformer() coreinformers.NamespaceInformer + RouteAdvertisementsInformer() rainformers.RouteAdvertisementsInformer + NodeCoreInformer() coreinformers.NodeInformer + PodCoreInformer() coreinformers.PodInformer + EgressIPInformer() egressipinformer.EgressIPInformer +} + // Interface is the main package entrypoint and provides network related // information to the rest of the project. type Interface interface { @@ -37,6 +58,11 @@ type Interface interface { // use GetActiveNetworkForNamespace. GetActiveNetworkForNamespaceFast(namespace string) util.NetInfo + // GetPrimaryNADForNamespace returns the full namespaced key of the + // primary NAD for the given namespace, if one exists. + // Returns default network if namespace has no primary UDN. + GetPrimaryNADForNamespace(namespace string) (string, error) + // GetNetwork returns the network of the given name or nil if unknown GetNetwork(name string) util.NetInfo @@ -48,9 +74,29 @@ type Interface interface { // DoWithLock takes care of locking and unlocking while iterating over all role primary user defined networks. DoWithLock(f func(network util.NetInfo) error) error GetActiveNetworkNamespaces(networkName string) ([]string, error) - // RegisterNADHandler allows external entities to register callback functions to be executed when - // a NAD is deleted/created/updated. These operations should be non-blocking and lightweight. - RegisterNADHandler(handler handlerFunc) error + + // GetNetInfoForNADKey returns a copy of the cached network info for the given NAD key, or nil if unknown. + // This is a cheap lookup that does not parse the NAD object; it relies on NAD controller state. + GetNetInfoForNADKey(nadKey string) util.NetInfo + // GetNetworkNameForNADKey returns the network name mapped to the NAD key, or empty if unknown. + // This uses NAD controller state and does not parse the NAD object. + GetNetworkNameForNADKey(nadKey string) string + // GetNADKeysForNetwork returns NAD keys mapped to the network name, or empty if unknown. + // This uses NAD controller state and does not parse the NAD object. + GetNADKeysForNetwork(networkName string) []string + // RegisterNADReconciler registers a reconciler to be notified of NAD changes. + RegisterNADReconciler(r NADReconciler) (uint64, error) + // DeRegisterNADReconciler removes a previously registered reconciler. + DeRegisterNADReconciler(id uint64) error + + // GetNetworkByID returns the network with the given ID or nil if not found. + // This is an O(1) lookup using an internal index. + GetNetworkByID(id int) util.NetInfo + + // NodeHasNetwork returns true if the given node has at least one pod/egress IP using any NAD + // for the specified network with Dynamic UDN. + // If Dynamic UDN is disabled, it always returns true. + NodeHasNetwork(node, networkName string) bool } // Controller handles the runtime of the package @@ -84,6 +130,7 @@ func NewForCluster( ovnClient, recorder, tunnelKeysAllocator, + "", ) } @@ -93,6 +140,10 @@ func NewForZone( cm ControllerManager, wf watchFactory, ) (Controller, error) { + z := zone + if zone == types.OvnDefaultZone { + z = "" + } return new( "zone-nad-controller", zone, @@ -102,6 +153,7 @@ func NewForZone( nil, nil, nil, + z, ) } @@ -120,6 +172,7 @@ func NewForNode( nil, nil, nil, + node, ) } @@ -135,8 +188,9 @@ func new( ovnClient *util.OVNClusterManagerClientset, recorder record.EventRecorder, tunnelKeysAllocator *id.TunnelKeysAllocator, + filterNADsOnNode string, ) (Controller, error) { - return newController(name, zone, node, cm, wf, ovnClient, recorder, tunnelKeysAllocator) + return newController(name, zone, node, cm, wf, ovnClient, recorder, tunnelKeysAllocator, filterNADsOnNode) } // ControllerManager manages controllers. Needs to be provided in order to build @@ -178,6 +232,9 @@ type BaseNetworkController interface { type NetworkController interface { BaseNetworkController Cleanup() error + // HandleNetworkRefChange is only used by nadControllers with Dynamic UDN + // to inform the network controller that a relevant NAD has become active or inactive. + HandleNetworkRefChange(node string, active bool) } // defaultNetworkManager assumes the default network is @@ -199,6 +256,10 @@ func (nm defaultNetworkManager) GetActiveNetworkForNamespace(string) (util.NetIn return &util.DefaultNetInfo{}, nil } +func (nm defaultNetworkManager) GetPrimaryNADForNamespace(_ string) (string, error) { + return types.DefaultNetworkName, nil +} + func (nm defaultNetworkManager) GetActiveNetworkForNamespaceFast(string) util.NetInfo { return &util.DefaultNetInfo{} } @@ -225,8 +286,28 @@ func (nm defaultNetworkManager) GetActiveNetwork(network string) util.NetInfo { return &util.DefaultNetInfo{} } -func (nm defaultNetworkManager) RegisterNADHandler(_ handlerFunc) error { - return nil +func (nm defaultNetworkManager) GetNetInfoForNADKey(_ string) util.NetInfo { return nil } + +func (nm defaultNetworkManager) GetNetworkNameForNADKey(_ string) string { return "" } + +func (nm defaultNetworkManager) GetNADKeysForNetwork(_ string) []string { return nil } + +func (nm defaultNetworkManager) RegisterNADReconciler(_ NADReconciler) (uint64, error) { + return 0, nil +} + +func (nm defaultNetworkManager) DeRegisterNADReconciler(_ uint64) error { return nil } + +func (nm defaultNetworkManager) GetNetworkByID(id int) util.NetInfo { + if id != types.DefaultNetworkID { + return nil + } + return &util.DefaultNetInfo{} +} + +func (nm defaultNetworkManager) NodeHasNetwork(_ string, _ string) bool { + // default network is never filtered + return true } var def Controller = &defaultNetworkManager{} diff --git a/go-controller/pkg/networkmanager/egressip_tracker.go b/go-controller/pkg/networkmanager/egressip_tracker.go new file mode 100644 index 0000000000..be941b9bcd --- /dev/null +++ b/go-controller/pkg/networkmanager/egressip_tracker.go @@ -0,0 +1,384 @@ +package networkmanager + +import ( + "fmt" + "reflect" + "sync" + "time" + + nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + v1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + egressipv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1" + egressiplisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/listers/egressip/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +// EgressIPTrackerController tracks which NADs must be present on which nodes +// due to EgressIP assignments. +type EgressIPTrackerController struct { + cacheMutex sync.Mutex // guards cache + name string + // node -> nad that has EIP + cache map[string]map[string]struct{} + + onNetworkRefChange func(nodeName, nadName string, present bool) + + primaryNADForNamespace func(namespace string) (string, error) + + nsLister v1.NamespaceLister + eipLister egressiplisters.EgressIPLister + nadLister nadlisters.NetworkAttachmentDefinitionLister + eipController controller.Controller + nsController controller.Controller + nadReconciler controller.Reconciler +} + +func NewEgressIPTrackerController( + name string, wf watchFactory, + onNetworkRefChange func(nodeName, nadName string, present bool), + primaryNADForNamespace func(namespace string) (string, error), +) *EgressIPTrackerController { + t := &EgressIPTrackerController{ + name: name, + cache: make(map[string]map[string]struct{}), + onNetworkRefChange: onNetworkRefChange, + nsLister: wf.NamespaceInformer().Lister(), + eipLister: wf.EgressIPInformer().Lister(), + nadLister: wf.NADInformer().Lister(), + primaryNADForNamespace: primaryNADForNamespace, + } + + if t.primaryNADForNamespace == nil { + t.primaryNADForNamespace = t.getPrimaryNADForNamespaceFromLister + } + + cfg := &controller.ControllerConfig[egressipv1.EgressIP]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: t.reconcileEgressIP, + ObjNeedsUpdate: t.egressIPNeedsUpdate, + MaxAttempts: controller.InfiniteAttempts, + Threadiness: 1, + Informer: wf.EgressIPInformer().Informer(), + Lister: wf.EgressIPInformer().Lister().List, + } + t.eipController = controller.NewController[egressipv1.EgressIP]("egressip-tracker", cfg) + + ncfg := &controller.ControllerConfig[corev1.Namespace]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: t.reconcileNamespace, + ObjNeedsUpdate: t.namespaceNeedsUpdate, + MaxAttempts: controller.InfiniteAttempts, + Threadiness: 1, + Informer: wf.NamespaceInformer().Informer(), + Lister: wf.NamespaceInformer().Lister().List, + } + t.nsController = controller.NewController[corev1.Namespace]("egressip-namespace-tracker", ncfg) + + t.nadReconciler = controller.NewReconciler( + fmt.Sprintf("%s-nad-reconciler", name), + &controller.ReconcilerConfig{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: t.reconcileNAD, + Threadiness: 1, + MaxAttempts: controller.InfiniteAttempts, + }, + ) + + return t +} + +func (t *EgressIPTrackerController) Start() error { + return controller.StartWithInitialSync(t.syncAll, t.eipController, t.nsController, t.nadReconciler) +} + +func (t *EgressIPTrackerController) Stop() { + controller.Stop(t.eipController, t.nsController, t.nadReconciler) +} + +func (t *EgressIPTrackerController) NodeHasNAD(node, nad string) bool { + t.cacheMutex.Lock() + defer t.cacheMutex.Unlock() + if _, ok := t.cache[node]; !ok { + return false + } + if _, ok := t.cache[node][nad]; !ok { + return false + } + return true +} + +func (t *EgressIPTrackerController) NADReconciler() controller.Reconciler { + return t.nadReconciler +} + +func (t *EgressIPTrackerController) egressIPNeedsUpdate(oldObj, newObj *egressipv1.EgressIP) bool { + if newObj == nil { + return false + } + if oldObj == nil { + return true // this is an Add + } + + if !reflect.DeepEqual(oldObj.Spec.NamespaceSelector, newObj.Spec.NamespaceSelector) { + return true + } + + if !reflect.DeepEqual(oldObj.Status.Items, newObj.Status.Items) { + return true + } + + return false +} + +func (t *EgressIPTrackerController) namespaceNeedsUpdate(oldObj, newObj *corev1.Namespace) bool { + if newObj == nil { + return false + } + if oldObj == nil { + return true // this is an Add + } + + // Only trigger reconcile if the labels (used by EgressIP selectors) change + return !reflect.DeepEqual(oldObj.Labels, newObj.Labels) +} + +// reconcileNAD determines if a NAD needs to reconcile, then triggers reconciliation +// via the namespace controller +func (t *EgressIPTrackerController) reconcileNAD(key string) error { + klog.V(5).Infof("%s - reconciling NAD key: %q", t.name, key) + namespace, _, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return fmt.Errorf("invalid NAD key %q: %v", key, err) + } + + // All NAD changes are funneled through namespace reconciliation to keep the cache consistent. + t.nsController.Reconcile(namespace) + return nil +} + +// reconcileEgressIP determines if an egress IP needs to reconcile, then triggers reconciliation +// via the namespace controller +func (t *EgressIPTrackerController) reconcileEgressIP(key string) error { + klog.V(5).Infof("%s - reconciling egress IP key: %q", t.name, key) + + eip, err := t.eipLister.Get(key) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get EgressIP %q from cache: %v", key, err) + } + if apierrors.IsNotFound(err) { + // EgressIP deleted → reconcile every namespace that had an active NAD cached + namespacesToReconcile := make(map[string]struct{}) + t.cacheMutex.Lock() + for _, nads := range t.cache { + for nad := range nads { + nsName, _, err := cache.SplitMetaNamespaceKey(nad) + if err != nil { + klog.Errorf("%s - Invalid NAD key in cache %q: %v", t.name, nad, err) + continue + } + namespacesToReconcile[nsName] = struct{}{} + } + } + t.cacheMutex.Unlock() + + for nsName := range namespacesToReconcile { + t.nsController.Reconcile(nsName) + } + return nil + } + + nsSelector, err := metav1.LabelSelectorAsSelector(&eip.Spec.NamespaceSelector) + if err != nil { + return fmt.Errorf("invalid namespaceSelector in EIP %s: %w", key, err) + } + nsList, err := t.nsLister.List(nsSelector) + if err != nil { + return fmt.Errorf("failed to list namespaces for EIP %s: %w", key, err) + } + + for _, ns := range nsList { + t.nsController.Reconcile(ns.Name) + } + + return nil +} + +func (t *EgressIPTrackerController) reconcileNamespace(key string) error { + var refChanges []refChange + klog.V(5).Infof("%s - reconciling namespace key: %q", t.name, key) + ns, err := t.nsLister.Get(key) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if apierrors.IsNotFound(err) { + // Namespace deleted → drop any cache + t.cacheMutex.Lock() + for node, nads := range t.cache { + for nad := range nads { + nadNamespace, _, err := cache.SplitMetaNamespaceKey(nad) + if err != nil { + klog.Errorf("%s - Invalid NAD key in cache %q: %v", t.name, nad, err) + delete(nads, nad) + } else if nadNamespace == key { + delete(nads, nad) + refChanges = append(refChanges, refChange{node, nad, false}) + } + } + if len(nads) == 0 { + delete(t.cache, node) + } + } + t.cacheMutex.Unlock() + if t.onNetworkRefChange != nil { + for _, callback := range refChanges { + t.onNetworkRefChange(callback.node, callback.nad, callback.active) + } + } + return nil + } + + primaryNAD, err := t.primaryNADForNamespace(ns.Name) + if err != nil { + if util.IsUnprocessedActiveNetworkError(err) { + // Namespace requires a primary network but none exists yet; NAD controller will requeue. + return nil + } + return fmt.Errorf("failed to get primary NAD for namespace %q: %w", ns.Name, err) + } + + if primaryNAD == types.DefaultNetworkName { + primaryNAD = "" + } + + // Gather the new set of (node,nad) pairs implied by this namespace's EIPs. Each namespace can + // have at most one primary NAD; if present we pin that NAD to every node that currently serves + // the namespace via an EgressIP assignment. + newActive := map[string]string{} // node -> nad + if primaryNAD != "" { + eips, err := t.eipLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("failed to list EgressIPs: %w", err) + } + for _, eip := range eips { + sel, err := metav1.LabelSelectorAsSelector(&eip.Spec.NamespaceSelector) + if err != nil { + return fmt.Errorf("invalid namespaceSelector in EIP %s: %w", eip.Name, err) + } + if sel.Matches(labels.Set(ns.Labels)) { + for _, st := range eip.Status.Items { + newActive[st.Node] = primaryNAD + } + } + } + } + + // Diff against cache + t.cacheMutex.Lock() + + // Removals first + for node, nads := range t.cache { + for nad := range nads { + nsName, _, err := cache.SplitMetaNamespaceKey(nad) + if err != nil { + klog.Errorf("%s - Invalid NAD key in cache %q: %v", t.name, nad, err) + delete(nads, nad) + } + if nsName == ns.Name { + if newActive[node] != nad { + delete(nads, nad) + refChanges = append(refChanges, refChange{node, nad, false}) + } + } + } + if len(nads) == 0 { + delete(t.cache, node) + } + } + + // Additions second + for node, nad := range newActive { + if _, ok := t.cache[node]; !ok { + t.cache[node] = map[string]struct{}{} + } + if _, exists := t.cache[node][nad]; !exists { + t.cache[node][nad] = struct{}{} + refChanges = append(refChanges, refChange{node, nad, true}) + } + } + t.cacheMutex.Unlock() + if t.onNetworkRefChange != nil { + for _, callback := range refChanges { + t.onNetworkRefChange(callback.node, callback.nad, callback.active) + } + } + return nil +} + +// syncAll builds the cache on initial controller start +// This is required because workers are started asynchronously and consumers of the tracker +// rely on the cache to be populated during start up +func (t *EgressIPTrackerController) syncAll() error { + start := time.Now() + defer func() { + klog.V(5).Infof("%s - syncAll took %v", t.name, time.Since(start)) + }() + + // handling all namespaces will handle setting up the egress IP tracker + namespaces, err := t.nsLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("syncAll: list Namespaces: %v", err) + } + + for _, ns := range namespaces { + nsName := ns.Name + if err := t.reconcileNamespace(nsName); err != nil { + klog.Errorf("%s - Failed to sync namespace %q: %v", t.name, nsName, err) + continue + } + } + + return nil +} + +// getPrimaryNADForNamespaceFromLister is a fallback resolver used in tests when no resolver is injected. +func (t *EgressIPTrackerController) getPrimaryNADForNamespaceFromLister(namespace string) (string, error) { + ns, err := t.nsLister.Get(namespace) + if err != nil { + return "", fmt.Errorf("failed to get namespace %q: %w", namespace, err) + } + if _, exists := ns.Labels[types.RequiredUDNNamespaceLabel]; !exists { + return types.DefaultNetworkName, nil + } + + nads, err := t.nadLister.NetworkAttachmentDefinitions(namespace).List(labels.Everything()) + if err != nil { + return "", fmt.Errorf("failed to list network attachment definitions: %w", err) + } + for _, nad := range nads { + if nad.Name == types.DefaultNetworkName { + continue + } + nadInfo, err := util.ParseNADInfo(nad) + if err != nil { + klog.Warningf("%s - Failed to parse network attachment definition %q: %v", t.name, nad.Name, err) + continue + } + if nadInfo.IsPrimaryNetwork() { + return util.GetNADName(nad.Namespace, nad.Name), nil + } + } + + // The namespace declared it needs a primary UDN but none exists yet. + return "", util.NewUnprocessedActiveNetworkError(namespace, "") +} diff --git a/go-controller/pkg/networkmanager/egressip_tracker_test.go b/go-controller/pkg/networkmanager/egressip_tracker_test.go new file mode 100644 index 0000000000..e82d2b9813 --- /dev/null +++ b/go-controller/pkg/networkmanager/egressip_tracker_test.go @@ -0,0 +1,333 @@ +package networkmanager + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + cnitypes "github.com/containernetworking/cni/pkg/types" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + egressipv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +func TestEgressIPTrackerControllerWithInformer(t *testing.T) { + type callbackEvent struct { + node string + nad string + active bool + } + + tests := []struct { + name string + nodeName string + namespace string + eipName string + labelKey string + labelValue string + newNodeName string + updateFn func(fakeClient *util.OVNKubeControllerClientset, g *gomega.WithT, tracker *EgressIPTrackerController) + expectAdds []callbackEvent + expectUpdates []callbackEvent + }{ + { + name: "basic EIP add/delete", + nodeName: "node1", + namespace: "ns1", + eipName: "eip1", + labelKey: "team", + labelValue: "a", + updateFn: func(fc *util.OVNKubeControllerClientset, g *gomega.WithT, _ *EgressIPTrackerController) { + err := fc.EgressIPClient.K8sV1().EgressIPs().Delete(context.Background(), "eip1", metav1.DeleteOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + }, + expectAdds: []callbackEvent{ + {"node1", "ns1/primary", true}, + }, + expectUpdates: []callbackEvent{ + {"node1", "ns1/primary", false}, + }, + }, + { + name: "namespace label change stops EIP", + nodeName: "node2", + namespace: "ns2", + eipName: "eip2", + labelKey: "team", + labelValue: "b", + updateFn: func(fc *util.OVNKubeControllerClientset, g *gomega.WithT, _ *EgressIPTrackerController) { + ns, err := fc.KubeClient.CoreV1().Namespaces().Get(context.Background(), "ns2", metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + ns.Labels = map[string]string{"team": "x"} + _, err = fc.KubeClient.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + }, + expectAdds: []callbackEvent{ + {"node2", "ns2/primary", true}, + }, + expectUpdates: []callbackEvent{ + {"node2", "ns2/primary", false}, + }, + }, + { + name: "EIP node reassignment", + nodeName: "node3", + newNodeName: "node4", + namespace: "ns3", + eipName: "eip3", + labelKey: "env", + labelValue: "prod", + updateFn: func(fc *util.OVNKubeControllerClientset, g *gomega.WithT, _ *EgressIPTrackerController) { + eip, err := fc.EgressIPClient.K8sV1().EgressIPs().Get(context.Background(), "eip3", metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + eip.Status.Items = []egressipv1.EgressIPStatusItem{ + {Node: "node4", EgressIP: "3.3.3.3"}, + } + _, err = fc.EgressIPClient.K8sV1().EgressIPs().Update(context.Background(), eip, metav1.UpdateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + }, + expectAdds: []callbackEvent{ + {"node3", "ns3/primary", true}, + }, + expectUpdates: []callbackEvent{ + {"node3", "ns3/primary", false}, + {"node4", "ns3/primary", true}, // new node add + }, + }, + { + name: "primary UDN change on namespace", + nodeName: "node5", + namespace: "ns5", + eipName: "eip5", + labelKey: "team", + labelValue: "blue", + updateFn: func(fc *util.OVNKubeControllerClientset, g *gomega.WithT, tracker *EgressIPTrackerController) { + // Simulate primary network change by replacing the NetInfo + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "new-primary", Type: "ovn-k8s-cni-overlay"}, + Topology: "layer3", + Role: "primary", + MTU: 1400, + NADName: "ns5/new-primary", + } + bytes, err := json.Marshal(netConf) + g.Expect(err).NotTo(gomega.HaveOccurred()) + err = fc.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions("ns5"). + Delete(context.Background(), "primary", metav1.DeleteOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + // Trigger the NAD create event + _, err = fc.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions("ns5"). + Create(context.Background(), &nadv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "new-primary", + Namespace: "ns5", + Labels: map[string]string{"role": "primary"}, + }, + Spec: nadv1.NetworkAttachmentDefinitionSpec{ + Config: string(bytes), + }, + }, metav1.CreateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Simulate NAD controller notification + g.Eventually(func() (string, error) { + return tracker.primaryNADForNamespace("ns5") + }, 2*time.Second, 100*time.Millisecond).Should(gomega.Equal(util.GetNADName("ns5", "new-primary"))) + tracker.nadReconciler.Reconcile(util.GetNADName("ns5", "new-primary")) + }, + expectAdds: []callbackEvent{ + {"node5", "ns5/primary", true}, + }, + expectUpdates: []callbackEvent{ + {"node5", "ns5/primary", false}, + {"node5", "ns5/new-primary", true}, + }, + }, + { + name: "multiple EgressIPs select same namespace triggers single callback", + nodeName: "node7", + namespace: "ns7", + eipName: "eip7a", // the first EgressIP + labelKey: "team", + labelValue: "shared", + updateFn: func(fc *util.OVNKubeControllerClientset, g *gomega.WithT, _ *EgressIPTrackerController) { + // Create a second EgressIP that selects the same namespace, + // has a different EgressIP name/address, but same node. + _, err := fc.EgressIPClient.K8sV1().EgressIPs().Create( + context.Background(), + &egressipv1.EgressIP{ + ObjectMeta: metav1.ObjectMeta{Name: "eip7b"}, + Spec: egressipv1.EgressIPSpec{ + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"team": "shared"}, + }, + }, + Status: egressipv1.EgressIPStatus{Items: []egressipv1.EgressIPStatusItem{ + {Node: "node7", EgressIP: "7.7.7.7"}, + }}, + }, + metav1.CreateOptions{}, + ) + g.Expect(err).NotTo(gomega.HaveOccurred()) + }, + expectAdds: []callbackEvent{ + // Only one initial callback is expected even if a second EgressIP + // later selects the same namespace and NAD on the same node. + {"node7", "ns7/primary", true}, + }, + expectUpdates: []callbackEvent{ + // No removals or additional adds, because the node+nad stays active. + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + err := config.PrepareTestConfig() + g.Expect(err).NotTo(gomega.HaveOccurred()) + config.OVNKubernetesFeature.EnableEgressIP = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + var got []callbackEvent + var gotMu sync.Mutex + + // Fake client and watch factory + fakeClient := util.GetOVNClientset().GetOVNKubeControllerClientset() + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClient) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + tracker := NewEgressIPTrackerController("test", wf, func(node, nad string, active bool) { + gotMu.Lock() + got = append(got, callbackEvent{node, nad, active}) + gotMu.Unlock() + }, nil) + + g.Expect(wf.Start()).To(gomega.Succeed()) + defer wf.Shutdown() + g.Expect(tracker.Start()).To(gomega.Succeed()) + defer tracker.Stop() + + // Create NAD + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "primary", Type: "ovn-k8s-cni-overlay"}, + Topology: "layer3", + Role: "primary", + MTU: 1400, + NADName: tt.namespace + "/primary", + } + bytes, err := json.Marshal(netConf) + g.Expect(err).NotTo(gomega.HaveOccurred()) + nad := &nadv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + UID: types.UID(tt.namespace), + Name: "primary", + Namespace: tt.namespace, + }, + Spec: nadv1.NetworkAttachmentDefinitionSpec{ + Config: string(bytes), + }, + } + _, err = fakeClient.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(tt.namespace). + Create(context.Background(), nad, metav1.CreateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Create nodes + _, err = fakeClient.KubeClient.CoreV1().Nodes().Create(context.Background(), + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: tt.nodeName}}, metav1.CreateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + if tt.newNodeName != "" { + _, err = fakeClient.KubeClient.CoreV1().Nodes().Create(context.Background(), + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: tt.newNodeName}}, metav1.CreateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + } + + // Create namespace matching selector + _, err = fakeClient.KubeClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.namespace, + Labels: map[string]string{ + tt.labelKey: tt.labelValue, + ovntypes.RequiredUDNNamespaceLabel: "", + }, + }, + }, metav1.CreateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Create EgressIP selecting the namespace + _, err = fakeClient.EgressIPClient.K8sV1().EgressIPs().Create(context.Background(), &egressipv1.EgressIP{ + ObjectMeta: metav1.ObjectMeta{Name: tt.eipName}, + Spec: egressipv1.EgressIPSpec{ + NamespaceSelector: metav1.LabelSelector{MatchLabels: map[string]string{tt.labelKey: tt.labelValue}}, + }, + Status: egressipv1.EgressIPStatus{Items: []egressipv1.EgressIPStatusItem{ + {Node: tt.nodeName, EgressIP: "1.1.1.1"}, + }}, + }, metav1.CreateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Expect add events + g.Eventually(func() []callbackEvent { + gotMu.Lock() + gotCopy := make([]callbackEvent, len(got)) + copy(gotCopy, got) + gotMu.Unlock() + return gotCopy + }, 3*time.Second, 100*time.Millisecond).Should(gomega.ConsistOf(tt.expectAdds)) + + g.Eventually(func(g gomega.Gomega) { + tracker.cacheMutex.Lock() + defer tracker.cacheMutex.Unlock() + for _, ev := range tt.expectAdds { + g.Expect(tracker.cache[ev.node]).To(gomega.HaveKey(ev.nad)) + } + }, 3*time.Second, 100*time.Millisecond).Should(gomega.Succeed()) + + // Apply the update (delete EIP, change label, or reassign node) + if tt.updateFn != nil { + tt.updateFn(fakeClient, g, tracker) + } + + expectedFinal := append(tt.expectAdds, tt.expectUpdates...) + + // Expect removal or new node events + g.Eventually(func() []callbackEvent { + gotMu.Lock() + gotCopy := make([]callbackEvent, len(got)) + copy(gotCopy, got) + gotMu.Unlock() + return gotCopy + }, 3*time.Second, 100*time.Millisecond).Should(gomega.ConsistOf(expectedFinal)) + + g.Consistently(func() []callbackEvent { + gotMu.Lock() + gotCopy := make([]callbackEvent, len(got)) + copy(gotCopy, got) + gotMu.Unlock() + return gotCopy + }, 500*time.Millisecond, 100*time.Millisecond).Should(gomega.ConsistOf(expectedFinal)) + + g.Eventually(func(g gomega.Gomega) { + tracker.cacheMutex.Lock() + defer tracker.cacheMutex.Unlock() + for _, ev := range tt.expectUpdates { + if !ev.active { // removal + g.Expect(tracker.cache[ev.node]).NotTo(gomega.HaveKey(ev.nad)) + } + } + }, 3*time.Second, 100*time.Millisecond).Should(gomega.Succeed()) + }) + } +} diff --git a/go-controller/pkg/networkmanager/fake.go b/go-controller/pkg/networkmanager/fake.go index d1ca4ad986..89ac6701b4 100644 --- a/go-controller/pkg/networkmanager/fake.go +++ b/go-controller/pkg/networkmanager/fake.go @@ -2,10 +2,14 @@ package networkmanager import ( "context" + "fmt" "sync" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" ) @@ -24,10 +28,12 @@ func (fnc *FakeNetworkController) Cleanup() error { return nil } -func (nc *FakeNetworkController) Reconcile(util.NetInfo) error { +func (fnc *FakeNetworkController) Reconcile(util.NetInfo) error { return nil } +func (fnc *FakeNetworkController) HandleNetworkRefChange(_ string, _ bool) {} + type FakeControllerManager struct{} func (fcm *FakeControllerManager) NewNetworkController(netInfo util.NetInfo) (NetworkController, error) { @@ -51,23 +57,42 @@ type FakeNetworkManager struct { // namespace -> netInfo // if netInfo is nil, it represents a namespace which contains the required UDN label but with no valid network. It will return invalid network error. PrimaryNetworks map[string]util.NetInfo - HandlerFuncs []handlerFunc + // nad key -> netInfo for non-primary lookups + NADNetworks map[string]util.NetInfo + Reconcilers []reconcilerRegistration + nextID uint64 // UDNNamespaces are a list of namespaces that require UDN for primary network UDNNamespaces sets.Set[string] } -func (fnm *FakeNetworkManager) RegisterNADHandler(h handlerFunc) error { +func (fnm *FakeNetworkManager) RegisterNADReconciler(r NADReconciler) (uint64, error) { fnm.Lock() defer fnm.Unlock() - fnm.HandlerFuncs = append(fnm.HandlerFuncs, h) - return nil + fnm.nextID++ + id := fnm.nextID + fnm.Reconcilers = append(fnm.Reconcilers, reconcilerRegistration{id: id, r: r}) + return id, nil +} + +func (fnm *FakeNetworkManager) DeRegisterNADReconciler(id uint64) error { + fnm.Lock() + defer fnm.Unlock() + for i, rec := range fnm.Reconcilers { + if rec.id == id { + fnm.Reconcilers = append(fnm.Reconcilers[:i], fnm.Reconcilers[i+1:]...) + return nil + } + } + return fmt.Errorf("reconciler not found") } func (fnm *FakeNetworkManager) TriggerHandlers(nadName string, info util.NetInfo, removed bool) { fnm.Lock() defer fnm.Unlock() - for _, h := range fnm.HandlerFuncs { - h(nadName, info, removed) + _ = info + _ = removed + for _, entry := range fnm.Reconcilers { + entry.r.Reconcile(nadName) } } @@ -87,6 +112,37 @@ func (fnm *FakeNetworkManager) GetActiveNetworkForNamespace(namespace string) (u return network, nil } +func (fnm *FakeNetworkManager) GetPrimaryNADForNamespace(namespace string) (string, error) { + fnm.Lock() + defer fnm.Unlock() + if primaryNetwork, ok := fnm.PrimaryNetworks[namespace]; ok { + if primaryNetwork == nil { + return "", util.NewInvalidPrimaryNetworkError(namespace) + } + var matches []string + for nadKey, netInfo := range fnm.NADNetworks { + if netInfo == nil || !netInfo.IsPrimaryNetwork() { + continue + } + nadNamespace, _, err := cache.SplitMetaNamespaceKey(nadKey) + if err != nil { + continue + } + if nadNamespace == namespace { + matches = append(matches, nadKey) + } + } + if len(matches) == 0 { + return "", util.NewInvalidPrimaryNetworkError(namespace) + } + if len(matches) > 1 { + return "", fmt.Errorf("multiple primary NADs found for namespace %q", namespace) + } + return matches[0], nil + } + return types.DefaultNetworkName, nil +} + func (fnm *FakeNetworkManager) GetActiveNetworkForNamespaceFast(namespace string) util.NetInfo { fnm.Lock() defer fnm.Unlock() @@ -112,11 +168,40 @@ func (fnm *FakeNetworkManager) GetActiveNetwork(networkName string) util.NetInfo return fnm.GetNetwork(networkName) } +func (fnm *FakeNetworkManager) GetNetInfoForNADKey(nadKey string) util.NetInfo { + fnm.Lock() + defer fnm.Unlock() + if netInfo, ok := fnm.NADNetworks[nadKey]; ok { + return netInfo + } + return nil +} + +func (fnm *FakeNetworkManager) GetNetworkNameForNADKey(nadKey string) string { + fnm.Lock() + defer fnm.Unlock() + if netInfo, ok := fnm.NADNetworks[nadKey]; ok { + return netInfo.GetNetworkName() + } + return "" +} + +func (fnm *FakeNetworkManager) GetNADKeysForNetwork(networkName string) []string { + fnm.Lock() + defer fnm.Unlock() + nadKeys := sets.New[string]() + for nadKey, netInfo := range fnm.NADNetworks { + if netInfo != nil && netInfo.GetNetworkName() == networkName { + nadKeys.Insert(nadKey) + } + } + return nadKeys.UnsortedList() +} + func (fnm *FakeNetworkManager) GetActiveNetworkNamespaces(networkName string) ([]string, error) { namespaces := make([]string, 0) for namespaceName, primaryNAD := range fnm.PrimaryNetworks { - nadNetworkName := primaryNAD.GetNADs()[0] - if nadNetworkName != networkName { + if primaryNAD == nil || primaryNAD.GetNetworkName() != networkName { continue } namespaces = append(namespaces, namespaceName) @@ -133,3 +218,18 @@ func (fnm *FakeNetworkManager) DoWithLock(f func(network util.NetInfo) error) er } return errors.Join(errs...) } + +func (fnm *FakeNetworkManager) GetNetworkByID(id int) util.NetInfo { + fnm.Lock() + defer fnm.Unlock() + for _, ni := range fnm.PrimaryNetworks { + if ni.GetNetworkID() == id { + return ni + } + } + return nil +} + +func (fnm *FakeNetworkManager) NodeHasNetwork(_, _ string) bool { + return !config.OVNKubernetesFeature.EnableDynamicUDNAllocation +} diff --git a/go-controller/pkg/networkmanager/nad_controller.go b/go-controller/pkg/networkmanager/nad_controller.go index 04435013cc..b282535f93 100644 --- a/go-controller/pkg/networkmanager/nad_controller.go +++ b/go-controller/pkg/networkmanager/nad_controller.go @@ -10,7 +10,6 @@ import ( nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" nadclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" - nadinformers "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1" nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" corev1 "k8s.io/api/core/v1" @@ -18,7 +17,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" - coreinformers "k8s.io/client-go/informers/core/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" @@ -26,9 +24,8 @@ import ( "k8s.io/klog/v2" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" - rainformers "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/informers/externalversions/routeadvertisements/v1" - userdefinednetworkinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/userdefinednetwork/v1" userdefinednetworklister "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/listers/userdefinednetwork/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -37,18 +34,6 @@ import ( utiludn "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/udn" ) -type watchFactory interface { - NADInformer() nadinformers.NetworkAttachmentDefinitionInformer - UserDefinedNetworkInformer() userdefinednetworkinformer.UserDefinedNetworkInformer - ClusterUserDefinedNetworkInformer() userdefinednetworkinformer.ClusterUserDefinedNetworkInformer - NamespaceInformer() coreinformers.NamespaceInformer - RouteAdvertisementsInformer() rainformers.RouteAdvertisementsInformer - NodeCoreInformer() coreinformers.NodeInformer -} - -// handlerFunc used as a callback that can be registered with nadController -type handlerFunc func(nadName string, info util.NetInfo, removed bool) - // nadController handles namespaced scoped NAD events and // manages cluster scoped networks defined in those NADs. NADs are mostly // referenced from pods to give them access to the network. Different NADs can @@ -57,8 +42,13 @@ type handlerFunc func(nadName string, info util.NetInfo, removed bool) // administration can lead to undefined behavior if referenced by running pods. type nadController struct { sync.RWMutex + // reconcilers keyed by registration ID. + reconcilers map[uint64]reconcilerRegistration + nextReconcilerID uint64 name string + stopChan chan struct{} + stopOnce sync.Once nadLister nadlisters.NetworkAttachmentDefinitionLister udnLister userdefinednetworklister.UserDefinedNetworkLister cudnLister userdefinednetworklister.ClusterUserDefinedNetworkLister @@ -73,6 +63,8 @@ type nadController struct { // nads to network mapping nads map[string]string + // nadsByNetwork tracks NAD keys grouped by network name. + nadsByNetwork map[string]sets.Set[string] // primaryNADs holds a mapping of namespace to NAD of primary UDNs primaryNADs map[string]string @@ -81,8 +73,24 @@ type nadController struct { networkIDAllocator id.Allocator tunnelKeysAllocator *id.TunnelKeysAllocator nadClient nadclientset.Interface - // handlerFuncs is a list of registered callbacks that is called during events - handlerFuncs []handlerFunc + + markedForRemoval map[string]time.Time + + // filterNADsOnNode is used by Dynamic UDN and refers the node name for which we consider that node local to + // where OVN-Kubernetes is running. + // For node/zone it is the name of the local node. + // For cluster-manager it is empty. + filterNADsOnNode string + + podTracker *PodTrackerController + egressIPTracker *EgressIPTrackerController + podReconcilerID uint64 + eipReconcilerID uint64 +} + +type reconcilerRegistration struct { + id uint64 + r NADReconciler } func newController( @@ -94,17 +102,43 @@ func newController( ovnClient *util.OVNClusterManagerClientset, recorder record.EventRecorder, tunnelKeysAllocator *id.TunnelKeysAllocator, + filterNADsOnNode string, ) (*nadController, error) { + networkController := newNetworkController(name, zone, node, cm, wf) c := &nadController{ name: fmt.Sprintf("[%s NAD controller]", name), + stopChan: make(chan struct{}), recorder: recorder, nadLister: wf.NADInformer().Lister(), nodeLister: wf.NodeCoreInformer().Lister(), - networkController: newNetworkController(name, zone, node, cm, wf), + networkController: networkController, + reconcilers: map[uint64]reconcilerRegistration{}, nads: map[string]string{}, + nadsByNetwork: map[string]sets.Set[string]{}, primaryNADs: map[string]string{}, - handlerFuncs: []handlerFunc{}, + markedForRemoval: map[string]time.Time{}, } + networkController.getNADKeysForNetwork = c.GetNADKeysForNetwork + + if cm != nil && config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + c.podTracker = NewPodTrackerController("pod-tracker", wf, c.OnNetworkRefChange, c.GetPrimaryNADForNamespace) + podID, err := c.RegisterNADReconciler(c.podTracker.NADReconciler()) + if err != nil { + return nil, fmt.Errorf("failed to register pod tracker NAD reconciler: %w", err) + } + c.podReconcilerID = podID + if config.OVNKubernetesFeature.EnableEgressIP { + c.egressIPTracker = NewEgressIPTrackerController("egress-ip-tracker", wf, c.OnNetworkRefChange, c.GetPrimaryNADForNamespace) + eipID, err := c.RegisterNADReconciler(c.egressIPTracker.NADReconciler()) + if err != nil { + return nil, fmt.Errorf("failed to register egress IP tracker NAD reconciler: %w", err) + } + c.eipReconcilerID = eipID + } + c.filterNADsOnNode = filterNADsOnNode + } + + c.networkController.nodeHasNetwork = c.NodeHasNetwork if ovnClient != nil { c.nadClient = ovnClient.NetworkAttchDefClient @@ -149,6 +183,161 @@ func newController( return c, nil } +func (c *nadController) nodeHasNAD(node, nad string) bool { + if !config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + return true + } + if c.podTracker != nil && c.podTracker.NodeHasNAD(node, nad) { + return true + } + if c.egressIPTracker != nil && c.egressIPTracker.NodeHasNAD(node, nad) { + return true + } + return false +} + +func (c *nadController) NodeHasNetwork(node, networkName string) bool { + if !config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + return true + } + if networkName == "" { + return false + } + if networkName == types.DefaultNetworkName { + return true + } + c.RLock() + nadSet := c.nadsByNetwork[networkName] + var nads []string + if len(nadSet) != 0 { + nads = nadSet.UnsortedList() + } + c.RUnlock() + for _, nad := range nads { + if c.nodeHasNAD(node, nad) { + return true + } + } + return false +} + +// addNADToNetworkLocked must be called with nadController locked +func (c *nadController) addNADToNetworkLocked(networkName, nadKey string) { + if networkName == "" { + return + } + if c.nadsByNetwork == nil { + c.nadsByNetwork = map[string]sets.Set[string]{} + } + nadSet := c.nadsByNetwork[networkName] + if nadSet == nil { + nadSet = sets.New[string]() + c.nadsByNetwork[networkName] = nadSet + } + nadSet.Insert(nadKey) +} + +// deleteNADFromNetworkLocked must be called with nadController locked +func (c *nadController) deleteNADFromNetworkLocked(networkName, nadKey string) { + if networkName == "" { + return + } + nadSet := c.nadsByNetwork[networkName] + if nadSet == nil { + return + } + nadSet.Delete(nadKey) + if len(nadSet) == 0 { + delete(c.nadsByNetwork, networkName) + } +} + +// OnNetworkRefChange is a callback function used to signal an action to this controller when +// a network needs to be added or removed or just updated. +// Used as a callback for pod/egress IP events when dynamic UDN allocation is enabled. +// This callback is invoked by pod/egress IP trackers and is blocking. Therefore it's work should be as lightweight +// as possible. +// The function handles: +// 1. Queuing local node event NADs to the NAD Controller for reconciliation later in the NAD Controller worker. +// 2. Queuing remote node event networks to the Network Manager for reconciliation later in the Network Manager worker. +// +// This function should never call into the trackers (i.e. nodeHasNAD) as it would cause deadlock. +func (c *nadController) OnNetworkRefChange(node, nadNamespacedName string, active bool) { + klog.V(4).Infof("Network change for zone controller triggered by pod/egress IP events "+ + "on node: %s, NAD: %s, active: %t", node, nadNamespacedName, active) + + namespace, name, err := cache.SplitMetaNamespaceKey(nadNamespacedName) + if err != nil { + klog.Errorf("Failed splitting key %q, falling back to normal network reconcile: %v", nadNamespacedName, err) + // fallback to regular reconcile + c.reconcile(nadNamespacedName) + return + } + + nadNetwork := c.GetNetInfoForNADKey(nadNamespacedName) + if nadNetwork == nil { + nad, err := c.nadLister.NetworkAttachmentDefinitions(namespace).Get(name) + if err != nil { + klog.Errorf("Failed to find NAD %q in informer, falling back to normal network reconcile: %v", nadNamespacedName, err) + // fallback to regular reconcile + c.reconcile(nadNamespacedName) + return + } + + ownerRef := metav1.GetControllerOf(nad) + if ownerRef == nil { + return + } + + if ownerRef.Kind != "ClusterUserDefinedNetwork" && ownerRef.Kind != "UserDefinedNetwork" { + return + } + + nadNetwork, err = util.ParseNADInfo(nad) + if err != nil || nadNetwork == nil { + klog.Errorf("Failed to parse NAD %q info, falling back to normal network reconcile: %v", nadNamespacedName, err) + // fallback to regular reconcile + c.reconcile(nadNamespacedName) + return + } + } else if !nadNetwork.IsUserDefinedNetwork() { + return + } + + isLocal := node == c.filterNADsOnNode + networkName := nadNetwork.GetNetworkName() + // Enqueue a network reconcile for remote nodes (non-blocking). + if !isLocal { + c.networkController.NotifyNetworkRefChange(networkName, node) + } + // Let the NAD controller handle lifecycle/teardown decisions asynchronously for local networks only. + if isLocal { + c.updateNADState(nadNamespacedName, active) + } + +} + +// filter should only be called if cm.filterNADsOnNode is set +func (c *nadController) filter(nad *nettypes.NetworkAttachmentDefinition) (bool, error) { + ownerRef := metav1.GetControllerOf(nad) + if ownerRef == nil { + return false, nil + } + + ourNode := c.filterNADsOnNode + + if ownerRef.Kind != "ClusterUserDefinedNetwork" && ownerRef.Kind != "UserDefinedNetwork" { + return false, nil + } + + // we don't support multiple nodes per zone, assume zone name is node name + if c.nodeHasNAD(ourNode, util.GetNADName(nad.Namespace, nad.Name)) { + return false, nil + } + + return true, nil +} + func (c *nadController) Interface() Interface { return c } @@ -165,6 +354,23 @@ func (c *nadController) Start() error { return err } + // Pod and Egress IP Trackers start and process existing pods/egress IPs. + // The trackers warm up their cache and trigger OnNetworkRefChange to queue keys + // to NAD Controller. + if c.podTracker != nil { + if err := c.podTracker.Start(); err != nil { + return fmt.Errorf("failed to start pod tracker: %w", err) + } + } + + if c.egressIPTracker != nil { + if err := c.egressIPTracker.Start(); err != nil { + return fmt.Errorf("failed to start egress ip tracker: %w", err) + } + } + + // NetworkController starts last and starts to process network keys to spin up network controllers. + // At this point the tracker cache's are warm to get accurate information for filtering. err = c.networkController.Start() if err != nil { return err @@ -176,26 +382,119 @@ func (c *nadController) Start() error { func (c *nadController) Stop() { klog.Infof("%s: shutting down", c.name) + c.stopOnce.Do(func() { + close(c.stopChan) + }) controller.Stop(c.controller) c.networkController.Stop() + if c.podReconcilerID != 0 { + if err := c.DeRegisterNADReconciler(c.podReconcilerID); err != nil { + klog.Warningf("Failed to deregister pod tracker NAD reconciler: %v", err) + } + } + if c.podTracker != nil { + c.podTracker.Stop() + } + if c.eipReconcilerID != 0 { + if err := c.DeRegisterNADReconciler(c.eipReconcilerID); err != nil { + klog.Warningf("Failed to deregister egress IP tracker NAD reconciler: %v", err) + } + } + if c.egressIPTracker != nil { + c.egressIPTracker.Stop() + } } -// RegisterNADHandler adds functions to be executed during NAD delete/update/add calls -// usage of this function should be restricted to lightweight, non-blocking operations -func (c *nadController) RegisterNADHandler(handler handlerFunc) error { +// RegisterNADReconciler registers a reconciler to receive NAD keys for reconciliation. +func (c *nadController) RegisterNADReconciler(r NADReconciler) (uint64, error) { c.Lock() defer c.Unlock() - c.handlerFuncs = append(c.handlerFuncs, handler) + if c.reconcilers == nil { + c.reconcilers = map[uint64]reconcilerRegistration{} + } + c.nextReconcilerID++ + id := c.nextReconcilerID + c.reconcilers[id] = reconcilerRegistration{id: id, r: r} + return id, nil +} + +// DeRegisterNADReconciler removes a previously registered reconciler by ID. +func (c *nadController) DeRegisterNADReconciler(id uint64) error { + c.Lock() + defer c.Unlock() + if _, ok := c.reconcilers[id]; !ok { + return fmt.Errorf("reconciler id %d not found", id) + } + delete(c.reconcilers, id) return nil } -// executeHandlers should always be done under lock -func (c *nadController) executeHandlers(nadName string, info util.NetInfo, removed bool) { - for _, handler := range c.handlerFuncs { - handler(nadName, info, removed) +// notifyReconcilers enqueues the NAD key to all registered reconcilers +// Must be called with nadController Mutex locked +func (c *nadController) notifyReconcilers(key string) { + for _, entry := range c.reconcilers { + entry.r.Reconcile(key) } } +func (c *nadController) reconcile(key string) { + c.controller.Reconcile(key) +} + +// updateNADState enqueues a sync for a given NAD. +// "active" defines if the network is actively being used by a dynamic resource +// - For local events, we either want to wait for grace period before tearing down an inactive network +// or clear any removal timer, but both conditions should lead to the network being reconciled (nad sync) +func (c *nadController) updateNADState(key string, active bool) { + if active { // if local and active, clear the mark for removal + c.removeMarkedForRemoval(key) + } else { // inactive start timer for removal + c.setMarkedForRemoval(key) + } + // always requeue to nad controller to syncNAD again + c.controller.Reconcile(key) +} + +func (c *nadController) setMarkedForRemoval(key string) { + c.Lock() + if _, ok := c.markedForRemoval[key]; ok { + c.Unlock() + return + } + removalTime := time.Now().Add(config.OVNKubernetesFeature.UDNDeletionGracePeriod) + c.markedForRemoval[key] = removalTime + c.Unlock() + + // ensure we reconcile later + stopCh := c.stopChan + go func() { + klog.V(5).Infof("Scheduling to remove nad %q after %v", key, removalTime) + timer := time.NewTimer(time.Until(removalTime)) + defer timer.Stop() + + select { + case <-stopCh: + return + case <-timer.C: + shouldReconcile := false + c.Lock() + if rt, ok := c.markedForRemoval[key]; ok && time.Now().After(rt) { + shouldReconcile = true + } + c.Unlock() + if shouldReconcile { + c.reconcile(key) + } + } + }() +} + +func (c *nadController) removeMarkedForRemoval(key string) { + c.Lock() + defer c.Unlock() + delete(c.markedForRemoval, key) +} + func (c *nadController) syncAll() (err error) { existingNADs, err := c.nadLister.List(labels.Everything()) if err != nil { @@ -206,6 +505,7 @@ func (c *nadController) syncAll() (err error) { key, err := cache.MetaNamespaceKeyFunc(nad) if err != nil { klog.Errorf("%s: failed to sync %v: %v", c.name, nad, err) + return nil } err = c.syncNAD(key, nad) if err != nil { @@ -294,16 +594,35 @@ func (c *nadController) sync(key string) error { return c.syncNAD(key, nad) } -func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefinition) error { +func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefinition) (syncErr error) { var nadNetworkName string var nadNetwork util.NetInfo var oldNetwork, ensureNetwork util.MutableNetInfo var err error + dynamicDelete := false + + c.Lock() + defer c.Unlock() namespace, _, err := cache.SplitMetaNamespaceKey(key) if err != nil { return fmt.Errorf("%s: failed splitting key %s: %v", c.name, key, err) } + previousNetworkName := c.nads[key] + + deleteTime, setforDeletion := c.markedForRemoval[key] + if setforDeletion && time.Now().After(deleteTime) { + // Grace period expired. Force a local teardown, but keep caches aligned to informer state. + klog.Infof("%s: NAD %q: marked for deletion and time has expired, will remove locally", c.name, key) + dynamicDelete = nad != nil + // Act like a delete for rendering/ensure paths + nad = nil + defer func() { + if syncErr == nil { + delete(c.markedForRemoval, key) + } + }() + } if nad != nil { nadNetwork, err = util.ParseNADInfo(nad) @@ -323,8 +642,10 @@ func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefin nadNetworkName = nadNetwork.GetNetworkName() } - c.Lock() - defer c.Unlock() + defer func() { + c.notifyReconcilers(key) // notify reconcilers after the sync runs with the latest information + }() + // We can only have one primary NAD per namespace primaryNAD := c.primaryNADs[namespace] if nadNetwork != nil && nadNetwork.IsPrimaryNetwork() && primaryNAD != "" && primaryNAD != key { @@ -353,7 +674,10 @@ func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefin // the NAD refers to an existing compatible network, ensure that // existing network holds a reference to this NAD ensureNetwork = currentNetwork - case sets.New(key).HasAll(currentNetwork.GetNADs()...): + case func() bool { + nadSet := c.nadsByNetwork[nadNetworkName] + return len(nadSet) == 1 && nadSet.Has(key) + }(): // the NAD is the only NAD referring to an existing incompatible // network, remove the reference from the old network and ensure that // existing network holds a reference to this NAD @@ -373,25 +697,45 @@ func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefin // remove the NAD reference from the old network and delete the network if // it is no longer referenced if oldNetwork != nil { + klog.V(5).Infof("%s: removing NAD %q reference for network %q", c.name, key, oldNetwork.GetNetworkName()) oldNetworkName := oldNetwork.GetNetworkName() oldNetwork.DeleteNADs(key) - if len(oldNetwork.GetNADs()) == 0 { + if !c.networkReferencedLocked(oldNetworkName, key) { c.networkController.DeleteNetwork(oldNetworkName) } else { c.networkController.EnsureNetwork(oldNetwork) } - c.executeHandlers(key, oldNetwork, true) + if !dynamicDelete && c.primaryNADs[namespace] == key { + delete(c.primaryNADs, namespace) + } } - if err := c.handleNetworkAnnotations(oldNetwork, ensureNetwork, nad); err != nil { - return err + // handleNetworkAnnotations prevents duplicated IDs from being allocated, so we call it even + // if the NAD/network is filtered by Dynamic UDN while we are ensuring the network. + // handleNetworkAnnotations also handles deletion and releasing based on NAD cache state + // IDs are not released during dynamicDeletes (going inactive) and are only released on a true + // NAD/network delete + if !dynamicDelete { + if err := c.handleNetworkAnnotations(ensureNetwork, nad, key, previousNetworkName); err != nil { + return err + } } // this was a nad delete if ensureNetwork == nil { - delete(c.nads, key) - if c.primaryNADs[namespace] == key { - delete(c.primaryNADs, namespace) + // On a true delete (incoming nad nil or expired grace period) we must clean caches, + // except for dynamicDelete where we keep informer-derived state. + if !dynamicDelete { + delete(c.markedForRemoval, key) + // clean up primary mapping even if we never had an oldNetwork rendered + if c.primaryNADs[namespace] == key { + delete(c.primaryNADs, namespace) + } + networkName := previousNetworkName + delete(c.nads, key) + if networkName != "" { + c.deleteNADFromNetworkLocked(networkName, key) + } } return err } @@ -400,13 +744,20 @@ func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefin // network, need to wait until cluster nad controller allocates an ID for // the network if ensureNetwork.GetNetworkID() == types.InvalidID { - klog.V(4).Infof("%s: will wait for cluster manager to allocate an ID before ensuring network %s", c.name, nadNetworkName) + klog.V(4).Infof("%s: will wait for cluster manager to allocate an ID before ensuring network %s, NAD: %s", + c.name, nadNetworkName, key) return nil } - // ensure the network is associated with the NAD - ensureNetwork.AddNADs(key) - c.nads[key] = ensureNetwork.GetNetworkName() + klog.V(5).Infof("%s: ensuring NAD %q reference for network %q with id %d", + c.name, key, ensureNetwork.GetNetworkName(), ensureNetwork.GetNetworkID()) + + networkName := ensureNetwork.GetNetworkName() + if previousNetworkName != "" && previousNetworkName != networkName { + c.deleteNADFromNetworkLocked(previousNetworkName, key) + } + c.nads[key] = networkName + c.addNADToNetworkLocked(networkName, key) // track primary NAD switch { case ensureNetwork.IsPrimaryNetwork(): @@ -417,9 +768,24 @@ func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefin } } - // reconcile the network - c.networkController.EnsureNetwork(ensureNetwork) - c.executeHandlers(key, ensureNetwork, false) + shouldNetworkExist := true + if c.filterNADsOnNode != "" { + shouldFilter, err := c.filter(nad) + if err != nil { + return fmt.Errorf("%s: failed filtering NAD %s: %w", c.name, key, err) + } + if shouldFilter { + shouldNetworkExist = false + } + } + if shouldNetworkExist { + // ensure the network is associated with the NAD + ensureNetwork.AddNADs(key) + // reconcile the network + c.networkController.EnsureNetwork(ensureNetwork) + } else { + klog.V(4).Infof("%s: Network is filtered and will not be rendered: %s", c.name, ensureNetwork.GetNetworkName()) + } return nil } @@ -429,7 +795,7 @@ func isOwnUpdate(manager string, managedFields []metav1.ManagedFieldsEntry) bool return util.IsLastUpdatedByManager(manager, managedFields) } -func (c *nadController) nadNeedsUpdate(oldNAD, newNAD *nettypes.NetworkAttachmentDefinition) bool { +func (c *nadController) nadNeedsUpdate(oldNAD, newNAD *nettypes.NetworkAttachmentDefinition) (needsUpdate bool) { if oldNAD == nil || newNAD == nil { return true } @@ -444,6 +810,28 @@ func (c *nadController) nadNeedsUpdate(oldNAD, newNAD *nettypes.NetworkAttachmen return false } + // notifyReconcilers during sync happens after netInfo is updated, so controllers receive the latest info, + // and it is safe to ignore own updates + defer func() { + if !needsUpdate { // ensure we send the NAD event to registered handlers anyway + var key string + var err error + if newNAD != nil { + key, err = cache.MetaNamespaceKeyFunc(newNAD) + if err != nil && oldNAD != nil { + key, err = cache.MetaNamespaceKeyFunc(oldNAD) + } + } + if err != nil || len(key) == 0 { + klog.Errorf("Failed to parse nad key during update, error: %v", err) + } else { + c.Lock() + defer c.Unlock() + c.notifyReconcilers(key) + } + } + }() + // also reconcile the network in case its route advertisements changed return !reflect.DeepEqual(oldNAD.Spec, newNAD.Spec) || oldNAD.Annotations[types.OvnRouteAdvertisementsKey] != newNAD.Annotations[types.OvnRouteAdvertisementsKey] || @@ -517,6 +905,35 @@ func (c *nadController) GetActiveNetworkForNamespaceFast(namespace string) util. return network } +// GetPrimaryNADForNamespace returns the full namespaced key of the +// primary NAD for the given namespace, if one exists. +// Returns default network if namespace has no primary UDN +func (c *nadController) GetPrimaryNADForNamespace(namespace string) (string, error) { + c.RLock() + primary := c.primaryNADs[namespace] + c.RUnlock() + if primary != "" { + return primary, nil + } + + // Double-check if the namespace *requires* a primary UDN. + ns, err := c.namespaceLister.Get(namespace) + if err != nil { + if apierrors.IsNotFound(err) { + // Namespace is gone — no primary NAD by definition. + return "", nil + } + return "", fmt.Errorf("failed to fetch namespace %q: %w", namespace, err) + } + if _, exists := ns.Labels[types.RequiredUDNNamespaceLabel]; exists { + // Namespace promises a primary UDN, but we haven't cached one yet. + return "", util.NewUnprocessedActiveNetworkError(namespace, "") + } + + // No required label: means default network only. + return types.DefaultNetworkName, nil +} + func (c *nadController) getActiveNetworkForNamespace(namespace string) (util.NetInfo, string) { c.RLock() defer c.RUnlock() @@ -552,6 +969,43 @@ func (c *nadController) GetNetwork(name string) util.NetInfo { return network } +func (c *nadController) GetNetInfoForNADKey(nadKey string) util.NetInfo { + c.RLock() + networkName := c.nads[nadKey] + c.RUnlock() + if networkName == "" { + return nil + } + network := c.networkController.getNetwork(networkName) + if network == nil && networkName == types.DefaultNetworkName { + return &util.DefaultNetInfo{} + } + if network == nil { + return nil + } + // Return a copy so callers can safely read fields without depending on controller locks. + return util.NewMutableNetInfo(network) +} + +func (c *nadController) GetNetworkNameForNADKey(nadKey string) string { + c.RLock() + defer c.RUnlock() + return c.nads[nadKey] +} + +func (c *nadController) GetNADKeysForNetwork(networkName string) []string { + if networkName == "" { + return nil + } + c.RLock() + defer c.RUnlock() + nadSet := c.nadsByNetwork[networkName] + if len(nadSet) == 0 { + return nil + } + return nadSet.UnsortedList() +} + func (c *nadController) GetActiveNetworkNamespaces(networkName string) ([]string, error) { if !util.IsNetworkSegmentationSupportEnabled() { return []string{"default"}, nil @@ -591,6 +1045,14 @@ func (c *nadController) DoWithLock(f func(network util.NetInfo) error) error { panic("NAD Controller broken consistency between primary NADs and cached NADs") } network := c.networkController.getNetwork(netName) + if network == nil { + // network may not always be rendered with Dynamic UDN + // otherwise this should never happen + if config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + continue + } + panic("NAD Controller broken consistency between primary NADs and network controller cache") + } n := util.NewMutableNetInfo(network) // update the returned netInfo copy to only have the primary NAD for this namespace n.SetNADs(primaryNAD) @@ -610,7 +1072,23 @@ func (c *nadController) DoWithLock(f func(network util.NetInfo) error) error { // If this is the NAD controller running in cluster manager then a new ID // is allocated and annotated on the NAD. The NAD controller running in // cluster manager also releases here the network ID of a network that is being deleted. -func (c *nadController) handleNetworkAnnotations(old util.NetInfo, new util.MutableNetInfo, nad *nettypes.NetworkAttachmentDefinition) (err error) { +func (c *nadController) handleNetworkAnnotations(new util.MutableNetInfo, nad *nettypes.NetworkAttachmentDefinition, nadKey, previousNetworkName string) (err error) { + newNetworkName := "" + if new != nil { + newNetworkName = new.GetNetworkName() + } + if previousNetworkName != "" && previousNetworkName != types.DefaultNetworkName && + (new == nil || previousNetworkName != newNetworkName) { + if c.networkReferencedLocked(previousNetworkName, nadKey) { + klog.V(5).Infof("%s: NADs still reference network %s; skipping ID release", c.name, previousNetworkName) + } else { + c.networkIDAllocator.ReleaseID(previousNetworkName) + if c.isClusterManagerMode() { + c.tunnelKeysAllocator.ReleaseKeys(previousNetworkName) + } + } + } + if new != nil && new.IsDefault() { return nil } @@ -620,6 +1098,9 @@ func (c *nadController) handleNetworkAnnotations(old util.NetInfo, new util.Muta // check if in cache first if new != nil { id = c.networkIDAllocator.GetID(new.GetNetworkName()) + if id != types.InvalidID { + klog.V(5).Infof("Previously cached network ID %d found for network: %s", id, new.GetNetworkName()) + } } if nad != nil && id == types.InvalidID { // check what ID is currently annotated @@ -629,6 +1110,7 @@ func (c *nadController) handleNetworkAnnotations(old util.NetInfo, new util.Muta if err != nil { return fmt.Errorf("failed to parse annotated network ID: %w", err) } + klog.V(5).Infof("Previously annotated network ID %d found for NAD: %s/%s", id, nad.Namespace, nad.Name) } } @@ -641,14 +1123,6 @@ func (c *nadController) handleNetworkAnnotations(old util.NetInfo, new util.Muta } } - // release old ID if the network is being deleted - if old != nil && !old.IsDefault() && len(old.GetNADs()) == 0 { - c.networkIDAllocator.ReleaseID(old.GetNetworkName()) - if c.isClusterManagerMode() { - c.tunnelKeysAllocator.ReleaseKeys(old.GetNetworkName()) - } - } - // nothing to allocate, delete case if new == nil { return nil @@ -765,6 +1239,26 @@ func (c *nadController) handleNetworkAnnotations(old util.NetInfo, new util.Muta return nil } +// networkReferencedLocked reports whether any NADs still reference a network. +// When excludeNAD is set, it is ignored in the reference check. +// Must be called with nadController locked. +func (c *nadController) networkReferencedLocked(networkName, excludeNAD string) bool { + if networkName == "" || networkName == types.DefaultNetworkName { + return false + } + nadSet := c.nadsByNetwork[networkName] + if len(nadSet) == 0 { + return false + } + if excludeNAD == "" { + return len(nadSet) > 0 + } + if nadSet.Has(excludeNAD) { + return len(nadSet) > 1 + } + return len(nadSet) > 0 +} + func (c *nadController) getNetworkIDFromNode(nadNetwork util.NetInfo) (int, error) { // check if the node has a legacy ID nodes, err := c.nodeLister.List(labels.Everything()) @@ -799,6 +1293,44 @@ func (c *nadController) GetActiveNetwork(network string) util.NetInfo { return state.controller } +func (c *nadController) GetNetworkByID(id int) util.NetInfo { + if id == types.InvalidID { + return nil + } + netInfo := c.networkController.GetNetworkByID(id) + if netInfo != nil { + return netInfo + } + c.RLock() + nadKeys := make([]string, 0, len(c.nads)) + for key := range c.nads { + nadKeys = append(nadKeys, key) + } + c.RUnlock() + // Handles the case where there is a cache miss. This is needed for filtered networks with + // Dynamic UDN as there will be no network manager cache entry. + // TODO (trozet): this is slow. We should optimize this by potentially storing the cache + // of netInfos even for filtered entries. + for _, key := range nadKeys { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + continue + } + nad, err := c.nadLister.NetworkAttachmentDefinitions(namespace).Get(name) + if err != nil { + continue + } + nadNetwork, err := util.ParseNADInfo(nad) + if err != nil || nadNetwork == nil { + continue + } + if nadNetwork.GetNetworkID() == id { + return nadNetwork + } + } + return nil +} + func (c *nadController) isClusterManagerMode() bool { return c.tunnelKeysAllocator != nil } diff --git a/go-controller/pkg/networkmanager/nad_controller_test.go b/go-controller/pkg/networkmanager/nad_controller_test.go index 3e6d89b59f..a794c07e01 100644 --- a/go-controller/pkg/networkmanager/nad_controller_test.go +++ b/go-controller/pkg/networkmanager/nad_controller_test.go @@ -8,9 +8,11 @@ import ( "strings" "sync" "testing" + "time" cnitypes "github.com/containernetworking/cni/pkg/types" nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" "github.com/onsi/gomega" "github.com/onsi/gomega/format" @@ -20,18 +22,149 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) +func TestNADNeedsUpdate_NotifiesReconcilersOnNoopUpdate(t *testing.T) { + g := gomega.NewWithT(t) + + keyCh := make(chan string, 1) + r := controller.NewReconciler("test-nad-reconciler", &controller.ReconcilerConfig{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: func(key string) error { + keyCh <- key + return nil + }, + Threadiness: 1, + MaxAttempts: controller.InfiniteAttempts, + }) + g.Expect(controller.Start(r)).To(gomega.Succeed()) + t.Cleanup(func() { controller.Stop(r) }) + + c := &nadController{name: "test-nad-controller"} + _, err := c.RegisterNADReconciler(r) + g.Expect(err).To(gomega.Succeed()) + + oldNAD := &nettypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "nad", ResourceVersion: "1"}, + Spec: nettypes.NetworkAttachmentDefinitionSpec{Config: `{"cniVersion":"0.3.1"}`}, + } + newNAD := oldNAD.DeepCopy() + newNAD.ResourceVersion = "2" + + needsUpdate := c.nadNeedsUpdate(oldNAD, newNAD) + g.Expect(needsUpdate).To(gomega.BeFalse()) + + g.Eventually(keyCh, time.Second).Should(gomega.Receive(gomega.Equal("ns/nad"))) +} + +func TestNADNeedsUpdate_DoesNotNotifyReconcilersOnRelevantUpdate(t *testing.T) { + g := gomega.NewWithT(t) + + keyCh := make(chan string, 1) + r := controller.NewReconciler("test-nad-reconciler", &controller.ReconcilerConfig{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: func(key string) error { + keyCh <- key + return nil + }, + Threadiness: 1, + MaxAttempts: controller.InfiniteAttempts, + }) + g.Expect(controller.Start(r)).To(gomega.Succeed()) + t.Cleanup(func() { controller.Stop(r) }) + + c := &nadController{name: "test-nad-controller"} + _, err := c.RegisterNADReconciler(r) + g.Expect(err).To(gomega.Succeed()) + + oldNAD := &nettypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "nad", ResourceVersion: "1"}, + Spec: nettypes.NetworkAttachmentDefinitionSpec{Config: `{"cniVersion":"0.3.1"}`}, + } + newNAD := oldNAD.DeepCopy() + newNAD.ResourceVersion = "2" + newNAD.Spec.Config = `{"cniVersion":"0.3.1","name":"changed"}` + + needsUpdate := c.nadNeedsUpdate(oldNAD, newNAD) + g.Expect(needsUpdate).To(gomega.BeTrue()) + + g.Consistently(keyCh, 200*time.Millisecond).ShouldNot(gomega.Receive()) +} + +func TestSyncNAD_NotifiesReconcilers(t *testing.T) { + g := gomega.NewWithT(t) + + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + t.Cleanup(func() { + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + }) + + keyCh := make(chan string, 1) + r := controller.NewReconciler("test-nad-reconciler-sync", &controller.ReconcilerConfig{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: func(key string) error { + keyCh <- key + return nil + }, + Threadiness: 1, + MaxAttempts: controller.InfiniteAttempts, + }) + g.Expect(controller.Start(r)).To(gomega.Succeed()) + t.Cleanup(func() { controller.Stop(r) }) + + nc := &networkController{ + networks: map[string]util.MutableNetInfo{}, + networkControllers: map[string]*networkControllerState{}, + } + netIDAlloc := id.NewIDAllocator("NetworkIDs", MaxNetworks) + g.Expect(netIDAlloc.ReserveID(types.DefaultNetworkName, types.DefaultNetworkID)).To(gomega.Succeed()) + + c := &nadController{ + name: "test-nad-controller", + networkController: nc, + nads: map[string]string{}, + primaryNADs: map[string]string{}, + networkIDAllocator: netIDAlloc, + } + _, err := c.RegisterNADReconciler(r) + g.Expect(err).To(gomega.Succeed()) + + nadNS := "ns" + nadName := "nad" + nadKey := nadNS + "/" + nadName + networkAPrimary := &ovncnitypes.NetConf{ + Topology: types.Layer2Topology, + NetConf: cnitypes.NetConf{ + Name: "networkAPrimary", + Type: "ovn-k8s-cni-overlay", + }, + Subnets: "10.1.130.0/24", + Role: types.NetworkRolePrimary, + MTU: 1400, + NADName: nadKey, + } + nad, err := buildNAD(nadName, nadNS, networkAPrimary) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // The NAD has no network ID annotation so syncNAD will not ensure the network, + // but it should still notify all reconcilers. + g.Expect(c.syncNAD(nadKey, nad)).To(gomega.Succeed()) + g.Eventually(keyCh, time.Second).Should(gomega.Receive(gomega.Equal(nadKey))) +} + type testNetworkController struct { util.ReconcilableNetInfo - tcm *testControllerManager + tcm *testControllerManager + handleRefChange func(node string, active bool) } func (tnc *testNetworkController) Start(context.Context) error { @@ -45,12 +178,14 @@ func (tnc *testNetworkController) Start(context.Context) error { func (tnc *testNetworkController) Stop() { tnc.tcm.Lock() defer tnc.tcm.Unlock() + fmt.Printf("stopping network: %s\n", testNetworkKey(tnc)) tnc.tcm.stopped = append(tnc.tcm.stopped, testNetworkKey(tnc)) } func (tnc *testNetworkController) Cleanup() error { tnc.tcm.Lock() defer tnc.tcm.Unlock() + fmt.Printf("cleaning up network: %s\n", testNetworkKey(tnc)) tnc.tcm.cleaned = append(tnc.tcm.cleaned, testNetworkKey(tnc)) return nil } @@ -59,12 +194,80 @@ func (tnc *testNetworkController) Reconcile(netInfo util.NetInfo) error { return util.ReconcileNetInfo(tnc.ReconcilableNetInfo, netInfo) } +func (tnc *testNetworkController) HandleNetworkRefChange(node string, active bool) { + if tnc.handleRefChange != nil { + tnc.handleRefChange(node, active) + } +} + // GomegaString is used to avoid printing embedded mutexes which can cause a // race func (tnc *testNetworkController) GomegaString() string { return format.Object(tnc.GetNetworkName(), 1) } +func TestSyncNAD_ForceDeleteKeepsCacheForExistingNAD(t *testing.T) { + g := gomega.NewWithT(t) + + key := "ns/nad" + ns := "ns" + nad := &nettypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nad", + Namespace: ns, + }, + } + + netIDAlloc := id.NewIDAllocator("NetworkIDs", MaxNetworks) + g.Expect(netIDAlloc.ReserveID(types.DefaultNetworkName, types.DefaultNetworkID)).To(gomega.Succeed()) + + c := &nadController{ + name: "test-nad-controller", + nads: map[string]string{key: "netA"}, + primaryNADs: map[string]string{ns: key}, + markedForRemoval: map[string]time.Time{key: time.Now().Add(-time.Minute)}, + networkIDAllocator: netIDAlloc, + networkController: &networkController{ + networks: map[string]util.MutableNetInfo{}, + networkControllers: map[string]*networkControllerState{}, + }, + } + + g.Expect(c.syncNAD(key, nad)).To(gomega.Succeed()) + g.Expect(c.nads).To(gomega.HaveKeyWithValue(key, "netA")) + g.Expect(c.primaryNADs).To(gomega.HaveKeyWithValue(ns, key)) + g.Expect(c.markedForRemoval).ToNot(gomega.HaveKey(key)) +} + +func TestSyncNAD_ForceDeleteRemovesCacheOnActualDelete(t *testing.T) { + g := gomega.NewWithT(t) + + key := "ns/nad" + ns := "ns" + + netIDAlloc := id.NewIDAllocator("NetworkIDs", MaxNetworks) + g.Expect(netIDAlloc.ReserveID(types.DefaultNetworkName, types.DefaultNetworkID)).To(gomega.Succeed()) + + c := &nadController{ + name: "test-nad-controller", + nads: map[string]string{key: "netA"}, + primaryNADs: map[string]string{ns: key}, + markedForRemoval: map[string]time.Time{key: time.Now().Add(-time.Minute)}, + networkIDAllocator: netIDAlloc, + networkController: &networkController{ + networks: map[string]util.MutableNetInfo{}, + networkControllers: map[string]*networkControllerState{}, + }, + } + + g.Expect(c.syncNAD(key, nil)).To(gomega.Succeed()) + g.Expect(c.nads).ToNot(gomega.HaveKey(key)) + g.Expect(c.primaryNADs).ToNot(gomega.HaveKey(ns)) + g.Expect(c.markedForRemoval).ToNot(gomega.HaveKey(key)) +} + +func ptrTo[T any](v T) *T { return &v } + func testNetworkKey(nInfo util.NetInfo) string { return nInfo.GetNetworkName() + " " + nInfo.TopologyType() } @@ -73,6 +276,41 @@ func networkFromTestNetworkKey(key string) string { return key[:strings.LastIndex(key, " ")] } +type fakeNADNamespaceLister struct { + nads map[string]*nettypes.NetworkAttachmentDefinition +} + +func (f *fakeNADNamespaceLister) List(_ labels.Selector) ([]*nettypes.NetworkAttachmentDefinition, error) { + result := []*nettypes.NetworkAttachmentDefinition{} + for _, nad := range f.nads { + result = append(result, nad) + } + return result, nil +} + +func (f *fakeNADNamespaceLister) Get(name string) (*nettypes.NetworkAttachmentDefinition, error) { + if nad, ok := f.nads[name]; ok { + return nad, nil + } + return nil, apierrors.NewNotFound(nettypes.Resource("networkattachmentdefinition"), name) +} + +type fakeNADLister struct { + nads map[string]*nettypes.NetworkAttachmentDefinition +} + +func (f *fakeNADLister) List(_ labels.Selector) ([]*nettypes.NetworkAttachmentDefinition, error) { + result := []*nettypes.NetworkAttachmentDefinition{} + for _, nad := range f.nads { + result = append(result, nad) + } + return result, nil +} + +func (f *fakeNADLister) NetworkAttachmentDefinitions(_ string) nadlisters.NetworkAttachmentDefinitionNamespaceLister { + return &fakeNADNamespaceLister{nads: f.nads} +} + type testControllerManager struct { sync.Mutex @@ -115,6 +353,10 @@ func (tcm *testControllerManager) Reconcile(string, util.NetInfo, util.NetInfo) return nil } +func (tcm *testControllerManager) Filter(*nettypes.NetworkAttachmentDefinition) (bool, error) { + return false, nil +} + type fakeNamespaceLister struct{} func (f *fakeNamespaceLister) List(labels.Selector) (ret []*corev1.Namespace, err error) { @@ -133,6 +375,92 @@ func (f *fakeNamespaceLister) Get(name string) (*corev1.Namespace, error) { } func TestNADController(t *testing.T) { + t.Run("filter respects node trackers", func(t *testing.T) { + if err := config.PrepareTestConfig(); err != nil { + t.Fatalf("prepare test config: %v", err) + } + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + + pt := &PodTrackerController{ + nodeNADToPodCache: map[string]map[string]map[string]struct{}{}, + } + pt.nodeNADToPodCache["node1"] = map[string]map[string]struct{}{ + "ns1/nad1": {"pod": {}}, + } + + cm := &nadController{ + filterNADsOnNode: "node1", + podTracker: pt, + } + + tests := []struct { + name string + nad *nettypes.NetworkAttachmentDefinition + expected bool + }{ + { + name: "no ownerRef", + nad: &nettypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1", Name: "nad1"}, + }, + expected: false, + }, + { + name: "unrelated ownerRef", + nad: &nettypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", Name: "nad1", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Deployment", + Controller: ptrTo(true), + }}, + }, + }, + expected: false, + }, + { + name: "UDN ownerRef but unused on node -> filtered", + nad: &nettypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns2", Name: "nad2", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "UserDefinedNetwork", + Controller: ptrTo(true), + }}, + }, + }, + expected: true, + }, + { + name: "UDN ownerRef and used on node -> not filtered", + nad: &nettypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", Name: "nad1", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "UserDefinedNetwork", + Controller: ptrTo(true), + }}, + }, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := cm.filter(tt.nad) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if got != tt.expected { + t.Fatalf("expected filter=%v got %v", tt.expected, got) + } + }) + } + }) + networkAPrimary := &ovncnitypes.NetConf{ Topology: types.Layer2Topology, NetConf: cnitypes.NetConf{ @@ -516,8 +844,20 @@ func TestNADController(t *testing.T) { g.Expect(err).ToNot(gomega.HaveOccurred()) netController := nadController.networkController - g.Expect(nadController.networkController.Start()).To(gomega.Succeed()) - defer nadController.networkController.Stop() + // Drive reconciliation only for networks touched by the NAD operation + // to avoid assertions against transient async queue states. + syncTouchedNetworks := func(nadKey, prevNetwork string) { + networkNames := sets.New[string]() + if prevNetwork != "" { + networkNames.Insert(prevNetwork) + } + if currNetwork := nadController.nads[nadKey]; currNetwork != "" { + networkNames.Insert(currNetwork) + } + for _, network := range networkNames.UnsortedList() { + g.Expect(netController.syncNetwork(network)).To(gomega.Succeed()) + } + } for _, args := range tt.args { namespace, name, err := cache.SplitMetaNamespaceKey(args.nad) @@ -532,12 +872,14 @@ func TestNADController(t *testing.T) { g.Expect(err).To(gomega.Or(gomega.Not(gomega.HaveOccurred()), gomega.MatchError(apierrors.IsAlreadyExists, "AlreadyExists"))) } + prevNetwork := nadController.nads[args.nad] err = nadController.syncNAD(args.nad, nad) if args.wantErr { g.Expect(err).To(gomega.HaveOccurred()) } else { g.Expect(err).NotTo(gomega.HaveOccurred()) } + syncTouchedNetworks(args.nad, prevNetwork) } meetsExpectations := func(g gomega.Gomega) { @@ -563,7 +905,8 @@ func TestNADController(t *testing.T) { g.Expect(netController.networks).To(gomega.HaveKey(name)) g.Expect(util.AreNetworksCompatible(netController.networks[name], netInfo)).To(gomega.BeTrue(), fmt.Sprintf("matching network config for network %s", name)) - g.Expect(netController.networks[name].GetNADs()).To(gomega.ConsistOf(expected.nads), + nadKeys := nadController.GetNADKeysForNetwork(name) + g.Expect(nadKeys).To(gomega.ConsistOf(expected.nads), fmt.Sprintf("matching NADs for network %s", name)) id, err := nadController.networkIDAllocator.AllocateID(name) g.Expect(err).ToNot(gomega.HaveOccurred()) @@ -578,8 +921,6 @@ func TestNADController(t *testing.T) { g.Expect(tcm.controllers).To(gomega.HaveKey(testNetworkKey)) g.Expect(util.AreNetworksCompatible(tcm.controllers[testNetworkKey], netInfo)).To(gomega.BeTrue(), fmt.Sprintf("matching network config for network %s", name)) - g.Expect(tcm.controllers[testNetworkKey].GetNADs()).To(gomega.ConsistOf(expected.nads), - fmt.Sprintf("matching NADs for network %s", name)) g.Expect(tcm.controllers[testNetworkKey].GetNetworkID()).To(gomega.Equal(id)) expectRunning = append(expectRunning, testNetworkKey) } @@ -591,7 +932,8 @@ func TestNADController(t *testing.T) { netInfoFound, err := nadController.GetActiveNetworkForNamespace(namespace) g.Expect(err).ToNot(gomega.HaveOccurred()) g.Expect(util.AreNetworksCompatible(netInfoFound, netInfo)).To(gomega.BeTrue()) - g.Expect(netInfoFound.GetNADs()).To(gomega.ConsistOf(expected.nads)) + nadKeys := nadController.GetNADKeysForNetwork(netInfoFound.GetNetworkName()) + g.Expect(nadKeys).To(gomega.ConsistOf(expected.nads)) } } tcm.Lock() @@ -617,12 +959,288 @@ func TestNADController(t *testing.T) { } } - g.Eventually(meetsExpectations).Should(gomega.Succeed()) - g.Consistently(meetsExpectations).Should(gomega.Succeed()) + meetsExpectations(g) }) } } +func TestNetworkGracePeriodCleanup(t *testing.T) { + g := gomega.NewWithT(t) + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + // Enable segmentation and grace period + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.UDNDeletionGracePeriod = 2 * time.Second // short grace period for test + tcm := &testControllerManager{ + controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, + } + fakeClient := util.GetOVNClientset().GetClusterManagerClientset() + fakeCtrl := &controller.FakeController{} + nadController := &nadController{ + nads: map[string]string{}, + primaryNADs: map[string]string{}, + networkController: newNetworkController("", "", "", tcm, nil), + networkIDAllocator: id.NewIDAllocator("NetworkIDs", MaxNetworks), + tunnelKeysAllocator: id.NewTunnelKeyAllocator("TunnelKeys"), + nadClient: fakeClient.NetworkAttchDefClient, + namespaceLister: &fakeNamespaceLister{}, + markedForRemoval: map[string]time.Time{}, + controller: fakeCtrl, + } + g.Expect(nadController.networkIDAllocator.ReserveID(types.DefaultNetworkName, types.DefaultNetworkID)).To(gomega.Succeed()) + g.Expect(nadController.networkController.Start()).To(gomega.Succeed()) + defer nadController.networkController.Stop() + // --- Step 1: Add a NAD --- + netConf := &ovncnitypes.NetConf{ + Topology: types.Layer2Topology, + NetConf: cnitypes.NetConf{ + Name: "networkAPrimary", + Type: "ovn-k8s-cni-overlay", + }, + Subnets: "10.1.130.0/24", + Role: types.NetworkRolePrimary, + MTU: 1400, + } + netConf.NADName = util.GetNADName("test", "nad1") + nad, err := buildNADWithAnnotations("nad1", "test", netConf, map[string]string{ + types.OvnNetworkIDAnnotation: "1", + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + // Create the NAD in the fake client so syncNAD can find it + _, err = fakeClient.NetworkAttchDefClient. + K8sCniCncfIoV1(). + NetworkAttachmentDefinitions(nad.Namespace). + Create(context.Background(), nad, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + err = nadController.syncNAD("test/nad1", nad) + g.Expect(err).ToNot(gomega.HaveOccurred()) + netInfo, err := util.NewNetInfo(netConf) + g.Expect(err).ToNot(gomega.HaveOccurred()) + // Should have been started + g.Eventually(func() []string { + tcm.Lock() + defer tcm.Unlock() + return append([]string(nil), tcm.started...) + }).WithTimeout(1*time.Second).Should(gomega.ContainElement(testNetworkKey(netInfo)), + "network should be started before we check grace period") + fakeCtrl.Lock() + numberOfReconciles := len(fakeCtrl.Reconciles) + fakeCtrl.Unlock() + // --- Step 2: Mark as inactive --- + // This triggers the grace-period timer, not immediate deletion. + nadController.updateNADState(util.GetNADName(nad.Namespace, nad.Name), false) + // updateNADState() also requeues immediately; capture that baseline first. + g.Eventually(func() int { + fakeCtrl.Lock() + defer fakeCtrl.Unlock() + return len(fakeCtrl.Reconciles) + }).WithTimeout(1 * time.Second).Should(gomega.Equal(numberOfReconciles + 1)) + reconcilesAfterImmediate := numberOfReconciles + 1 + // --- Step 3: Verify that within the grace period, cleanup has NOT happened --- + g.Consistently(func() []string { + tcm.Lock() + defer tcm.Unlock() + return append([]string(nil), tcm.cleaned...) + }).WithTimeout(1*time.Second).Should(gomega.BeEmpty(), + "cleanup should not happen before grace period ends") + // --- Step 4: Verify a *second* reconcile only AFTER grace period expires --- + g.Eventually(func() int { + fakeCtrl.Lock() + defer fakeCtrl.Unlock() + return len(fakeCtrl.Reconciles) + }).WithTimeout(5 * time.Second).Should(gomega.Equal(reconcilesAfterImmediate + 1)) +} + +func TestFilteredNADDeleteReleasesNetworkID(t *testing.T) { + g := gomega.NewWithT(t) + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + + tcm := &testControllerManager{ + controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, + } + nadController := &nadController{ + nads: map[string]string{}, + primaryNADs: map[string]string{}, + networkController: newNetworkController("", "", "", tcm, nil), + networkIDAllocator: id.NewIDAllocator("NetworkIDs", MaxNetworks), + filterNADsOnNode: "node1", + } + g.Expect(nadController.networkIDAllocator.ReserveID(types.DefaultNetworkName, types.DefaultNetworkID)).To(gomega.Succeed()) + + nadKey := util.GetNADName("ns1", "nad1") + netConf := &ovncnitypes.NetConf{ + Topology: types.Layer2Topology, + NetConf: cnitypes.NetConf{ + Name: "filtered-net", + Type: "ovn-k8s-cni-overlay", + }, + Subnets: "10.1.130.0/24", + Role: types.NetworkRolePrimary, + MTU: 1400, + NADName: nadKey, + } + nad, err := buildNADWithAnnotations("nad1", "ns1", netConf, map[string]string{ + types.OvnNetworkIDAnnotation: "2", + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + controller := true + nad.OwnerReferences = []metav1.OwnerReference{ + { + Kind: "UserDefinedNetwork", + Name: "udn1", + Controller: &controller, + }, + } + + netInfo, err := util.NewNetInfo(netConf) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Dynamic UDN is on and node is filtered, expect ID to be reserved anyway + g.Expect(nadController.syncNAD(nadKey, nad)).To(gomega.Succeed()) + g.Expect(nadController.networkIDAllocator.GetID(netInfo.GetNetworkName())).To(gomega.Equal(2)) + + // Simulate a delete on filtered network and makes sure it still releases the ID + g.Expect(nadController.syncNAD(nadKey, nil)).To(gomega.Succeed()) + g.Expect(nadController.networkIDAllocator.GetID(netInfo.GetNetworkName())).To(gomega.Equal(types.InvalidID)) +} + +func TestFilteredAndActiveNADDeleteRetainsIDUntilNoRefs(t *testing.T) { + g := gomega.NewWithT(t) + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + + tcm := &testControllerManager{ + controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, + } + pt := &PodTrackerController{ + nodeNADToPodCache: map[string]map[string]map[string]struct{}{}, + } + pt.nodeNADToPodCache["node1"] = map[string]map[string]struct{}{ + "ns1/nad1": {"pod": {}}, + } + + nadController := &nadController{ + nads: map[string]string{}, + primaryNADs: map[string]string{}, + networkController: newNetworkController("", "", "", tcm, nil), + networkIDAllocator: id.NewIDAllocator("NetworkIDs", MaxNetworks), + filterNADsOnNode: "node1", + podTracker: pt, + } + g.Expect(nadController.networkIDAllocator.ReserveID(types.DefaultNetworkName, types.DefaultNetworkID)).To(gomega.Succeed()) + + nadKey1 := util.GetNADName("ns1", "nad1") + nadKey2 := util.GetNADName("ns2", "nad2") + netConf := &ovncnitypes.NetConf{ + Topology: types.Layer2Topology, + NetConf: cnitypes.NetConf{ + Name: "shared-net", + Type: "ovn-k8s-cni-overlay", + }, + Subnets: "10.1.130.0/24", + Role: types.NetworkRolePrimary, + MTU: 1400, + NADName: nadKey1, + } + netConf2 := *netConf + netConf2.NADName = nadKey2 + + nad1, err := buildNADWithAnnotations("nad1", "ns1", netConf, map[string]string{ + types.OvnNetworkIDAnnotation: "2", + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + nad2, err := buildNADWithAnnotations("nad2", "ns2", &netConf2, map[string]string{ + types.OvnNetworkIDAnnotation: "2", + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + nad1.OwnerReferences = []metav1.OwnerReference{{ + Kind: "UserDefinedNetwork", + Name: "udn1", + Controller: ptrTo(true), + }} + nad2.OwnerReferences = []metav1.OwnerReference{{ + Kind: "UserDefinedNetwork", + Name: "udn2", + Controller: ptrTo(true), + }} + + // Active NAD should render, filtered NAD should not, but both should reserve ID. + g.Expect(nadController.syncNAD(nadKey1, nad1)).To(gomega.Succeed()) + g.Expect(nadController.syncNAD(nadKey2, nad2)).To(gomega.Succeed()) + g.Expect(nadController.networkIDAllocator.GetID(netConf.Name)).To(gomega.Equal(2)) + + // Delete active NAD; filtered NAD still references the network, so ID stays reserved. + g.Expect(nadController.syncNAD(nadKey1, nil)).To(gomega.Succeed()) + g.Expect(nadController.networkIDAllocator.GetID(netConf.Name)).To(gomega.Equal(2)) + + // Delete filtered NAD; now no refs remain, so ID is released. + g.Expect(nadController.syncNAD(nadKey2, nil)).To(gomega.Succeed()) + g.Expect(nadController.networkIDAllocator.GetID(netConf.Name)).To(gomega.Equal(types.InvalidID)) +} + +func TestDynamicDeleteDoesNotReleaseNetworkID(t *testing.T) { + g := gomega.NewWithT(t) + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + + tcm := &testControllerManager{ + controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, + } + nadController := &nadController{ + nads: map[string]string{}, + primaryNADs: map[string]string{}, + networkController: newNetworkController("", "", "", tcm, nil), + networkIDAllocator: id.NewIDAllocator("NetworkIDs", MaxNetworks), + } + g.Expect(nadController.networkIDAllocator.ReserveID(types.DefaultNetworkName, types.DefaultNetworkID)).To(gomega.Succeed()) + + nadKey := util.GetNADName("ns1", "nad1") + netConf := &ovncnitypes.NetConf{ + Topology: types.Layer2Topology, + NetConf: cnitypes.NetConf{ + Name: "dyn-net", + Type: "ovn-k8s-cni-overlay", + }, + Subnets: "10.1.130.0/24", + Role: types.NetworkRolePrimary, + MTU: 1400, + NADName: nadKey, + } + + nad, err := buildNADWithAnnotations("nad1", "ns1", netConf, map[string]string{ + types.OvnNetworkIDAnnotation: "2", + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Initial sync reserves the ID. + g.Expect(nadController.syncNAD(nadKey, nad)).To(gomega.Succeed()) + g.Expect(nadController.networkIDAllocator.GetID(netConf.Name)).To(gomega.Equal(2)) + + // Simulate inactive transition via expired grace period. + nadController.markedForRemoval = map[string]time.Time{nadKey: time.Now().Add(-time.Minute)} + g.Expect(nadController.syncNAD(nadKey, nad)).To(gomega.Succeed()) + g.Expect(nadController.networkIDAllocator.GetID(netConf.Name)).To(gomega.Equal(2)) +} + func TestSyncAll(t *testing.T) { const nodeNetworkID = 1337 type mode string @@ -1016,7 +1634,8 @@ func TestSyncAll(t *testing.T) { g.Expect(info.GetNetworkID()).To(gomega.Equal(1)) // Both NADs should now be part of the same network - g.Expect(info.GetNADs()).To(gomega.HaveLen(2)) + nadKeys := controller.Interface().GetNADKeysForNetwork(info.GetNetworkName()) + g.Expect(nadKeys).To(gomega.HaveLen(2)) // NAD2 should now have the inherited ID nad2, _ := fakeClient.NetworkAttchDefClient.K8sCniCncfIoV1(). @@ -1122,3 +1741,108 @@ func buildNADWithAnnotations(name, namespace string, network *ovncnitypes.NetCon nad.Annotations = annotations return nad, nil } + +func TestOnNetworkRefChangeNotifiesNetworkController(t *testing.T) { + tests := []struct { + name string + notifyActive bool + nodeHasNetworkActive bool + }{ + { + name: "active notification with no refs", + notifyActive: true, + nodeHasNetworkActive: false, + }, + { + name: "inactive notification with refs", + notifyActive: false, + nodeHasNetworkActive: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + err := config.PrepareTestConfig() + g.Expect(err).ToNot(gomega.HaveOccurred()) + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + + // Build fake NAD with UDN owner reference and overlay topology. + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: "udn-net", + Type: "ovn-k8s-cni-overlay", + }, + Topology: types.Layer3Topology, + Role: types.NetworkRolePrimary, + NADName: "ns1/primary", + } + nad, err := buildNAD("primary", "ns1", netConf) + g.Expect(err).ToNot(gomega.HaveOccurred()) + nad.OwnerReferences = []metav1.OwnerReference{{ + Kind: "UserDefinedNetwork", + Name: "udn", + Controller: ptrTo(true), + }} + + nadLister := &fakeNADLister{ + nads: map[string]*nettypes.NetworkAttachmentDefinition{ + "primary": nad, + }, + } + nodeName := "node1" + + tcm := &testControllerManager{ + controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, + } + + nc := &nadController{ + nads: map[string]string{}, + primaryNADs: map[string]string{}, + networkController: newNetworkController("", "", "", tcm, nil), + networkIDAllocator: id.NewIDAllocator("NetworkIDs", MaxNetworks), + tunnelKeysAllocator: id.NewTunnelKeyAllocator("TunnelKeys"), + nadLister: nadLister, + } + err = nc.networkIDAllocator.ReserveID(types.DefaultNetworkName, types.DefaultNetworkID) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + nadNetwork, err := util.ParseNADInfo(nad) + g.Expect(err).ToNot(gomega.HaveOccurred()) + networkName := nadNetwork.GetNetworkName() + mutableNetInfo := util.NewMutableNetInfo(nadNetwork) + mutableNetInfo.SetNADs(util.GetNADName(nad.Namespace, nad.Name)) + nc.networkController.setNetwork(networkName, mutableNetInfo) + nc.networkController.nodeHasNetwork = func(_, _ string) bool { return tt.nodeHasNetworkActive } + var gotNode string + var gotActive bool + var callCount int + testController := &testNetworkController{ + ReconcilableNetInfo: util.NewReconcilableNetInfo(nadNetwork), + tcm: tcm, + handleRefChange: func(node string, active bool) { + gotNode = node + gotActive = active + callCount++ + }, + } + nc.networkController.networkControllers[networkName] = &networkControllerState{ + controller: testController, + } + + // Trigger network ref change. + nc.OnNetworkRefChange(nodeName, util.GetNADName(nad.Namespace, nad.Name), tt.notifyActive) + err = nc.networkController.syncNetwork(networkName) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + g.Expect(callCount).To(gomega.Equal(1)) + g.Expect(gotNode).To(gomega.Equal(nodeName)) + g.Expect(gotActive).To(gomega.Equal(tt.nodeHasNetworkActive)) + }) + } +} diff --git a/go-controller/pkg/networkmanager/network_controller.go b/go-controller/pkg/networkmanager/network_controller.go index 023e566f37..0f43c9dfee 100644 --- a/go-controller/pkg/networkmanager/network_controller.go +++ b/go-controller/pkg/networkmanager/network_controller.go @@ -29,12 +29,15 @@ import ( func newNetworkController(name, zone, node string, cm ControllerManager, wf watchFactory) *networkController { nc := &networkController{ - name: fmt.Sprintf("[%s network controller]", name), - node: node, - zone: zone, - cm: cm, - networks: map[string]util.MutableNetInfo{}, - networkControllers: map[string]*networkControllerState{}, + name: fmt.Sprintf("[%s network controller]", name), + node: node, + zone: zone, + cm: cm, + networks: map[string]util.MutableNetInfo{}, + networksByID: map[int]string{}, + networkControllers: map[string]*networkControllerState{}, + pendingNetworkRefNodes: map[string]sets.Set[string]{}, + networkRefStates: map[string]map[string]bool{}, } // this controller does not feed from an informer, networks are manually @@ -110,7 +113,23 @@ type networkController struct { cm ControllerManager networks map[string]util.MutableNetInfo + networksByID map[int]string // reverse index: network ID -> network name for O(1) lookup networkControllers map[string]*networkControllerState + // getNADKeysForNetwork returns NAD keys associated with a network name. + getNADKeysForNetwork func(networkName string) []string + + // networkRefLock is used to protect pendingNetworkRefNodes and networkRefStates access + networkRefLock sync.Mutex + // pendingNetworkRefNodes is used to temporarily hold network name -> nodes mapping for future reconciliation + // Dynamic UDN asks network controller to reconcile and stores nodes that have changed state in this map + pendingNetworkRefNodes map[string]sets.Set[string] + // networkRefStates is used to hold network name -> node (active/inactive) state as it is currently configured + // This is used by Dynamic UDN to avoid calling the network controller's HandleNetworkRefChange for nodes that have + // not changed state. + networkRefStates map[string]map[string]bool + // nodeHasNetwork is a function used to query if a node has a resource (pod or egress IP) that would inform the + // network controller to either add or remove the remote resources for that network. + nodeHasNetwork func(node, networkName string) bool } // Start will cleanup stale networks that have not been ensured via @@ -168,11 +187,21 @@ func (c *networkController) DeleteNetwork(network string) { func (c *networkController) setNetwork(network string, netInfo util.MutableNetInfo) { c.Lock() defer c.Unlock() + // Remove old ID mapping if network existed + if old, exists := c.networks[network]; exists { + if oldID := old.GetNetworkID(); oldID != types.InvalidID { + delete(c.networksByID, oldID) + } + } if netInfo == nil { delete(c.networks, network) return } c.networks[network] = netInfo + // Add new ID mapping + if id := netInfo.GetNetworkID(); id != types.InvalidID { + c.networksByID[id] = network + } } func (c *networkController) getNetwork(network string) util.MutableNetInfo { @@ -215,10 +244,108 @@ func (c *networkController) getReconcilableNetworkState(network string) (Reconci if network == types.DefaultNetworkName { return c.cm.GetDefaultNetworkController(), false } - state := c.getNetworkState(network) + c.RLock() + defer c.RUnlock() + state := c.networkControllers[network] + if state == nil { + return nil, false + } return state.controller, state.stoppedAndDeleting } +func (c *networkController) getControllerForNotify(networkName string) (NetworkController, bool) { + c.RLock() + defer c.RUnlock() + state := c.networkControllers[networkName] + if state == nil || state.controller == nil || state.stoppedAndDeleting { + return nil, false + } + return state.controller, true +} + +// NotifyNetworkRefChange enqueues a network reconcile so node-level state can be recomputed. +func (c *networkController) NotifyNetworkRefChange(networkName, node string) { + c.queueNetworkRefChange(networkName, node) + c.networkReconciler.Reconcile(networkName) +} + +func (c *networkController) queueNetworkRefChange(networkName, node string) { + if networkName == "" || node == "" { + return + } + c.networkRefLock.Lock() + defer c.networkRefLock.Unlock() + if c.pendingNetworkRefNodes == nil { + c.pendingNetworkRefNodes = map[string]sets.Set[string]{} + } + nodes := c.pendingNetworkRefNodes[networkName] + if nodes == nil { + nodes = sets.New[string]() + c.pendingNetworkRefNodes[networkName] = nodes + } + nodes.Insert(node) +} + +func (c *networkController) popPendingNetworkRefNodes(networkName string) []string { + c.networkRefLock.Lock() + defer c.networkRefLock.Unlock() + nodes := c.pendingNetworkRefNodes[networkName] + if len(nodes) == 0 { + return nil + } + delete(c.pendingNetworkRefNodes, networkName) + return nodes.UnsortedList() +} + +func (c *networkController) clearPendingNetworkRefNodes(networkName string) { + c.networkRefLock.Lock() + defer c.networkRefLock.Unlock() + delete(c.pendingNetworkRefNodes, networkName) +} + +func (c *networkController) clearNetworkRefState(networkName string) { + c.networkRefLock.Lock() + defer c.networkRefLock.Unlock() + delete(c.networkRefStates, networkName) +} + +func (c *networkController) shouldHandleNetworkRefChange(networkName, node string, active bool) bool { + c.networkRefLock.Lock() + defer c.networkRefLock.Unlock() + if c.networkRefStates == nil { + c.networkRefStates = map[string]map[string]bool{} + } + nodeStates := c.networkRefStates[networkName] + if nodeStates == nil { + nodeStates = map[string]bool{} + c.networkRefStates[networkName] = nodeStates + } + prev, ok := nodeStates[node] + if ok && prev == active { + return false + } + nodeStates[node] = active + return true +} + +func (c *networkController) reconcilePendingNetworkRefChanges(networkName string) { + ctrl, ok := c.getControllerForNotify(networkName) + if !ok || c.nodeHasNetwork == nil { + return + } + nodes := c.popPendingNetworkRefNodes(networkName) + if len(nodes) == 0 { + return + } + for _, node := range nodes { + active := c.nodeHasNetwork(node, networkName) + if !c.shouldHandleNetworkRefChange(networkName, node, active) { + continue + } + ctrl.HandleNetworkRefChange(node, active) + } +} + func (c *networkController) getAllNetworkStates() []*networkControllerState { c.RLock() defer c.RUnlock() @@ -298,23 +425,22 @@ func (c *networkController) syncNetwork(network string) error { } ensureNetwork := !compatible || util.DoesNetworkNeedReconciliation(have, want) - if !ensureNetwork { - // no network changes - return nil - } + if ensureNetwork { + // inform controller manager of upcoming changes so other controllers are + // aware + err = c.cm.Reconcile(network, have, want) + if err != nil { + return fmt.Errorf("failed to reconcile controller manager for network %s: %w", network, err) + } - // inform controller manager of upcoming changes so other controllers are - // aware - err = c.cm.Reconcile(network, have, want) - if err != nil { - return fmt.Errorf("failed to reconcile controller manager for network %s: %w", network, err) + // ensure the network controller + err = c.ensureNetwork(want) + if err != nil { + return fmt.Errorf("%s: failed to ensure network %s: %w", c.name, network, err) + } } - // ensure the network controller - err = c.ensureNetwork(want) - if err != nil { - return fmt.Errorf("%s: failed to ensure network %s: %w", c.name, network, err) - } + c.reconcilePendingNetworkRefChanges(network) return nil } @@ -352,22 +478,31 @@ func (c *networkController) ensureNetwork(network util.MutableNetInfo) error { } func (c *networkController) deleteNetwork(network string) error { - have := c.getNetworkState(network) - if have.controller == nil { + c.Lock() + have := c.networkControllers[network] + if have == nil || have.controller == nil { + c.Unlock() + c.clearPendingNetworkRefNodes(network) + c.clearNetworkRefState(network) return nil } + ctrl := have.controller + alreadyStopping := have.stoppedAndDeleting + have.stoppedAndDeleting = true + c.Unlock() - if !have.stoppedAndDeleting { - have.controller.Stop() + if !alreadyStopping { + ctrl.Stop() } - have.stoppedAndDeleting = true - err := have.controller.Cleanup() + err := ctrl.Cleanup() if err != nil { return fmt.Errorf("%s: failed to cleanup network %s: %w", c.name, network, err) } c.setNetworkState(network, nil) + c.clearPendingNetworkRefNodes(network) + c.clearNetworkRefState(network) return nil } @@ -385,9 +520,12 @@ func (c *networkController) setAdvertisements(network util.MutableNetInfo) error if !c.hasRouteAdvertisements() { return nil } + if c.getNADKeysForNetwork == nil { + return fmt.Errorf("missing NAD resolver for network %q", network.GetNetworkName()) + } raNames := sets.New[string]() - for _, nadNamespacedName := range network.GetNADs() { + for _, nadNamespacedName := range c.getNADKeysForNetwork(network.GetNetworkName()) { namespace, name, err := cache.SplitMetaNamespaceKey(nadNamespacedName) if err != nil { return err @@ -542,3 +680,17 @@ func (c *networkController) getRunningNetwork(id int) string { } return "" } + +// GetNetworkByID returns the network with the given ID or nil if not found. +// This is an O(1) lookup using an internal index. +func (c *networkController) GetNetworkByID(id int) util.NetInfo { + if id == types.DefaultNetworkID { + return &util.DefaultNetInfo{} + } + c.RLock() + defer c.RUnlock() + if name, ok := c.networksByID[id]; ok { + return c.networks[name] + } + return nil +} diff --git a/go-controller/pkg/networkmanager/network_controller_test.go b/go-controller/pkg/networkmanager/network_controller_test.go index 4f3e85b8aa..9ee6d389ba 100644 --- a/go-controller/pkg/networkmanager/network_controller_test.go +++ b/go-controller/pkg/networkmanager/network_controller_test.go @@ -210,6 +210,13 @@ func TestSetAdvertisements(t *testing.T) { mutableNetInfo := util.NewMutableNetInfo(netInfo) mutableNetInfo.AddNADs(testNADName) + nm.getNADKeysForNetwork = func(networkName string) []string { + if networkName == mutableNetInfo.GetNetworkName() { + return []string{testNADName} + } + return nil + } + nm.EnsureNetwork(mutableNetInfo) meetsExpectations := func(g gomega.Gomega) { @@ -240,3 +247,158 @@ func TestSetAdvertisements(t *testing.T) { }) } } + +func TestNetworkControllerReconcilePendingNetworkRefChange(t *testing.T) { + g := gomega.NewWithT(t) + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + t.Cleanup(func() { + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + }) + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableRouteAdvertisements = false + + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: "udn-net", + Type: "ovn-k8s-cni-overlay", + }, + Topology: types.Layer3Topology, + Role: types.NetworkRolePrimary, + NADName: "ns1/primary", + Subnets: "10.128.0.0/14", + } + netInfo, err := util.NewNetInfo(netConf) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + tests := []struct { + name string + nodeHasNetwork bool + }{ + { + name: "active", + nodeHasNetwork: true, + }, + { + name: "inactive", + nodeHasNetwork: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + tcm := &testControllerManager{ + controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, + } + nm := newNetworkController("", "", "", tcm, nil) + nm.nodeHasNetwork = func(_, _ string) bool { return tt.nodeHasNetwork } + + networkName := netInfo.GetNetworkName() + mutableNetInfo := util.NewMutableNetInfo(netInfo) + mutableNetInfo.SetNADs(netConf.NADName) + nm.setNetwork(networkName, mutableNetInfo) + + var gotNode string + var gotActive bool + var callCount int + testController := &testNetworkController{ + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), + tcm: tcm, + handleRefChange: func(node string, active bool) { + gotNode = node + gotActive = active + callCount++ + }, + } + nm.networkControllers[networkName] = &networkControllerState{ + controller: testController, + } + + nm.NotifyNetworkRefChange(networkName, "node1") + err := nm.syncNetwork(networkName) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + g.Expect(callCount).To(gomega.Equal(1)) + g.Expect(gotNode).To(gomega.Equal("node1")) + g.Expect(gotActive).To(gomega.Equal(tt.nodeHasNetwork)) + + nm.NotifyNetworkRefChange(networkName, "node1") + err = nm.syncNetwork(networkName) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(callCount).To(gomega.Equal(1)) + }) + } +} + +func TestNetworkControllerClearsPendingNetworkRefOnDelete(t *testing.T) { + g := gomega.NewWithT(t) + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + t.Cleanup(func() { + g.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + }) + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableRouteAdvertisements = false + + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: "udn-net", + Type: "ovn-k8s-cni-overlay", + }, + Topology: types.Layer3Topology, + Role: types.NetworkRolePrimary, + NADName: "ns1/primary", + Subnets: "10.128.0.0/14", + } + netInfo, err := util.NewNetInfo(netConf) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + tcm := &testControllerManager{ + controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, + } + nm := newNetworkController("", "", "", tcm, nil) + nm.nodeHasNetwork = func(_, _ string) bool { return true } + + networkName := netInfo.GetNetworkName() + mutableNetInfo := util.NewMutableNetInfo(netInfo) + mutableNetInfo.SetNADs(netConf.NADName) + nm.setNetwork(networkName, mutableNetInfo) + + var callCount int + testController := &testNetworkController{ + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), + tcm: tcm, + handleRefChange: func(string, bool) { + callCount++ + }, + } + nm.networkControllers[networkName] = &networkControllerState{ + controller: testController, + } + + nm.NotifyNetworkRefChange(networkName, "node1") + err = nm.deleteNetwork(networkName) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(callCount).To(gomega.Equal(0)) + + var followupCalls int + followupController := &testNetworkController{ + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), + tcm: tcm, + handleRefChange: func(string, bool) { + followupCalls++ + }, + } + nm.networkControllers[networkName] = &networkControllerState{ + controller: followupController, + } + nm.setNetwork(networkName, mutableNetInfo) + err = nm.syncNetwork(networkName) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(followupCalls).To(gomega.Equal(0)) +} diff --git a/go-controller/pkg/networkmanager/pod_tracker.go b/go-controller/pkg/networkmanager/pod_tracker.go new file mode 100644 index 0000000000..4a300dd099 --- /dev/null +++ b/go-controller/pkg/networkmanager/pod_tracker.go @@ -0,0 +1,443 @@ +package networkmanager + +import ( + "fmt" + "sync" + + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + v1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +type nodeNAD struct { + node string + nads []string +} + +type refChange struct { + node string + nad string + active bool +} + +// PodTrackerController tracks which pods are using which NADs on which nodes and notifies subscribers +// when the first pod for a node/NAD appears or the last one disappears. +type PodTrackerController struct { + // cacheMutex guards nodeNADToPodCache and podToNodeNAD + cacheMutex sync.Mutex + name string + // primaryNADForNamespace resolves the primary NAD for a namespace (cached in NAD controller) + primaryNADForNamespace func(namespace string) (string, error) + // nodeNADToPodCache holds a mapping of node -> NAD namespaced name -> pod namespaced name + nodeNADToPodCache map[string]map[string]map[string]struct{} + // podToNodeNAD is the reverse index: pod key -> (node, NAD) + podToNodeNAD map[string]nodeNAD + // callback when a node+NAD goes active/inactive + onNetworkRefChange func(node, nad string, active bool) + podController controller.Controller + nadReconciler controller.Reconciler + podLister v1.PodLister + nadLister nadlisters.NetworkAttachmentDefinitionLister + namespaceLister v1.NamespaceLister +} + +func NewPodTrackerController( + name string, + wf watchFactory, + onNetworkRefChange func(node, nad string, active bool), + primaryNADForNamespace func(namespace string) (string, error), +) *PodTrackerController { + p := &PodTrackerController{ + name: name, + nodeNADToPodCache: make(map[string]map[string]map[string]struct{}), + podToNodeNAD: make(map[string]nodeNAD), + onNetworkRefChange: onNetworkRefChange, + podLister: wf.PodCoreInformer().Lister(), + nadLister: wf.NADInformer().Lister(), + namespaceLister: wf.NamespaceInformer().Lister(), + primaryNADForNamespace: primaryNADForNamespace, + } + + if p.primaryNADForNamespace == nil { + p.primaryNADForNamespace = p.getPrimaryNADForNamespaceFromLister + } + + cfg := &controller.ControllerConfig[corev1.Pod]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: p.reconcile, + ObjNeedsUpdate: p.needUpdate, + MaxAttempts: controller.InfiniteAttempts, + Threadiness: 1, + Informer: wf.PodCoreInformer().Informer(), + Lister: wf.PodCoreInformer().Lister().List, + } + p.podController = controller.NewController[corev1.Pod](p.name, cfg) + + // Reconciler fed by NAD controller to refresh cache when NADs change. + p.nadReconciler = controller.NewReconciler( + fmt.Sprintf("%s-nad-reconciler", name), + &controller.ReconcilerConfig{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: func(key string) error { + return p.requeuePodsForNAD(key) + }, + Threadiness: 1, + MaxAttempts: controller.InfiniteAttempts, + }, + ) + return p +} + +func (c *PodTrackerController) Start() error { + klog.Infof("Starting %s controller", c.name) + return controller.StartWithInitialSync(c.syncAll, c.podController, c.nadReconciler) +} + +func (c *PodTrackerController) Stop() { + klog.Infof("Stopping %s controller", c.name) + controller.Stop(c.podController, c.nadReconciler) +} + +func (c *PodTrackerController) NodeHasNAD(node, nad string) bool { + c.cacheMutex.Lock() + defer c.cacheMutex.Unlock() + if _, ok := c.nodeNADToPodCache[node]; !ok { + return false + } + if _, ok := c.nodeNADToPodCache[node][nad]; !ok { + return false + } + return len(c.nodeNADToPodCache[node][nad]) > 0 +} + +// getNADsForPod resolves the primary and secondary networks for a pod. +func (c *PodTrackerController) getNADsForPod(pod *corev1.Pod) ([]string, error) { + var nadList []string + + requiresUDN := false + // check if required UDN label is on namespace + ns, err := c.namespaceLister.Get(pod.Namespace) + if err != nil { + return nil, fmt.Errorf("failed to get namespace %q: %w", pod.Namespace, err) + } + if _, exists := ns.Labels[types.RequiredUDNNamespaceLabel]; exists { + requiresUDN = true + } + + // Primary NAD from namespace + primaryNAD, err := c.primaryNADForNamespace(pod.Namespace) + if err != nil { + return nil, err + } + if len(primaryNAD) > 0 && primaryNAD != types.DefaultNetworkName { + nadList = append(nadList, primaryNAD) + } else if requiresUDN { + return nil, util.NewInvalidPrimaryNetworkError(pod.Namespace) + } + + // Secondary NADs from pod annotation + networks, err := util.GetK8sPodAllNetworkSelections(pod) + if err != nil { + return nil, fmt.Errorf("failed to parse network annotations for pod %s/%s: %v", pod.Namespace, pod.Name, err) + } + for _, net := range networks { + ns := net.Namespace + if ns == "" { + ns = pod.Namespace + } + nadList = append(nadList, fmt.Sprintf("%s/%s", ns, net.Name)) + } + + klog.V(5).Infof("%s - tracked NADS for pod %q: %#v", c.name, pod.Name, nadList) + + return nadList, nil +} + +// getPrimaryNADForNamespaceFromLister is a fallback resolver used in tests when no resolver is injected. +func (c *PodTrackerController) getPrimaryNADForNamespaceFromLister(namespace string) (string, error) { + ns, err := c.namespaceLister.Get(namespace) + if err != nil { + return "", fmt.Errorf("failed to get namespace %q: %w", namespace, err) + } + if _, hasLabel := ns.Labels[types.RequiredUDNNamespaceLabel]; !hasLabel { + return types.DefaultNetworkName, nil + } + + nads, err := c.nadLister.NetworkAttachmentDefinitions(namespace).List(labels.Everything()) + if err != nil { + return "", fmt.Errorf("failed to list network attachment definitions: %w", err) + } + for _, nad := range nads { + if nad.Name == types.DefaultNetworkName { + continue + } + nadInfo, err := util.ParseNADInfo(nad) + if err != nil { + klog.Warningf("Failed to parse network attachment definition %q: %v", nad.Name, err) + continue + } + if nadInfo.IsPrimaryNetwork() { + return util.GetNADName(nad.Namespace, nad.Name), nil + } + } + return "", util.NewUnprocessedActiveNetworkError(namespace, "") +} + +// syncAll builds the cache on initial controller start +// This is required because workers are started asynchronously and consumers of the tracker +// rely on the cache to be populated during start up +func (c *PodTrackerController) syncAll() error { + klog.Infof("%s: warming up cache with existing pods", c.name) + + pods, err := c.podLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("failed to list pods: %v", err) + } + + for _, pod := range pods { + if pod.Spec.NodeName == "" || pod.DeletionTimestamp != nil { + continue + } + + nadList, err := c.getNADsForPod(pod) + if err != nil { + klog.Errorf("Pod Tracker sync - Failed to get nads for pod %s/%s: %v", pod.Namespace, pod.Name, err) + continue + } + if len(nadList) == 0 { + continue + } + + c.addPodToCache(pod, pod.Spec.NodeName, nadList) + } + + klog.Infof("%s: cache warmup complete with %d pods", c.name, len(pods)) + return nil +} + +// NADReconciler returns the reconciler that should be registered with the NAD controller. +func (c *PodTrackerController) NADReconciler() controller.Reconciler { + return c.nadReconciler +} + +// requeuePodsForNAD enqueues pods in the NAD's namespace so they get retried +// once the NAD (or its primary designation) becomes available. +func (c *PodTrackerController) requeuePodsForNAD(key string) error { + namespace, _, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return fmt.Errorf("failed to split meta namespace key %q: %v", key, err) + } + if namespace == "" { + return nil + } + + pods, err := c.podLister.Pods(namespace).List(labels.Everything()) + if err != nil { + return fmt.Errorf("failed to list pods in namespace %q: %v", namespace, err) + } + for _, pod := range pods { + if pod == nil { + continue + } + key, err := cache.MetaNamespaceKeyFunc(pod) + if err != nil { + klog.Warningf("%s: failed to build key for pod %s/%s: %v", c.name, pod.Namespace, pod.Name, err) + continue + } + // Use rate-limited enqueue to avoid hot-looping on a flood of pods + c.podController.ReconcileRateLimited(key) + } + return nil +} + +// needUpdate return true when the pod has been created or updated. +func (c *PodTrackerController) needUpdate(old, new *corev1.Pod) bool { + // ignore pods that only want hostNetwork with no NADs + if new != nil && util.PodWantsHostNetwork(new) { + if _, ok := new.Annotations[nadv1.NetworkAttachmentAnnot]; !ok { + return false + } + } + + // Ignore adds/updates while the pod is still unscheduled; we'll react once it gets a node. + if new != nil && new.Spec.NodeName == "" { + return false + } + + // Add + if old == nil { + return true + } + + if new == nil { + return false + } + + // Ignore updates while the pod is still unscheduled; we'll react once it gets a node. + if new.Spec.NodeName == "" { + return false + } + + // If the node assignment changed (including unscheduled -> scheduled), reconcile. + if old.Spec.NodeName != new.Spec.NodeName { + return true + } + + // If the network attachment annotations changed, reconcile. + oldAnno := old.Annotations[nadv1.NetworkAttachmentAnnot] + newAnno := new.Annotations[nadv1.NetworkAttachmentAnnot] + return oldAnno != newAnno +} + +// reconcile notify subscribers with the request namespace key following namespace events. +func (c *PodTrackerController) reconcile(key string) error { + klog.V(5).Infof("%s reconcile called for pod %s", c.name, key) + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return fmt.Errorf("failed to split meta namespace key %q: %v", key, err) + } + + pod, err := c.podLister.Pods(namespace).Get(name) + if err != nil { + if apierrors.IsNotFound(err) { + // Pod deleted → cleanup cache + c.deletePodFromCache(key) + return nil + } + return fmt.Errorf("failed to get pod %q from cache: %v", key, err) + } + + // If pod is terminating → remove from cache + if pod.DeletionTimestamp != nil { + c.deletePodFromCache(key) + return nil + } + + // Ignore pods that are not yet scheduled; callbacks/cache should only be for scheduled pods. + if pod.Spec.NodeName == "" { + return nil + } + + nadList, err := c.getNADsForPod(pod) + if err != nil { + return err + } + if len(nadList) == 0 { + c.deletePodFromCache(key) + return nil + } + + // Track pod under its node for each NAD + c.addPodToCache(pod, pod.Spec.NodeName, nadList) + + return nil +} + +func (c *PodTrackerController) addPodToCache(pod *corev1.Pod, node string, nads []string) { + c.cacheMutex.Lock() + klog.V(5).Infof("%s - addPodToCache for pod %s/%s, node: %s, nads: %#v", c.name, pod.Namespace, pod.Name, node, nads) + + key := fmt.Sprintf("%s/%s", pod.Namespace, pod.Name) + + // Short-circuit if identical node/NAD set already cached + if existing, found := c.podToNodeNAD[key]; found && existing.node == node && equalStringSets(existing.nads, nads) { + c.cacheMutex.Unlock() + return + } + + // First clean up any existing entry for this pod + refChanges := c.deletePodFromCacheLocked(key) + + for _, nad := range nads { + if _, ok := c.nodeNADToPodCache[node]; !ok { + c.nodeNADToPodCache[node] = make(map[string]map[string]struct{}) + } + if _, ok := c.nodeNADToPodCache[node][nad]; !ok { + c.nodeNADToPodCache[node][nad] = make(map[string]struct{}) + } + before := len(c.nodeNADToPodCache[node][nad]) + c.nodeNADToPodCache[node][nad][key] = struct{}{} + // Only fire on the 0 -> 1 transition; repeated adds for the same NAD are ignored + if before == 0 { + // 0 → 1 transition + refChanges = append(refChanges, refChange{node, nad, true}) + } + } + + c.podToNodeNAD[key] = nodeNAD{node: node, nads: append([]string(nil), nads...)} + c.cacheMutex.Unlock() + if c.onNetworkRefChange != nil { + for _, callback := range refChanges { + c.onNetworkRefChange(callback.node, callback.nad, callback.active) + } + } +} + +func (c *PodTrackerController) deletePodFromCache(key string) { + klog.V(5).Infof("%s - deletePodFromCache for pod %s", c.name, key) + c.cacheMutex.Lock() + changes := c.deletePodFromCacheLocked(key) + c.cacheMutex.Unlock() + + if c.onNetworkRefChange != nil { + for _, ev := range changes { + c.onNetworkRefChange(ev.node, ev.nad, ev.active) + } + } +} + +func (c *PodTrackerController) deletePodFromCacheLocked(key string) []refChange { + var refChanges []refChange + loc, ok := c.podToNodeNAD[key] + if !ok { + return nil + } + + for _, nad := range loc.nads { + if _, ok := c.nodeNADToPodCache[loc.node][nad]; ok { + before := len(c.nodeNADToPodCache[loc.node][nad]) + delete(c.nodeNADToPodCache[loc.node][nad], key) + after := len(c.nodeNADToPodCache[loc.node][nad]) + if before == 1 && after == 0 { + // 1 → 0 transition + refChanges = append(refChanges, refChange{loc.node, nad, false}) + } + if after == 0 { + delete(c.nodeNADToPodCache[loc.node], nad) + } + } + } + if len(c.nodeNADToPodCache[loc.node]) == 0 { + delete(c.nodeNADToPodCache, loc.node) + } + + delete(c.podToNodeNAD, key) + return refChanges +} + +func equalStringSets(a, b []string) bool { + if len(a) != len(b) { + return false + } + set := make(map[string]struct{}, len(a)) + for _, v := range a { + set[v] = struct{}{} + } + for _, v := range b { + if _, ok := set[v]; !ok { + return false + } + } + return true +} diff --git a/go-controller/pkg/networkmanager/pod_tracker_test.go b/go-controller/pkg/networkmanager/pod_tracker_test.go new file mode 100644 index 0000000000..863ff06ec8 --- /dev/null +++ b/go-controller/pkg/networkmanager/pod_tracker_test.go @@ -0,0 +1,382 @@ +package networkmanager + +import ( + "context" + "encoding/json" + "sync" + "testing" + + cnitypes "github.com/containernetworking/cni/pkg/types" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +func TestPodTrackerControllerWithInformerAndDelete(t *testing.T) { + type callbackEvent struct { + node string + nad string + active bool + } + + tests := []struct { + name string + nodeName string + podName string + namespace string + annotations map[string]string + hasPrimaryUDN bool + createPod bool + deletePod bool + expectedNADs []string + expectEvents []callbackEvent + }{ + { + name: "pod with primary and secondary NADs triggers callback on add", + nodeName: "node1", + podName: "pod1", + namespace: "testns", + annotations: map[string]string{nadv1.NetworkAttachmentAnnot: `[ {"name": "sec1", "namespace": "testns"} ]`}, + hasPrimaryUDN: true, + createPod: true, + expectedNADs: []string{"testns/primary", "testns/sec1"}, + expectEvents: []callbackEvent{ + {"node1", "testns/primary", true}, + {"node1", "testns/sec1", true}, + }, + }, + { + name: "pod with primary and secondary NADs triggers deletion callback on last pod removal", + nodeName: "node2", + podName: "pod2", + namespace: "testns", + annotations: map[string]string{nadv1.NetworkAttachmentAnnot: `[ {"name": "sec1", "namespace": "testns"} ]`}, + hasPrimaryUDN: true, + createPod: true, + deletePod: true, + expectedNADs: nil, + expectEvents: []callbackEvent{ + {"node2", "testns/primary", true}, // first pod add + {"node2", "testns/sec1", true}, + {"node2", "testns/primary", false}, // last pod delete + {"node2", "testns/sec1", false}, + }, + }, + { + name: "pod with default network and secondary NADs", + nodeName: "node3", + podName: "pod3", + namespace: "testns", + annotations: map[string]string{nadv1.NetworkAttachmentAnnot: `[ {"name": "secA", "namespace": "testns"}, {"name": "secB", "namespace": "testns"} ]`}, + hasPrimaryUDN: false, // default -> no primary UDN + createPod: true, + expectedNADs: []string{"testns/secA", "testns/secB"}, + expectEvents: []callbackEvent{ + {"node3", "testns/secA", true}, + {"node3", "testns/secB", true}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + err := config.PrepareTestConfig() + g.Expect(err).NotTo(gomega.HaveOccurred()) + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableInterconnect = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + + // Track callback events + var events []callbackEvent + var eventsMu sync.Mutex + + // Setup fake client + watch factory + fakeClient := util.GetOVNClientset().GetOVNKubeControllerClientset() + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClient) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Create PodTrackerController with dummy callback + ptc := NewPodTrackerController("test-pod-tracker", wf, func(node, nad string, active bool) { + eventsMu.Lock() + events = append(events, callbackEvent{node, nad, active}) + eventsMu.Unlock() + }, nil) + + // Start informers + err = wf.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer wf.Shutdown() + + // Start pod controller + g.Expect(ptc.Start()).Should(gomega.Succeed()) + defer ptc.Stop() + + nsLabel := map[string]string{} + if tt.hasPrimaryUDN { + nsLabel = map[string]string{ovntypes.RequiredUDNNamespaceLabel: ""} + } + // Create namespace + _, err = fakeClient.KubeClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.namespace, + Labels: nsLabel, + }, + }, metav1.CreateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + if tt.hasPrimaryUDN { + // Create Primary NAD + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "primary", Type: "ovn-k8s-cni-overlay"}, + Topology: "layer3", + Role: "primary", + MTU: 1400, + NADName: "testns/primary", + } + bytes, err := json.Marshal(netConf) + if err != nil { + t.Fatalf("failed to marshal netconf: %v", err) + } + nad := &nadv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + UID: types.UID(tt.namespace), + Name: "primary", + Namespace: tt.namespace, + }, + Spec: nadv1.NetworkAttachmentDefinitionSpec{ + Config: string(bytes), + }, + } + if _, err := fakeClient.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(tt.namespace). + Create(context.Background(), nad, metav1.CreateOptions{}); err != nil { + t.Fatalf("failed to create NAD: %v", err) + } + } + + // Create node + _, err = fakeClient.KubeClient.CoreV1().Nodes().Create(context.Background(), &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: tt.nodeName}, + }, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + key := tt.namespace + "/" + tt.podName + + if tt.createPod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.podName, + Namespace: tt.namespace, + Annotations: tt.annotations, + }, + Spec: corev1.PodSpec{NodeName: tt.nodeName}, + } + _, err = fakeClient.KubeClient.CoreV1().Pods(tt.namespace).Create(context.Background(), pod, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Wait for the controller to process the ADD and populate reverse entry + g.Eventually(func() bool { + ptc.cacheMutex.Lock() + _, ok := ptc.podToNodeNAD[key] + ptc.cacheMutex.Unlock() + return ok + }, "2s", "50ms").Should(gomega.BeTrue(), "pod add was not processed by controller") + } + + if tt.deletePod { + // Now delete; do this *after* we've observed the add + err = fakeClient.KubeClient.CoreV1().Pods(tt.namespace).Delete(context.Background(), tt.podName, metav1.DeleteOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Wait for the controller to process the DELETE and remove the reverse entry + g.Eventually(func() bool { + ptc.cacheMutex.Lock() + _, ok := ptc.podToNodeNAD[key] + ptc.cacheMutex.Unlock() + return !ok + }, "2s", "50ms").Should(gomega.BeTrue(), "pod delete was not processed by controller") + } + + // Now assert final cache + events (allowing the controller a moment to deliver callbacks) + g.Eventually(func(g gomega.Gomega) { + ptc.cacheMutex.Lock() + defer ptc.cacheMutex.Unlock() + + if tt.expectedNADs == nil { + g.Expect(ptc.podToNodeNAD).ToNot(gomega.HaveKey(key)) + } else { + g.Expect(ptc.podToNodeNAD).To(gomega.HaveKey(key)) + for _, nad := range tt.expectedNADs { + g.Expect(ptc.nodeNADToPodCache[tt.nodeName]).To(gomega.HaveKey(nad)) + g.Expect(ptc.nodeNADToPodCache[tt.nodeName][nad]).To(gomega.HaveKey(key)) + } + } + + // Verify callback events equal expected sequence + eventsMu.Lock() + defer eventsMu.Unlock() + g.Expect(events).To(gomega.ConsistOf(tt.expectEvents)) + }, "2s", "50ms").Should(gomega.Succeed()) + }) + } +} + +func TestPodTrackerControllerSyncAll(t *testing.T) { + g := gomega.NewWithT(t) + err := config.PrepareTestConfig() + g.Expect(err).NotTo(gomega.HaveOccurred()) + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableInterconnect = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + + // Setup fake client + watch factory + fakeClient := util.GetOVNClientset().GetOVNKubeControllerClientset() + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClient) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Track callback events + var events []struct { + node string + nad string + active bool + } + var eventsMu sync.Mutex + + // Create PodTrackerController + ptc := NewPodTrackerController("test-pod-tracker", wf, func(node, nad string, active bool) { + eventsMu.Lock() + events = append(events, struct { + node string + nad string + active bool + }{node, nad, active}) + eventsMu.Unlock() + }, nil) + + // Start informers + err = wf.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer wf.Shutdown() + + // Start pod controller + g.Expect(ptc.Start()).Should(gomega.Succeed()) + defer ptc.Stop() + + // Create NAD + namespace := "testns" + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "primary", Type: "ovn-k8s-cni-overlay"}, + Topology: "layer3", + Role: "primary", + MTU: 1400, + NADName: "testns/primary", + } + bytes, err := json.Marshal(netConf) + g.Expect(err).ToNot(gomega.HaveOccurred()) + nad := &nadv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + UID: types.UID(namespace), + Name: "primary", + Namespace: namespace, + }, + Spec: nadv1.NetworkAttachmentDefinitionSpec{ + Config: string(bytes), + }, + } + // Create namespace + _, err = fakeClient.KubeClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Labels: map[string]string{ovntypes.RequiredUDNNamespaceLabel: ""}, + }, + }, metav1.CreateOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Create NAD + _, err = fakeClient.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespace). + Create(context.Background(), nad, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Create a node + _, err = fakeClient.KubeClient.CoreV1().Nodes().Create(context.Background(), &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "nodeX"}, + }, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Create a pod with primary + secondary NADs + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "podX", + Namespace: "testns", + Annotations: map[string]string{ + nadv1.NetworkAttachmentAnnot: `[ {"name": "sec1", "namespace": "testns"} ]`, + }, + }, + Spec: corev1.PodSpec{NodeName: "nodeX"}, + } + _, err = fakeClient.KubeClient.CoreV1().Pods("testns").Create(context.Background(), pod, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + key := "testns/podX" + + // Wait for add + g.Eventually(func() bool { + ptc.cacheMutex.Lock() + _, ok := ptc.podToNodeNAD[key] + ptc.cacheMutex.Unlock() + return ok + }, "2s", "50ms").Should(gomega.BeTrue()) + + // Manually clear controller state to simulate stale cache + ptc.cacheMutex.Lock() + ptc.nodeNADToPodCache = make(map[string]map[string]map[string]struct{}) + ptc.podToNodeNAD = make(map[string]nodeNAD) + ptc.cacheMutex.Unlock() + + // Call syncAll to rebuild state + g.Expect(ptc.syncAll()).To(gomega.Succeed()) + + // Verify that syncAll restored the pod->NAD mappings + g.Eventually(func(g gomega.Gomega) { + ptc.cacheMutex.Lock() + defer ptc.cacheMutex.Unlock() + g.Expect(ptc.podToNodeNAD).To(gomega.HaveKey(key)) + g.Expect(ptc.nodeNADToPodCache["nodeX"]).To(gomega.HaveKey("testns/primary")) + g.Expect(ptc.nodeNADToPodCache["nodeX"]).To(gomega.HaveKey("testns/sec1")) + }, "2s", "50ms").Should(gomega.Succeed()) + + // Verify callbacks included active=true rebuild events + g.Eventually(func() []struct { + node string + nad string + active bool + } { + eventsMu.Lock() + defer eventsMu.Unlock() + return append([]struct { + node string + nad string + active bool + }(nil), events...) + }, "2s", "50ms").Should(gomega.ContainElements( + struct { + node, nad string + active bool + }{"nodeX", "testns/primary", true}, + struct { + node, nad string + active bool + }{"nodeX", "testns/sec1", true}, + )) +} diff --git a/go-controller/pkg/node/base_node_network_controller_dpu.go b/go-controller/pkg/node/base_node_network_controller_dpu.go index 973e3d2937..db79e35c39 100644 --- a/go-controller/pkg/node/base_node_network_controller_dpu.go +++ b/go-controller/pkg/node/base_node_network_controller_dpu.go @@ -21,20 +21,20 @@ import ( // Check if the Pod is ready so that we can add its associated DPU to br-int. // If true, return its dpuConnDetails, otherwise return nil -func (bnnc *BaseNodeNetworkController) podReadyToAddDPU(pod *corev1.Pod, nadName string) *util.DPUConnectionDetails { +func (bnnc *BaseNodeNetworkController) podReadyToAddDPU(pod *corev1.Pod, nadKey string) *util.DPUConnectionDetails { if bnnc.name != pod.Spec.NodeName { klog.V(5).Infof("Pod %s/%s is not scheduled on this node %s", pod.Namespace, pod.Name, bnnc.name) return nil } - dpuCD, err := util.UnmarshalPodDPUConnDetails(pod.Annotations, nadName) + dpuCD, err := util.UnmarshalPodDPUConnDetails(pod.Annotations, nadKey) if err != nil { if !util.IsAnnotationNotSetError(err) { klog.Errorf("Failed to get DPU annotation for pod %s/%s NAD %s: %v", - pod.Namespace, pod.Name, nadName, err) + pod.Namespace, pod.Name, nadKey, err) } else { klog.V(5).Infof("DPU connection details annotation still not found for %s/%s for NAD %s", - pod.Namespace, pod.Name, nadName) + pod.Namespace, pod.Name, nadKey) } return nil } @@ -43,11 +43,11 @@ func (bnnc *BaseNodeNetworkController) podReadyToAddDPU(pod *corev1.Pod, nadName } func (bnnc *BaseNodeNetworkController) addDPUPodForNAD(pod *corev1.Pod, dpuCD *util.DPUConnectionDetails, - netName, nadName string, getter cni.PodInfoGetter) error { - podDesc := fmt.Sprintf("pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadName) + netName, nadKey string, getter cni.PodInfoGetter) error { + podDesc := fmt.Sprintf("pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadKey) klog.Infof("Adding %s on DPU", podDesc) podInterfaceInfo, err := cni.PodAnnotation2PodInfo(pod.Annotations, nil, - string(pod.UID), "", nadName, netName, config.Default.MTU) + string(pod.UID), "", nadKey, netName, config.Default.MTU) if err != nil { return fmt.Errorf("failed to get pod interface information of %s: %v. retrying", podDesc, err) } @@ -58,14 +58,14 @@ func (bnnc *BaseNodeNetworkController) addDPUPodForNAD(pod *corev1.Pod, dpuCD *u return nil } -func (bnnc *BaseNodeNetworkController) delDPUPodForNAD(pod *corev1.Pod, dpuCD *util.DPUConnectionDetails, nadName string, podDeleted bool) error { +func (bnnc *BaseNodeNetworkController) delDPUPodForNAD(pod *corev1.Pod, dpuCD *util.DPUConnectionDetails, nadKey string, podDeleted bool) error { var errs []error - podDesc := fmt.Sprintf("pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadName) + podDesc := fmt.Sprintf("pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadKey) klog.Infof("Deleting %s from DPU", podDesc) // no need to unset connection status annotation if pod is deleted anyway if !podDeleted { - err := bnnc.updatePodDPUConnStatusWithRetry(pod, nil, nadName) + err := bnnc.updatePodDPUConnStatusWithRetry(pod, nil, nadKey) if err != nil { errs = append(errs, fmt.Errorf("failed to remove the old DPU connection status annotation for %s: %v", podDesc, err)) } @@ -74,7 +74,7 @@ func (bnnc *BaseNodeNetworkController) delDPUPodForNAD(pod *corev1.Pod, dpuCD *u if err != nil { errs = append(errs, fmt.Errorf("failed to get old VF representor for %s, dpuConnDetail %+v Representor port may have been deleted: %v", podDesc, dpuCD, err)) } else { - err = bnnc.delRepPort(pod, dpuCD, vfRepName, nadName) + err = bnnc.delRepPort(pod, dpuCD, vfRepName, nadKey) if err != nil { errs = append(errs, fmt.Errorf("failed to delete VF representor for %s: %v", podDesc, err)) } @@ -104,7 +104,6 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) return bnnc.watchFactory.AddPodHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { var activeNetwork util.NetInfo - var err error pod := obj.(*corev1.Pod) klog.V(5).Infof("Add for Pod: %s/%s for network %s", pod.Namespace, pod.Name, netName) @@ -117,6 +116,19 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) nadToDPUCDMap := map[string]*util.DPUConnectionDetails{} if bnnc.IsUserDefinedNetwork() { if bnnc.IsPrimaryNetwork() { + // check to see if the primary NAD is even applicable to our controller + foundNamespaceNAD, err := bnnc.networkManager.GetPrimaryNADForNamespace(pod.Namespace) + if err != nil { + klog.Errorf("Failed to get primary network NAD for namespace %s: %v", pod.Namespace, err) + return + } + if foundNamespaceNAD == types.DefaultNetworkName { + return + } + networkName := bnnc.networkManager.GetNetworkNameForNADKey(foundNamespaceNAD) + if networkName != "" && networkName != netName { + return + } activeNetwork, err = bnnc.networkManager.GetActiveNetworkForNamespace(pod.Namespace) if err != nil { klog.Errorf("Failed looking for the active network for namespace %s: %v", pod.Namespace, err) @@ -124,7 +136,13 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) } } - on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, bnnc.GetNetInfo(), activeNetwork) + on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork( + pod, + bnnc.GetNetInfo(), + activeNetwork, + bnnc.networkManager.GetNetworkNameForNADKey, + bnnc.networkManager.GetPrimaryNADForNamespace, + ) if err != nil || !on { if err != nil { // configuration error, no need to retry, do not return error @@ -136,21 +154,21 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) } return } - for nadName := range networkMap { - nadToDPUCDMap[nadName] = nil + for nadKey := range networkMap { + nadToDPUCDMap[nadKey] = nil } } else { nadToDPUCDMap[types.DefaultNetworkName] = nil } - for nadName := range nadToDPUCDMap { - dpuCD := bnnc.podReadyToAddDPU(pod, nadName) + for nadKey := range nadToDPUCDMap { + dpuCD := bnnc.podReadyToAddDPU(pod, nadKey) if dpuCD != nil { - err := bnnc.addDPUPodForNAD(pod, dpuCD, netName, nadName, clientSet) + err := bnnc.addDPUPodForNAD(pod, dpuCD, netName, nadKey, clientSet) if err != nil { klog.Errorf("Error adding pod %s/%s for for network %s: %v", pod.Namespace, pod.Name, bnnc.GetNetworkName(), err) } else { - nadToDPUCDMap[nadName] = dpuCD + nadToDPUCDMap[nadKey] = dpuCD } } } @@ -167,9 +185,9 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) return } nadToDPUCDMap := v.(map[string]*util.DPUConnectionDetails) - for nadName := range nadToDPUCDMap { - oldDPUCD := nadToDPUCDMap[nadName] - newDPUCD := bnnc.podReadyToAddDPU(newPod, nadName) + for nadKey := range nadToDPUCDMap { + oldDPUCD := nadToDPUCDMap[nadKey] + newDPUCD := bnnc.podReadyToAddDPU(newPod, nadKey) if !dpuConnectionDetailChanged(oldDPUCD, newDPUCD) { continue } @@ -178,21 +196,21 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) klog.Infof("Deleting the old VF since either kubelet issued cmdDEL or assigned a new VF or "+ "the sandbox id itself changed. Old connection details (%v), New connection details (%v)", oldDPUCD, newDPUCD) - err := bnnc.delDPUPodForNAD(oldPod, oldDPUCD, nadName, false) + err := bnnc.delDPUPodForNAD(oldPod, oldDPUCD, nadKey, false) if err != nil { klog.Errorf("Error deleting pod %s/%s for for network %s: %v", oldPod.Namespace, oldPod.Name, bnnc.GetNetworkName(), err) } - nadToDPUCDMap[nadName] = nil + nadToDPUCDMap[nadKey] = nil } if newDPUCD != nil { klog.Infof("Adding VF during update because either during Pod Add we failed to add VF or "+ "connection details weren't present or the VF ID has changed. Old connection details (%v), "+ "New connection details (%v)", oldDPUCD, newDPUCD) - err := bnnc.addDPUPodForNAD(newPod, newDPUCD, netName, nadName, clientSet) + err := bnnc.addDPUPodForNAD(newPod, newDPUCD, netName, nadKey, clientSet) if err != nil { klog.Errorf("Error adding pod %s/%s for for network %s: %v", newPod.Namespace, newPod.Name, bnnc.GetNetworkName(), err) } else { - nadToDPUCDMap[nadName] = newDPUCD + nadToDPUCDMap[nadKey] = newDPUCD } } } @@ -209,9 +227,9 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) klog.V(5).Infof("Delete for Pod: %s/%s for network %s", pod.Namespace, pod.Name, netName) nadToDPUCDMap := v.(map[string]*util.DPUConnectionDetails) bnnc.podNADToDPUCDMap.Delete(pod.UID) - for nadName, dpuCD := range nadToDPUCDMap { + for nadKey, dpuCD := range nadToDPUCDMap { if dpuCD != nil { - err := bnnc.delDPUPodForNAD(pod, dpuCD, nadName, true) + err := bnnc.delDPUPodForNAD(pod, dpuCD, nadKey, true) if err != nil { klog.Errorf("Error deleting pod %s/%s for for network %s: %v", pod.Namespace, pod.Name, bnnc.GetNetworkName(), err) } @@ -223,15 +241,15 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) // updatePodDPUConnStatusWithRetry update the pod annotion with the givin connection details func (bnnc *BaseNodeNetworkController) updatePodDPUConnStatusWithRetry(origPod *corev1.Pod, - dpuConnStatus *util.DPUConnectionStatus, nadName string) error { + dpuConnStatus *util.DPUConnectionStatus, nadKey string) error { podDesc := fmt.Sprintf("pod %s/%s", origPod.Namespace, origPod.Name) - klog.Infof("Updating pod %s with connection status (%+v) for NAD %s", podDesc, dpuConnStatus, nadName) + klog.Infof("Updating pod %s with connection status (%+v) for NAD %s", podDesc, dpuConnStatus, nadKey) err := util.UpdatePodDPUConnStatusWithRetry( bnnc.watchFactory.PodCoreInformer().Lister(), bnnc.Kube, origPod, dpuConnStatus, - nadName, + nadKey, ) if util.IsAnnotationAlreadySetError(err) { return nil @@ -243,8 +261,8 @@ func (bnnc *BaseNodeNetworkController) updatePodDPUConnStatusWithRetry(origPod * // addRepPort adds the representor of the VF to the ovs bridge func (bnnc *BaseNodeNetworkController) addRepPort(pod *corev1.Pod, dpuCD *util.DPUConnectionDetails, ifInfo *cni.PodInterfaceInfo, getter cni.PodInfoGetter) error { - nadName := ifInfo.NADName - podDesc := fmt.Sprintf("pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadName) + nadKey := ifInfo.NADKey + podDesc := fmt.Sprintf("pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadKey) vfRepName, err := util.GetSriovnetOps().GetVfRepresentorDPU(dpuCD.PfId, dpuCD.VfId) if err != nil { klog.Infof("Failed to get VF representor for %s dpuConnDetail %+v: %v", podDesc, dpuCD, err) @@ -261,48 +279,48 @@ func (bnnc *BaseNodeNetworkController) addRepPort(pod *corev1.Pod, dpuCD *util.D } klog.Infof("Adding VF representor %s for %s", vfRepName, podDesc) - err = cni.ConfigureOVS(context.TODO(), pod.Namespace, pod.Name, vfRepName, ifInfo, dpuCD.SandboxId, vfPciAddress, getter) + err = cni.ConfigureOVS(context.TODO(), pod.Namespace, pod.Name, "", vfRepName, ifInfo, dpuCD.SandboxId, vfPciAddress, getter) if err != nil { // Note(adrianc): we are lenient with cleanup in this method as pod is going to be retried anyway. - _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadName) + _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadKey) return err } klog.Infof("Port %s added to bridge br-int", vfRepName) link, err := util.GetNetLinkOps().LinkByName(vfRepName) if err != nil { - _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadName) + _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadKey) return fmt.Errorf("failed to get link device for interface %s", vfRepName) } if err = util.GetNetLinkOps().LinkSetMTU(link, ifInfo.MTU); err != nil { - _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadName) + _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadKey) return fmt.Errorf("failed to setup representor port. failed to set MTU for interface %s", vfRepName) } if err = util.GetNetLinkOps().LinkSetUp(link); err != nil { - _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadName) + _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadKey) return fmt.Errorf("failed to setup representor port. failed to set link up for interface %s", vfRepName) } // Update connection-status annotation // TODO(adrianc): we should update Status in case of error as well connStatus := util.DPUConnectionStatus{Status: util.DPUConnectionStatusReady, Reason: ""} - err = bnnc.updatePodDPUConnStatusWithRetry(pod, &connStatus, nadName) + err = bnnc.updatePodDPUConnStatusWithRetry(pod, &connStatus, nadKey) if err != nil { _ = util.GetNetLinkOps().LinkSetDown(link) - _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadName) + _ = bnnc.delRepPort(pod, dpuCD, vfRepName, nadKey) return fmt.Errorf("failed to setup representor port. failed to set pod annotations. %v", err) } return nil } // delRepPort delete the representor of the VF from the ovs bridge -func (bnnc *BaseNodeNetworkController) delRepPort(pod *corev1.Pod, dpuCD *util.DPUConnectionDetails, vfRepName, nadName string) error { +func (bnnc *BaseNodeNetworkController) delRepPort(pod *corev1.Pod, dpuCD *util.DPUConnectionDetails, vfRepName, nadKey string) error { //TODO(adrianc): handle: clearPodBandwidth(pr.SandboxID), pr.deletePodConntrack() - podDesc := fmt.Sprintf("pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadName) + podDesc := fmt.Sprintf("pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadKey) klog.Infof("Delete VF representor %s for %s", vfRepName, podDesc) - ifExists, sandbox, expectedNADName, err := util.GetOVSPortPodInfo(vfRepName) + ifExists, sandbox, expectedNADKey, err := util.GetOVSPortPodInfo(vfRepName) if err != nil { return err } @@ -313,8 +331,8 @@ func (bnnc *BaseNodeNetworkController) delRepPort(pod *corev1.Pod, dpuCD *util.D if sandbox != dpuCD.SandboxId { return fmt.Errorf("OVS port %s was added for sandbox (%s), expecting (%s)", vfRepName, sandbox, dpuCD.SandboxId) } - if expectedNADName != nadName { - return fmt.Errorf("OVS port %s was added for NAD (%s), expecting (%s)", vfRepName, expectedNADName, nadName) + if expectedNADKey != nadKey { + return fmt.Errorf("OVS port %s was added for NAD key (%s), expecting (%s)", vfRepName, expectedNADKey, nadKey) } // Set link down for representor port link, err := util.GetNetLinkOps().LinkByName(vfRepName) diff --git a/go-controller/pkg/node/base_node_network_controller_dpu_test.go b/go-controller/pkg/node/base_node_network_controller_dpu_test.go index d372d49777..9cac5497c8 100644 --- a/go-controller/pkg/node/base_node_network_controller_dpu_test.go +++ b/go-controller/pkg/node/base_node_network_controller_dpu_test.go @@ -159,7 +159,7 @@ var _ = Describe("Node DPU tests", func() { Egress: -1, IsDPUHostMode: true, NetName: types.DefaultNetworkName, - NADName: types.DefaultNetworkName, + NADKey: types.DefaultNetworkName, PodUID: "a-pod", } diff --git a/go-controller/pkg/node/controllers/egressip/egressip.go b/go-controller/pkg/node/controllers/egressip/egressip.go index dd1f15f3c9..08726875a3 100644 --- a/go-controller/pkg/node/controllers/egressip/egressip.go +++ b/go-controller/pkg/node/controllers/egressip/egressip.go @@ -1534,6 +1534,12 @@ func generateIPTablesSNATRuleArg(srcIP net.IP, isIPv6 bool, infName, snatIP stri func isEgressIPOnLink(linkIndex, ipFamily int, assignedEIPs sets.Set[string]) (bool, error) { link, err := netlink.LinkByIndex(linkIndex) if err != nil { + // If the link doesn't exist, there can't be an EgressIP on it. + // This can happen when a route is added/deleted causing the interface + // to momentarily disappear or change its index. + if util.GetNetLinkOps().IsLinkNotFoundError(err) { + return false, nil + } return false, err } addresses, err := netlink.AddrList(link, ipFamily) diff --git a/go-controller/pkg/node/controllers/egressip/egressip_test.go b/go-controller/pkg/node/controllers/egressip/egressip_test.go index 7de412fdc2..2f55b62a6a 100644 --- a/go-controller/pkg/node/controllers/egressip/egressip_test.go +++ b/go-controller/pkg/node/controllers/egressip/egressip_test.go @@ -1834,3 +1834,113 @@ func getEIPIPVersions(eip *egressipv1.EgressIP) (bool, bool) { } return v4, v6 } + +var _ = ginkgo.Describe("isEgressIPOnLink", func() { + ginkgo.It("returns false when link does not exist", func() { + defer ginkgo.GinkgoRecover() + if ovntest.NoRoot() { + ginkgo.Skip("Test requires root privileges") + } + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + // Use a non-existent link index + nonExistentLinkIndex := 99999 + assignedEIPs := sets.New[string]("192.168.1.100") + + // Create a test namespace to ensure we're in a clean network environment + testNS, err := testutils.NewNS() + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + defer func() { + gomega.Expect(testNS.Close()).Should(gomega.Succeed()) + gomega.Expect(testutils.UnmountNS(testNS)).Should(gomega.Succeed()) + }() + + err = testNS.Do(func(ns.NetNS) error { + // Call isEgressIPOnLink with a non-existent link index + // This should return false, nil instead of an error + result, err := isEgressIPOnLink(nonExistentLinkIndex, netlink.FAMILY_V4, assignedEIPs) + if err != nil { + return fmt.Errorf("isEgressIPOnLink should not return error for non-existent link, got: %v", err) + } + if result { + return fmt.Errorf("isEgressIPOnLink should return false for non-existent link") + } + return nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("returns true when EgressIP is on link", func() { + defer ginkgo.GinkgoRecover() + if ovntest.NoRoot() { + ginkgo.Skip("Test requires root privileges") + } + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + testNS, cleanupFn, err := setupFakeTestNode(nodeConfig{ + linkConfigs: []linkConfig{ + {dummyLink1Name, []address{{dummy1IPv4CIDR, false}, {egressIP1IPV4CIDR, true}}}, + }, + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + defer func() { + gomega.Expect(cleanupFn()).Should(gomega.Succeed()) + }() + + err = testNS.Do(func(ns.NetNS) error { + link, err := netlink.LinkByName(dummyLink1Name) + if err != nil { + return err + } + assignedEIPs := sets.New[string](egressIP1IPV4) + result, err := isEgressIPOnLink(link.Attrs().Index, netlink.FAMILY_V4, assignedEIPs) + if err != nil { + return fmt.Errorf("isEgressIPOnLink should not return error: %v", err) + } + if !result { + return fmt.Errorf("isEgressIPOnLink should return true when EgressIP is on link") + } + return nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("returns false when no EgressIP is on link", func() { + defer ginkgo.GinkgoRecover() + if ovntest.NoRoot() { + ginkgo.Skip("Test requires root privileges") + } + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + testNS, cleanupFn, err := setupFakeTestNode(nodeConfig{ + linkConfigs: []linkConfig{ + {dummyLink1Name, []address{{dummy1IPv4CIDR, false}}}, + }, + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + defer func() { + gomega.Expect(cleanupFn()).Should(gomega.Succeed()) + }() + + err = testNS.Do(func(ns.NetNS) error { + link, err := netlink.LinkByName(dummyLink1Name) + if err != nil { + return err + } + // Use an EgressIP that is NOT on the link + assignedEIPs := sets.New[string](egressIP1IPV4) + result, err := isEgressIPOnLink(link.Attrs().Index, netlink.FAMILY_V4, assignedEIPs) + if err != nil { + return fmt.Errorf("isEgressIPOnLink should not return error: %v", err) + } + if result { + return fmt.Errorf("isEgressIPOnLink should return false when EgressIP is not on link") + } + return nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +}) diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index b5ac2fd93c..512aa6fb53 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -42,6 +42,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/managementport" nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/ovspinning" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/podresourcesapi" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" nodetypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/apbroute" @@ -972,7 +973,7 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error { defer nc.wg.Done() nodeController.Run(nc.stopChan) }() - } else { + } else if config.OvnKubeNode.Mode != types.NodeModeDPUHost { // attempt to cleanup the possibly stale bridge _, stderr, err := util.RunOVSVsctl("--if-exists", "del-br", "br-ext") if err != nil { @@ -1081,7 +1082,17 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error { nc.wg.Add(1) go func() { defer nc.wg.Done() - ovspinning.Run(nc.stopChan) + podResClient, err := podresourcesapi.New() + if err != nil { + klog.Errorf("Failed to initialize PodResourcesAPI client: %v", err) + return + } + defer func() { + if err := podResClient.Close(); err != nil { + klog.V(4).Infof("Error closing PodResourcesAPI client: %v", err) + } + }() + ovspinning.Run(ctx, nc.stopChan, podResClient) }() klog.Infof("Default node network controller initialized and ready.") @@ -1413,9 +1424,23 @@ func (nc *DefaultNodeNetworkController) syncNodes(objs []interface{}) error { } // validateVTEPInterfaceMTU checks if the MTU of the interface that has ovn-encap-ip is big -// enough to carry the `config.Default.MTU` and the Geneve header. If the MTU is not big -// enough, it will return an error +// enough to carry the `config.Default.MTU` and the Geneve header (if overlay transport is used). +// If the MTU is not big enough, it will return an error func (nc *DefaultNodeNetworkController) validateVTEPInterfaceMTU() error { + // calc required MTU + var requiredMTU int + if config.Gateway.SingleNode || config.Default.Transport == types.NetworkTransportNoOverlay { + requiredMTU = config.Default.MTU + } else { + if config.IPv4Mode && !config.IPv6Mode { + // we run in single-stack IPv4 only + requiredMTU = config.Default.MTU + types.GeneveHeaderLengthIPv4 + } else { + // we run in single-stack IPv6 or dual-stack mode + requiredMTU = config.Default.MTU + types.GeneveHeaderLengthIPv6 + } + } + // OVN allows `external_ids:ovn-encap-ip` to be a list of IPs separated by comma ovnEncapIps := strings.Split(config.Default.EffectiveEncapIP, ",") for _, ip := range ovnEncapIps { @@ -1428,20 +1453,6 @@ func (nc *DefaultNodeNetworkController) validateVTEPInterfaceMTU() error { return fmt.Errorf("could not get MTU for the interface with address %s: %w", ovnEncapIP, err) } - // calc required MTU - var requiredMTU int - if config.Gateway.SingleNode { - requiredMTU = config.Default.MTU - } else { - if config.IPv4Mode && !config.IPv6Mode { - // we run in single-stack IPv4 only - requiredMTU = config.Default.MTU + types.GeneveHeaderLengthIPv4 - } else { - // we run in single-stack IPv6 or dual-stack mode - requiredMTU = config.Default.MTU + types.GeneveHeaderLengthIPv6 - } - } - if mtu < requiredMTU { return fmt.Errorf("MTU (%d) of network interface %s is too small for specified overlay MTU (%d)", mtu, interfaceName, requiredMTU) diff --git a/go-controller/pkg/node/egressip/gateway_egressip.go b/go-controller/pkg/node/egressip/gateway_egressip.go index 27700e026e..83657404b8 100644 --- a/go-controller/pkg/node/egressip/gateway_egressip.go +++ b/go-controller/pkg/node/egressip/gateway_egressip.go @@ -209,115 +209,94 @@ func (g *BridgeEIPAddrManager) GetCache() *MarkIPsCache { } func (g *BridgeEIPAddrManager) AddEgressIP(eip *egressipv1.EgressIP) (bool, error) { - var isUpdated bool - if !util.IsEgressIPMarkSet(eip.Annotations) { - return isUpdated, nil + ip, pktMark, shouldSkip, err := g.parseAndValidateEIP(eip) + if err != nil { + return false, fmt.Errorf("failed to add EgressIP gateway config because unable to parse and validate EgressIP: %v", err) } - for _, status := range eip.Status.Items { - if status.Node != g.nodeName { - continue - } - ip, pktMark, err := parseEIPMarkIP(eip.Annotations, status.EgressIP) - if err != nil { - return isUpdated, fmt.Errorf("failed to add EgressIP gateway config because unable to extract config from EgressIP obj: %v", err) - } - // must always add to cache before adding IP because we want to inform node ip handler that this is not a valid node IP - g.cache.insertMarkIP(pktMark, ip) - if err = g.addIPToAnnotation(ip); err != nil { - return isUpdated, fmt.Errorf("failed to add EgressIP gateway config because unable to add EgressIP IP to Node annotation: %v", err) - } - if err = g.addIPBridge(ip); err != nil { - return isUpdated, fmt.Errorf("failed to add EgressIP gateway config because failed to add address to link: %v", err) - } - isUpdated = true - break // no need to continue as only one EIP IP is assigned to a node + if shouldSkip { + return false, nil } - return isUpdated, nil + // must always add to cache before adding IP because we want to inform node ip handler that this is not a valid node IP + g.cache.insertMarkIP(pktMark, ip) + if err = g.addIPToAnnotation(ip); err != nil { + return false, fmt.Errorf("failed to add EgressIP gateway config because unable to add EgressIP IP to Node annotation: %v", err) + } + if err = g.addIPBridge(ip); err != nil { + return false, fmt.Errorf("failed to add EgressIP gateway config because failed to add address to link: %v", err) + } + return true, nil } func (g *BridgeEIPAddrManager) UpdateEgressIP(oldEIP, newEIP *egressipv1.EgressIP) (bool, error) { + // Parse and validate old EgressIP + oldIP, oldMark, oldSkip, err := g.parseAndValidateEIP(oldEIP) + if err != nil { + return false, fmt.Errorf("unable to parse and validate old EgressIP: %v", err) + } + + // Parse and validate new EgressIP + newIP, newMark, newSkip, err := g.parseAndValidateEIP(newEIP) + if err != nil { + return false, fmt.Errorf("unable to parse and validate new EgressIP: %v", err) + } + + // Both should be skipped - no change + if oldSkip && newSkip { + return false, nil + } + + // Both exist and are the same - no change + if !oldSkip && !newSkip && oldIP.Equal(newIP) { + return false, nil + } + var isUpdated bool - // at most, one status item for this node will be found. - for _, oldStatus := range oldEIP.Status.Items { - if oldStatus.Node != g.nodeName { - continue - } - if !util.IsEgressIPMarkSet(oldEIP.Annotations) { - // this scenario may occur during upgrade from when ovn-k didn't apply marks to EIP objs - break - } - if util.IsItemInSlice(newEIP.Status.Items, oldStatus) { - // if one status entry exists in both status items, then nothing needs to be done because no status update. - // also, because at most only one status item can be assigned to a node, we can return early. - return isUpdated, nil - } - ip, pktMark, err := parseEIPMarkIP(oldEIP.Annotations, oldStatus.EgressIP) - if err != nil { - return isUpdated, fmt.Errorf("failed to update EgressIP SNAT for ext bridge cache because unable to extract config from old EgressIP obj: %v", err) - } - if err = g.deleteIPBridge(ip); err != nil { + + // Delete old if it exists and is different from new + if !oldSkip { + if err = g.deleteIPBridge(oldIP); err != nil { return isUpdated, fmt.Errorf("failed to update EgressIP gateway config because failed to delete address from link: %v", err) } - g.cache.deleteMarkIP(pktMark, ip) - if err = g.deleteIPsFromAnnotation(ip); err != nil { + g.cache.deleteMarkIP(oldMark, oldIP) + if err = g.deleteIPsFromAnnotation(oldIP); err != nil { return isUpdated, fmt.Errorf("failed to update EgressIP gateway config because unable to delete EgressIP IP from Node annotation: %v", err) } isUpdated = true - break } - for _, newStatus := range newEIP.Status.Items { - if newStatus.Node != g.nodeName { - continue - } - if !util.IsEgressIPMarkSet(newEIP.Annotations) { - // this scenario may occur during upgrade from when ovn-k didn't apply marks to EIP objs - return isUpdated, nil - } - ip, pktMark, err := parseEIPMarkIP(newEIP.Annotations, newStatus.EgressIP) - if err != nil { - return isUpdated, fmt.Errorf("failed to update EgressIP gateway config because unable to extract config from EgressIP obj: %v", err) - } - // must always add to OF cache before adding IP because we want to inform node ip handler that this is not a valid node IP - g.cache.insertMarkIP(pktMark, ip) - if err = g.addIPToAnnotation(ip); err != nil { + + // Add new if it exists + if !newSkip { + // must always add to cache before adding IP because we want to inform node ip handler that this is not a valid node IP + g.cache.insertMarkIP(newMark, newIP) + if err = g.addIPToAnnotation(newIP); err != nil { return isUpdated, fmt.Errorf("failed to update EgressIP gateway config because unable to add EgressIP IP to Node annotation: %v", err) } - if err = g.addIPBridge(ip); err != nil { + if err = g.addIPBridge(newIP); err != nil { return isUpdated, fmt.Errorf("failed to update EgressIP gateway config because failed to add address to link: %v", err) } isUpdated = true - break } + return isUpdated, nil } func (g *BridgeEIPAddrManager) DeleteEgressIP(eip *egressipv1.EgressIP) (bool, error) { - var isUpdated bool - if !util.IsEgressIPMarkSet(eip.Annotations) { - return isUpdated, nil + ip, pktMark, shouldSkip, err := g.parseAndValidateEIP(eip) + if err != nil { + return false, fmt.Errorf("failed to delete EgressIP gateway config because unable to parse and validate EgressIP: %v", err) } - for _, status := range eip.Status.Items { - if status.Node != g.nodeName { - continue - } - if !util.IsEgressIPMarkSet(eip.Annotations) { - continue - } - ip, pktMark, err := parseEIPMarkIP(eip.Annotations, status.EgressIP) - if err != nil { - return isUpdated, fmt.Errorf("failed to delete EgressIP gateway config because unable to extract config from EgressIP obj: %v", err) - } - if err = g.deleteIPBridge(ip); err != nil { - return isUpdated, fmt.Errorf("failed to delete EgressIP gateway config because failed to delete address from link: %v", err) - } - g.cache.deleteMarkIP(pktMark, ip) - if err = g.deleteIPsFromAnnotation(ip); err != nil { - return isUpdated, fmt.Errorf("failed to delete EgressIP gateway config because failed to delete EgressIP IP from Node annotation: %v", err) - } - isUpdated = true - break // no need to continue as only one EIP IP is assigned per node + // Skip secondary network IPs. Cleanup of stale secondary IPs from old code is handled in SyncEgressIP. + if shouldSkip { + return false, nil } - return isUpdated, nil + if err = g.deleteIPBridge(ip); err != nil { + return false, fmt.Errorf("failed to delete EgressIP gateway config because failed to delete address from link: %v", err) + } + g.cache.deleteMarkIP(pktMark, ip) + if err = g.deleteIPsFromAnnotation(ip); err != nil { + return false, fmt.Errorf("failed to delete EgressIP gateway config because failed to delete EgressIP IP from Node annotation: %v", err) + } + return true, nil } func (g *BridgeEIPAddrManager) SyncEgressIP(objs []interface{}) error { @@ -332,27 +311,21 @@ func (g *BridgeEIPAddrManager) SyncEgressIP(objs []interface{}) error { if !ok { return fmt.Errorf("expected EgressIP type but received %T", obj) } - // This may happen during upgrade when node controllers upgrade before cluster manager upgrades when cluster manager doesn't contain func - // to add a pkt mark to EgressIPs. - if !util.IsEgressIPMarkSet(eip.Annotations) { + ip, pktMark, shouldSkip, err := g.parseAndValidateEIP(eip) + if err != nil { + klog.Errorf("Failed to sync EgressIP %s gateway config because unable to parse and validate EgressIP: %v", eip.Name, err) continue } - for _, status := range eip.Status.Items { - if status.Node != g.nodeName { - continue - } - if ip, pktMark, err := parseEIPMarkIP(eip.Annotations, status.EgressIP); err != nil { - klog.Errorf("Failed to sync EgressIP %s gateway config because unable to extract config from EIP obj: %v", eip.Name, err) - } else { - configs.insert(pktMark, ip) - if err = g.addIPToAnnotation(ip); err != nil { - return fmt.Errorf("failed to sync EgressIP gateway config because unable to add EgressIP IP to Node annotation: %v", err) - } - if err = g.addIPBridge(ip); err != nil { - return fmt.Errorf("failed to sync EgressIP gateway config because failed to add address to link: %v", err) - } - } - break + if shouldSkip { + // Skip IPs not on OVN network (i.e., secondary network IPs) + continue + } + configs.insert(pktMark, ip) + if err = g.addIPToAnnotation(ip); err != nil { + return fmt.Errorf("failed to sync EgressIP gateway config because unable to add EgressIP IP to Node annotation: %v", err) + } + if err = g.addIPBridge(ip); err != nil { + return fmt.Errorf("failed to sync EgressIP gateway config because failed to add address to link: %v", err) } } ipsToDel := make([]net.IP, 0) @@ -361,12 +334,12 @@ func (g *BridgeEIPAddrManager) SyncEgressIP(objs []interface{}) error { continue } if err = g.deleteIPBridge(annotIP); err != nil { - klog.Errorf("Failed to delete stale EgressIP IP %s from gateway: %v", annotIP, err) - continue + return fmt.Errorf("failed to delete stale EgressIP IP %s from bridge: %v", annotIP, err) } ipsToDel = append(ipsToDel, annotIP) } if len(ipsToDel) > 0 { + klog.V(5).Infof("Deleting stale EgressIP IPs from Node annotation: %v", getIPsStr(ipsToDel...)) if err = g.deleteIPsFromAnnotation(ipsToDel...); err != nil { return fmt.Errorf("failed to delete EgressIP IPs from Node annotation: %v", err) } @@ -486,23 +459,80 @@ func (g *BridgeEIPAddrManager) getAnnotationIPs() ([]net.IP, error) { return ips, nil } -func parseEIPMarkIP(annotations map[string]string, eip string) (net.IP, util.EgressIPMark, error) { - pktMark, err := util.ParseEgressIPMark(annotations) +// isOVNNetworkIP checks if the given IP belongs to the OVN (primary) network +// Returns true if the IP is on the OVN network, false if it's on a secondary network +func (g *BridgeEIPAddrManager) isOVNNetworkIP(ip net.IP) (bool, error) { + node, err := g.nodeLister.Get(g.nodeName) if err != nil { - return nil, pktMark, fmt.Errorf("failed to extract packet mark from EgressIP annotations: %v", err) + return false, fmt.Errorf("failed to get node %s: %v", g.nodeName, err) } - // status update and pkt mark should be configured as one operation by cluster manager + nodePrimaryIPs, err := util.ParseNodePrimaryIfAddr(node) + if err != nil { + return false, fmt.Errorf("failed to parse node primary interface address for node %s: %v", g.nodeName, err) + } + return util.IsOVNNetwork(nodePrimaryIPs, ip), nil +} + +// parseAndValidateEIP parses and validates an EgressIP for this node +// Returns: +// - ip: the parsed IP address +// - pktMark: the parsed packet mark +// - shouldSkip: true if this EgressIP should be skipped (e.g., no mark set, not assigned to this node, belongs to secondary network) +// - error: any error encountered during parsing/validation +func (g *BridgeEIPAddrManager) parseAndValidateEIP(eip *egressipv1.EgressIP) (net.IP, util.EgressIPMark, bool, error) { + var pktMark util.EgressIPMark + + // Check if packet mark is set + // This scenario may occur during upgrade from when ovn-k didn't apply marks to EIP objs + if !util.IsEgressIPMarkSet(eip.Annotations) { + return nil, pktMark, true, nil + } + + // Find the EgressIP assigned to this node + var eipAddr string + for _, status := range eip.Status.Items { + if status.Node == g.nodeName { + eipAddr = status.EgressIP + break + } + } + if eipAddr == "" { + return nil, pktMark, true, nil + } + + // Parse the IP address + ip := net.ParseIP(eipAddr) + if ip == nil { + return nil, pktMark, false, fmt.Errorf("failed to parse EgressIP %s", eipAddr) + } + + // Parse the packet mark + var err error + pktMark, err = util.ParseEgressIPMark(eip.Annotations) + if err != nil { + return nil, pktMark, false, fmt.Errorf("failed to extract packet mark from EgressIP annotations: %v", err) + } + + // Validate packet mark if !pktMark.IsAvailable() { - return nil, pktMark, fmt.Errorf("packet mark is not set") + return nil, pktMark, false, fmt.Errorf("packet mark is not set") } if !pktMark.IsValid() { - return nil, pktMark, fmt.Errorf("packet mark is not valid") + return nil, pktMark, false, fmt.Errorf("packet mark is not valid") } - ip := net.ParseIP(eip) - if ip == nil { - return nil, pktMark, fmt.Errorf("invalid IP") + + // Check if this IP belongs to the OVN (primary) network + isOVN, err := g.isOVNNetworkIP(ip) + if err != nil { + return nil, pktMark, false, fmt.Errorf("failed to check if IP is OVN network: %w", err) + } + if !isOVN { + // Skip IPs not on OVN network (i.e., secondary network IPs) + klog.V(5).Infof("Skipping EgressIP %s on bridge %s because it does not belong to the OVN network", ip.String(), g.bridgeName) + return ip, pktMark, true, nil } - return ip, pktMark, nil + + return ip, pktMark, false, nil } func getIPsStr(ips ...net.IP) []string { diff --git a/go-controller/pkg/node/egressip/gateway_egressip_test.go b/go-controller/pkg/node/egressip/gateway_egressip_test.go index 1fe48a6f5b..6493cb968a 100644 --- a/go-controller/pkg/node/egressip/gateway_egressip_test.go +++ b/go-controller/pkg/node/egressip/gateway_egressip_test.go @@ -1,6 +1,7 @@ package egressip import ( + "encoding/json" "fmt" "net" "strings" @@ -127,6 +128,25 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, egressip.GetNetlinkAddress(net.ParseIP(ipV4Addr), bridgeLinkIndex))).Should(gomega.BeTrue()) }) + + ginkgo.It("doesn't configure when EgressIP is on a secondary host network", func() { + // Setup a node with host-cidrs annotation containing a secondary network subnet + secondarySubnet := "10.10.10.0/24" + secondaryIP := "10.10.10.5" + + nlMock.On("AddrAdd", nlLinkMock, egressip.GetNetlinkAddress(net.ParseIP(secondaryIP), bridgeLinkIndex)).Return(nil) + addrMgr, stopFn := initBridgeEIPAddrManagerWithHostCIDRs(nodeName, bridgeName, emptyAnnotation, []string{secondarySubnet}) + defer stopFn() + eip := getEIPAssignedToNode(nodeName, mark, secondaryIP) + isUpdated, err := addrMgr.AddEgressIP(eip) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should not error for secondary network IP") + gomega.Expect(isUpdated).Should(gomega.BeFalse(), "should not update for secondary network IP") + node, err := addrMgr.nodeLister.Get(nodeName) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") + gomega.Expect(parseEIPsFromAnnotation(node)).ShouldNot(gomega.ConsistOf(secondaryIP), "secondary IP should not be in annotation") + gomega.Expect(nlMock.AssertNotCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, + egressip.GetNetlinkAddress(net.ParseIP(secondaryIP), bridgeLinkIndex))).Should(gomega.BeTrue(), "should not add IP to bridge") + }) }) ginkgo.Context("update EgressIP", func() { @@ -198,9 +218,11 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err = addrMgr.UpdateEgressIP(assignedEIP1, assignedEIP2) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeTrue()) - node, err := addrMgr.nodeLister.Get(nodeName) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") - gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr2)) + gomega.Eventually(func() []string { + node, err := addrMgr.nodeLister.Get(nodeName) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") + return parseEIPsFromAnnotation(node) + }).Should(gomega.ConsistOf(ipV4Addr2)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, egressip.GetNetlinkAddress(net.ParseIP(ipV4Addr), bridgeLinkIndex))).Should(gomega.BeTrue()) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -292,9 +314,11 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { eipAssigned2 := getEIPAssignedToNode(nodeName, mark2, ipV4Addr2) err := addrMgr.SyncEgressIP([]interface{}{eipAssigned1, eipAssigned2}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process valid EgressIPs") - node, err := addrMgr.nodeLister.Get(nodeName) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") - gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr, ipV4Addr2)) + gomega.Eventually(func() []string { + node, err := addrMgr.nodeLister.Get(nodeName) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") + return parseEIPsFromAnnotation(node) + }).Should(gomega.ConsistOf(ipV4Addr, ipV4Addr2)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, egressip.GetNetlinkAddress(net.ParseIP(ipV4Addr), bridgeLinkIndex))).Should(gomega.BeTrue()) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -313,16 +337,75 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.BeEmpty()) }) + + ginkgo.It("cleans up mistakenly configured secondary network EgressIP", func() { + // Setup: secondary network IP that was mistakenly configured by old buggy code + secondaryIP := "10.10.10.5" + secondarySubnet := "10.10.10.0/24" + + nlLinkMock.On("Attrs").Return(&netlink.LinkAttrs{Name: bridgeName, Index: bridgeLinkIndex}, nil) + nlMock.On("LinkByName", bridgeName).Return(nlLinkMock, nil) + nlMock.On("LinkByIndex", bridgeLinkIndex).Return(nlLinkMock, nil) + nlMock.On("LinkList").Return([]netlink.Link{nlLinkMock}, nil) + nlMock.On("AddrList", nlLinkMock, 0).Return([]netlink.Addr{}, nil) + nlMock.On("AddrDel", nlLinkMock, egressip.GetNetlinkAddress(net.ParseIP(secondaryIP), bridgeLinkIndex)).Return(nil) + nlMock.On("AddrAdd", nlLinkMock, egressip.GetNetlinkAddress(net.ParseIP(ipV4Addr), bridgeLinkIndex)).Return(nil) + + // Initialize with host-cidrs that includes the secondary network and mistakenly configured secondary IP + addrMgr, stopFn := initBridgeEIPAddrManagerWithHostCIDRs(nodeName, bridgeName, generateAnnotFromIPs(secondaryIP), []string{secondarySubnet}) + defer stopFn() + + // Simulate mistaken configuration by old buggy code: IP is in cache + secondaryEIP := getEIPAssignedToNode(nodeName, mark, secondaryIP) + pktMark, err := util.ParseEgressIPMark(secondaryEIP.Annotations) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + addrMgr.cache.insertMarkIP(pktMark, net.ParseIP(secondaryIP)) + + // Verify mistaken state exists + gomega.Expect(addrMgr.cache.IsIPPresent(net.ParseIP(secondaryIP))).Should(gomega.BeTrue(), "IP should be in cache (mistaken state)") + node, err := addrMgr.nodeLister.Get(nodeName) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(secondaryIP), "IP should be in annotation (mistaken state)") + + // Sync with a valid OVN network EgressIP - should clean up the secondary IP and add the new one + validEIP := getEIPAssignedToNode(nodeName, mark2, ipV4Addr) + err = addrMgr.SyncEgressIP([]interface{}{validEIP}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should sync and clean up mistaken secondary network EgressIP") + + // Verify cleanup: secondary IP removed from cache, annotation, and bridge + gomega.Expect(addrMgr.cache.IsIPPresent(net.ParseIP(secondaryIP))).Should(gomega.BeFalse(), "secondary IP should be removed from cache") + node, err = addrMgr.nodeLister.Get(nodeName) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr), "only valid OVN IP should be in annotation") + gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrDel", nlLinkMock, + egressip.GetNetlinkAddress(net.ParseIP(secondaryIP), bridgeLinkIndex))).Should(gomega.BeTrue(), "should delete secondary IP from bridge") + gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, + egressip.GetNetlinkAddress(net.ParseIP(ipV4Addr), bridgeLinkIndex))).Should(gomega.BeTrue(), "should add valid OVN IP to bridge") + }) }) }) func initBridgeEIPAddrManager(nodeName, bridgeName string, bridgeEIPAnnot string) (*BridgeEIPAddrManager, func()) { + return initBridgeEIPAddrManagerWithHostCIDRs(nodeName, bridgeName, bridgeEIPAnnot, nil) +} + +// initBridgeEIPAddrManagerWithHostCIDRs is a variant of initBridgeEIPAddrManager that sets the host-cidrs annotation +func initBridgeEIPAddrManagerWithHostCIDRs(nodeName, bridgeName string, bridgeEIPAnnot string, hostCIDRs []string) (*BridgeEIPAddrManager, func()) { node := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{Name: nodeName, Annotations: map[string]string{}}, } if bridgeEIPAnnot != "" { node.Annotations[util.OVNNodeBridgeEgressIPs] = bridgeEIPAnnot } + // Add OVN network annotation - required for isOVNNetworkIP to work + node.Annotations[util.OvnNodeIfAddr] = `{"ipv4":"192.168.1.10/24"}` + if len(hostCIDRs) > 0 { + // Add OVN (primary) network to host-cidrs + hostCIDRs = append(hostCIDRs, "192.168.1.0/24") + cidrsJSON, err := json.Marshal(hostCIDRs) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + node.Annotations[util.OVNNodeHostCIDRs] = string(cidrsJSON) + } client := fake.NewSimpleClientset(node) watchFactory, err := factory.NewNodeWatchFactory(&util.OVNNodeClientset{KubeClient: client}, nodeName) gomega.Expect(watchFactory.Start()).Should(gomega.Succeed(), "watch factory should start") diff --git a/go-controller/pkg/node/gateway.go b/go-controller/pkg/node/gateway.go index 4a7416c64e..552cdf6a7a 100644 --- a/go-controller/pkg/node/gateway.go +++ b/go-controller/pkg/node/gateway.go @@ -234,8 +234,21 @@ func (g *gateway) DeleteEndpointSlice(epSlice *discovery.EndpointSlice) error { return utilerrors.Join(errors...) } +// canHandleBridgeEgressIP returns true if this node should handle EgressIP +// configuration on the bridge. Returns false if: +// - Network segmentation (UDN) is not enabled +// - Interconnect is not enabled +// - Gateway mode is disabled +// - Running in DPU-host mode (EgressIP is handled by ovnkube on the DPU where OVS runs) +func canHandleBridgeEgressIP() bool { + return util.IsNetworkSegmentationSupportEnabled() && + config.OVNKubernetesFeature.EnableInterconnect && + config.Gateway.Mode != config.GatewayModeDisabled && + config.OvnKubeNode.Mode != types.NodeModeDPUHost +} + func (g *gateway) AddEgressIP(eip *egressipv1.EgressIP) error { - if !util.IsNetworkSegmentationSupportEnabled() || !config.OVNKubernetesFeature.EnableInterconnect || config.Gateway.Mode == config.GatewayModeDisabled { + if !canHandleBridgeEgressIP() { return nil } isSyncRequired, err := g.bridgeEIPAddrManager.AddEgressIP(eip) @@ -251,7 +264,7 @@ func (g *gateway) AddEgressIP(eip *egressipv1.EgressIP) error { } func (g *gateway) UpdateEgressIP(oldEIP, newEIP *egressipv1.EgressIP) error { - if !util.IsNetworkSegmentationSupportEnabled() || !config.OVNKubernetesFeature.EnableInterconnect || config.Gateway.Mode == config.GatewayModeDisabled { + if !canHandleBridgeEgressIP() { return nil } isSyncRequired, err := g.bridgeEIPAddrManager.UpdateEgressIP(oldEIP, newEIP) @@ -267,7 +280,7 @@ func (g *gateway) UpdateEgressIP(oldEIP, newEIP *egressipv1.EgressIP) error { } func (g *gateway) DeleteEgressIP(eip *egressipv1.EgressIP) error { - if !util.IsNetworkSegmentationSupportEnabled() || !config.OVNKubernetesFeature.EnableInterconnect || config.Gateway.Mode == config.GatewayModeDisabled { + if !canHandleBridgeEgressIP() { return nil } isSyncRequired, err := g.bridgeEIPAddrManager.DeleteEgressIP(eip) @@ -283,7 +296,7 @@ func (g *gateway) DeleteEgressIP(eip *egressipv1.EgressIP) error { } func (g *gateway) SyncEgressIP(eips []interface{}) error { - if !util.IsNetworkSegmentationSupportEnabled() || !config.OVNKubernetesFeature.EnableInterconnect || config.Gateway.Mode == config.GatewayModeDisabled { + if !canHandleBridgeEgressIP() { return nil } if err := g.bridgeEIPAddrManager.SyncEgressIP(eips); err != nil { diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index 9b49f54adc..f0eb9094d6 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -75,7 +75,7 @@ func getGatewayNextHops() ([]net.IP, string, error) { } } gatewayIntf := config.Gateway.Interface - if gatewayIntf != "" { + if gatewayIntf != "" && config.OvnKubeNode.Mode != types.NodeModeDPUHost { if bridgeName, _, err := util.RunOVSVsctl("port-to-br", gatewayIntf); err == nil { // This is an OVS bridge's internal port gatewayIntf = bridgeName @@ -475,6 +475,10 @@ func (nc *DefaultNodeNetworkController) initGatewayDPUHostPreStart(kubeNodeIP ne return err } + // In DPU-host mode, bridgeEIPAddrManager is not initialized because: + // - There's no OVS on the host (it runs on the DPU) + // - Traffic is handled on the DPU which has the EgressIP configuration + // - There's no openflow manager to use the mark-to-IP cache nc.Gateway = &gateway{ initFunc: func() error { return nil }, readyFunc: func() (bool, error) { return true, nil }, diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index b06df1331b..de5d1ee235 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -892,6 +892,9 @@ func (npw *nodePortWatcher) AddService(service *corev1.Service) error { netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(service.Namespace) if err != nil { + if util.IsInvalidPrimaryNetworkError(err) { + return nil + } return fmt.Errorf("error getting active network for service %s in namespace %s: %w", service.Name, service.Namespace, err) } @@ -974,6 +977,9 @@ func (npw *nodePortWatcher) UpdateService(old, new *corev1.Service) error { netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(new.Namespace) if err != nil { + if util.IsInvalidPrimaryNetworkError(err) { + return utilerrors.Join(errors...) + } return fmt.Errorf("error getting active network for service %s in namespace %s: %w", new.Name, new.Namespace, err) } @@ -1405,11 +1411,6 @@ func (npw *nodePortWatcher) DeleteEndpointSlice(epSlice *discovery.EndpointSlice } localEndpoints := npw.GetLocalEligibleEndpointAddresses(epSlices, svc) if svcConfig, exists := npw.updateServiceInfo(*namespacedName, nil, &hasLocalHostNetworkEp, localEndpoints); exists { - netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(namespacedName.Namespace) - if err != nil { - return fmt.Errorf("error getting active network for service %s/%s: %w", namespacedName.Namespace, namespacedName.Name, err) - } - // Lock the cache mutex here so we don't miss a service delete during an endpoint delete // we have to do this because deleting and adding iptables rules is slow. npw.serviceInfoLock.Lock() @@ -1418,6 +1419,28 @@ func (npw *nodePortWatcher) DeleteEndpointSlice(epSlice *discovery.EndpointSlice if err = delServiceRules(svcConfig.service, svcConfig.localEndpoints, npw); err != nil { errors = append(errors, err) } + + // Get network info after deleting old rules, before adding new ones. + // This ensures old rules are cleaned up even if namespace/network is deleted, + // and allows graceful handling of deletion race conditions. + netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(namespacedName.Namespace) + if err != nil { + // If the namespace was deleted, skip adding new service rules + if apierrors.IsNotFound(err) { + klog.V(5).Infof("Namespace not found for service %s/%s during endpoint slice delete, skipping adding service rules", + namespacedName.Namespace, namespacedName.Name) + return utilerrors.Join(errors...) + } + // If the UDN was deleted, skip adding new service rules + if util.IsInvalidPrimaryNetworkError(err) { + klog.V(5).Infof("Skipping addServiceRules for %s/%s during endpoint slice delete: primary network invalid: %v", + namespacedName.Namespace, namespacedName.Name, err) + return utilerrors.Join(errors...) + } + errors = append(errors, fmt.Errorf("error getting active network for service %s/%s: %w", namespacedName.Namespace, namespacedName.Name, err)) + return utilerrors.Join(errors...) + } + if err = addServiceRules(svcConfig.service, netInfo, localEndpoints, hasLocalHostNetworkEp, npw); err != nil { errors = append(errors, err) } @@ -1543,6 +1566,9 @@ func (npwipt *nodePortWatcherIptables) AddService(service *corev1.Service) error netInfo, err := npwipt.networkManager.GetActiveNetworkForNamespace(service.Namespace) if err != nil { + if util.IsInvalidPrimaryNetworkError(err) { + return nil + } return fmt.Errorf("error getting active network for service %s in namespace %s: %w", service.Name, service.Namespace, err) } @@ -1571,6 +1597,9 @@ func (npwipt *nodePortWatcherIptables) UpdateService(old, new *corev1.Service) e if util.ServiceTypeHasClusterIP(new) && util.IsClusterIPSet(new) { netInfo, err := npwipt.networkManager.GetActiveNetworkForNamespace(new.Namespace) if err != nil { + if util.IsInvalidPrimaryNetworkError(err) { + return utilerrors.Join(errors...) + } return fmt.Errorf("error getting active network for service %s in namespace %s: %w", new.Name, new.Namespace, err) } diff --git a/go-controller/pkg/node/gateway_shared_intf_test.go b/go-controller/pkg/node/gateway_shared_intf_test.go new file mode 100644 index 0000000000..065b7c52ad --- /dev/null +++ b/go-controller/pkg/node/gateway_shared_intf_test.go @@ -0,0 +1,258 @@ +//go:build linux +// +build linux + +package node + +import ( + "fmt" + + nadfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" + + corev1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/fake" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + adminpolicybasedrouteclient "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/clientset/versioned/fake" + udnfakeclient "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/fake" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Note: Local mocks are used instead of FakeNetworkManager to test specific error conditions +// (NotFound, InvalidPrimaryNetworkError) from GetActiveNetworkForNamespace. FakeNetworkManager +// doesn't support error injection. And the tests here are not dependent on the methods that +// FakeNetworkManager implements. If more node tests need this, we will enhance FakeNetworkManager. + +// mockNetworkManagerWithNamespaceNotFoundError simulates namespace deletion race condition +type mockNetworkManagerWithNamespaceNotFoundError struct { + networkmanager.Interface +} + +func (m *mockNetworkManagerWithNamespaceNotFoundError) GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { + notFoundErr := apierrors.NewNotFound(schema.GroupResource{Resource: "namespaces"}, namespace) + return nil, fmt.Errorf("failed to get namespace %q: %w", namespace, notFoundErr) +} + +// mockNetworkManagerWithInvalidPrimaryNetworkError simulates UDN deletion scenario +type mockNetworkManagerWithInvalidPrimaryNetworkError struct { + networkmanager.Interface +} + +func (m *mockNetworkManagerWithInvalidPrimaryNetworkError) GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { + return nil, util.NewInvalidPrimaryNetworkError(namespace) +} + +// mockNetworkManagerWithError tests that non-graceful errors are properly propagated +type mockNetworkManagerWithError struct { + networkmanager.Interface +} + +func (m *mockNetworkManagerWithError) GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { + return nil, fmt.Errorf("network lookup failed for namespace %q", namespace) +} + +// verifyIPTablesRule checks if an iptables rule exists and asserts the expected state +func verifyIPTablesRule(ipt util.IPTablesHelper, serviceIP string, servicePort, nodePort int32, shouldExist bool, message string) { + exists, err := ipt.Exists("nat", "OVN-KUBE-NODEPORT", + "-p", "TCP", "-m", "addrtype", "--dst-type", "LOCAL", + "--dport", fmt.Sprintf("%d", nodePort), "-j", "DNAT", + "--to-destination", fmt.Sprintf("%s:%d", serviceIP, servicePort)) + Expect(err).NotTo(HaveOccurred()) + if shouldExist { + Expect(exists).To(BeTrue(), message) + } else { + Expect(exists).To(BeFalse(), message) + } +} + +// setupServiceAndEndpointSliceWithRules creates a service and endpoint slice, adds them to npw, +// and verifies iptables rules are created. Returns the created endpoint slice. +func setupServiceAndEndpointSliceWithRules(npw *nodePortWatcher, ipt util.IPTablesHelper, svcName, namespace, serviceIP, endpointIP string, servicePort, nodePort int32, annotations map[string]string) *discovery.EndpointSlice { + // Create service + service := newService(svcName, namespace, serviceIP, + []corev1.ServicePort{{ + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: servicePort, + TargetPort: intstr.FromInt(int(servicePort) + 8000), // e.g., 80 -> 8080 + NodePort: nodePort, + }}, + corev1.ServiceTypeNodePort, nil, corev1.ServiceStatus{}, false, false) + + // Create endpoint slice with endpoints + epPortName := "http" + epPortValue := servicePort + 8000 // Match targetPort + epPortProtocol := corev1.ProtocolTCP + epSlice := newEndpointSlice( + svcName, + namespace, + []discovery.Endpoint{ + { + Addresses: []string{endpointIP}, + }, + }, + []discovery.EndpointPort{ + { + Name: &epPortName, + Protocol: &epPortProtocol, + Port: &epPortValue, + }, + }, + ) + + // Apply annotations if provided + if len(annotations) > 0 { + if epSlice.Annotations == nil { + epSlice.Annotations = make(map[string]string) + } + for k, v := range annotations { + epSlice.Annotations[k] = v + } + } + + // Add service and endpoint slice + err := npw.AddService(service) + Expect(err).NotTo(HaveOccurred()) + + err = npw.AddEndpointSlice(epSlice) + Expect(err).NotTo(HaveOccurred()) + + // Verify iptables rules were created + verifyIPTablesRule(ipt, serviceIP, servicePort, nodePort, true, "iptables rule should exist before deletion") + + return epSlice +} + +var _ = Describe("DeleteEndpointSlice", func() { + var ( + fakeClient *util.OVNNodeClientset + watcher *factory.WatchFactory + npw *nodePortWatcher + iptV4 util.IPTablesHelper + iptV6 util.IPTablesHelper + ) + + const ( + nodeName = "test-node" + testNamespace = "test-namespace" + testService = "test-service" + ) + + BeforeEach(func() { + var err error + // Restore global default values before each test + Expect(config.PrepareTestConfig()).To(Succeed()) + config.Gateway.Mode = config.GatewayModeLocal + config.IPv4Mode = true + config.IPv6Mode = false + + fakeClient = &util.OVNNodeClientset{ + KubeClient: fake.NewSimpleClientset(), + } + fakeClient.AdminPolicyRouteClient = adminpolicybasedrouteclient.NewSimpleClientset() + fakeClient.NetworkAttchDefClient = nadfake.NewSimpleClientset() + fakeClient.UserDefinedNetworkClient = udnfakeclient.NewSimpleClientset() + + watcher, err = factory.NewNodeWatchFactory(fakeClient, nodeName) + Expect(err).NotTo(HaveOccurred()) + err = watcher.Start() + Expect(err).NotTo(HaveOccurred()) + + // Initialize nodePortWatcher with default network manager + iptV4, iptV6 = util.SetFakeIPTablesHelpers() + npw = initFakeNodePortWatcher(iptV4, iptV6) + npw.watchFactory = watcher + npw.networkManager = networkmanager.Default().Interface() + + // Initialize nodeIPManager (required for GetLocalEligibleEndpointAddresses) + k := &kube.Kube{KClient: fakeClient.KubeClient} + npw.nodeIPManager = newAddressManagerInternal(nodeName, k, nil, watcher, nil, false) + }) + + AfterEach(func() { + watcher.Shutdown() + }) + + Context("when UDN is deleted before processing endpoint slice", func() { + It("should execute delServiceRules and gracefully skip addServiceRules", func() { + // Setup service and endpoint slice with iptables rules + // Add UDN annotation to simulate a mirrored UDN EndpointSlice + epSlice := setupServiceAndEndpointSliceWithRules(npw, iptV4, testService, testNamespace, "10.96.0.2", "10.244.0.2", 80, 30081, + map[string]string{types.UserDefinedNetworkEndpointSliceAnnotation: "test-udn"}) + + // Replace network manager with one that returns InvalidPrimaryNetworkError + // This simulates UDN deletion scenario + npw.networkManager = &mockNetworkManagerWithInvalidPrimaryNetworkError{} + + // Call DeleteEndpointSlice - should not return error + err := npw.DeleteEndpointSlice(epSlice) + + // Should gracefully handle UDN deletion (no error) + Expect(err).NotTo(HaveOccurred()) + + // iptables rules should be deleted even when UDN is deleted + verifyIPTablesRule(iptV4, "10.96.0.2", 80, 30081, false, "iptables rule should be deleted even when UDN is deleted") + }) + }) + + Context("when network lookup returns other errors", func() { + It("should execute delServiceRules but return error from network lookup", func() { + // Setup service and endpoint slice with iptables rules + epSlice := setupServiceAndEndpointSliceWithRules(npw, iptV4, testService, testNamespace, "10.96.0.3", "10.244.0.3", 80, 30082, nil) + + // Replace network manager with one that returns a generic error + npw.networkManager = &mockNetworkManagerWithError{} + + // Call DeleteEndpointSlice - should return error + err := npw.DeleteEndpointSlice(epSlice) + + // Should return error for other types of failures + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("error getting active network")) + Expect(err.Error()).To(ContainSubstring(testNamespace)) + Expect(err.Error()).To(ContainSubstring(testService)) + + // iptables rules should still be deleted even when error is returned + verifyIPTablesRule(iptV4, "10.96.0.3", 80, 30082, false, "iptables rule should be deleted even when error occurs") + }) + }) + + Context("when service does not exist in cache", func() { + It("should return nil without error", func() { + // Create endpoint slice (but no service in cache) + epSlice := newEndpointSlice(testService, testNamespace, nil, nil) + + // Call DeleteEndpointSlice when service not in cache + err := npw.DeleteEndpointSlice(epSlice) + + // Should return nil (no-op when not in cache) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when namespace is deleted before processing endpoint slice", func() { + It("should clean up old rules even when namespace is gone", func() { + // Setup service and endpoint slice with iptables rules + epSlice := setupServiceAndEndpointSliceWithRules(npw, iptV4, testService, testNamespace, "10.96.0.10", "10.244.0.5", 80, 30090, nil) + + // Simulate namespace not found error + npw.networkManager = &mockNetworkManagerWithNamespaceNotFoundError{} + err := npw.DeleteEndpointSlice(epSlice) + // Verify no error (graceful handling) + Expect(err).NotTo(HaveOccurred()) + + // iptables rules should be deleted even though namespace lookup failed + verifyIPTablesRule(iptV4, "10.96.0.10", 80, 30090, false, "iptables rule should be deleted even when namespace lookup fails") + }) + }) +}) diff --git a/go-controller/pkg/node/iprulemanager/ip_rule_manager.go b/go-controller/pkg/node/iprulemanager/ip_rule_manager.go index 56c527c4ca..308cc23adc 100644 --- a/go-controller/pkg/node/iprulemanager/ip_rule_manager.go +++ b/go-controller/pkg/node/iprulemanager/ip_rule_manager.go @@ -2,6 +2,7 @@ package iprulemanager import ( "fmt" + "net" "sync" "time" @@ -190,9 +191,6 @@ func (rm *Controller) reconcile() error { } found = false for _, ruleWanted := range rm.rules { - if ruleWanted.rule.Priority != priority { - continue - } if areNetlinkRulesEqual(ruleWanted.rule, &ruleFound) { found = true break @@ -213,15 +211,42 @@ func (rm *Controller) reconcile() error { } func areNetlinkRulesEqual(r1, r2 *netlink.Rule) bool { - return r1.String() == r2.String() + if r1.Priority != r2.Priority { + return false + } + if r1.Table != r2.Table { + return false + } + if r1.Type != r2.Type { + return false + } + if r1.Mark != r2.Mark { + return false + } + + return areIPNetsEqual(r1.Src, r2.Src) && areIPNetsEqual(r1.Dst, r2.Dst) +} + +func areIPNetsEqual(n1, n2 *net.IPNet) bool { + if n1 == nil && n2 == nil { + return true + } + if n1 == nil || n2 == nil { + return false + } + + if !n1.IP.Equal(n2.IP) { + return false + } + + n1ones, n1bits := n1.Mask.Size() + n2ones, n2bits := n2.Mask.Size() + return n1ones == n2ones && n1bits == n2bits } func isNetlinkRuleInSlice(rules []netlink.Rule, candidate *netlink.Rule) (bool, *netlink.Rule) { for _, r := range rules { r := r - if r.Priority != candidate.Priority { - continue - } if areNetlinkRulesEqual(&r, candidate) { return true, &r } diff --git a/go-controller/pkg/node/linkmanager/link_network_manager.go b/go-controller/pkg/node/linkmanager/link_network_manager.go index d43442e4d8..964f5765cb 100644 --- a/go-controller/pkg/node/linkmanager/link_network_manager.go +++ b/go-controller/pkg/node/linkmanager/link_network_manager.go @@ -1,11 +1,13 @@ package linkmanager import ( + "errors" "fmt" "sync" "time" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" @@ -152,7 +154,7 @@ func (c *Controller) DelAddress(address netlink.Addr) error { c.mu.Lock() defer c.mu.Unlock() if err := util.GetNetLinkOps().AddrDel(link, &address); err != nil { - if !util.GetNetLinkOps().IsLinkNotFoundError(err) { + if !util.GetNetLinkOps().IsLinkNotFoundError(err) && !errors.Is(err, unix.EADDRNOTAVAIL) { return fmt.Errorf("failed to delete address %s: %v", address.String(), err) } } diff --git a/go-controller/pkg/node/ovspinning/ovspinning_linux.go b/go-controller/pkg/node/ovspinning/ovspinning_linux.go index 1f4cf78290..f40ce48b8f 100644 --- a/go-controller/pkg/node/ovspinning/ovspinning_linux.go +++ b/go-controller/pkg/node/ovspinning/ovspinning_linux.go @@ -5,6 +5,7 @@ package ovspinning import ( "bytes" + "context" "fmt" "os" "path/filepath" @@ -15,7 +16,12 @@ import ( "github.com/fsnotify/fsnotify" "golang.org/x/sys/unix" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/klog/v2" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" + "k8s.io/utils/cpuset" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -25,11 +31,13 @@ var tickDuration time.Duration = 1 * time.Second var getOvsVSwitchdPIDFn func() (string, error) = util.GetOvsVSwitchdPID var getOvsDBServerPIDFn func() (string, error) = util.GetOvsDBServerPID var featureEnablerFile string = "/etc/openvswitch/enable_dynamic_cpu_affinity" +var kubeletConfigFilePath = "/host/etc/kubernetes/kubelet.conf" // Run monitors OVS daemon's processes (ovs-vswitchd and ovsdb-server) and sets their CPU affinity // masks to that of the current process. // This feature is enabled by the presence of a non-empty file in the path `/etc/openvswitch/enable_dynamic_cpu_affinity` -func Run(stopCh <-chan struct{}) { +// we're passing the podResCli from the caller, so we could support unit-tests +func Run(ctx context.Context, stopCh <-chan struct{}, podResCli podresourcesapi.PodResourcesListerClient) { // The file must be present at startup to enable the feature isFeatureEnabled, err := isFileNotEmpty(featureEnablerFile) @@ -61,6 +69,20 @@ func Run(stopCh <-chan struct{}) { defer fileWatcher.Close() } + // we only need to check reservedSystemCPUs once at startup. + // any change to KubeletConfig file triggers a node reboot, which also restarts the ovnkube-node pod. + // as a result, this logic is re-executed automatically after every change. + reservedCPUs, err := getReservedCPUs(kubeletConfigFilePath) + if err != nil { + klog.Warningf("Failed to get reservedSystemCPUs from kubelet config file on: %q: err=%v\n.Falling back to detect reserved from system", kubeletConfigFilePath, err) + reservedCPUs, err = getReservedCPUsFallback(ctx, podResCli) + if err != nil { + klog.Warningf("Fallback method to obtain reservedSystemCPUs failed. err=%v", err) + return + } + } + klog.Infof("OVS CPU dynamic pinning reservedSystemCPUs set: %s", reservedCPUs) + ticker := time.NewTicker(tickDuration) defer ticker.Stop() @@ -100,13 +122,18 @@ func Run(stopCh <-chan struct{}) { if !isFeatureEnabled { continue } - - err := setOvsVSwitchdCPUAffinity() + cpus, err := getNonPinnedCPUs(ctx, podResCli) + if err != nil { + klog.Warningf("Error while trying to get system non pinned CPUs: %v", err) + } + // add reservedSystemCPUs as well, because PodResourcesAPI does not count for them. + cpus = cpus.Union(reservedCPUs) + err = setOvsVSwitchdCPUAffinity(&cpus) if err != nil { klog.Warningf("Error while aligning ovs-vswitchd CPUs to current process: %v", err) } - err = setOvsDBServerCPUAffinity() + err = setOvsDBServerCPUAffinity(&cpus) if err != nil { klog.Warningf("Error while aligning ovsdb-server CPUs to current process: %v", err) } @@ -114,15 +141,15 @@ func Run(stopCh <-chan struct{}) { } } -func createFileWatcherFor(filename string) (*fsnotify.Watcher, error) { +func createFileWatcherFor(path string) (*fsnotify.Watcher, error) { fileWatcher, err := fsnotify.NewWatcher() if err != nil { return nil, fmt.Errorf("failed to create filesystem watcher: %w", err) } - err = fileWatcher.Add(filename) + err = fileWatcher.Add(path) if err != nil { - return nil, fmt.Errorf("unable to watch [%s] file: %w", filename, err) + return nil, fmt.Errorf("unable to watch [%s] path: %w", path, err) } return fileWatcher, nil @@ -141,7 +168,7 @@ func isFileNotEmpty(filename string) (bool, error) { return f.Size() > 0, nil } -func setOvsVSwitchdCPUAffinity() error { +func setOvsVSwitchdCPUAffinity(set *cpuset.CPUSet) error { ovsVSwitchdPID, err := getOvsVSwitchdPIDFn() if err != nil { @@ -149,10 +176,10 @@ func setOvsVSwitchdCPUAffinity() error { } klog.V(5).Infof("Managing ovs-vswitchd[%s] daemon CPU affinity", ovsVSwitchdPID) - return setProcessCPUAffinity(ovsVSwitchdPID) + return setProcessCPUAffinity(ovsVSwitchdPID, set) } -func setOvsDBServerCPUAffinity() error { +func setOvsDBServerCPUAffinity(set *cpuset.CPUSet) error { ovsDBserverPID, err := getOvsDBServerPIDFn() if err != nil { @@ -160,21 +187,43 @@ func setOvsDBServerCPUAffinity() error { } klog.V(5).Infof("Managing ovsdb-server[%s] daemon CPU affinity", ovsDBserverPID) - return setProcessCPUAffinity(ovsDBserverPID) + return setProcessCPUAffinity(ovsDBserverPID, set) } -// setProcessCPUAffinity sets the CPU affinity of the given process to the same affinity as the current process -func setProcessCPUAffinity(targetPIDStr string) error { +// setProcessCPUAffinity sets the CPU affinity of a target process and all its threads +// to the specified CPU set. If the provided CPU set is empty, it falls back to using +// the current process's CPU affinity as the desired affinity. +// +// The function operates at the thread level, iterating through all threads (tasks) +// of the target process and setting their individual CPU affinities. This ensures +// that both the main process and any spawned threads are properly pinned to the +// specified CPUs. +// +// Parameters: +// - targetPIDStr: string representation of the target process ID +// - set: pointer to the desired CPU set; if empty, current process affinity is used +// +// Returns: +// - error: any error encountered during PID conversion, affinity retrieval, or setting +// +// The function skips setting affinity if the target process already has the desired +// CPU affinity. Individual thread affinity setting failures are logged as warnings +// but don't stop the overall operation. +func setProcessCPUAffinity(targetPIDStr string, set *cpuset.CPUSet) error { targetPID, err := strconv.Atoi(targetPIDStr) if err != nil { return fmt.Errorf("can't convert PID[%s] to integer: %w", targetPIDStr, err) } - var currentProcessCPUs unix.CPUSet - err = unix.SchedGetaffinity(os.Getpid(), ¤tProcessCPUs) - if err != nil { - return fmt.Errorf("can't get own CPU affinity") + desiredProcessCPUs := convertCPUSet(set) + if set.IsEmpty() { + selfPID := os.Getpid() + klog.InfoS("Given CPU set is empty, setting self CPU affinity", "selfPID", selfPID, "targetPID", targetPID) + err = unix.SchedGetaffinity(selfPID, &desiredProcessCPUs) + if err != nil { + return fmt.Errorf("can't get own CPU affinity") + } } var targetProcessCPUs unix.CPUSet @@ -183,8 +232,8 @@ func setProcessCPUAffinity(targetPIDStr string) error { return fmt.Errorf("can't get process (PID:%d) CPU affinity: %w", targetPID, err) } - if currentProcessCPUs == targetProcessCPUs { - klog.V(5).Infof("Process[%d] CPU affinity already match current process's affinity %s", targetPID, printCPUSet(currentProcessCPUs)) + if desiredProcessCPUs == targetProcessCPUs { + klog.V(5).Infof("Process[%d] CPU affinity already matches desired process affinity %s", targetPID, printCPUSet(desiredProcessCPUs)) return nil } @@ -193,12 +242,12 @@ func setProcessCPUAffinity(targetPIDStr string) error { return fmt.Errorf("can't get tasks of PID(%d):%w", targetPID, err) } - klog.Infof("Setting CPU affinity of PID(%d) (ntasks=%d) to %s, was %s", targetPID, len(taskIDs), printCPUSet(currentProcessCPUs), printCPUSet(targetProcessCPUs)) + klog.Infof("Setting CPU affinity of PID(%d) (ntasks=%d) to %s, was %s", targetPID, len(taskIDs), printCPUSet(desiredProcessCPUs), printCPUSet(targetProcessCPUs)) for _, taskID := range taskIDs { - err = unix.SchedSetaffinity(taskID, ¤tProcessCPUs) + err = unix.SchedSetaffinity(taskID, &desiredProcessCPUs) if err != nil { // The task may have been stopped, don't break the loop and continue setting CPU affinity on other tasks. - klog.Warningf("Error while setting CPU affinity of task(%d) PID(%d) to %s: %v", taskID, targetPID, printCPUSet(currentProcessCPUs), err) + klog.Warningf("Error while setting CPU affinity of task(%d) PID(%d) to %s: %v", taskID, targetPID, printCPUSet(desiredProcessCPUs), err) } } @@ -270,3 +319,152 @@ func getThreadsOfProcess(pid int) ([]int, error) { return ret, nil } + +func convertCPUSet(k8sSet *cpuset.CPUSet) unix.CPUSet { + var uSet unix.CPUSet + for _, cpu := range k8sSet.List() { + uSet.Set(cpu) + } + return uSet +} + +// getNonPinnedCPUs calculates and returns all allocatable CPUs on the node which are not +// exclusively pinned to any container. IOW it returns the CPUs that are dedicated for +// Burstable and BestEffort QoS containers +func getNonPinnedCPUs(ctx context.Context, podResCli podresourcesapi.PodResourcesListerClient) (cpuset.CPUSet, error) { + // Get allocatable CPUs + allocatableResp, err := podResCli.GetAllocatableResources(ctx, &podresourcesapi.AllocatableResourcesRequest{}) + if err != nil { + return cpuset.CPUSet{}, fmt.Errorf("GetAllocatableResources failed: %w", err) + } + allocatableCPUs := cpuset.New(convertInt64ToInt(allocatableResp.CpuIds)...) + + // List pod resources and collect used CPUs + listCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + listResp, err := podResCli.List(listCtx, &podresourcesapi.ListPodResourcesRequest{}) + if err != nil { + return cpuset.CPUSet{}, fmt.Errorf("ListPodResources failed: %w", err) + } + + usedCPUs := cpuset.New() + for _, pod := range listResp.PodResources { + for _, container := range pod.Containers { + usedCPUs = usedCPUs.Union(cpuset.New(convertInt64ToInt(container.CpuIds)...)) + } + } + + // Calculate the difference + availableCPUs := allocatableCPUs.Difference(usedCPUs) + return availableCPUs, nil +} + +func convertInt64ToInt(int64s []int64) []int { + ints := make([]int, len(int64s)) + for i, v := range int64s { + ints[i] = int(v) + } + return ints +} + +// getReservedCPUs reads a kubelet configuration file and extracts the ReservedSystemCPUs setting. +// It parses the kubelet config YAML/JSON file at the given path and returns the set of CPUs +// that are reserved for system use according to the kubelet configuration. +// +// Parameters: +// - path: filesystem path to the kubelet configuration file +// +// Returns: +// - cpuset.CPUSet: the set of CPUs reserved for system use +// - error: any error encountered while reading or parsing the configuration +// +// Note: An empty ReservedSystemCPUs field in the config is not considered an error, +// it simply returns an empty CPU set. +func getReservedCPUs(path string) (cpuset.CPUSet, error) { + scheme := runtime.NewScheme() + codecs := serializer.NewCodecFactory(scheme) + + if err := kubeletconfigv1beta1.AddToScheme(scheme); err != nil { + return cpuset.CPUSet{}, fmt.Errorf("failed to add kubelet config scheme: %w", err) + } + + data, err := os.ReadFile(path) + if err != nil { + return cpuset.CPUSet{}, fmt.Errorf("failed to read file: %s: %w", path, err) + } + + obj, _, err := codecs.UniversalDecoder(kubeletconfigv1beta1.SchemeGroupVersion).Decode(data, nil, nil) + if err != nil { + return cpuset.CPUSet{}, fmt.Errorf("failed to decode kubelet config: %w", err) + } + + kc, ok := obj.(*kubeletconfigv1beta1.KubeletConfiguration) + if !ok { + return cpuset.CPUSet{}, fmt.Errorf("decoded object is not a KubeletConfiguration") + } + + // kc.ReservedSystemCPUs could be empty. it's not a desired state, but not considered as an error either. + cset, err := cpuset.Parse(kc.ReservedSystemCPUs) + if err != nil { + return cpuset.CPUSet{}, fmt.Errorf("failed to parse reservedSystemCPUs: %w", err) + } + + return cset, nil +} + +// getReservedCPUsFallback determines the set of reserved CPUs by calculating the difference +// between online CPUs and allocatable CPUs. This method serves as a fallback when the +// kubelet configuration file is not available or cannot be parsed. +// +// The logic is: Reserved CPUs = Online CPUs - Allocatable CPUs +// This works because reserved CPUs are those that are online but not available for +// pod allocation by the kubelet. +// +// Parameters: +// - ctx: context for the operation +// - podResCli: client for querying the kubelet's pod resources API +// +// Returns: +// - cpuset.CPUSet: the set of CPUs reserved for system use +// - error: any error encountered while querying CPU information +func getReservedCPUsFallback(ctx context.Context, podResCli podresourcesapi.PodResourcesListerClient) (cpuset.CPUSet, error) { + onlineCPUs, err := getOnlineCPUs() + if err != nil { + return cpuset.CPUSet{}, fmt.Errorf("failed to get onlineCPUs CPUs %w", err) + } + allocatableCPUs, err := getAllocatableCPUs(ctx, podResCli) + if err != nil { + return cpuset.CPUSet{}, err + } + // online - allocatable is the reserved set + return onlineCPUs.Difference(allocatableCPUs), nil + +} + +// getOnlineCPUs retrieves the set of CPUs that are currently online on the system. +// It reads from the Linux sysfs interface at /sys/devices/system/cpu/online which +// contains a comma-separated list or range of CPU IDs that are currently online +// and available for use by the kernel. +// +// Returns: +// - cpuset.CPUSet: the set of CPUs that are currently online +// - error: any error encountered while reading the sysfs file or parsing the CPU list +// +// Example sysfs content: "0-3,8-11" (CPUs 0,1,2,3,8,9,10,11 are online) +func getOnlineCPUs() (cpuset.CPUSet, error) { + onlineCPUList, err := os.ReadFile("/sys/devices/system/cpu/online") + if err != nil { + return cpuset.CPUSet{}, err + } + return cpuset.Parse(strings.TrimSpace(string(onlineCPUList))) +} + +func getAllocatableCPUs(ctx context.Context, podResCli podresourcesapi.PodResourcesListerClient) (cpuset.CPUSet, error) { + getCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + allocatableResp, err := podResCli.GetAllocatableResources(getCtx, &podresourcesapi.AllocatableResourcesRequest{}) + if err != nil { + return cpuset.CPUSet{}, fmt.Errorf("GetAllocatableResources failed: %w", err) + } + return cpuset.New(convertInt64ToInt(allocatableResp.CpuIds)...), nil +} diff --git a/go-controller/pkg/node/ovspinning/ovspinning_linux_test.go b/go-controller/pkg/node/ovspinning/ovspinning_linux_test.go index 0c6c19cc89..4df7c8ec7f 100644 --- a/go-controller/pkg/node/ovspinning/ovspinning_linux_test.go +++ b/go-controller/pkg/node/ovspinning/ovspinning_linux_test.go @@ -8,104 +8,173 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "runtime" "sync" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" "k8s.io/klog/v2" + kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1" + "k8s.io/utils/cpuset" + + mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/kubelet/pkg/apis/podresources/v1" ) func TestAlignCPUAffinity(t *testing.T) { - ovsDBPid, ovsDBStop := mockOvsdbProcess(t) - defer ovsDBStop() - - ovsVSwitchdPid, ovsVSwitchdStop := mockOvsVSwitchdProcess(t) - defer ovsVSwitchdStop() - - defer setTickDuration(20 * time.Millisecond)() - defer mockFeatureEnableFile(t, "1")() - - var wg sync.WaitGroup - stopCh := make(chan struct{}) - defer func() { - close(stopCh) - wg.Wait() - }() - - wg.Add(1) - go func() { - // Be sure the system under test goroutine is finished before cleaning - defer wg.Done() - Run(stopCh) - }() - - var initialCPUset unix.CPUSet - err := unix.SchedGetaffinity(os.Getpid(), &initialCPUset) - require.NoError(t, err) - - defer func() { - // Restore any previous CPU affinity value it was in place before the test - err = unix.SchedSetaffinity(os.Getpid(), &initialCPUset) - assert.NoError(t, err) - }() - - assert.Greater(t, runtime.NumCPU(), 1) - - for i := 0; i < runtime.NumCPU(); i++ { - var tmpCPUset unix.CPUSet - tmpCPUset.Set(i) - err = unix.SchedSetaffinity(os.Getpid(), &tmpCPUset) - require.NoError(t, err) - - klog.Infof("Test CPU Affinity %x", tmpCPUset) - - assertPIDHasSchedAffinity(t, ovsVSwitchdPid, tmpCPUset) - assertPIDHasSchedAffinity(t, ovsDBPid, tmpCPUset) + testCases := []struct { + name string + allocatableCPUs []int64 + reservedCPUs []int + usedCPUs [][]int64 + }{ + { + name: "Simple split with some used", + allocatableCPUs: []int64{0, 1}, + reservedCPUs: []int{2, 3}, + usedCPUs: [][]int64{{1}}, + }, + { + name: "All allocatable used", + allocatableCPUs: []int64{0, 1}, + reservedCPUs: []int{2, 3}, + usedCPUs: [][]int64{{0}, {1}}, + }, + { + name: "No used CPUs", + allocatableCPUs: []int64{0, 1}, + reservedCPUs: []int{2, 3}, + usedCPUs: [][]int64{}, + }, + { + name: "Partial usage with multiple containers", + allocatableCPUs: []int64{0, 1, 2}, + reservedCPUs: []int{3}, + usedCPUs: [][]int64{{0}, {2}}, + }, + { + name: "low cpu capacity: Simple split with some used", + allocatableCPUs: []int64{1, 2, 3}, + reservedCPUs: []int{0}, + usedCPUs: [][]int64{{3}}, + }, + { + name: "Empty. should use self affinity", + allocatableCPUs: []int64{}, + reservedCPUs: []int{}, + usedCPUs: [][]int64{}, + }, } - // Disable the feature by making the enabler file empty - err = os.WriteFile(featureEnablerFile, []byte(""), 0) - require.NoError(t, err) - - var tmpCPUset unix.CPUSet - tmpCPUset.Set(0) - err = unix.SchedSetaffinity(os.Getpid(), &tmpCPUset) - require.NoError(t, err) - - assertNeverPIDHasSchedAffinity(t, ovsVSwitchdPid, tmpCPUset) - assertNeverPIDHasSchedAffinity(t, ovsDBPid, tmpCPUset) - - // Enable the feature back by putting contents in the enabler file - err = os.WriteFile(featureEnablerFile, []byte("1"), 0) - require.NoError(t, err) - - assertPIDHasSchedAffinity(t, ovsVSwitchdPid, tmpCPUset) - assertPIDHasSchedAffinity(t, ovsDBPid, tmpCPUset) - - // Disable the feature by deleting the enabler file - klog.Infof("Remove the enabler file to disable the feature") - err = os.Remove(featureEnablerFile) - require.NoError(t, err) - - tmpCPUset.Set(1) - err = unix.SchedSetaffinity(os.Getpid(), &tmpCPUset) - require.NoError(t, err) - - assertNeverPIDHasSchedAffinity(t, ovsVSwitchdPid, tmpCPUset) - assertNeverPIDHasSchedAffinity(t, ovsDBPid, tmpCPUset) - - // Re-enable the feature back by recreating the enabler file - klog.Infof("Re-enable the feature") - err = os.WriteFile(featureEnablerFile, []byte("1"), 0) - require.NoError(t, err) - - assertPIDHasSchedAffinity(t, ovsVSwitchdPid, tmpCPUset) - assertPIDHasSchedAffinity(t, ovsDBPid, tmpCPUset) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + expectedCPUs := calculateExpectedCPUs(tc.allocatableCPUs, tc.reservedCPUs, tc.usedCPUs) + // check that we can run this test on the tested machine + numCPUs := runtime.NumCPU() + if !expectedCPUsValid(expectedCPUs.List(), numCPUs) { + t.Skipf("Skipping test case %q: CPU ID out of range for this machine (have %d CPUs)", tc.name, numCPUs) + } + + ovsDBPid, ovsDBStop := mockOvsdbProcess(t) + defer ovsDBStop() + + ovsVSwitchdPid, ovsVSwitchdStop := mockOvsVSwitchdProcess(t) + defer ovsVSwitchdStop() + + defer setTickDuration(20 * time.Millisecond)() + defer mockFeatureEnableFile(t, "1")() + defer mockKubeletConfigFile(t, cpuset.New(tc.reservedCPUs...))() + + var wg sync.WaitGroup + stopCh := make(chan struct{}) + defer func() { + close(stopCh) + wg.Wait() + }() + + wg.Add(1) + go func() { + // Be sure the system under test goroutine is finished before cleaning + defer wg.Done() + mockClient := mocks.NewPodResourcesListerClient(t) + mockClient.On("GetAllocatableResources", mock.Anything, mock.Anything).Return( + &kubeletpodresourcesv1.AllocatableResourcesResponse{CpuIds: tc.allocatableCPUs}, nil) + mockClient.On("List", mock.Anything, mock.Anything).Return( + buildListPodResourcesResponse(tc.usedCPUs), nil) + Run(context.Background(), stopCh, mockClient) + }() + + expectedUnixCPUSet := convertCPUSet(&expectedCPUs) + if expectedCPUs.IsEmpty() { + klog.Info("expectedCPUs is empty, using running process's self-affinity") + // use self-affinity + var pidSelfCPUs unix.CPUSet + err := unix.SchedGetaffinity(os.Getpid(), &pidSelfCPUs) + require.NoError(t, err) + + expectedCPUs, err = convertUnixCPUSetToK8sCPUSet(pidSelfCPUs) + assert.NoError(t, err) + expectedUnixCPUSet = pidSelfCPUs + + } + klog.Infof("Test CPU Affinity %s", expectedCPUs) + + assertPIDHasSchedAffinity(t, ovsVSwitchdPid, expectedUnixCPUSet) + assertPIDHasSchedAffinity(t, ovsDBPid, expectedUnixCPUSet) + + // Disable the feature by making the enabler file empty + err := os.WriteFile(featureEnablerFile, []byte(""), 0) + require.NoError(t, err) + + // wait for the ovspinning loop to stabilize and stop running + time.Sleep(1 * time.Second) + + var tmpCPUset unix.CPUSet + tmpCPUset.Set(0) + assert.NoError(t, unix.SchedSetaffinity(ovsVSwitchdPid, &tmpCPUset)) + assert.NoError(t, unix.SchedSetaffinity(ovsDBPid, &tmpCPUset)) + + // Should not set the affinity back + assertNeverPIDHasSchedAffinity(t, ovsVSwitchdPid, expectedUnixCPUSet) + assertNeverPIDHasSchedAffinity(t, ovsDBPid, expectedUnixCPUSet) + + // Enable the feature back by putting contents in the enabler file + err = os.WriteFile(featureEnablerFile, []byte("1"), 0) + require.NoError(t, err) + + assertPIDHasSchedAffinity(t, ovsVSwitchdPid, expectedUnixCPUSet) + assertPIDHasSchedAffinity(t, ovsDBPid, expectedUnixCPUSet) + + // Disable the feature by deleting the enabler file + klog.Infof("Remove the enabler file to disable the feature") + err = os.Remove(featureEnablerFile) + require.NoError(t, err) + + // wait for the ovspinning loop to stabilize and stop running + time.Sleep(1 * time.Second) + + tmpCPUset.Set(1) + assert.NoError(t, unix.SchedSetaffinity(ovsVSwitchdPid, &tmpCPUset)) + assert.NoError(t, unix.SchedSetaffinity(ovsDBPid, &tmpCPUset)) + + // Should not set the affinity back + assertNeverPIDHasSchedAffinity(t, ovsVSwitchdPid, expectedUnixCPUSet) + assertNeverPIDHasSchedAffinity(t, ovsDBPid, expectedUnixCPUSet) + + // Re-enable the feature back by recreating the enabler file + klog.Infof("Re-enable the feature") + err = os.WriteFile(featureEnablerFile, []byte("1"), 0) + require.NoError(t, err) + + assertPIDHasSchedAffinity(t, ovsVSwitchdPid, expectedUnixCPUSet) + assertPIDHasSchedAffinity(t, ovsDBPid, expectedUnixCPUSet) + }) + } } func TestIsFileNotEmpty(t *testing.T) { @@ -129,12 +198,12 @@ func TestIsFileNotEmpty(t *testing.T) { func TestPrintCPUSetAll(t *testing.T) { var x unix.CPUSet - for i := 0; i < 16; i++ { + for i := 0; i < 4; i++ { x.Set(i) } assert.Equal(t, - "0-15", + "0-3", printCPUSet(x), ) @@ -146,19 +215,110 @@ func TestPrintCPUSetAll(t *testing.T) { func TestPrintCPUSetRanges(t *testing.T) { var x unix.CPUSet + x.Set(0) x.Set(2) x.Set(3) - x.Set(6) - x.Set(7) - x.Set(8) - x.Set(14) assert.Equal(t, - "2-3,6-8,14", + "0,2-3", printCPUSet(x), ) } +func TestGetReservedCPUs(t *testing.T) { + tests := []struct { + name string + yamlContent string + expectError bool + expectedCPUSet string + expectedIsEmpty bool + }{ + { + name: "valid config", + yamlContent: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +reservedSystemCPUs: "0-1,3" +`, + expectError: false, + expectedCPUSet: "0-1,3", + }, + { + name: "empty reservedSystemCPUs", + yamlContent: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +`, + expectError: false, + expectedIsEmpty: true, + }, + { + name: "invalid cpuset string", + yamlContent: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +reservedSystemCPUs: "not-a-valid-range" +`, + expectError: true, + }, + { + name: "invalid YAML format", + yamlContent: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +reservedSystemCPUs: [0,1 +`, + expectError: true, + }, + { + name: "file not found", + yamlContent: "", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var path string + if tt.name == "file not found" { + path = "/nonexistent/path.yaml" + } else { + path = writeTempFile(t, tt.yamlContent) + } + + cset, err := getReservedCPUs(path) + if tt.expectError { + if err == nil { + t.Errorf("expected error, got none") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if tt.expectedIsEmpty && !cset.IsEmpty() { + t.Errorf("expected empty cpuset, got %s", cset.String()) + } else if tt.expectedCPUSet != "" { + expected, _ := cpuset.Parse(tt.expectedCPUSet) + if !cset.Equals(expected) { + t.Errorf("expected cpuset %s, got %s", expected.String(), cset.String()) + } + } + }) + } +} + +func writeTempFile(t *testing.T, content string) string { + t.Helper() + tmp := t.TempDir() + path := filepath.Join(tmp, "kubelet-config.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("failed to write temp file: %v", err) + } + return path +} + func mockOvsdbProcess(t *testing.T) (int, func()) { t.Helper() ctx, stopCmd := context.WithCancel(context.Background()) @@ -227,7 +387,29 @@ func mockFeatureEnableFile(t *testing.T, data string) func() { return func() { featureEnablerFile = previousValue - os.Remove(f.Name()) + _ = os.Remove(f.Name()) + } +} + +func mockKubeletConfigFile(t *testing.T, reservedCPUs cpuset.CPUSet) func() { + t.Helper() + f, err := os.CreateTemp("", "kubelet.conf") + require.NoError(t, err) + + previousValue := kubeletConfigFilePath + kubeletConfigFilePath = f.Name() + + data := fmt.Sprintf(` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +reservedSystemCPUs: %q +`, reservedCPUs) + err = os.WriteFile(kubeletConfigFilePath, []byte(data), 0) + assert.NoError(t, err) + + return func() { + kubeletConfigFilePath = previousValue + _ = os.Remove(f.Name()) } } @@ -239,7 +421,7 @@ func assertPIDHasSchedAffinity(t *testing.T, pid int, expectedCPUSet unix.CPUSet assert.NoError(t, err) return actual == expectedCPUSet - }, time.Second, 10*time.Millisecond, "pid[%d] Expected CPUSet %0x != Actual CPUSet %0x", pid, expectedCPUSet, actual) + }, 2*time.Second, 10*time.Millisecond, "pid[%d] Expected CPUSet %#x != Actual CPUSet %#x", pid, expectedCPUSet, actual) tasks, err := getThreadsOfProcess(pid) require.NoError(t, err) @@ -260,7 +442,79 @@ func assertNeverPIDHasSchedAffinity(t *testing.T, pid int, targetCPUSet unix.CPU assert.Never(t, func() bool { err := unix.SchedGetaffinity(pid, &actual) assert.NoError(t, err) - return actual == targetCPUSet - }, time.Second, 10*time.Millisecond, "pid[%d] == Actual CPUSet %0x expected to be different than %0x", pid, actual, targetCPUSet) + }, 1*time.Second, 10*time.Millisecond, "pid[%d] == Actual CPUSet %#x expected to be different than %#x", pid, actual, targetCPUSet) +} + +// convertUnixCPUSetToK8sCPUSet converts a unix.CPUSet to a k8s.io/utils/cpuset.CPUSet +func convertUnixCPUSetToK8sCPUSet(unixSet unix.CPUSet) (cpuset.CPUSet, error) { + var cpus []int + const maxCPUs = 1024 // Maximum CPUs supported by unix.CPUSet (CPU_SETSIZE on Linux) + for i := 0; i < maxCPUs; i++ { + if unixSet.IsSet(i) { + cpus = append(cpus, i) + } + } + if len(cpus) == 0 { + return cpuset.CPUSet{}, fmt.Errorf("no CPUs found in unix.CPUSet") + } + return cpuset.New(cpus...), nil +} + +// calculateExpectedCPUs computes (allocatable ∪ reserved) - used +func calculateExpectedCPUs(allocatableCPUs []int64, reservedCPUs []int, usedCPUs [][]int64) cpuset.CPUSet { + // Convert slices to CPUSet + allocSet := cpuset.New(convertInt64ToInt(allocatableCPUs)...) + reservedSet := cpuset.New(reservedCPUs...) + + unionSet := allocSet.Union(reservedSet) + + // Flatten usedCPUs and build a CPUSet + var flatUsed []int + for _, grp := range usedCPUs { + flatUsed = append(flatUsed, convertInt64ToInt(grp)...) + } + usedSet := cpuset.New(flatUsed...) + + // Final result: (alloc ∪ reserved) - used + return unionSet.Difference(usedSet) +} + +func expectedCPUsValid(cpus []int, max int) bool { + for _, cpu := range cpus { + if cpu >= max { + return false + } + } + return true +} + +// buildListPodResourcesResponse builds a ListPodResourcesResponse from usedCPUs test data. +// usedCPUs is a slice of CPU ID slices, one per container. +func buildListPodResourcesResponse(usedCPUs [][]int64) *kubeletpodresourcesv1.ListPodResourcesResponse { + var podResources []*kubeletpodresourcesv1.PodResources + + if len(usedCPUs) > 0 { + var containers []*kubeletpodresourcesv1.ContainerResources + for i, containerCPUs := range usedCPUs { + if len(containerCPUs) > 0 { + containers = append(containers, &kubeletpodresourcesv1.ContainerResources{ + Name: fmt.Sprintf("container-%d", i), + CpuIds: containerCPUs, + }) + } + } + + if len(containers) > 0 { + podResources = append(podResources, &kubeletpodresourcesv1.PodResources{ + Name: "test-pod", + Namespace: "default", + Containers: containers, + }) + } + } + + return &kubeletpodresourcesv1.ListPodResourcesResponse{ + PodResources: podResources, + } } diff --git a/go-controller/pkg/node/ovspinning/ovspinning_noop.go b/go-controller/pkg/node/ovspinning/ovspinning_noop.go index 3e668c6fb0..e3cf1d8ba7 100644 --- a/go-controller/pkg/node/ovspinning/ovspinning_noop.go +++ b/go-controller/pkg/node/ovspinning/ovspinning_noop.go @@ -4,9 +4,13 @@ package ovspinning import ( + "context" + "k8s.io/klog/v2" + + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" ) -func Run(_ <-chan struct{}) { +func Run(_ context.Context, _ <-chan struct{}, _ podresourcesapi.PodResourcesListerClient) { klog.Infof("OVS CPU pinning is supported on linux platform only") } diff --git a/go-controller/pkg/node/podresourcesapi/podresourcesapi.go b/go-controller/pkg/node/podresourcesapi/podresourcesapi.go new file mode 100644 index 0000000000..7dc5f2dc9d --- /dev/null +++ b/go-controller/pkg/node/podresourcesapi/podresourcesapi.go @@ -0,0 +1,38 @@ +package podresourcesapi + +import ( + "fmt" + "path/filepath" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" +) + +const KubeletSocketPath = "/var/lib/kubelet/pod-resources/kubelet.sock" + +var _ podresourcesapi.PodResourcesListerClient = (*PodResClient)(nil) + +type PodResClient struct { + podresourcesapi.PodResourcesListerClient + conn *grpc.ClientConn +} + +// New initializes a new podresources client with the given socket path. +func New() (*PodResClient, error) { + socketPath := fmt.Sprintf("unix://%s", filepath.Clean(KubeletSocketPath)) + + conn, err := grpc.NewClient(socketPath, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("failed to connect to podresources socket: %w", err) + } + + client := podresourcesapi.NewPodResourcesListerClient(conn) + return &PodResClient{conn: conn, PodResourcesListerClient: client}, nil +} + +// Close closes the gRPC connection +func (c *PodResClient) Close() error { + return c.conn.Close() +} diff --git a/go-controller/pkg/node/udn_isolation.go b/go-controller/pkg/node/udn_isolation.go index 6a24afd89d..7c41105948 100644 --- a/go-controller/pkg/node/udn_isolation.go +++ b/go-controller/pkg/node/udn_isolation.go @@ -357,7 +357,11 @@ func (m *UDNHostIsolationManager) runKubeletRestartTracker(ctx context.Context) klog.Errorf("Error closing dbus connection for UDN isolation: %v", err) } return - case signal := <-signalChan: + case signal, ok := <-signalChan: + if !ok || signal == nil { + // Channel was closed, connection is shutting down + return + } klog.V(5).Infof("D-Bus event received: %#v", signal) // Extract unit name from path unitPath := signal.Path diff --git a/go-controller/pkg/node/user_defined_node_network_controller.go b/go-controller/pkg/node/user_defined_node_network_controller.go index 9e88801082..5170a24c28 100644 --- a/go-controller/pkg/node/user_defined_node_network_controller.go +++ b/go-controller/pkg/node/user_defined_node_network_controller.go @@ -127,6 +127,10 @@ func (nc *UserDefinedNodeNetworkController) Cleanup() error { return nil } +// HandleNetworkRefChange satisfies the NetworkController interface. UDN node controllers only +// manage local node state, so NAD reference changes for remote nodes are ignored. +func (nc *UserDefinedNodeNetworkController) HandleNetworkRefChange(_ string, _ bool) {} + func (nc *UserDefinedNodeNetworkController) shouldReconcileNetworkChange(old, new util.NetInfo) bool { wasUDNNetworkAdvertisedAtNode := util.IsPodNetworkAdvertisedAtNode(old, nc.name) isUDNNetworkAdvertisedAtNode := util.IsPodNetworkAdvertisedAtNode(new, nc.name) diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index 1a2ef4d8dc..f67f3b3972 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -14,6 +14,7 @@ import ( corev1 "k8s.io/api/core/v1" knet "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -89,6 +90,8 @@ type BaseNetworkController struct { // network information util.ReconcilableNetInfo + nadKeysLock sync.Mutex + lastNADKeys sets.Set[string] // retry framework for pods retryPods *ovnretry.RetryFramework @@ -204,7 +207,8 @@ func (oc *BaseNetworkController) reconcile(netInfo util.NetInfo, setNodeFailed f return true }) reconcileRoutes := oc.routeImportManager != nil && oc.routeImportManager.NeedsReconciliation(netInfo) - reconcilePendingPods := !oc.IsDefault() && !oc.ReconcilableNetInfo.EqualNADs(netInfo.GetNADs()...) + nadKeys := oc.networkManager.GetNADKeysForNetwork(netInfo.GetNetworkName()) + reconcilePendingPods := !oc.IsDefault() && oc.updateNADKeysChanged(nadKeys) reconcileNamespaces := sets.NewString() if oc.IsPrimaryNetwork() { // since CanServeNamespace filters out namespace events for namespaces unknown @@ -219,12 +223,21 @@ func (oc *BaseNetworkController) reconcile(netInfo util.NetInfo, setNodeFailed f if err != nil { return fmt.Errorf("failed to reconcile network information for network %s: %v", oc.GetNetworkName(), err) } - oc.doReconcile(reconcileRoutes, reconcilePendingPods, reconcileNodes, setNodeFailed, reconcileNamespaces.List()) return nil } +func (oc *BaseNetworkController) updateNADKeysChanged(nadKeys []string) bool { + oc.nadKeysLock.Lock() + defer oc.nadKeysLock.Unlock() + + next := sets.New(nadKeys...) + changed := oc.lastNADKeys == nil || !next.Equal(oc.lastNADKeys) + oc.lastNADKeys = next + return changed +} + // doReconcile performs the reconciliation after the controller NetInfo has already being // updated with the changes. What needs to be reconciled should already be known and // provided on the arguments of the method. This method returns no error and logs them @@ -246,11 +259,13 @@ func (oc *BaseNetworkController) doReconcile(reconcileRoutes, reconcilePendingPo klog.Infof("Failed to get node %s for reconciling network %s: %v", nodeName, oc.GetNetworkName(), err) continue } + klog.V(5).Infof("Requesting to add node %s to network %s", nodeName, oc.GetNetworkName()) err = oc.retryNodes.AddRetryObjWithAddNoBackoff(node) if err != nil { klog.Errorf("Failed to retry node %s for network %s: %v", nodeName, oc.GetNetworkName(), err) } } + if len(reconcileNodes) > 0 { oc.retryNodes.RequestRetryObjs() } @@ -303,23 +318,56 @@ func (oc *BaseUserDefinedNetworkController) FilterOutResource(objType reflect.Ty klog.Errorf("Failed to cast the provided object to a namespace") return false } - return !util.CanServeNamespace(oc.GetNetInfo(), ns.Name) + return oc.shouldFilterNamespace(ns.Name) case factory.PodType: pod, ok := obj.(*corev1.Pod) if !ok { klog.Errorf("Failed to cast the provided object to a pod") return false } - return !util.CanServeNamespace(oc.GetNetInfo(), pod.GetNamespace()) + return oc.shouldFilterNamespace(pod.GetNamespace()) default: return false } } +func (oc *BaseUserDefinedNetworkController) shouldFilterNamespace(namespace string) bool { + if !oc.IsPrimaryNetwork() || oc.networkManager == nil { + return !util.CanServeNamespace(oc.GetNetInfo(), namespace) + } + + nadKey, err := oc.networkManager.GetPrimaryNADForNamespace(namespace) + if err != nil { + if util.IsUnprocessedActiveNetworkError(err) { + return false + } + if util.IsInvalidPrimaryNetworkError(err) { + return true + } + return false + } + if nadKey == types.DefaultNetworkName { + return true + } + + networkName := oc.networkManager.GetNetworkNameForNADKey(nadKey) + if networkName == "" { + return !util.CanServeNamespace(oc.GetNetInfo(), namespace) + } + return networkName != oc.GetNetworkName() +} + func getNetworkControllerName(netName string) string { return netName + "-network-controller" } +func (bnc *BaseNetworkController) getNetworkNameForNADKeyFunc() func(nadKey string) string { + if bnc.networkManager == nil || !bnc.GetNetInfo().IsUserDefinedNetwork() { + return nil + } + return bnc.networkManager.GetNetworkNameForNADKey +} + // NewCommonNetworkControllerInfo creates CommonNetworkControllerInfo shared by controllers func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeOVN, wf *factory.WatchFactory, recorder record.EventRecorder, nbClient libovsdbclient.Client, sbClient libovsdbclient.Client, @@ -344,11 +392,11 @@ func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeO }, nil } -func (bnc *BaseNetworkController) GetLogicalPortName(pod *corev1.Pod, nadName string) string { +func (bnc *BaseNetworkController) GetLogicalPortName(pod *corev1.Pod, nadKey string) string { if !bnc.IsUserDefinedNetwork() { return util.GetLogicalPortName(pod.Namespace, pod.Name) } else { - return util.GetUserDefinedNetworkLogicalPortName(pod.Namespace, pod.Name, nadName) + return util.GetUserDefinedNetworkLogicalPortName(pod.Namespace, pod.Name, nadKey) } } @@ -586,7 +634,7 @@ func (bnc *BaseNetworkController) createNodeLogicalSwitch(nodeName string, hostS } err := libovsdbops.CreateOrUpdateLogicalSwitch(bnc.nbClient, &logicalSwitch, &logicalSwitch.OtherConfig, - &logicalSwitch.LoadBalancerGroup) + &logicalSwitch.LoadBalancerGroup, &logicalSwitch.ExternalIDs) if err != nil { return fmt.Errorf("failed to add logical switch %+v: %v", logicalSwitch, err) } @@ -944,12 +992,12 @@ func (bnc *BaseNetworkController) doesNetworkRequireIPAM() bool { return util.DoesNetworkRequireIPAM(bnc.GetNetInfo()) } -func (bnc *BaseNetworkController) getPodNADNames(pod *corev1.Pod) []string { +func (bnc *BaseNetworkController) getPodNADKeys(pod *corev1.Pod) []string { if !bnc.IsUserDefinedNetwork() { return []string{types.DefaultNetworkName} } - podNadNames, _ := util.PodNadNames(pod, bnc.GetNetInfo()) - return podNadNames + podNADKeys, _ := util.PodNADKeys(pod, bnc.GetNetInfo(), bnc.networkManager.GetNetworkNameForNADKey) + return podNADKeys } func (bnc *BaseNetworkController) getClusterPortGroupDbIDs(base string) *libovsdbops.DbObjectIDs { @@ -992,7 +1040,12 @@ func (bnc *BaseNetworkController) isLocalZoneNode(node *corev1.Node) bool { // GetNetworkRole returns the role of this controller's network for the given pod func (bnc *BaseNetworkController) GetNetworkRole(pod *corev1.Pod) (string, error) { - role, err := util.GetNetworkRole(bnc.GetNetInfo(), bnc.networkManager.GetActiveNetworkForNamespace, pod) + role, err := util.GetNetworkRole( + bnc.GetNetInfo(), + bnc.networkManager.GetPrimaryNADForNamespace, + bnc.networkManager.GetNetworkNameForNADKey, + pod, + ) if err != nil { if util.IsUnprocessedActiveNetworkError(err) { bnc.recordPodErrorEvent(pod, err) @@ -1007,6 +1060,39 @@ func (bnc *BaseNetworkController) isLayer2Interconnect() bool { return config.OVNKubernetesFeature.EnableInterconnect && bnc.TopologyType() == types.Layer2Topology } +// HandleNetworkRefChange enqueues node reconciliation when a NAD reference becomes active/inactive. +func (bnc *BaseNetworkController) HandleNetworkRefChange(nodeName string, active bool) { + if bnc.retryNodes == nil || bnc.watchFactory == nil { + return + } + var node *corev1.Node + var err error + if active { + node, err = bnc.watchFactory.GetNode(nodeName) + if err != nil { + klog.V(4).Infof("%s: skipping network ref change for node %s: %v", bnc.controllerName, nodeName, err) + return + } + } else { + // Prefer the cached node for deletes; if it is gone, fall back to a stub with just the name. + node, err = bnc.watchFactory.GetNode(nodeName) + if err != nil { + node = &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}} + } + } + if active { + if err := bnc.retryNodes.AddRetryObjWithAddNoBackoff(node); err != nil { + klog.V(4).Infof("%s: failed to enqueue add for node %s: %v", bnc.controllerName, nodeName, err) + } + } else { + if err := bnc.retryNodes.AddRetryObjWithDeleteNoBackoff(node); err != nil { + klog.V(4).Infof("%s: failed to enqueue delete for node %s: %v", bnc.controllerName, nodeName, err) + } + } + // Nudge the queue so newly enqueued work is processed promptly. + bnc.retryNodes.RequestRetryObjs() +} + func (bnc *BaseNetworkController) nodeZoneClusterChanged(oldNode, newNode *corev1.Node) bool { // Check if the annotations have changed. Use network topology and local params to skip unnecessary checks @@ -1141,6 +1227,7 @@ func (bnc *BaseNetworkController) newNetworkQoSController() error { bnc.watchFactory.PodCoreInformer(), bnc.watchFactory.NodeCoreInformer(), nadInformer, + bnc.networkManager, bnc.addressSetFactory, bnc.isPodScheduledinLocalZone, bnc.zone, diff --git a/go-controller/pkg/ovn/base_network_controller_multipolicy.go b/go-controller/pkg/ovn/base_network_controller_multipolicy.go index 1c22ff0fb9..60874d1b6b 100644 --- a/go-controller/pkg/ovn/base_network_controller_multipolicy.go +++ b/go-controller/pkg/ovn/base_network_controller_multipolicy.go @@ -54,7 +54,8 @@ func (bsnc *BaseUserDefinedNetworkController) shouldApplyMultiPolicy(mpolicy *mn networkName = substrings[1] networkNamespace = substrings[0] } - if bsnc.HasNAD(util.GetNADName(networkNamespace, networkName)) { + nadKey := util.GetNADName(networkNamespace, networkName) + if bsnc.networkManager.GetNetworkNameForNADKey(nadKey) == bsnc.GetNetworkName() { return true } } diff --git a/go-controller/pkg/ovn/base_network_controller_namespace.go b/go-controller/pkg/ovn/base_network_controller_namespace.go index f980cd35b7..af8abd152d 100644 --- a/go-controller/pkg/ovn/base_network_controller_namespace.go +++ b/go-controller/pkg/ovn/base_network_controller_namespace.go @@ -396,6 +396,7 @@ func (bnc *BaseNetworkController) getAllNamespacePodAddresses(ns string) []net.I } var ips []net.IP + resolver := bnc.getNetworkNameForNADKeyFunc() // Get all the pods in the namespace and append their IP to the address_set existingPods, err := bnc.watchFactory.GetPods(ns) if err != nil { @@ -404,7 +405,7 @@ func (bnc *BaseNetworkController) getAllNamespacePodAddresses(ns string) []net.I ips = make([]net.IP, 0, len(existingPods)) for _, pod := range existingPods { if !util.PodWantsHostNetwork(pod) && !util.PodCompleted(pod) && util.PodScheduled(pod) { - podIPs, err := util.GetPodIPsOfNetwork(pod, bnc.GetNetInfo()) + podIPs, err := util.GetPodIPsOfNetwork(pod, bnc.GetNetInfo(), resolver) if err != nil { klog.Warningf("Failed to get IPs for pod %s/%s: %v", pod.Namespace, pod.Name, err) continue @@ -448,7 +449,7 @@ func (bnc *BaseNetworkController) getNamespacePortGroupName(namespace string) st // failure indicates it should be retried later. func (bsnc *BaseNetworkController) removeRemoteZonePodFromNamespaceAddressSet(pod *corev1.Pod) error { podDesc := fmt.Sprintf("pod %s/%s/%s", bsnc.GetNetworkName(), pod.Namespace, pod.Name) - podIfAddrs, err := util.GetPodCIDRsWithFullMask(pod, bsnc.GetNetInfo()) + podIfAddrs, err := util.GetPodCIDRsWithFullMask(pod, bsnc.GetNetInfo(), bsnc.getNetworkNameForNADKeyFunc()) if err != nil { // maybe the pod is not scheduled yet or addLSP has not happened yet, so it doesn't have IPs. // let us ignore deletion failures for podIPs not found because diff --git a/go-controller/pkg/ovn/base_network_controller_pods.go b/go-controller/pkg/ovn/base_network_controller_pods.go index 4f1db7300a..6903541f73 100644 --- a/go-controller/pkg/ovn/base_network_controller_pods.go +++ b/go-controller/pkg/ovn/base_network_controller_pods.go @@ -36,12 +36,12 @@ import ( ) func (bnc *BaseNetworkController) allocatePodIPs(pod *corev1.Pod, - annotations *util.PodAnnotation, nadName string) (expectedLogicalPortName string, err error) { + annotations *util.PodAnnotation, nadKey string) (expectedLogicalPortName string, err error) { switchName, err := bnc.getExpectedSwitchName(pod) if err != nil { return "", err } - return bnc.allocatePodIPsOnSwitch(pod, annotations, nadName, switchName) + return bnc.allocatePodIPsOnSwitch(pod, annotations, nadKey, switchName) } var errNodeNotFound = errors.New("node not found") @@ -50,7 +50,7 @@ var errNodeNotFound = errors.New("node not found") // a specified switch, this switch can be different than the one the pod is // attachted to, for example hypershift kubevirt provider live migration. func (bnc *BaseNetworkController) allocatePodIPsOnSwitch(pod *corev1.Pod, - annotations *util.PodAnnotation, nadName string, switchName string) (expectedLogicalPortName string, err error) { + annotations *util.PodAnnotation, nadKey string, switchName string) (expectedLogicalPortName string, err error) { // Completed pods will be allocated as well to avoid having their IPs // allocated to other pods before we make sure that we have released them @@ -68,7 +68,7 @@ func (bnc *BaseNetworkController) allocatePodIPsOnSwitch(pod *corev1.Pod, return "", nil } - expectedLogicalPortName = bnc.GetLogicalPortName(pod, nadName) + expectedLogicalPortName = bnc.GetLogicalPortName(pod, nadKey) // For IPAM-less networks (e.g., localnet without subnet ), skip IP allocation. // The logical port should still be tracked to prevent deletion during sync. @@ -193,7 +193,7 @@ func (bnc *BaseNetworkController) lookupPortUUIDAndSwitchName(logicalPort string } func (bnc *BaseNetworkController) deletePodLogicalPort(pod *corev1.Pod, portInfo *lpInfo, - nadName string) (*lpInfo, error) { + nadKey string) (*lpInfo, error) { var portUUID, switchName, logicalPort string var podIfAddrs []*net.IPNet @@ -202,12 +202,12 @@ func (bnc *BaseNetworkController) deletePodLogicalPort(pod *corev1.Pod, portInfo return nil, err } - podDesc := fmt.Sprintf("pod %s/%s/%s", nadName, pod.Namespace, pod.Name) - logicalPort = bnc.GetLogicalPortName(pod, nadName) + podDesc := fmt.Sprintf("pod %s/%s/%s", nadKey, pod.Namespace, pod.Name) + logicalPort = bnc.GetLogicalPortName(pod, nadKey) if portInfo == nil { // If ovnkube-master restarts, it is also possible the Pod's logical switch port // is not re-added into the cache. Delete logical switch port anyway. - annotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + annotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { if util.IsAnnotationNotSetError(err) { // if the annotation doesn’t exist, that’s not an error. It means logical port does not need to be deleted. @@ -244,7 +244,7 @@ func (bnc *BaseNetworkController) deletePodLogicalPort(pod *corev1.Pod, portInfo podDesc, expectedSwitchName, switchName, portUUID) } - shouldRelease, err := bnc.shouldReleaseDeletedPod(pod, switchName, nadName, podIfAddrs) + shouldRelease, err := bnc.shouldReleaseDeletedPod(pod, switchName, nadKey, podIfAddrs) if err != nil { return nil, fmt.Errorf("unable to determine if ip should be released: %v", err) } @@ -295,7 +295,7 @@ func (bnc *BaseNetworkController) deletePodLogicalPort(pod *corev1.Pod, portInfo // findPodWithIPAddresses finds any pods with the same IPs in a running state on the cluster // If nodeName is provided, pods only belonging to the same node will be checked, unless this pod has // potentially live migrated. -func findPodWithIPAddresses(watchFactory *factory.WatchFactory, netInfo util.NetInfo, needleIPs []net.IP, nodeName string) (*corev1.Pod, error) { +func findPodWithIPAddresses(watchFactory *factory.WatchFactory, netInfo util.NetInfo, needleIPs []net.IP, nodeName string, getNetworkNameForNADKey func(nadKey string) string) (*corev1.Pod, error) { allPods, err := watchFactory.GetAllPods() if err != nil { return nil, fmt.Errorf("unable to get pods: %w", err) @@ -317,7 +317,7 @@ func findPodWithIPAddresses(watchFactory *factory.WatchFactory, netInfo util.Net } // check if the pod addresses match in the OVN annotation - haystackPodAddrs, err := util.GetPodIPsOfNetwork(p, netInfo) + haystackPodAddrs, err := util.GetPodIPsOfNetwork(p, netInfo, getNetworkNameForNADKey) if err != nil { continue } @@ -341,7 +341,7 @@ func (bnc *BaseNetworkController) canReleasePodIPs(podIfAddrs []*net.IPNet, node needleIPs = append(needleIPs, podIPNet.IP) } - collidingPod, err := findPodWithIPAddresses(bnc.watchFactory, bnc.GetNetInfo(), needleIPs, nodeName) + collidingPod, err := findPodWithIPAddresses(bnc.watchFactory, bnc.GetNetInfo(), needleIPs, nodeName, bnc.getNetworkNameForNADKeyFunc()) if err != nil { return false, fmt.Errorf("unable to determine if pod IPs: %#v are in use by another pod :%w", podIfAddrs, err) @@ -432,9 +432,9 @@ func (bnc *BaseNetworkController) getExpectedSwitchName(pod *corev1.Pod) (string // to the same virtual machine, for normal pods it will unmarshal and return // it, also there returned boolean will be true if the pod subnet belong to // controller's zone. -func (bnc *BaseNetworkController) ensurePodAnnotation(pod *corev1.Pod, nadName string) (*util.PodAnnotation, bool, error) { +func (bnc *BaseNetworkController) ensurePodAnnotation(pod *corev1.Pod, nadKey string) (*util.PodAnnotation, bool, error) { if kubevirt.IsPodLiveMigratable(pod) { - podAnnotation, err := kubevirt.EnsurePodAnnotationForVM(bnc.watchFactory, bnc.kube, pod, nadName) + podAnnotation, err := kubevirt.EnsurePodAnnotationForVM(bnc.watchFactory, bnc.kube, pod, nadKey) if err != nil { return nil, false, err } @@ -446,19 +446,19 @@ func (bnc *BaseNetworkController) ensurePodAnnotation(pod *corev1.Pod, nadName s _, zoneContainsPodSubnet := kubevirt.ZoneContainsPodSubnet(bnc.lsManager, podAnnotation.IPs) return podAnnotation, zoneContainsPodSubnet, nil } - podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { return nil, true, nil } return podAnnotation, true, nil } -func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *corev1.Pod, nadName string, +func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *corev1.Pod, nadKey string, network *nadapi.NetworkSelectionElement, enable *bool) (ops []ovsdb.Operation, lsp *nbdb.LogicalSwitchPort, podAnnotation *util.PodAnnotation, newlyCreatedPort bool, err error) { var ls *nbdb.LogicalSwitch - podDesc := fmt.Sprintf("%s/%s/%s", nadName, pod.Namespace, pod.Name) + podDesc := fmt.Sprintf("%s/%s/%s", nadKey, pod.Namespace, pod.Name) switchName, err := bnc.getExpectedSwitchName(pod) if err != nil { return nil, nil, nil, false, fmt.Errorf("[%s] failed geting expected switch name when adding logical switch port: %v", podDesc, err) @@ -483,7 +483,7 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *corev1.Pod, nadNa return nil, nil, nil, false, err } - portName := bnc.GetLogicalPortName(pod, nadName) + portName := bnc.GetLogicalPortName(pod, nadKey) klog.Infof("[%s] creating logical port %s for pod on switch %s", podDesc, portName, switchName) var addresses []string @@ -538,15 +538,6 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *corev1.Pod, nadNa if !lspExist || len(existingLSP.Options["iface-id-ver"]) != 0 { lsp.Options["iface-id-ver"] = string(pod.UID) } - // Bind the port to the node's chassis; prevents ping-ponging between - // chassis if ovnkube-node isn't running correctly and hasn't cleared - // out iface-id for an old instance of this pod, and the pod got - // rescheduled. - - if !config.Kubernetes.DisableRequestedChassis { - lsp.Options[libovsdbops.RequestedChassis] = pod.Spec.NodeName - } - // let's calculate if this network controller's role for this pod // and pass that information while determining the podAnnotations networkRole, err := bnc.GetNetworkRole(pod) @@ -559,6 +550,28 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *corev1.Pod, nadNa return nil, nil, nil, false, nil } + // Bind the port to the node's chassis. + // For IC this is required for Layer 2 networks with remote ports. + // For Legacy with OVN Central Mode it prevents ping-ponging between + // chassis if ovnkube-node isn't running correctly and hasn't cleared + // out iface-id for an old instance of this pod, and the pod got + // rescheduled. + var node *corev1.Node + if !config.Kubernetes.DisableRequestedChassis { + node, err = bnc.watchFactory.GetNode(pod.Spec.NodeName) + if err != nil { + return nil, nil, nil, false, err + } + chassisID, err := util.ParseNodeChassisIDAnnotation(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + return nil, nil, nil, false, ovntypes.NewSuppressedError(err) + } + return nil, nil, nil, false, err + } + lsp.Options[libovsdbops.RequestedChassis] = chassisID + } + // Although we have different code to allocate the pod annotation for the // default network and user-defined networks, at the time of this writing they // are functionally equivalent and the only reason to keep them separated is @@ -567,9 +580,9 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *corev1.Pod, nadNa // functionally equivalent going forward. var annotationUpdated bool if bnc.IsUserDefinedNetwork() { - podAnnotation, annotationUpdated, err = bnc.allocatePodAnnotationForUserDefinedNetwork(pod, existingLSP, nadName, network, networkRole) + podAnnotation, annotationUpdated, err = bnc.allocatePodAnnotationForUserDefinedNetwork(pod, existingLSP, nadKey, network, networkRole) } else { - podAnnotation, annotationUpdated, err = bnc.allocatePodAnnotation(pod, existingLSP, podDesc, nadName, network, networkRole) + podAnnotation, annotationUpdated, err = bnc.allocatePodAnnotation(pod, existingLSP, podDesc, nadKey, network, networkRole) } if err != nil { @@ -600,7 +613,7 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *corev1.Pod, nadNa lsp.ExternalIDs = map[string]string{"namespace": pod.Namespace, "pod": "true"} if bnc.IsUserDefinedNetwork() { lsp.ExternalIDs[ovntypes.NetworkExternalID] = bnc.GetNetworkName() - lsp.ExternalIDs[ovntypes.NADExternalID] = nadName + lsp.ExternalIDs[ovntypes.NADExternalID] = nadKey lsp.ExternalIDs[ovntypes.TopologyExternalID] = bnc.TopologyType() } @@ -631,13 +644,13 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *corev1.Pod, nadNa return ops, lsp, podAnnotation, annotationUpdated && !lspExist, nil } -func (bnc *BaseNetworkController) updatePodAnnotationWithRetry(origPod *corev1.Pod, podInfo *util.PodAnnotation, nadName string) error { +func (bnc *BaseNetworkController) updatePodAnnotationWithRetry(origPod *corev1.Pod, podInfo *util.PodAnnotation, nadKey string) error { return util.UpdatePodAnnotationWithRetry( bnc.watchFactory.PodCoreInformer().Lister(), bnc.kube, origPod, podInfo, - nadName, + nadKey, ) } @@ -800,14 +813,15 @@ func calculateStaticMAC(podDesc string, mac string) (net.HardwareAddr, error) { } // allocatePodAnnotation and update the corresponding pod annotation. -func (bnc *BaseNetworkController) allocatePodAnnotation(pod *corev1.Pod, existingLSP *nbdb.LogicalSwitchPort, podDesc, nadName string, network *nadapi.NetworkSelectionElement, networkRole string) (*util.PodAnnotation, bool, error) { +func (bnc *BaseNetworkController) allocatePodAnnotation(pod *corev1.Pod, existingLSP *nbdb.LogicalSwitchPort, podDesc, + nadKey string, network *nadapi.NetworkSelectionElement, networkRole string) (*util.PodAnnotation, bool, error) { var releaseIPs bool var podMac net.HardwareAddr var podIfAddrs []*net.IPNet switchName := pod.Spec.NodeName - podAnnotation, zoneContainsPodSubnet, err := bnc.ensurePodAnnotation(pod, nadName) + podAnnotation, zoneContainsPodSubnet, err := bnc.ensurePodAnnotation(pod, nadKey) if err != nil { return nil, false, fmt.Errorf("unable to ensure pod annotation: %v", err) } @@ -870,7 +884,7 @@ func (bnc *BaseNetworkController) allocatePodAnnotation(pod *corev1.Pod, existin } else if bnc.doesNetworkRequireIPAM() { if err = bnc.lsManager.AllocateIPs(switchName, podIfAddrs); err != nil && err != ipallocator.ErrAllocated { klog.Warningf("Unable to allocate IPs %s found on existing OVN port: %s, for pod %s on switch: %s"+ - " error: %v", util.JoinIPNetIPs(podIfAddrs, " "), bnc.GetLogicalPortName(pod, nadName), podDesc, switchName, err) + " error: %v", util.JoinIPNetIPs(podIfAddrs, " "), bnc.GetLogicalPortName(pod, nadKey), podDesc, switchName, err) needsNewMacOrIPAllocation = true } @@ -932,7 +946,7 @@ func (bnc *BaseNetworkController) allocatePodAnnotation(pod *corev1.Pod, existin klog.V(5).Infof("Annotation values: ip=%v ; mac=%s ; gw=%s", podIfAddrs, podMac, podAnnotation.Gateways) annoStart := time.Now() - err = bnc.updatePodAnnotationWithRetry(pod, podAnnotation, nadName) + err = bnc.updatePodAnnotationWithRetry(pod, podAnnotation, nadKey) podAnnoTime := time.Since(annoStart) klog.Infof("[%s] addLogicalPort annotation time took %v", podDesc, podAnnoTime) if err != nil { @@ -946,7 +960,7 @@ func (bnc *BaseNetworkController) allocatePodAnnotation(pod *corev1.Pod, existin // allocatePodAnnotationForUserDefinedNetwork and update the corresponding pod // annotation. func (bnc *BaseNetworkController) allocatePodAnnotationForUserDefinedNetwork(pod *corev1.Pod, lsp *nbdb.LogicalSwitchPort, - nadName string, network *nadapi.NetworkSelectionElement, networkRole string) (*util.PodAnnotation, bool, error) { + nadKey string, network *nadapi.NetworkSelectionElement, networkRole string) (*util.PodAnnotation, bool, error) { switchName, err := bnc.getExpectedSwitchName(pod) if err != nil { return nil, false, err @@ -955,11 +969,11 @@ func (bnc *BaseNetworkController) allocatePodAnnotationForUserDefinedNetwork(pod // In certain configurations, pod IP allocation is handled from cluster // manager so wait for it to allocate the IPs if !bnc.allocatesPodAnnotation() { - podAnnotation, _ := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + podAnnotation, _ := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if !util.IsValidPodAnnotation(podAnnotation) { return nil, false, ovntypes.NewSuppressedError(fmt.Errorf( "failed to get PodAnnotation for %s/%s/%s, cluster manager might have not allocated it yet", - nadName, pod.Namespace, pod.Name)) + nadKey, pod.Namespace, pod.Name)) } return podAnnotation, false, nil @@ -974,14 +988,14 @@ func (bnc *BaseNetworkController) allocatePodAnnotationForUserDefinedNetwork(pod mac, ips, err := bnc.getPortAddresses(switchName, lsp) if err != nil { return nil, false, fmt.Errorf("failed to get pod addresses for pod %s/%s/%s on node %s, err: %v", - nadName, pod.Namespace, pod.Name, switchName, err) + nadKey, pod.Namespace, pod.Name, switchName, err) } network.MacRequest = mac.String() network.IPRequest = util.StringSlice(ips) reallocate = true klog.V(5).Infof("Will attempt to use LSP IP addresses %v and mac %s for pod %s/%s/%s", - network.IPRequest, network.MacRequest, nadName, pod.Namespace, pod.Name) + network.IPRequest, network.MacRequest, nadKey, pod.Namespace, pod.Name) } var ipAllocator subnetipallocator.NamedAllocator @@ -991,12 +1005,13 @@ func (bnc *BaseNetworkController) allocatePodAnnotationForUserDefinedNetwork(pod node, err := bnc.watchFactory.GetNode(pod.Spec.NodeName) if err != nil { return nil, false, fmt.Errorf("failed to get pod %s/%s/%s node %q: %w", - nadName, pod.Namespace, pod.Name, pod.Spec.NodeName, err) + nadKey, pod.Namespace, pod.Name, pod.Spec.NodeName, err) } updatedPod, podAnnotation, err := bnc.podAnnotationAllocator.AllocatePodAnnotation( ipAllocator, node, pod, + nadKey, network, reallocate, networkRole, @@ -1010,12 +1025,12 @@ func (bnc *BaseNetworkController) allocatePodAnnotationForUserDefinedNetwork(pod } if updatedPod != nil { - klog.V(5).Infof("Allocated IP addresses %v, mac address %s, gateways %v and routes %s for pod %s/%s on nad %s", + klog.V(5).Infof("Allocated IP addresses %v, mac address %s, gateways %v and routes %s for pod %s/%s on NAD key %s", util.StringSlice(podAnnotation.IPs), podAnnotation.MAC, util.StringSlice(podAnnotation.Gateways), util.StringSlice(podAnnotation.Routes), - pod.Namespace, pod.Name, nadName, + pod.Namespace, pod.Name, nadKey, ) return podAnnotation, true, nil @@ -1038,7 +1053,7 @@ func (bnc *BaseNetworkController) allocatesPodAnnotation() bool { return true } -func (bnc *BaseNetworkController) shouldReleaseDeletedPod(pod *corev1.Pod, switchName, nad string, podIfAddrs []*net.IPNet) (bool, error) { +func (bnc *BaseNetworkController) shouldReleaseDeletedPod(pod *corev1.Pod, switchName, nadKey string, podIfAddrs []*net.IPNet) (bool, error) { var err error if !bnc.IsUserDefinedNetwork() && kubevirt.IsPodLiveMigratable(pod) { allVMPodsAreCompleted, err := kubevirt.AllVMPodsAreCompleted(bnc.watchFactory.PodCoreInformer().Lister(), pod) @@ -1059,11 +1074,11 @@ func (bnc *BaseNetworkController) shouldReleaseDeletedPod(pod *corev1.Pod, switc return true, nil } - if bnc.wasPodReleasedBeforeStartup(string(pod.UID), nad) { - klog.Infof("Completed pod %s/%s was already released for nad %s before startup", + if bnc.wasPodReleasedBeforeStartup(string(pod.UID), nadKey) { + klog.Infof("Completed pod %s/%s was already released for NAD key %s before startup", pod.Namespace, pod.Name, - nad, + nadKey, ) return false, nil } @@ -1103,10 +1118,10 @@ func (bnc *BaseNetworkController) shouldReleaseDeletedPod(pod *corev1.Pod, switc } if err != nil { - return false, fmt.Errorf("cannot determine if IPs are safe to release for completed pod %s/%s on nad %s: %w", + return false, fmt.Errorf("cannot determine if IPs are safe to release for completed pod %s/%s on NAD key %s: %w", pod.Namespace, pod.Name, - nad, + nadKey, err) } @@ -1162,7 +1177,7 @@ func (bnc *BaseNetworkController) trackPodsReleasedBeforeStartup(podAnnotations for _, pod := range pods { uid := string(pod.UID) - for nad, annotation := range podAnnotations[pod] { + for nadKey, annotation := range podAnnotations[pod] { ips := []string{} for _, ipnet := range annotation.IPs { // normalize ips to their 16 octect string representation @@ -1176,46 +1191,46 @@ func (bnc *BaseNetworkController) trackPodsReleasedBeforeStartup(podAnnotations } if !util.PodCompleted(pod) { // this should not happen, but let's log it just in case - klog.Errorf("Non completed pod %s/%s shares ips %s from NAD %s with some other non completed pod", + klog.Errorf("Non completed pod %s/%s shares ips %s from NAD key %s with some other non completed pod", pod.Namespace, pod.Name, util.StringSlice(annotation.IPs), - nad, + nadKey, ) continue } // otherwise consider the IPs of this NAD already released for the pod - if bnc.releasedPodsBeforeStartup[nad] == nil { - bnc.releasedPodsBeforeStartup[nad] = sets.New(uid) + if bnc.releasedPodsBeforeStartup[nadKey] == nil { + bnc.releasedPodsBeforeStartup[nadKey] = sets.New(uid) } else { - bnc.releasedPodsBeforeStartup[nad].Insert(uid) + bnc.releasedPodsBeforeStartup[nadKey].Insert(uid) } } } } // forgetPodReleasedBeforeStartup stops tracking a released pod on the specified NAD -func (bnc *BaseNetworkController) forgetPodReleasedBeforeStartup(uid, nad string) { +func (bnc *BaseNetworkController) forgetPodReleasedBeforeStartup(uid, nadKey string) { bnc.releasedPodsOnStartupMutex.Lock() defer bnc.releasedPodsOnStartupMutex.Unlock() - if bnc.releasedPodsBeforeStartup[nad] == nil { + if bnc.releasedPodsBeforeStartup[nadKey] == nil { return } - bnc.releasedPodsBeforeStartup[nad].Delete(uid) - if bnc.releasedPodsBeforeStartup[nad].Len() == 0 { - delete(bnc.releasedPodsBeforeStartup, nad) + bnc.releasedPodsBeforeStartup[nadKey].Delete(uid) + if bnc.releasedPodsBeforeStartup[nadKey].Len() == 0 { + delete(bnc.releasedPodsBeforeStartup, nadKey) } } // wasPodReleasedBeforeStartup returns whether a pod has been considered released on // startup for the specific NAD -func (bnc *BaseNetworkController) wasPodReleasedBeforeStartup(uid, nad string) bool { +func (bnc *BaseNetworkController) wasPodReleasedBeforeStartup(uid, nadKey string) bool { bnc.releasedPodsOnStartupMutex.Lock() defer bnc.releasedPodsOnStartupMutex.Unlock() - if bnc.releasedPodsBeforeStartup[nad] == nil { + if bnc.releasedPodsBeforeStartup[nadKey] == nil { return false } - return bnc.releasedPodsBeforeStartup[nad].Has(uid) + return bnc.releasedPodsBeforeStartup[nadKey].Has(uid) } func (bnc *BaseNetworkController) isNonHostSubnetSwitch(switchName string) bool { diff --git a/go-controller/pkg/ovn/base_network_controller_policy.go b/go-controller/pkg/ovn/base_network_controller_policy.go index 8ee14de88a..79a46449ae 100644 --- a/go-controller/pkg/ovn/base_network_controller_policy.go +++ b/go-controller/pkg/ovn/base_network_controller_policy.go @@ -597,9 +597,9 @@ func (bnc *BaseNetworkController) getNewLocalPolicyPorts(np *networkPolicy, continue } - nadNames := bnc.getPodNADNames(pod) - for _, nadName := range nadNames { - logicalPortName := bnc.GetLogicalPortName(pod, nadName) + nadKeys := bnc.getPodNADKeys(pod) + for _, nadKey := range nadKeys { + logicalPortName := bnc.GetLogicalPortName(pod, nadKey) if _, ok := np.localPods.Load(logicalPortName); ok { // port is already added for this policy continue @@ -608,11 +608,11 @@ func (bnc *BaseNetworkController) getNewLocalPolicyPorts(np *networkPolicy, // Return error for retry if // 1. getting pod LSP from the cache fails, // 2. the gotten LSP is scheduled for removal (stateful-sets). - portInfo, err := bnc.logicalPortCache.get(pod, nadName) + portInfo, err := bnc.logicalPortCache.get(pod, nadKey) if err != nil { - klog.Warningf("Failed to get get LSP for pod %s/%s NAD %s for networkPolicy %s, err: %v", - pod.Namespace, pod.Name, nadName, np.name, err) - errs = append(errs, fmt.Errorf("unable to get port info for pod %s/%s NAD %s", pod.Namespace, pod.Name, nadName)) + klog.Warningf("Failed to get get LSP for pod %s/%s NAD key %s for networkPolicy %s, err: %v", + pod.Namespace, pod.Name, nadKey, np.name, err) + errs = append(errs, fmt.Errorf("unable to get port info for pod %s/%s NAD key %s", pod.Namespace, pod.Name, nadKey)) continue } @@ -620,7 +620,7 @@ func (bnc *BaseNetworkController) getNewLocalPolicyPorts(np *networkPolicy, if !portInfo.expires.IsZero() { klog.Warningf("Stale LSP %s for network policy %s found in cache", portInfo.name, np.name) - errs = append(errs, fmt.Errorf("unable to get port info for pod %s/%s NAD %s", pod.Namespace, pod.Name, nadName)) + errs = append(errs, fmt.Errorf("unable to get port info for pod %s/%s NAD key %s", pod.Namespace, pod.Name, nadKey)) continue } @@ -646,9 +646,9 @@ func (bnc *BaseNetworkController) getExistingLocalPolicyPorts(np *networkPolicy, for _, obj := range objs { pod := obj.(*corev1.Pod) - nadNames := bnc.getPodNADNames(pod) - for _, nadName := range nadNames { - logicalPortName := bnc.GetLogicalPortName(pod, nadName) + nadKeys := bnc.getPodNADKeys(pod) + for _, nadKey := range nadKeys { + logicalPortName := bnc.GetLogicalPortName(pod, nadKey) loadedPortUUID, ok := np.localPods.Load(logicalPortName) if !ok { // port is already deleted for this policy diff --git a/go-controller/pkg/ovn/base_network_controller_user_defined.go b/go-controller/pkg/ovn/base_network_controller_user_defined.go index ee074f1969..238daee738 100644 --- a/go-controller/pkg/ovn/base_network_controller_user_defined.go +++ b/go-controller/pkg/ovn/base_network_controller_user_defined.go @@ -260,13 +260,31 @@ func (bsnc *BaseUserDefinedNetworkController) ensurePodForUserDefinedNetwork(pod var activeNetwork util.NetInfo if bsnc.IsPrimaryNetwork() { + // check to see if the primary NAD is even applicable to our controller + foundNamespaceNAD, err := bsnc.networkManager.GetPrimaryNADForNamespace(pod.Namespace) + if err != nil { + return fmt.Errorf("failed to get primary network namespace NAD: %w", err) + } + if foundNamespaceNAD == types.DefaultNetworkName { + return nil + } + networkName := bsnc.networkManager.GetNetworkNameForNADKey(foundNamespaceNAD) + if networkName != "" && networkName != bsnc.GetNetworkName() { + return nil + } activeNetwork, err = bsnc.networkManager.GetActiveNetworkForNamespace(pod.Namespace) if err != nil { return fmt.Errorf("failed looking for the active network at namespace '%s': %w", pod.Namespace, err) } } - on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, bsnc.GetNetInfo(), activeNetwork) + on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork( + pod, + bsnc.GetNetInfo(), + activeNetwork, + bsnc.networkManager.GetNetworkNameForNADKey, + bsnc.networkManager.GetPrimaryNADForNamespace, + ) if err != nil { bsnc.recordPodErrorEvent(pod, err) // configuration error, no need to retry, do not return error @@ -289,9 +307,9 @@ func (bsnc *BaseUserDefinedNetworkController) ensurePodForUserDefinedNetwork(pod } var errs []error - for nadName, network := range networkMap { - if err = bsnc.addLogicalPortToNetworkForNAD(pod, nadName, switchName, network, kubevirtLiveMigrationStatus); err != nil { - errs = append(errs, fmt.Errorf("failed to add logical port of Pod %s/%s for NAD %s: %w", pod.Namespace, pod.Name, nadName, err)) + for nadKey, network := range networkMap { + if err = bsnc.addLogicalPortToNetworkForNAD(pod, nadKey, switchName, network, kubevirtLiveMigrationStatus); err != nil { + errs = append(errs, fmt.Errorf("failed to add logical port of Pod %s/%s for NAD key %s: %w", pod.Namespace, pod.Name, nadKey, err)) } } if len(errs) != 0 { @@ -300,15 +318,15 @@ func (bsnc *BaseUserDefinedNetworkController) ensurePodForUserDefinedNetwork(pod return nil } -func (bsnc *BaseUserDefinedNetworkController) addLogicalPortToNetworkForNAD(pod *corev1.Pod, nadName, switchName string, +func (bsnc *BaseUserDefinedNetworkController) addLogicalPortToNetworkForNAD(pod *corev1.Pod, nadKey, switchName string, network *nadapi.NetworkSelectionElement, kubevirtLiveMigrationStatus *kubevirt.LiveMigrationStatus, ) error { var libovsdbExecuteTime time.Duration start := time.Now() defer func() { - klog.Infof("[%s/%s] addLogicalPort for NAD %s took %v, libovsdb time %v", - pod.Namespace, pod.Name, nadName, time.Since(start), libovsdbExecuteTime) + klog.Infof("[%s/%s] addLogicalPort for NAD key %s took %v, libovsdb time %v", + pod.Namespace, pod.Name, nadKey, time.Since(start), libovsdbExecuteTime) }() var err error @@ -333,7 +351,7 @@ func (bsnc *BaseUserDefinedNetworkController) addLogicalPortToNetworkForNAD(pod requiresLogicalPort := isLocalPod || bsnc.isLayer2Interconnect() if requiresLogicalPort { - ops, lsp, podAnnotation, newlyCreated, err = bsnc.addLogicalPortToNetwork(pod, nadName, network, lspEnabled) + ops, lsp, podAnnotation, newlyCreated, err = bsnc.addLogicalPortToNetwork(pod, nadKey, network, lspEnabled) if err != nil { return err } @@ -345,7 +363,7 @@ func (bsnc *BaseUserDefinedNetworkController) addLogicalPortToNetworkForNAD(pod // Not needed for layer3 networks as in that case the whole node switch // is removed // No need to release IPs as those are allocated from cluster manager - logicalPort := bsnc.GetLogicalPortName(pod, nadName) + logicalPort := bsnc.GetLogicalPortName(pod, nadKey) expectedSwitchName, err := bsnc.getExpectedSwitchName(pod) if err != nil { return err @@ -354,21 +372,21 @@ func (bsnc *BaseUserDefinedNetworkController) addLogicalPortToNetworkForNAD(pod if err != nil { return err } - bsnc.logicalPortCache.remove(pod, nadName) + bsnc.logicalPortCache.remove(pod, nadKey) } if shouldHandleLiveMigration && kubevirtLiveMigrationStatus.IsTargetDomainReady() && // At localnet there is no source pod remote LSP so it should be skipped (bsnc.TopologyType() != types.LocalnetTopology || bsnc.isPodScheduledinLocalZone(kubevirtLiveMigrationStatus.SourcePod)) { - ops, err = bsnc.disableLiveMigrationSourceLSPOps(kubevirtLiveMigrationStatus, nadName, ops) + ops, err = bsnc.disableLiveMigrationSourceLSPOps(kubevirtLiveMigrationStatus, nadKey, ops) if err != nil { return fmt.Errorf("failed to create LSP ops for source pod during Live-migration status: %w", err) } } if podAnnotation == nil { - podAnnotation, err = util.UnmarshalPodAnnotation(pod.Annotations, nadName) + podAnnotation, err = util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { return err } @@ -403,7 +421,7 @@ func (bsnc *BaseUserDefinedNetworkController) addLogicalPortToNetworkForNAD(pod txOkCallBack() if lsp != nil { - _ = bsnc.logicalPortCache.add(pod, switchName, nadName, lsp.UUID, podAnnotation.MAC, podAnnotation.IPs) + _ = bsnc.logicalPortCache.add(pod, switchName, nadKey, lsp.UUID, podAnnotation.MAC, podAnnotation.IPs) if bsnc.requireDHCP(pod) { if err := bsnc.ensureDHCP(pod, podAnnotation, lsp); err != nil { return err @@ -437,7 +455,7 @@ func (bsnc *BaseUserDefinedNetworkController) removePodForUserDefinedNetwork(pod // for a specific NAD belongs to this network, Pod's logical port might already be created half-way // without its lpInfo cache being created; need to deleted resources created for that NAD as well. - // So, first get all nadNames from pod annotation, but handle NADs belong to this network only. + // So, first get all nadKeys from pod annotation, but handle NADs belong to this network only. podNetworks, err := util.UnmarshalPodAnnotationAllNetworks(pod.Annotations) if err != nil { return err @@ -448,13 +466,14 @@ func (bsnc *BaseUserDefinedNetworkController) removePodForUserDefinedNetwork(pod } var alreadyProcessed bool - for nadName, podAnnotation := range podNetworks { - if !bsnc.HasNAD(nadName) { + for nadKey, podAnnotation := range podNetworks { + networkName := bsnc.networkManager.GetNetworkNameForNADKey(nadKey) + if networkName == "" || networkName != bsnc.GetNetworkName() { continue } // pod has a network managed by this controller - klog.Infof("Deleting pod: %s for network %s, NAD: %s", podDesc, bsnc.GetNetworkName(), nadName) + klog.Infof("Deleting pod: %s for network %s, NAD key: %s", podDesc, bsnc.GetNetworkName(), nadKey) // handle remote pod clean up but only do this one time if !hasLogicalPort && !alreadyProcessed { @@ -475,12 +494,12 @@ func (bsnc *BaseUserDefinedNetworkController) removePodForUserDefinedNetwork(pod } if kubevirt.IsPodAllowedForMigration(pod, bsnc.GetNetInfo()) { - if err = bsnc.enableSourceLSPFailedLiveMigration(pod, nadName, podAnnotation.MAC, podAnnotation.IPs); err != nil { + if err = bsnc.enableSourceLSPFailedLiveMigration(pod, nadKey, podAnnotation.MAC, podAnnotation.IPs); err != nil { return err } } - bsnc.logicalPortCache.remove(pod, nadName) - pInfo, err := bsnc.deletePodLogicalPort(pod, portInfoMap[nadName], nadName) + bsnc.logicalPortCache.remove(pod, nadKey) + pInfo, err := bsnc.deletePodLogicalPort(pod, portInfoMap[nadKey], nadKey) if err != nil { return err } @@ -492,13 +511,13 @@ func (bsnc *BaseUserDefinedNetworkController) removePodForUserDefinedNetwork(pod // do not release IP address unless we have validated no other pod is using it if pInfo == nil || len(pInfo.ips) == 0 { - bsnc.forgetPodReleasedBeforeStartup(string(pod.UID), nadName) + bsnc.forgetPodReleasedBeforeStartup(string(pod.UID), nadKey) continue } // if we allow for persistent IPs, then we need to check if this pod has an IPAM Claim if bsnc.allowPersistentIPs() { - hasIPAMClaim, err := bsnc.hasIPAMClaim(pod, nadName) + hasIPAMClaim, err := bsnc.hasIPAMClaim(pod, nadKey) if err != nil { return fmt.Errorf("unable to determine if pod %s has IPAM Claim: %w", podDesc, err) } @@ -518,15 +537,15 @@ func (bsnc *BaseUserDefinedNetworkController) removePodForUserDefinedNetwork(pod return err } - bsnc.forgetPodReleasedBeforeStartup(string(pod.UID), nadName) + bsnc.forgetPodReleasedBeforeStartup(string(pod.UID), nadKey) } return nil } // hasIPAMClaim determines whether a pod's IPAM is being handled by IPAMClaim CR. -// pod passed should already be validated as having a network connection to nadName -func (bsnc *BaseUserDefinedNetworkController) hasIPAMClaim(pod *corev1.Pod, nadNamespacedName string) (bool, error) { +// pod passed should already be validated as having a network connection to nadKey +func (bsnc *BaseUserDefinedNetworkController) hasIPAMClaim(pod *corev1.Pod, nadKey string) (bool, error) { if !bsnc.AllowsPersistentIPs() { return false, nil } @@ -550,19 +569,21 @@ func (bsnc *BaseUserDefinedNetworkController) hasIPAMClaim(pod *corev1.Pod, nadN } } else { // secondary network the IPAM claim reference is on the network selection element - nadKeys := strings.Split(nadNamespacedName, "/") - if len(nadKeys) != 2 { - return false, fmt.Errorf("invalid NAD name %s", nadNamespacedName) - } - nadNamespace := nadKeys[0] - nadName := nadKeys[1] - allNetworks, err := util.GetK8sPodAllNetworkSelections(pod) + on, networkMap, err := util.GetUDNPodNADToNetworkMapping( + pod, + bsnc.GetNetInfo(), + bsnc.networkManager.GetNetworkNameForNADKey, + ) if err != nil { - return false, err + return false, fmt.Errorf("failed to get network mapping for pod %s/%s on network %s: %v", + pod.Namespace, pod.Name, bsnc.GetNetworkName(), err) } - for _, network := range allNetworks { - if network.Namespace == nadNamespace && network.Name == nadName { - // found network selection element, check if it has IPAM + if !on { + klog.Warningf("Pod %s/%s is not scheduled on network %s", pod.Namespace, pod.Name, bsnc.GetNetworkName()) + return false, nil + } + for key, network := range networkMap { + if key == nadKey { if len(network.IPAMClaimReference) > 0 { ipamClaimName = network.IPAMClaimReference wasPersistentIPRequested = true @@ -603,6 +624,18 @@ func (bsnc *BaseUserDefinedNetworkController) syncPodsForUserDefinedNetwork(pods var activeNetwork util.NetInfo var err error if bsnc.IsPrimaryNetwork() { + // check to see if the primary NAD is even applicable to our controller + foundNamespaceNAD, err := bsnc.networkManager.GetPrimaryNADForNamespace(pod.Namespace) + if err != nil { + return fmt.Errorf("failed to get primary network namespace NAD: %w", err) + } + if foundNamespaceNAD == types.DefaultNetworkName { + continue + } + networkName := bsnc.networkManager.GetNetworkNameForNADKey(foundNamespaceNAD) + if networkName != "" && networkName != bsnc.GetNetworkName() { + continue + } activeNetwork, err = bsnc.networkManager.GetActiveNetworkForNamespace(pod.Namespace) if err != nil { if apierrors.IsNotFound(err) { @@ -616,7 +649,13 @@ func (bsnc *BaseUserDefinedNetworkController) syncPodsForUserDefinedNetwork(pods } } - on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, bsnc.GetNetInfo(), activeNetwork) + on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork( + pod, + bsnc.GetNetInfo(), + activeNetwork, + bsnc.networkManager.GetNetworkNameForNADKey, + bsnc.networkManager.GetPrimaryNADForNamespace, + ) if err != nil || !on { if err != nil { bsnc.recordPodErrorEvent(pod, err) @@ -629,11 +668,11 @@ func (bsnc *BaseUserDefinedNetworkController) syncPodsForUserDefinedNetwork(pods isLocalPod := bsnc.isPodScheduledinLocalZone(pod) hasRemotePort := !isLocalPod || bsnc.isLayer2Interconnect() - for nadName := range networkMap { - annotations, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + for nadKey := range networkMap { + annotations, err := util.UnmarshalPodAnnotation(pod.Annotations, nadKey) if err != nil { if !util.IsAnnotationNotSetError(err) { - klog.Errorf("Failed to get pod annotation of pod %s/%s for NAD %s", pod.Namespace, pod.Name, nadName) + klog.Errorf("Failed to get pod annotation of pod %s/%s for NAD key %s", pod.Namespace, pod.Name, nadKey) } continue } @@ -641,7 +680,7 @@ func (bsnc *BaseUserDefinedNetworkController) syncPodsForUserDefinedNetwork(pods if bsnc.allocatesPodAnnotation() && isLocalPod { // only keep track of IPs/ports that have been allocated by this // controller - expectedLogicalPortName, err := bsnc.allocatePodIPs(pod, annotations, nadName) + expectedLogicalPortName, err := bsnc.allocatePodIPs(pod, annotations, nadKey) if err != nil { return err } @@ -652,11 +691,11 @@ func (bsnc *BaseUserDefinedNetworkController) syncPodsForUserDefinedNetwork(pods if annotatedLocalPods[pod] == nil { annotatedLocalPods[pod] = map[string]*util.PodAnnotation{} } - annotatedLocalPods[pod][nadName] = annotations + annotatedLocalPods[pod][nadKey] = annotations } else if hasRemotePort { // keep also track of remote ports created for layer2 on // interconnect - expectedLogicalPorts[bsnc.GetLogicalPortName(pod, nadName)] = true + expectedLogicalPorts[bsnc.GetLogicalPortName(pod, nadKey)] = true } } } @@ -941,9 +980,9 @@ func (bsnc *BaseUserDefinedNetworkController) requireDHCP(pod *corev1.Pod) bool } func (bsnc *BaseUserDefinedNetworkController) setPodLogicalSwitchPortAddressesAndEnabledField( - pod *corev1.Pod, nadName string, mac string, ips []string, enabled bool, ops []ovsdb.Operation, + pod *corev1.Pod, nadKey string, mac string, ips []string, enabled bool, ops []ovsdb.Operation, ) ([]ovsdb.Operation, *nbdb.LogicalSwitchPort, error) { - lsp := &nbdb.LogicalSwitchPort{Name: bsnc.GetLogicalPortName(pod, nadName)} + lsp := &nbdb.LogicalSwitchPort{Name: bsnc.GetLogicalPortName(pod, nadKey)} lsp.Enabled = ptr.To(enabled) customFields := []libovsdbops.ModelUpdateField{ libovsdbops.LogicalSwitchPortEnabled, @@ -978,14 +1017,14 @@ func (bsnc *BaseUserDefinedNetworkController) setPodLogicalSwitchPortAddressesAn func (bsnc *BaseUserDefinedNetworkController) disableLiveMigrationSourceLSPOps( kubevirtLiveMigrationStatus *kubevirt.LiveMigrationStatus, - nadName string, ops []ovsdb.Operation, + nadKey string, ops []ovsdb.Operation, ) ([]ovsdb.Operation, error) { // closing the sourcePod lsp to ensure traffic goes to the now ready targetPod. - ops, _, err := bsnc.setPodLogicalSwitchPortAddressesAndEnabledField(kubevirtLiveMigrationStatus.SourcePod, nadName, "", nil, false, ops) + ops, _, err := bsnc.setPodLogicalSwitchPortAddressesAndEnabledField(kubevirtLiveMigrationStatus.SourcePod, nadKey, "", nil, false, ops) return ops, err } -func (bsnc *BaseUserDefinedNetworkController) enableSourceLSPFailedLiveMigration(pod *corev1.Pod, nadName string, mac string, ips []string) error { +func (bsnc *BaseUserDefinedNetworkController) enableSourceLSPFailedLiveMigration(pod *corev1.Pod, nadKey string, mac string, ips []string) error { kubevirtLiveMigrationStatus, err := kubevirt.DiscoverLiveMigrationStatus(bsnc.watchFactory, pod) if err != nil { return fmt.Errorf("failed to discover Live-migration status after pod termination: %w", err) @@ -996,7 +1035,7 @@ func (bsnc *BaseUserDefinedNetworkController) enableSourceLSPFailedLiveMigration return nil } // make sure sourcePod lsp is enabled if migration failed after DomainReady was set. - ops, sourcePodLsp, err := bsnc.setPodLogicalSwitchPortAddressesAndEnabledField(kubevirtLiveMigrationStatus.SourcePod, nadName, mac, ips, true, nil) + ops, sourcePodLsp, err := bsnc.setPodLogicalSwitchPortAddressesAndEnabledField(kubevirtLiveMigrationStatus.SourcePod, nadKey, mac, ips, true, nil) if err != nil { return fmt.Errorf("failed to set source Pod lsp to enabled after migration failed: %w", err) } diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy.go b/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy.go index b1d345bcf4..d13771ffd5 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy.go @@ -354,7 +354,7 @@ func (c *Controller) expandRulePeers(rule *gressRule) error { if util.PodWantsHostNetwork(pod) || util.PodCompleted(pod) || !util.PodScheduled(pod) { continue } - podIPs, err := util.GetPodIPsOfNetwork(pod, &util.DefaultNetInfo{}) + podIPs, err := util.GetPodIPsOfNetwork(pod, &util.DefaultNetInfo{}, nil) if err != nil { if errors.Is(err, util.ErrNoPodIPFound) { // we ignore podIPsNotFound error here because onANPPodUpdate @@ -472,7 +472,7 @@ func (c *Controller) convertANPSubjectToLSPs(anp *adminNetworkPolicyState) ([]*n continue } // we need to collect podIP:cPort information - podIPs, err := util.GetPodIPsOfNetwork(pod, &util.DefaultNetInfo{}) + podIPs, err := util.GetPodIPsOfNetwork(pod, &util.DefaultNetInfo{}, nil) if err != nil { if errors.Is(err, util.ErrNoPodIPFound) { // we ignore podIPsNotFound error here because onANPPodUpdate diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy_controller.go b/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy_controller.go index ed0ad36356..7dc2e62787 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy_controller.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy_controller.go @@ -514,8 +514,8 @@ func (c *Controller) onANPPodUpdate(oldObj, newObj interface{}) { // zones. Rest of the cases we may return oldPodLabels := labels.Set(oldPod.Labels) newPodLabels := labels.Set(newPod.Labels) - oldPodIPs, _ := util.GetPodIPsOfNetwork(oldPod, &util.DefaultNetInfo{}) - newPodIPs, _ := util.GetPodIPsOfNetwork(newPod, &util.DefaultNetInfo{}) + oldPodIPs, _ := util.GetPodIPsOfNetwork(oldPod, &util.DefaultNetInfo{}, nil) + newPodIPs, _ := util.GetPodIPsOfNetwork(newPod, &util.DefaultNetInfo{}, nil) oldPodRunning := util.PodRunning(oldPod) newPodRunning := util.PodRunning(newPod) oldPodCompleted := util.PodCompleted(oldPod) diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/status.go b/go-controller/pkg/ovn/controller/admin_network_policy/status.go index 5ffb2fcc2d..828f159370 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/status.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/status.go @@ -46,6 +46,22 @@ const ( policyNotReadyReason = "SetupFailed" ) +// doesStatusNeedAnUpdate compares the existing condition with the new condition +// and returns true if an update is needed, false if the status is already in the desired state. +// This helps avoid unnecessary API server calls when the status hasn't changed. +func doesStatusNeedAnUpdate(existingCondition *metav1.Condition, newCondition metav1.Condition) bool { + if existingCondition == nil { + return true // condition doesn't exist yet, needs to be created + } + // Check if Status, Reason, and Message are all the same - if so, no update needed + if existingCondition.Status == newCondition.Status && + existingCondition.Reason == newCondition.Reason && + existingCondition.Message == newCondition.Message { + return false + } + return true +} + // updateANPStatusToReady updates the status of the policy to reflect that it is ready // Each zone's ovnkube-controller will call this, hence let's update status using server-side-apply func (c *Controller) updateANPStatusToReady(anpName string) error { @@ -59,8 +75,6 @@ func (c *Controller) updateANPStatusToReady(anpName string) error { if err != nil { return fmt.Errorf("unable to update the status of ANP %s, err: %v", anpName, err) } - klog.V(5).Infof("Patched the status of ANP %v with condition type %v/%v", - anpName, policyReadyStatusType+c.zone, metav1.ConditionTrue) return nil } @@ -83,8 +97,6 @@ func (c *Controller) updateANPStatusToNotReady(anpName, message string) error { if err != nil { return fmt.Errorf("unable update the status of ANP %s, err: %v", anpName, err) } - klog.V(3).Infof("Patched the status of ANP %v with condition type %v/%v and reason %s/%s", - anpName, policyReadyStatusType+c.zone, metav1.ConditionFalse, policyNotReadyReason, message) return nil } @@ -94,6 +106,10 @@ func (c *Controller) updateANPZoneStatusCondition(newCondition metav1.Condition, return err } existingCondition := meta.FindStatusCondition(anp.Status.Conditions, newCondition.Type) + if !doesStatusNeedAnUpdate(existingCondition, newCondition) { + // status is already in the desired state, skip the update to reduce API server load + return nil + } if existingCondition == nil { newCondition.LastTransitionTime = metav1.NewTime(time.Now()) } else { @@ -109,6 +125,10 @@ func (c *Controller) updateANPZoneStatusCondition(newCondition metav1.Condition, WithStatus(anpapiapply.AdminNetworkPolicyStatus().WithConditions(newCondition)) _, err = c.anpClientSet.PolicyV1alpha1().AdminNetworkPolicies(). ApplyStatus(context.TODO(), applyObj, metav1.ApplyOptions{FieldManager: c.zone, Force: true}) + if err == nil { + klog.V(5).Infof("Patched the status of ANP %s with condition type %s/%s, reason %s, message: %s", + anpName, newCondition.Type, newCondition.Status, newCondition.Reason, newCondition.Message) + } return err } @@ -125,8 +145,6 @@ func (c *Controller) updateBANPStatusToReady(banpName string) error { if err != nil { return fmt.Errorf("unable to update the status of BANP %s, err: %v", banpName, err) } - klog.V(5).Infof("Patched the status of BANP %v with condition type %v/%v", - banpName, policyReadyStatusType+c.zone, metav1.ConditionTrue) return nil } @@ -146,8 +164,6 @@ func (c *Controller) updateBANPStatusToNotReady(banpName, message string) error if err != nil { return fmt.Errorf("unable update the status of BANP %s, err: %v", banpName, err) } - klog.V(3).Infof("Patched the status of BANP %v with condition type %v/%v and reason %s", - banpName, policyReadyStatusType+c.zone, metav1.ConditionFalse, policyNotReadyReason) return nil } @@ -157,6 +173,10 @@ func (c *Controller) updateBANPZoneStatusCondition(newCondition metav1.Condition return err } existingCondition := meta.FindStatusCondition(banp.Status.Conditions, newCondition.Type) + if !doesStatusNeedAnUpdate(existingCondition, newCondition) { + // status is already in the desired state, skip the update to reduce API server load + return nil + } if existingCondition == nil { newCondition.LastTransitionTime = metav1.NewTime(time.Now()) } else { @@ -172,5 +192,9 @@ func (c *Controller) updateBANPZoneStatusCondition(newCondition metav1.Condition WithStatus(anpapiapply.BaselineAdminNetworkPolicyStatus().WithConditions(newCondition)) _, err = c.anpClientSet.PolicyV1alpha1().BaselineAdminNetworkPolicies(). ApplyStatus(context.TODO(), applyObj, metav1.ApplyOptions{FieldManager: c.zone, Force: true}) + if err == nil { + klog.V(5).Infof("Patched the status of BANP %s with condition type %s/%s, reason %s, message: %s", + banpName, newCondition.Type, newCondition.Status, newCondition.Reason, newCondition.Message) + } return err } diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/status_test.go b/go-controller/pkg/ovn/controller/admin_network_policy/status_test.go index 6a28fa60d3..02d2ece268 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/status_test.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/status_test.go @@ -147,6 +147,177 @@ func newANPControllerWithDBSetup(dbSetup libovsdbtest.TestSetup, initANPs anpapi return controller, nil } +func TestDoesStatusNeedAnUpdate(t *testing.T) { + tests := []struct { + name string + existingCondition *metav1.Condition + newCondition metav1.Condition + expectedResult bool + }{ + { + name: "nil existing condition should need update", + existingCondition: nil, + newCondition: metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionTrue, + Reason: "SetupSucceeded", + Message: "success", + }, + expectedResult: true, + }, + { + name: "same status, reason, message should not need update", + existingCondition: &metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionTrue, + Reason: "SetupSucceeded", + Message: "success", + }, + newCondition: metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionTrue, + Reason: "SetupSucceeded", + Message: "success", + }, + expectedResult: false, + }, + { + name: "different status should need update", + existingCondition: &metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionFalse, + Reason: "SetupFailed", + Message: "error", + }, + newCondition: metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionTrue, + Reason: "SetupSucceeded", + Message: "success", + }, + expectedResult: true, + }, + { + name: "different reason should need update", + existingCondition: &metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionTrue, + Reason: "OldReason", + Message: "success", + }, + newCondition: metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionTrue, + Reason: "NewReason", + Message: "success", + }, + expectedResult: true, + }, + { + name: "different message should need update", + existingCondition: &metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionTrue, + Reason: "SetupSucceeded", + Message: "old message", + }, + newCondition: metav1.Condition{ + Type: "Ready-In-Zone-test", + Status: metav1.ConditionTrue, + Reason: "SetupSucceeded", + Message: "new message", + }, + expectedResult: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := doesStatusNeedAnUpdate(tt.existingCondition, tt.newCondition) + if result != tt.expectedResult { + t.Errorf("doesStatusNeedAnUpdate() = %v, want %v", result, tt.expectedResult) + } + }) + } +} + +func TestStatusUpdateSkippedWhenUnchanged(t *testing.T) { + g := gomega.NewGomegaWithT(t) + controller, err := newANPController( + anpapi.AdminNetworkPolicyList{ + Items: []anpapi.AdminNetworkPolicy{initialANP}, + }, + anpapi.BaselineAdminNetworkPolicyList{ + Items: []anpapi.BaselineAdminNetworkPolicy{initialBANP}, + }, + ) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // First call - should make an API call to set status to Ready + err = controller.updateANPStatusToReady(initialANP.Name) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Wait for the status to be reflected in the lister + g.Eventually(func() int { + latestANP, err := controller.anpLister.Get(initialANP.Name) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return len(latestANP.Status.Conditions) + }).Should(gomega.Equal(1)) + + // Get the number of actions after first update + actionsAfterANPFirstUpdate := len(controller.anpClientSet.(*anpfake.Clientset).Actions()) + + // Second call with same status - should NOT make an API call + err = controller.updateANPStatusToReady(initialANP.Name) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify no new actions were added (ApplyStatus was skipped) + actionsAfterANPSecondUpdate := len(controller.anpClientSet.(*anpfake.Clientset).Actions()) + g.Expect(actionsAfterANPSecondUpdate).To(gomega.Equal(actionsAfterANPFirstUpdate), + "Expected no new API calls when status is unchanged, but got %d new actions", + actionsAfterANPSecondUpdate-actionsAfterANPFirstUpdate) + + // Third call with different status (NotReady) - SHOULD make an API call + err = controller.updateANPStatusToNotReady(initialANP.Name, "something went wrong") + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify a new action WAS added (ApplyStatus was called) + actionsAfterANPThirdUpdate := len(controller.anpClientSet.(*anpfake.Clientset).Actions()) + g.Expect(actionsAfterANPThirdUpdate).To(gomega.Equal(actionsAfterANPFirstUpdate+1), + "Expected 1 new API call when status changed to NotReady, but got %d new actions", + actionsAfterANPThirdUpdate-actionsAfterANPSecondUpdate) + + // Now test BANP + err = controller.updateBANPStatusToReady(initialBANP.Name) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + g.Eventually(func() int { + latestBANP, err := controller.banpLister.Get(initialBANP.Name) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return len(latestBANP.Status.Conditions) + }).Should(gomega.Equal(1)) + + actionsAfterBANPFirstUpdate := len(controller.anpClientSet.(*anpfake.Clientset).Actions()) + + // Second call with same status - should NOT make an API call + err = controller.updateBANPStatusToReady(initialBANP.Name) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + actionsAfterBANPSecondUpdate := len(controller.anpClientSet.(*anpfake.Clientset).Actions()) + g.Expect(actionsAfterBANPSecondUpdate).To(gomega.Equal(actionsAfterBANPFirstUpdate), + "Expected no new API calls when BANP status is unchanged") + + // Third call with different status (NotReady) - SHOULD make an API call + err = controller.updateBANPStatusToNotReady(initialBANP.Name, "something went wrong") + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify a new action WAS added (ApplyStatus was called) + actionsAfterBANPThirdUpdate := len(controller.anpClientSet.(*anpfake.Clientset).Actions()) + g.Expect(actionsAfterBANPThirdUpdate).To(gomega.Equal(actionsAfterBANPFirstUpdate+1), + "Expected 1 new API call when BANP status changed to NotReady, but got %d new actions", + actionsAfterBANPThirdUpdate-actionsAfterBANPSecondUpdate) +} + func TestAddOrUpdateAdminNetworkPolicyStatus(t *testing.T) { anpName := "harry-potter" banpName := "jon-snow" diff --git a/go-controller/pkg/ovn/controller/apbroute/repair.go b/go-controller/pkg/ovn/controller/apbroute/repair.go index 75c50765b1..56867f82e5 100644 --- a/go-controller/pkg/ovn/controller/apbroute/repair.go +++ b/go-controller/pkg/ovn/controller/apbroute/repair.go @@ -155,7 +155,7 @@ func (c *ExternalGatewayMasterController) Repair() error { // if pod had no ECMP routes we need to make sure we remove logical route policy for local gw mode if !podHasAnyECMPRoutes { for _, ovnRoute := range ovnRoutes { - node := strings.TrimPrefix(ovnRoute.router, types.GWRouterPrefix) + node := util.GetWorkerFromGatewayRouter(ovnRoute.router) if err := c.nbClient.delHybridRoutePolicyForPod(net.ParseIP(podIP), node); err != nil { return fmt.Errorf("error while removing hybrid policy for pod IP: %s, on node: %s, error: %v", podIP, node, err) diff --git a/go-controller/pkg/ovn/controller/egressfirewall/egressfirewall.go b/go-controller/pkg/ovn/controller/egressfirewall/egressfirewall.go index 9729d7e4b7..d537804f56 100644 --- a/go-controller/pkg/ovn/controller/egressfirewall/egressfirewall.go +++ b/go-controller/pkg/ovn/controller/egressfirewall/egressfirewall.go @@ -108,9 +108,11 @@ type matchTarget struct { type matchKind int type cacheEntry struct { - pgName string - hasNodeSelector bool - stale bool + pgName string + hasNodeSelector bool + subnetsKey string + efResourceVersion string + logHash string } type EFController struct { @@ -129,9 +131,11 @@ type EFController struct { namespaceLister corelisters.NamespaceLister efLister v2.EgressFirewallLister - controller controller.Controller - nodeController controller.Controller - networkManager networkmanager.Interface + controller controller.Controller + nodeController controller.Controller + networkManager networkmanager.Interface + nadReconciler networkmanager.NADReconciler + nadReconcilerID uint64 // dnsNameResolver is used for resolving the IP addresses of DNS names // used in egress firewall rules dnsNameResolver dnsnameresolver.DNSNameResolver @@ -197,6 +201,19 @@ func NewEFController( nodeControllerConfig, ) + // this controller does not feed from an informer, nads are added + // to the queue by NAD Controller + nadReconcilerConfig := &controller.ReconcilerConfig{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: c.syncNAD, + Threadiness: 1, + MaxAttempts: controller.InfiniteAttempts, + } + c.nadReconciler = controller.NewReconciler( + c.name+"-NAD", + nadReconcilerConfig, + ) + return c, nil } @@ -205,6 +222,30 @@ func (oc *EFController) Reconcile(key string) { oc.controller.Reconcile(key) } +func (oc *EFController) syncNAD(key string) error { + startTime := time.Now() + controllerName := oc.name + "-NAD" + klog.V(5).Infof("%s: sync NAD %s", controllerName, key) + defer func() { + klog.V(4).Infof("%s: finished syncing NAD %s, took %v", controllerName, key, time.Since(startTime)) + }() + + namespace, _, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + klog.Errorf("%s: failed splitting key %s: %v", controllerName, key, err) + return nil + } + + // Only reconcile namespace EF for primary network changes. Secondary NADs are irrelevant for EF. + if ni := oc.networkManager.GetNetInfoForNADKey(key); ni != nil && !ni.IsPrimaryNetwork() { + return nil + } + // On delete, NAD controller has already removed the NAD from its cache, so we can't tell if it was + // primary or secondary. Reconcile anyway as a safe fallback. + oc.controller.Reconcile(namespace + "/" + egressFirewallName) + return nil +} + // initialSync deletes stale db entries for previous versions of Egress Firewall implementation and removes // stale db entries for Egress Firewalls that don't exist anymore. // Egress firewall implementation had many versions, the latest one makes no difference for gateway modes, and creates @@ -239,42 +280,56 @@ func (oc *EFController) initialSync() error { return err } + getPGPredicate := func(acl *nbdb.ACL) func(item *nbdb.PortGroup) bool { + return func(item *nbdb.PortGroup) bool { + if len(item.ACLs) == 0 { + return false + } + if item.ExternalIDs[libovsdbops.OwnerTypeKey.String()] != libovsdbops.NamespaceOwnerType { + return false + } + if acl.ExternalIDs[libovsdbops.ObjectNameKey.String()] != item.ExternalIDs[libovsdbops.ObjectNameKey.String()] { + return false + } + + for _, aclUUID := range item.ACLs { + if acl.UUID == aclUUID { + return true + } + } + return false + } + } + var deletedNSACLs = map[string][]*nbdb.ACL{} + // iterate EF ACLs, build cache with existing entries, track stale entries to be removed for _, acl := range efACLs { namespace := acl.ExternalIDs[libovsdbops.ObjectNameKey.String()] if !existingEFNamespaces[namespace] { deletedNSACLs[namespace] = append(deletedNSACLs[namespace], acl) + continue + } + // Seed cache from NBDB for namespaces that have an EF and associated ACL/PG. + if _, cached := oc.cache.Load(namespace); cached { + continue + } + foundPGs, err := libovsdbops.FindPortGroupsWithPredicate(oc.nbClient, getPGPredicate(acl)) + if err != nil { + return fmt.Errorf("failed to search for port groups during egress firewall cache seed: %w", err) + } + if len(foundPGs) > 0 { + oc.cache.Store(namespace, &cacheEntry{pgName: foundPGs[0].Name}) } } + // remove stale ACLs err = batching.BatchMap[*nbdb.ACL](aclChangePGBatchSize, deletedNSACLs, func(batchNsACLs map[string][]*nbdb.ACL) error { var ops []ovsdb.Operation var err error for _, acls := range batchNsACLs { // find the port group that has the stale acl for _, acl := range acls { - p := func(item *nbdb.PortGroup) bool { - if len(item.ACLs) == 0 { - return false - } - // should be namespace port group - if item.ExternalIDs[libovsdbops.OwnerTypeKey.String()] != libovsdbops.NamespaceOwnerType { - return false - } - - // name should match PG name - if acl.ExternalIDs[libovsdbops.ObjectNameKey.String()] != item.ExternalIDs[libovsdbops.ObjectNameKey.String()] { - return false - } - - for _, aclUUID := range item.ACLs { - if acl.UUID == aclUUID { - return true - } - } - return false - } - foundPGs, err := libovsdbops.FindPortGroupsWithPredicate(oc.nbClient, p) + foundPGs, err := libovsdbops.FindPortGroupsWithPredicate(oc.nbClient, getPGPredicate(acl)) if err != nil { return fmt.Errorf("failed to search for port groups during egress firewall ACL sync: %w", err) } @@ -317,55 +372,24 @@ func (oc *EFController) initialSync() error { func (oc *EFController) Start() error { klog.Infof("Starting EgressFirewall controller") - if err := controller.StartWithInitialSync(oc.initialSync, oc.controller, oc.nodeController); err != nil { + id, err := oc.networkManager.RegisterNADReconciler(oc.nadReconciler) + if err != nil { return err } - return oc.networkManager.RegisterNADHandler(oc.handleNetworkEvent) + oc.nadReconcilerID = id + return controller.StartWithInitialSync(oc.initialSync, oc.controller, oc.nodeController, oc.nadReconciler) } func (oc *EFController) Stop() { klog.Infof("%s: shutting down", oc.name) - controller.Stop(oc.nodeController, oc.controller) -} - -func (oc *EFController) handleNetworkEvent(nadName string, info util.NetInfo, removed bool) { - if info != nil && !info.IsPrimaryNetwork() { // egressFirewall only supported for primary network - return - } - if removed { // delete case - namespace, _, err := cache.SplitMetaNamespaceKey(nadName) - if err != nil { - klog.Errorf("%s: failed splitting key %s: %v", oc.name, nadName, err) - return + if oc.nadReconcilerID != 0 { + if err := oc.networkManager.DeRegisterNADReconciler(oc.nadReconcilerID); err != nil { + klog.Warningf("%s: failed to deregister NAD reconciler: %v", oc.name, err) } - oc.cache.LockKey(namespace) - entry, ok := oc.cache.Load(namespace) - if !ok { - oc.cache.UnlockKey(namespace) - return // no cache entry exists, so nothing to remove - } - entry.stale = true - oc.cache.UnlockKey(namespace) - klog.V(3).Infof("NAD removed for egress firewall in namespace: %q. Will sync.", namespace) - oc.controller.Reconcile(fmt.Sprintf("%s/%s", namespace, egressFirewallName)) - return - } - // add/update case - namespace, _, err := cache.SplitMetaNamespaceKey(nadName) - if err != nil { - klog.Errorf("%s: failed splitting key %s: %v", oc.name, nadName, err) - return - } - ef, err := oc.efLister.EgressFirewalls(namespace).Get(egressFirewallName) - if err != nil || ef == nil { - return - } - key, err := cache.MetaNamespaceKeyFunc(ef) - if err != nil { - return } - klog.V(3).Infof("NAD add/update for egress firewall in namespace: %q. Will sync.", namespace) - oc.controller.Reconcile(key) + controller.Stop(oc.nodeController, oc.controller, oc.nadReconciler) + oc.nadReconciler = nil + oc.nadReconcilerID = 0 } func (oc *EFController) sync(key string) (updateErr error) { @@ -374,44 +398,80 @@ func (oc *EFController) sync(key string) (updateErr error) { if err != nil { return fmt.Errorf("invalid resource key for egress firewall: %s", key) } - shouldRemove := false - shouldAddUpdate := false + oc.cache.LockKey(namespace) + defer oc.cache.UnlockKey(namespace) + + existingEntry, _ := oc.cache.Load(namespace) + var newEntry *cacheEntry ef, err := oc.efLister.EgressFirewalls(namespace).Get(efName) if err != nil { if !apierrors.IsNotFound(err) { return err } - shouldRemove = true } else { - shouldAddUpdate = true + skipStatusUpdate := false + defer func() { + if skipStatusUpdate { + return + } + if statusErr := oc.setEgressFirewallStatus(ef, updateErr); statusErr != nil { + updateErr = utilerrors.Join(updateErr, fmt.Errorf("failed to update egress firewall status %s, error: %w", + GetEgressFirewallNamespacedName(ef), statusErr)) + } + }() + + activeNetwork, netErr := oc.networkManager.GetActiveNetworkForNamespace(namespace) + if netErr != nil { + if util.IsUnprocessedActiveNetworkError(netErr) { + klog.V(5).Infof("Skipping egress firewall %s/%s: primary network not ready: %v", namespace, efName, netErr) + skipStatusUpdate = true + return nil + } + if util.IsInvalidPrimaryNetworkError(netErr) { + // Namespace requires P-UDN, but it does not exist. Remove EF config and surface error in status. + updateErr = netErr + } else { + return fmt.Errorf("failed to get active network for egress firewall %s/%s namespace: %w", + namespace, efName, netErr) + } + } else { + aclLoggingLevels, logErr := oc.getNamespaceACLLogging(namespace) + if logErr != nil { + return fmt.Errorf("failed to get acl logging levels for egress firewall %s/%s: %w", + namespace, efName, logErr) + } + ownerController := activeNetwork.GetNetworkName() + "-network-controller" + newEntry = &cacheEntry{ + pgName: libovsdbutil.GetPortGroupName(getNamespacePortGroupDbIDs(namespace, ownerController)), + subnetsKey: subnetsKeyForNetInfo(activeNetwork), + efResourceVersion: ef.ResourceVersion, + logHash: aclLogHash(aclLoggingLevels), + } + } } - oc.cache.LockKey(namespace) - defer oc.cache.UnlockKey(namespace) - - if entry, ok := oc.cache.Load(namespace); ok { - if entry.stale { - shouldRemove = true - } + // If nothing relevant changed since last apply, skip update. + if existingEntry != nil && existingEntry.hasNodeSelector { + // Node selector rules depend on node state; always recalculate. + } else if entriesEqual(existingEntry, newEntry) { + return updateErr } - if shouldRemove { - // delete case - entry, ok := oc.cache.Load(namespace) - if !ok { - return nil - } - klog.Infof("Removing egress firewall %s", key) - pgName := entry.pgName + // Delete existing state if there is no desired state. + if newEntry == nil { + pgName := existingEntry.pgName + if pgName != "" { + klog.Infof("Removing egress firewall %s", key) - p := libovsdbops.GetPredicate[*nbdb.ACL](oc.GetEgressFirewallACLDbIDsNoRule(namespace), nil) - invalidACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) - if err != nil { - return fmt.Errorf("error finding ACLs for egress firewall %s: %w", key, err) - } - if err := libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{pgName}, invalidACLs...); err != nil { - return fmt.Errorf("error deleting stale ACLs for egress firewall %s: %w", key, err) + p := libovsdbops.GetPredicate[*nbdb.ACL](oc.GetEgressFirewallACLDbIDsNoRule(namespace), nil) + invalidACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) + if err != nil { + return fmt.Errorf("error finding ACLs for egress firewall %s: %w", key, err) + } + if err := libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{pgName}, invalidACLs...); err != nil { + return fmt.Errorf("error deleting stale ACLs for egress firewall %s: %w", key, err) + } } oc.cache.Delete(namespace) if err := oc.dnsNameResolver.Delete(namespace); err != nil { @@ -422,37 +482,18 @@ func (oc *EFController) sync(key string) (updateErr error) { metrics.UpdateEgressFirewallRuleCount(float64(-numRules.(uint32))) oc.ruleCounter.Delete(key) } else { - klog.Errorf("Unable to decrement egress firewall rule count, cache miss for key: %s", key) + klog.V(4).Infof("Unable to decrement egress firewall rule count, cache miss for key: %s", key) } metrics.DecrementEgressFirewallCount() + return updateErr } - if !shouldAddUpdate { - return nil + // Apply desired state. + if err := oc.addEgressFirewall(ef, newEntry); err != nil { + updateErr = err + return updateErr } - - // add/update case - defer func() { - if statusErr := oc.setEgressFirewallStatus(ef, updateErr); statusErr != nil { - updateErr = utilerrors.Join(updateErr, fmt.Errorf("failed to update egress firewall status %s, error: %w", - GetEgressFirewallNamespacedName(ef), statusErr)) - } - }() - - pgName, err := oc.getNamespacePortGroupName(ef.Namespace) - if err != nil { - return fmt.Errorf("failed to get portgroup for egress firewall %s/%s namespace: %w", - ef.Namespace, ef.Name, err) - } - - entry := &cacheEntry{ - pgName: pgName, - } - updateErr = oc.addEgressFirewall(ef, entry) - if updateErr != nil { - return - } - oc.cache.Store(namespace, entry) + oc.cache.Store(namespace, newEntry) // remove stale ACLs p := libovsdbops.GetPredicate[*nbdb.ACL](oc.GetEgressFirewallACLDbIDsNoRule(namespace), func(acl *nbdb.ACL) bool { @@ -469,19 +510,73 @@ func (oc *EFController) sync(key string) (updateErr error) { } return false }) - var invalidACLs []*nbdb.ACL - invalidACLs, updateErr = libovsdbops.FindACLsWithPredicate(oc.nbClient, p) - if updateErr != nil { - updateErr = fmt.Errorf("error finding ACLs for egress firewall %s: %w", ef.Name, updateErr) + invalidACLs, findErr := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) + if findErr != nil { + updateErr = fmt.Errorf("error finding ACLs for egress firewall %s: %w", ef.Name, findErr) return } - if removalErr := libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{pgName}, invalidACLs...); removalErr != nil { + if removalErr := libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{newEntry.pgName}, invalidACLs...); removalErr != nil { updateErr = fmt.Errorf("error deleting stale ACLs for egress firewall %s: %w", ef.Name, removalErr) } + // If the port-group changed, remove the (now updated) ACLs from the previous port-group so + // we don't keep stale references around. + if existingEntry != nil && existingEntry.pgName != "" && existingEntry.pgName != newEntry.pgName { + p := libovsdbops.GetPredicate[*nbdb.ACL](oc.GetEgressFirewallACLDbIDsNoRule(namespace), nil) + acls, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) + if err != nil { + updateErr = utilerrors.Join(updateErr, fmt.Errorf("error finding ACLs for egress firewall %s/%s: %w", namespace, efName, err)) + } else if err := libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{existingEntry.pgName}, acls...); err != nil { + updateErr = utilerrors.Join(updateErr, fmt.Errorf("error deleting stale ACL refs for egress firewall %s/%s: %w", namespace, efName, err)) + } + } + + // Clean up any DNS address sets that are no longer referenced by ACLs. + if err := oc.dnsNameResolver.DeleteStaleAddrSets(oc.nbClient); err != nil { + updateErr = utilerrors.Join(updateErr, fmt.Errorf("error deleting stale DNS address sets for egress firewall %s/%s: %w", + namespace, efName, err)) + } + return } +func subnetsKeyForNetInfo(netInfo util.NetInfo) string { + if netInfo == nil { + return "" + } + subnets := netInfo.Subnets() + if len(subnets) == 0 { + return "" + } + keys := make([]string, 0, len(subnets)) + for _, s := range subnets { + keys = append(keys, s.String()) + } + slices.Sort(keys) + return strings.Join(keys, ",") +} + +func entriesEqual(a, b *cacheEntry) bool { + switch { + case a == nil && b == nil: + return true + case a == nil || b == nil: + return false + default: + return a.pgName == b.pgName && + a.subnetsKey == b.subnetsKey && + a.efResourceVersion == b.efResourceVersion && + a.logHash == b.logHash + } +} + +func aclLogHash(levels *libovsdbutil.ACLLoggingLevels) string { + if levels == nil { + return "" + } + return fmt.Sprintf("%s|%s", levels.Allow, levels.Deny) +} + func (oc *EFController) buildEgressFirewallConstruct(egressFirewall *egressfirewallapi.EgressFirewall, entry *cacheEntry) (*egressFirewall, error) { ef := cloneEgressFirewall(egressFirewall) var errorList []error @@ -995,7 +1090,7 @@ func getNamespacePortGroupDbIDs(ns string, controller string) *libovsdbops.DbObj func (oc *EFController) getNamespacePortGroupName(namespace string) (string, error) { activeNetwork, err := oc.networkManager.GetActiveNetworkForNamespace(namespace) if err != nil { - return "", fmt.Errorf("failed to get active network for namespace %s: %v", namespace, err) + return "", fmt.Errorf("failed to get active network for namespace %s: %w", namespace, err) } ownerController := activeNetwork.GetNetworkName() + "-network-controller" return libovsdbutil.GetPortGroupName(getNamespacePortGroupDbIDs(namespace, ownerController)), nil diff --git a/go-controller/pkg/ovn/controller/egressfirewall/egressfirewall_sync_test.go b/go-controller/pkg/ovn/controller/egressfirewall/egressfirewall_sync_test.go new file mode 100644 index 0000000000..eb280d5109 --- /dev/null +++ b/go-controller/pkg/ovn/controller/egressfirewall/egressfirewall_sync_test.go @@ -0,0 +1,191 @@ +package egressfirewall + +import ( + "context" + "sync" + "testing" + + cnitypes "github.com/containernetworking/cni/pkg/types" + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + libovsdbclient "github.com/ovn-kubernetes/libovsdb/client" + "github.com/ovn-kubernetes/libovsdb/ovsdb" + + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + egressfirewallapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1" + egressfirewalllisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/listers/egressfirewall/v1" + libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" + libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + fakenetworkmanager "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/syncmap" + libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +type noopDNSNameResolver struct{} + +func (noopDNSNameResolver) Add(string, string) (addressset.AddressSet, error) { return nil, nil } +func (noopDNSNameResolver) Delete(string) error { return nil } +func (noopDNSNameResolver) Run() error { return nil } +func (noopDNSNameResolver) Shutdown() {} +func (noopDNSNameResolver) DeleteStaleAddrSets(libovsdbclient.Client) error { return nil } + +type panicTransactClient struct { + libovsdbclient.Client +} + +func (p *panicTransactClient) Transact(context.Context, ...ovsdb.Operation) ([]ovsdb.OperationResult, error) { + panic("unexpected Transact call") +} + +func mustNetInfo(t *testing.T, name, subnets string) util.NetInfo { + t.Helper() + ni, err := util.NewNetInfo(&ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: name}, + Topology: types.Layer3Topology, + Subnets: subnets, + Role: types.NetworkRolePrimary, + MTU: 1400, + }) + require.NoError(t, err) + return ni +} + +func TestEFControllerSync_UpdatesOnSubnetChangeAndSkipsWhenUnchanged(t *testing.T) { + require.NoError(t, config.PrepareTestConfig()) + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + + const ( + namespace = "namespace1" + udnName = "udn-test" + zone = "global" + ) + + netInfo1 := mustNetInfo(t, udnName, "10.128.0.0/14") + // Keep the subnet intersecting the destination CIDR, but change it so we can verify the ACL match + // is updated (rather than just removing the exclusion entirely). + netInfo2 := mustNetInfo(t, udnName, "10.128.0.0/15") + + networkManager := &fakenetworkmanager.FakeNetworkManager{ + PrimaryNetworks: map[string]util.NetInfo{ + namespace: netInfo1, + }, + } + + ownerController := udnName + "-network-controller" + pgName := libovsdbutil.GetPortGroupName(getNamespacePortGroupDbIDs(namespace, ownerController)) + + initialDB := libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.PortGroup{ + Name: pgName, + ExternalIDs: map[string]string{ + libovsdbops.OwnerTypeKey.String(): libovsdbops.NamespaceOwnerType, + libovsdbops.OwnerControllerKey.String(): ownerController, + libovsdbops.ObjectNameKey.String(): namespace, + }, + }, + }, + } + nbClient, _, cleanup, err := libovsdbtest.NewNBSBTestHarness(initialDB) + require.NoError(t, err) + t.Cleanup(cleanup.Cleanup) + + nsIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + require.NoError(t, nsIndexer.Add(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}})) + namespaceLister := corelisters.NewNamespaceLister(nsIndexer) + + efIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + ef := &egressfirewallapi.EgressFirewall{ + ObjectMeta: metav1.ObjectMeta{ + Name: egressFirewallName, + Namespace: namespace, + ResourceVersion: "1", + }, + Spec: egressfirewallapi.EgressFirewallSpec{ + Egress: []egressfirewallapi.EgressFirewallRule{ + { + Type: egressfirewallapi.EgressFirewallRuleAllow, + To: egressfirewallapi.EgressFirewallDestination{ + CIDRSelector: "10.128.1.0/24", + }, + }, + }, + }, + Status: egressfirewallapi.EgressFirewallStatus{ + Messages: []string{types.GetZoneStatus(zone, EgressFirewallAppliedCorrectly)}, + }, + } + require.NoError(t, efIndexer.Add(ef)) + efLister := egressfirewalllisters.NewEgressFirewallLister(efIndexer) + + oc := &EFController{ + name: "test", + zone: zone, + cache: syncmap.NewSyncMap[*cacheEntry](), + nbClient: nbClient, + kube: nil, // status updates are no-op in this test due to pre-seeded status message + namespaceLister: namespaceLister, + efLister: efLister, + networkManager: networkManager, + ruleCounter: sync.Map{}, + dnsNameResolver: noopDNSNameResolver{}, + } + + // Pre-seed rule counter so status updates don't affect global metrics. + oc.ruleCounter.Store(namespace+"/"+egressFirewallName, uint32(len(ef.Spec.Egress))) + + // First sync creates ACLs and stores cache entry. + err = oc.sync(namespace + "/" + egressFirewallName) + require.NoError(t, err) + + p := libovsdbops.GetPredicate[*nbdb.ACL](oc.GetEgressFirewallACLDbIDs(namespace, 0), nil) + acls, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) + require.NoError(t, err) + require.Len(t, acls, 1) + require.Contains(t, acls[0].Match, "ip4.dst != 10.128.0.0/14") + + // Update netInfo subnets (same network name => same PG name), then sync again. + networkManager.Lock() + networkManager.PrimaryNetworks[namespace] = netInfo2 + networkManager.Unlock() + + err = oc.sync(namespace + "/" + egressFirewallName) + require.NoError(t, err) + + acls, err = libovsdbops.FindACLsWithPredicate(oc.nbClient, p) + require.NoError(t, err) + require.Len(t, acls, 1) + require.NotContains(t, acls[0].Match, "ip4.dst != 10.128.0.0/14") + require.Contains(t, acls[0].Match, "ip4.dst != 10.128.0.0/15") + + // Now that netInfo, EF, and PG are stable, ensure we skip OVN updates. + oc.nbClient = &panicTransactClient{Client: nbClient} + require.NotPanics(t, func() { + err = oc.sync(namespace + "/" + egressFirewallName) + }) + require.NoError(t, err) + + // No further changes; ensure match is still the updated one. + acls, err = libovsdbops.FindACLsWithPredicate(nbClient, p) + require.NoError(t, err) + require.Len(t, acls, 1) + require.NotContains(t, acls[0].Match, "ip4.dst != 10.128.0.0/14") + require.Contains(t, acls[0].Match, "ip4.dst != 10.128.0.0/15") + + // Sanity: the controller cache is present and matches current pg/subnets. + entry, ok := oc.cache.Load(namespace) + require.True(t, ok) + require.Equal(t, pgName, entry.pgName) + require.Equal(t, subnetsKeyForNetInfo(netInfo2), entry.subnetsKey) +} diff --git a/go-controller/pkg/ovn/controller/network_qos/network_qos.go b/go-controller/pkg/ovn/controller/network_qos/network_qos.go index 04b9ceef9d..b754f074fa 100644 --- a/go-controller/pkg/ovn/controller/network_qos/network_qos.go +++ b/go-controller/pkg/ovn/controller/network_qos/network_qos.go @@ -25,6 +25,7 @@ import ( crdtypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/types" udnv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) func (c *Controller) processNextNQOSWorkItem(wg *sync.WaitGroup) bool { @@ -380,9 +381,23 @@ func (c *Controller) networkManagedByMe(networkSelectors crdtypes.NetworkSelecto return false, nil } for _, nad := range selectedNads { - nadKey := joinMetaNamespaceAndName(nad.Namespace, nad.Name) - if ((nadKey == types.DefaultNetworkName) && c.IsDefault()) || - (!c.IsDefault() && c.HasNAD(nadKey)) { + networkName := util.GetAnnotatedNetworkName(nad) + if networkName == "" { + nadInfo, err := util.ParseNADInfo(nad) + if err == nil && nadInfo != nil { + networkName = nadInfo.GetNetworkName() + } + } + if networkName == "" { + continue + } + if networkName == types.DefaultNetworkName && c.IsDefault() { + return true, nil + } + if c.IsDefault() { + continue + } + if networkName == c.GetNetworkName() { return true, nil } } diff --git a/go-controller/pkg/ovn/controller/network_qos/network_qos_controller.go b/go-controller/pkg/ovn/controller/network_qos/network_qos_controller.go index 3a75fc0f30..797fb930d2 100644 --- a/go-controller/pkg/ovn/controller/network_qos/network_qos_controller.go +++ b/go-controller/pkg/ovn/controller/network_qos/network_qos_controller.go @@ -28,6 +28,7 @@ import ( networkqosinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1/apis/informers/externalversions/networkqos/v1alpha1" networkqoslister "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1/apis/listers/networkqos/v1alpha1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/syncmap" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -49,7 +50,8 @@ type Controller struct { // (values are default-network-controller, secondary-network-controller etc..) controllerName string util.NetInfo - nqosClientSet networkqosclientset.Interface + networkManager networkmanager.Interface + nqosClientSet networkqosclientset.Interface // libovsdb northbound client interface nbClient libovsdbclient.Client @@ -135,13 +137,19 @@ func NewController( podInformer corev1informers.PodInformer, nodeInformer corev1informers.NodeInformer, nadInformer nadinformerv1.NetworkAttachmentDefinitionInformer, + networkManager networkmanager.Interface, addressSetFactory addressset.AddressSetFactory, isPodScheduledinLocalZone func(*corev1.Pod) bool, zone string) (*Controller, error) { + if netInfo.IsUserDefinedNetwork() && networkManager == nil { + return nil, fmt.Errorf("network manager is required for network %q", netInfo.GetNetworkName()) + } + c := &Controller{ controllerName: controllerName, NetInfo: netInfo, + networkManager: networkManager, nbClient: nbClient, nqosClientSet: nqosClient, addressSetFactory: addressSetFactory, @@ -485,8 +493,9 @@ func (c *Controller) onNQOSPodUpdate(oldObj, newObj interface{}) { // zones. Rest of the cases we may return oldPodLabels := labels.Set(oldPod.Labels) newPodLabels := labels.Set(newPod.Labels) - oldPodIPs, _ := util.GetPodIPsOfNetwork(oldPod, c.NetInfo) - newPodIPs, _ := util.GetPodIPsOfNetwork(newPod, c.NetInfo) + resolver := c.podNetworkResolver() + oldPodIPs, _ := util.GetPodIPsOfNetwork(oldPod, c.NetInfo, resolver) + newPodIPs, _ := util.GetPodIPsOfNetwork(newPod, c.NetInfo, resolver) oldPodCompleted := util.PodCompleted(oldPod) newPodCompleted := util.PodCompleted(newPod) if labels.Equals(oldPodLabels, newPodLabels) && @@ -500,6 +509,13 @@ func (c *Controller) onNQOSPodUpdate(oldObj, newObj interface{}) { c.nqosPodQueue.Add(newEventData(oldPod, newPod)) } +func (c *Controller) podNetworkResolver() func(nadKey string) string { + if !c.NetInfo.IsUserDefinedNetwork() { + return nil + } + return c.networkManager.GetNetworkNameForNADKey +} + // onNQOSPodDelete queues the pod for processing. func (c *Controller) onNQOSPodDelete(obj interface{}) { pod, ok := obj.(*corev1.Pod) diff --git a/go-controller/pkg/ovn/controller/network_qos/network_qos_pod.go b/go-controller/pkg/ovn/controller/network_qos/network_qos_pod.go index 625a8549ee..98f4608404 100644 --- a/go-controller/pkg/ovn/controller/network_qos/network_qos_pod.go +++ b/go-controller/pkg/ovn/controller/network_qos/network_qos_pod.go @@ -56,7 +56,7 @@ func (c *Controller) syncNetworkQoSPod(eventData *eventData[*corev1.Pod]) error // - match source: add the ip to source address set, bind qos rule to the switch // - match dest: add the ip to the destination address set func (c *Controller) setPodForNQOS(pod *corev1.Pod, nqosState *networkQoSState, namespace *corev1.Namespace, addressSetMap map[string]sets.Set[string]) error { - addresses, err := getPodAddresses(pod, c.NetInfo) + addresses, err := getPodAddresses(pod, c.NetInfo, c.podNetworkResolver()) if err == nil && len(addresses) == 0 { // pod either is not attached to this network, or hasn't been annotated with addresses yet, return without retry klog.V(6).Infof("Pod %s/%s doesn't have addresses on network %s, skip NetworkQoS processing", pod.Namespace, pod.Name, c.GetNetworkName()) diff --git a/go-controller/pkg/ovn/controller/network_qos/network_qos_test.go b/go-controller/pkg/ovn/controller/network_qos/network_qos_test.go index 4b79f1198b..34f79690cb 100644 --- a/go-controller/pkg/ovn/controller/network_qos/network_qos_test.go +++ b/go-controller/pkg/ovn/controller/network_qos/network_qos_test.go @@ -25,6 +25,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" ovnk8stesting "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" @@ -305,13 +306,13 @@ func tableEntrySetup(enableInterconnect bool) { fakeNQoSClient = ovnClientset.NetworkQoSClient initEnv(ovnClientset, initialDB) // init controller for default network - initNetworkQoSController(&util.DefaultNetInfo{}, defaultAddrsetFactory, defaultControllerName, enableInterconnect) + initNetworkQoSController(&util.DefaultNetInfo{}, nil, defaultAddrsetFactory, defaultControllerName, enableInterconnect) // init controller for stream nad streamImmutableNadInfo, err := util.ParseNADInfo(nad) Expect(err).NotTo(HaveOccurred()) streamNadInfo := util.NewMutableNetInfo(streamImmutableNadInfo) streamNadInfo.AddNADs("default/stream") - initNetworkQoSController(streamNadInfo, streamAddrsetFactory, streamControllerName, enableInterconnect) + initNetworkQoSController(streamNadInfo, []string{"default/stream"}, streamAddrsetFactory, streamControllerName, enableInterconnect) } var _ = AfterEach(func() { @@ -803,7 +804,7 @@ var _ = Describe("NetworkQoS Controller", func() { Expect(err).NotTo(HaveOccurred()) localnetNadInfo := util.NewMutableNetInfo(localnetImmutableNadInfo) localnetNadInfo.AddNADs("default/netwk1") - ctrl := initNetworkQoSController(localnetNadInfo, addressset.NewFakeAddressSetFactory("netwk1-controller"), "netwk1-controller", enableInterconnect) + ctrl := initNetworkQoSController(localnetNadInfo, []string{"default/netwk1"}, addressset.NewFakeAddressSetFactory("netwk1-controller"), "netwk1-controller", enableInterconnect) lsName := ctrl.getLogicalSwitchName("dummy") Expect(lsName).To(Equal("netwk1_ovn_localnet_switch")) } @@ -815,7 +816,7 @@ var _ = Describe("NetworkQoS Controller", func() { Expect(err).NotTo(HaveOccurred()) layer2NadInfo := util.NewMutableNetInfo(layer2ImmutableNadInfo) layer2NadInfo.AddNADs("default/netwk2") - ctrl := initNetworkQoSController(layer2NadInfo, addressset.NewFakeAddressSetFactory("netwk2-controller"), "netwk2-controller", enableInterconnect) + ctrl := initNetworkQoSController(layer2NadInfo, []string{"default/netwk2"}, addressset.NewFakeAddressSetFactory("netwk2-controller"), "netwk2-controller", enableInterconnect) lsName := ctrl.getLogicalSwitchName("dummy") Expect(lsName).To(Equal("netwk2_ovn_layer2_switch")) } @@ -890,7 +891,7 @@ var _ = Describe("NetworkQoS Controller", func() { // Wrap the NetInfo with our custom implementation that returns true for IsPrimaryNetwork() primNetWrapper := &primaryNetInfoWrapper{NetInfo: primaryNadInfo} - initNetworkQoSController(primNetWrapper, addressset.NewFakeAddressSetFactory("primary-controller"), "primary-controller", enableInterconnect) + initNetworkQoSController(primNetWrapper, nil, addressset.NewFakeAddressSetFactory("primary-controller"), "primary-controller", enableInterconnect) // Ensure app1 namespace exists before testing primary networks ns, err := fakeKubeClient.CoreV1().Namespaces().Get(context.TODO(), app1Namespace, metav1.GetOptions{}) @@ -1013,7 +1014,7 @@ var _ = Describe("NetworkQoS Controller", func() { // Wrap the NetInfo with our custom implementation that returns true for IsUserDefinedNetwork() secNetWrapper := &secondaryNetInfoWrapper{NetInfo: secondaryNadInfo} - initNetworkQoSController(secNetWrapper, addressset.NewFakeAddressSetFactory("secondary-controller"), "secondary-controller", enableInterconnect) + initNetworkQoSController(secNetWrapper, []string{"default/secondary"}, addressset.NewFakeAddressSetFactory("secondary-controller"), "secondary-controller", enableInterconnect) // Ensure app3 namespace exists before testing secondary networks ns, err := fakeKubeClient.CoreV1().Namespaces().Get(context.TODO(), app3Namespace, metav1.GetOptions{}) @@ -1205,7 +1206,20 @@ func initEnv(clientset *util.OVNClientset, initialDB *libovsdbtest.TestSetup) { streamAddrsetFactory = addressset.NewFakeAddressSetFactory("stream-network-controller") } -func initNetworkQoSController(netInfo util.NetInfo, addrsetFactory addressset.AddressSetFactory, controllerName string, enableInterconnect bool) *Controller { +func initNetworkQoSController(netInfo util.NetInfo, nadKeys []string, addrsetFactory addressset.AddressSetFactory, controllerName string, enableInterconnect bool) *Controller { + var networkMgr networkmanager.Interface + if netInfo.IsUserDefinedNetwork() { + if len(nadKeys) == 0 { + panic(fmt.Sprintf("missing NAD keys for user-defined network %q", netInfo.GetNetworkName())) + } + fakeNetworkMgr := &networkmanager.FakeNetworkManager{ + NADNetworks: map[string]util.NetInfo{}, + } + for _, nadKey := range nadKeys { + fakeNetworkMgr.NADNetworks[nadKey] = netInfo + } + networkMgr = fakeNetworkMgr + } nqosController, err := NewController( controllerName, netInfo, @@ -1217,6 +1231,7 @@ func initNetworkQoSController(netInfo util.NetInfo, addrsetFactory addressset.Ad watchFactory.PodCoreInformer(), watchFactory.NodeCoreInformer(), watchFactory.NADInformer(), + networkMgr, addrsetFactory, func(pod *corev1.Pod) bool { return pod.Spec.NodeName == "node1" || !enableInterconnect diff --git a/go-controller/pkg/ovn/controller/network_qos/utils.go b/go-controller/pkg/ovn/controller/network_qos/utils.go index 7d58a7ad3e..eaafe69152 100644 --- a/go-controller/pkg/ovn/controller/network_qos/utils.go +++ b/go-controller/pkg/ovn/controller/network_qos/utils.go @@ -31,7 +31,7 @@ func GetNetworkQoSAddrSetDbIDs(nqosNamespace, nqosName, ruleIndex, ipBlockIndex, }) } -func getPodAddresses(pod *corev1.Pod, networkInfo ovnkutil.NetInfo) ([]string, error) { +func getPodAddresses(pod *corev1.Pod, networkInfo ovnkutil.NetInfo, resolver func(nadKey string) string) ([]string, error) { // check annotation "k8s.ovn.org/pod-networks" before calling GetPodIPsOfNetwork, // as it's no easy to check if the error is caused by missing annotation, while // we don't want to return error for such case as it will trigger retry @@ -40,7 +40,7 @@ func getPodAddresses(pod *corev1.Pod, networkInfo ovnkutil.NetInfo) ([]string, e // pod hasn't been annotated yet, return nil to avoid retry return nil, nil } - ips, err := ovnkutil.GetPodIPsOfNetwork(pod, networkInfo) + ips, err := ovnkutil.GetPodIPsOfNetwork(pod, networkInfo, resolver) if err != nil { return nil, err } diff --git a/go-controller/pkg/ovn/controller/networkconnect/controller.go b/go-controller/pkg/ovn/controller/networkconnect/controller.go new file mode 100644 index 0000000000..a66c8ec266 --- /dev/null +++ b/go-controller/pkg/ovn/controller/networkconnect/controller.go @@ -0,0 +1,416 @@ +package networkconnect + +import ( + "fmt" + "reflect" + "sync" + "time" + + nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + libovsdbclient "github.com/ovn-kubernetes/libovsdb/client" + + controllerutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" + networkconnectlisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1/apis/listers/clusternetworkconnect/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +const ( + controllerName = "ovnkube-network-connect-controller" +) + +// Controller manages network connectivity between (C)UDNs based on ClusterNetworkConnect CRs. +// It runs in each ovnkube-controller and creates OVN topology: +// - 1 connect-router for each CNC +// - ports connecting each selected (C)UDN network routers to connect-router +// - logical router policies on each selected (C)UDN's network router steering traffic to the connect-router +// - logical router static routes on the connect-router routing traffic to the corresponding selected (C)UDNs +type Controller struct { + // zone is the name of the zone that this controller manages + zone string + + // nbClient is the libovsdb northbound client interface + nbClient libovsdbclient.Client + + // wf is the watch factory for accessing informers + wf *factory.WatchFactory + + // listers + cncLister networkconnectlisters.ClusterNetworkConnectLister + nodeLister corev1listers.NodeLister + nadLister nadlisters.NetworkAttachmentDefinitionLister + + // networkManager provides access to network information + networkManager networkmanager.Interface + + // cncController handles ClusterNetworkConnect events + cncController controllerutil.Controller + + // nodeController handles Node events (for updating routes when nodes change) + nodeController controllerutil.Controller + + // nadReconciler handles NAD-triggered CNC requeues + nadReconciler networkmanager.NADReconciler + nadReconcilerID uint64 + + // Single global lock protecting all controller state + sync.RWMutex + + // cncCache holds the state for each CNC keyed by CNC name + cncCache map[string]*networkConnectState +} + +// networkConnectState tracks the state of a single ClusterNetworkConnect +type networkConnectState struct { + // name of the ClusterNetworkConnect + name string + // tunnelID for the connect router + tunnelID int + // connectedNetworks is the set of owner keys (e.g., "layer3_1", "layer2_2") for networks + // connected by this CNC. Used to track OVN resources created and detect NAD matching changes. + connectedNetworks sets.Set[string] +} + +// NewController creates a new network connect controller for ovnkube-controller. +func NewController( + zone string, + nbClient libovsdbclient.Client, + wf *factory.WatchFactory, + networkManager networkmanager.Interface, +) *Controller { + cncLister := wf.ClusterNetworkConnectInformer().Lister() + nodeLister := wf.NodeCoreInformer().Lister() + nadLister := wf.NADInformer().Lister() + + c := &Controller{ + zone: zone, + nbClient: nbClient, + wf: wf, + cncLister: cncLister, + nodeLister: nodeLister, + nadLister: nadLister, + networkManager: networkManager, + cncCache: make(map[string]*networkConnectState), + } + + cncCfg := &controllerutil.ControllerConfig[networkconnectv1.ClusterNetworkConnect]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Informer: wf.ClusterNetworkConnectInformer().Informer(), + Lister: cncLister.List, + Reconcile: c.reconcileCNC, + ObjNeedsUpdate: cncNeedsUpdate, + Threadiness: 1, + } + c.cncController = controllerutil.NewController( + "ovnkube-network-connect-controller", + cncCfg, + ) + + nodeCfg := &controllerutil.ControllerConfig[corev1.Node]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Informer: wf.NodeCoreInformer().Informer(), + Lister: nodeLister.List, + Reconcile: c.reconcileNode, + ObjNeedsUpdate: nodeNeedsUpdate, + Threadiness: 1, + } + c.nodeController = controllerutil.NewController( + "ovnkube-network-connect-node-controller", + nodeCfg, + ) + + // this controller does not feed from an informer, nads are added + // to the queue by NAD Controller + nadReconcilerConfig := &controllerutil.ReconcilerConfig{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: c.syncNAD, + Threadiness: 1, + MaxAttempts: controllerutil.InfiniteAttempts, + } + c.nadReconciler = controllerutil.NewReconciler( + "ovnkube-network-connect-nad", + nadReconcilerConfig, + ) + + return c +} + +// Start starts the controller. +func (c *Controller) Start() error { + klog.Infof("Starting ovnkube network connect controller for zone %s", c.zone) + if c.nadReconciler == nil { + return controllerutil.Start( + c.cncController, + c.nodeController, + ) + } + id, err := c.networkManager.RegisterNADReconciler(c.nadReconciler) + if err != nil { + return err + } + c.nadReconcilerID = id + return controllerutil.Start( + c.cncController, + c.nodeController, + c.nadReconciler, + ) +} + +// Stop stops the controller. +func (c *Controller) Stop() { + if c.nadReconcilerID != 0 { + if err := c.networkManager.DeRegisterNADReconciler(c.nadReconcilerID); err != nil { + klog.Warningf("Failed to deregister CNC NAD reconciler: %v", err) + } + } + if c.nadReconciler != nil { + controllerutil.Stop( + c.cncController, + c.nodeController, + c.nadReconciler, + ) + } else { + controllerutil.Stop( + c.cncController, + c.nodeController, + ) + } + c.nadReconciler = nil + c.nadReconcilerID = 0 + klog.Infof("Stopped ovnkube network connect controller for zone %s", c.zone) +} + +func (c *Controller) syncNAD(key string) error { + if c.nadLister == nil || c.networkManager == nil { + return nil + } + nadNetwork := c.networkManager.GetNetInfoForNADKey(key) + if nadNetwork == nil { + return nil + } + networkID := nadNetwork.GetNetworkID() + if networkID == types.InvalidID { + return nil + } + cncs, err := c.cncLister.List(labels.Everything()) + if err != nil { + return err + } + for _, cnc := range cncs { + subnets, err := util.ParseNetworkConnectSubnetAnnotation(cnc) + if err != nil { + klog.Warningf("Failed parsing CNC %s subnet annotation: %v", cnc.Name, err) + continue + } + shouldReconcile := false + for owner := range subnets { + _, ownerNetworkID, err := util.ParseNetworkOwner(owner) + if err != nil { + continue + } + if ownerNetworkID == networkID { + shouldReconcile = true + break + } + } + if shouldReconcile { + c.cncController.Reconcile(cnc.Name) + } + } + return nil +} + +// cncNeedsUpdate determines if a CNC update requires reconciliation. +func cncNeedsUpdate(oldObj, newObj *networkconnectv1.ClusterNetworkConnect) bool { + // Always process create/delete + if oldObj == nil || newObj == nil { + return true + } + + // Process if annotations changed (subnet or tunnel key updates from cluster manager) + // this event is triggered when the cluster manager updates the annotations on the CNC object + // based on CNC or NAD or namespace changes. + // Since we watch for the annotation updates on CNC objects, we don't need to directly + // watch for NAD or namespace changes or even CNC network selectors changes as part of this controller. + if util.NetworkConnectSubnetAnnotationChanged(oldObj, newObj) || util.NetworkConnectTunnelKeyAnnotationsChanged(oldObj, newObj) { + return true + } + + // Process if connectivity changed + if !reflect.DeepEqual(oldObj.Spec.Connectivity, newObj.Spec.Connectivity) { + return true + } + + return false +} + +// nodeNeedsUpdate determines if a node update requires reconciliation. +func nodeNeedsUpdate(oldObj, newObj *corev1.Node) bool { + // Always process create/delete + if oldObj == nil || newObj == nil { + return true + } + + // Process if zone annotation changed (affects router port creation for l3 network connectivity + // since the port needs to be toggled between a remote and local port on the connect router) + if util.NodeZoneAnnotationChanged(oldObj, newObj) { + return true + } + + // Process if node subnet annotation changed (affects static routes) + if util.NodeSubnetAnnotationChanged(oldObj, newObj) { + return true + } + + // Process if node ID annotation changed (the only supported scenario + // is when the node is added and get's an annotation update once the nodeID is allocated) + return util.NodeIDAnnotationChanged(oldObj, newObj) && oldObj.Annotations[util.OvnNodeID] == "" +} + +// reconcileCNC reconciles a ClusterNetworkConnect object. +func (c *Controller) reconcileCNC(key string) error { + c.Lock() + defer c.Unlock() + + startTime := time.Now() + _, cncName, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return err + } + + klog.V(5).Infof("Reconciling CNC %s", cncName) + defer func() { + klog.V(4).Infof("Reconciling CNC %s took %v", cncName, time.Since(startTime)) + }() + + cnc, err := c.cncLister.Get(cncName) + if err != nil { + if apierrors.IsNotFound(err) { + // CNC was deleted, clean up OVN resources + return c.cleanupCNC(cncName) + } + return err + } + + return c.syncCNC(cnc) +} + +// reconcileNode reconciles node changes that might affect network connectivity. +func (c *Controller) reconcileNode(key string) error { + startTime := time.Now() + klog.V(5).Infof("Reconciling node %s for network connect", key) + defer func() { + klog.V(4).Infof("Reconciling node %s for network connect took %v", key, time.Since(startTime)) + }() + + _, err := c.nodeLister.Get(key) + if err != nil { + if apierrors.IsNotFound(err) { + // Node was deleted, requeue all CNCs to update routes + return c.requeueAllCNCs() + } + return err + } + + // Requeue all CNCs to update routes for this node. + // We process ALL nodes (not just our zone) because the connect-router + // needs static routes to all node subnets across all zones. + // TODO (trozet): we should check what changed in the node and only reconcile affected CNCs + return c.requeueAllCNCs() +} + +// requeueAllCNCs requeues all CNCs for reconciliation. +func (c *Controller) requeueAllCNCs() error { + cncs, err := c.cncLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("failed to list CNCs: %v", err) + } + + for _, cnc := range cncs { + c.cncController.Reconcile(cnc.Name) + } + return nil +} + +// syncCNC synchronizes the OVN topology for a CNC. +func (c *Controller) syncCNC(cnc *networkconnectv1.ClusterNetworkConnect) error { + // Get or create CNC state + cncState, exists := c.cncCache[cnc.Name] + if !exists { + // this means its CNC create event + cncState = &networkConnectState{ + name: cnc.Name, + connectedNetworks: sets.New[string](), + } + c.cncCache[cnc.Name] = cncState + } + // STEP1: Create the connect router for the CNC using tunnel ID from CNC annotation + // This is always a one time operation - Every CNC has exactly one connect router. + // tunnelID is set by cluster manager during CNC creation and is considered immutable + if cncState.tunnelID == 0 { + // Parse tunnel key from annotation (set by cluster manager) + tunnelID, err := util.ParseNetworkConnectTunnelKeyAnnotation(cnc) + if err != nil { + return fmt.Errorf("failed to parse tunnel key annotation for CNC %s: %v", cnc.Name, err) + } + if tunnelID == 0 { + klog.V(4).Infof("CNC %s does not have tunnel key annotation yet, waiting for cluster manager", cnc.Name) + // we don't return error here because we want to wait for the cluster manager to set the annotation + // and cncUpdate event will trigger the reconciliation again. + return nil + } + // Create the connect router + if err := c.ensureConnectRouter(cnc, tunnelID); err != nil { + return fmt.Errorf("failed to ensure connect router for CNC %s: %v", cnc.Name, err) + } + cncState.tunnelID = tunnelID + } + allocatedSubnets, err := util.ParseNetworkConnectSubnetAnnotation(cnc) + if err != nil { + return fmt.Errorf("failed to parse subnet annotation for CNC %s: %w", cnc.Name, err) + } + if err := c.syncNetworkConnections(cnc, allocatedSubnets); err != nil { + return fmt.Errorf("failed to sync network connections for CNC %s: %v", cnc.Name, err) + } + return nil +} + +// cleanupCNC removes OVN resources for a deleted CNC. +func (c *Controller) cleanupCNC(cncName string) error { + klog.V(4).Infof("Cleaning up CNC %s", cncName) + + _, exists := c.cncCache[cncName] + if !exists { + klog.V(4).Infof("CNC %s not found in cache, nothing to clean up", cncName) + return nil + } + + // Cleanup network connections + if err := c.cleanupNetworkConnections(cncName); err != nil { + return fmt.Errorf("failed to cleanup network connections for CNC %s: %v", cncName, err) + } + + // Remove the connect router + if err := c.deleteConnectRouter(cncName); err != nil { + return fmt.Errorf("failed to delete connect router for CNC %s: %v", cncName, err) + } + + // Remove from cache + delete(c.cncCache, cncName) + klog.V(4).Infof("Cleaned up CNC %s", cncName) + + return nil +} diff --git a/go-controller/pkg/ovn/controller/networkconnect/controller_components_test.go b/go-controller/pkg/ovn/controller/networkconnect/controller_components_test.go new file mode 100644 index 0000000000..3a08c31901 --- /dev/null +++ b/go-controller/pkg/ovn/controller/networkconnect/controller_components_test.go @@ -0,0 +1,617 @@ +package networkconnect + +import ( + "context" + "fmt" + "strconv" + "sync" + "testing" + "time" + + nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + controllerutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +// Test for cncNeedsUpdate function +func TestCNCNeedsUpdate(t *testing.T) { + tests := []struct { + name string + oldObj *networkconnectv1.ClusterNetworkConnect + newObj *networkconnectv1.ClusterNetworkConnect + expected bool + }{ + { + name: "create event (oldObj nil)", + oldObj: nil, + newObj: &networkconnectv1.ClusterNetworkConnect{}, + expected: true, + }, + { + name: "delete event (newObj nil)", + oldObj: &networkconnectv1.ClusterNetworkConnect{}, + newObj: nil, + expected: true, + }, + { + name: "no changes", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + Connectivity: []networkconnectv1.ConnectivityType{networkconnectv1.PodNetwork}, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + Connectivity: []networkconnectv1.ConnectivityType{networkconnectv1.PodNetwork}, + }, + }, + expected: false, + }, + { + name: "connectivity changed", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + Connectivity: []networkconnectv1.ConnectivityType{networkconnectv1.PodNetwork}, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + Connectivity: []networkconnectv1.ConnectivityType{networkconnectv1.PodNetwork, networkconnectv1.ClusterIPServiceNetwork}, + }, + }, + expected: true, + }, + { + name: "irrelevant annotations changed - should not trigger update", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"key": "value1"}, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"key": "value2"}, + }, + }, + expected: false, + }, + { + name: "subnet annotation changed", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/network-connect-subnet": `{"layer3_1":{"ipv4":"192.168.0.0/24"}}`}, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/network-connect-subnet": `{"layer3_1":{"ipv4":"192.168.0.0/24"},"layer3_2":{"ipv4":"192.168.1.0/24"}}`}, + }, + }, + expected: true, + }, + { + name: "tunnel key annotation changed", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/connect-router-tunnel-key": "12345"}, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/connect-router-tunnel-key": "67890"}, + }, + }, + expected: true, + }, + { + name: "relevant annotations unchanged", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "k8s.ovn.org/network-connect-subnet": `{"layer3_1":{"ipv4":"192.168.0.0/24"}}`, + "k8s.ovn.org/connect-router-tunnel-key": "12345", + }, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "k8s.ovn.org/network-connect-subnet": `{"layer3_1":{"ipv4":"192.168.0.0/24"}}`, + "k8s.ovn.org/connect-router-tunnel-key": "12345", + }, + }, + }, + expected: false, + }, + { + name: "network selectors changed - CUDN selector added", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + NetworkSelectors: []types.NetworkSelector{}, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + NetworkSelectors: []types.NetworkSelector{ + { + NetworkSelectionType: types.ClusterUserDefinedNetworks, + ClusterUserDefinedNetworkSelector: &types.ClusterUserDefinedNetworkSelector{ + NetworkSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + }, + }, + }, + }, + expected: false, + }, + { + name: "network selectors changed - PUDN selector label changed", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + NetworkSelectors: []types.NetworkSelector{ + { + NetworkSelectionType: types.PrimaryUserDefinedNetworks, + PrimaryUserDefinedNetworkSelector: &types.PrimaryUserDefinedNetworkSelector{ + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "dev"}, + }, + }, + }, + }, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + NetworkSelectors: []types.NetworkSelector{ + { + NetworkSelectionType: types.PrimaryUserDefinedNetworks, + PrimaryUserDefinedNetworkSelector: &types.PrimaryUserDefinedNetworkSelector{ + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "prod"}, + }, + }, + }, + }, + }, + }, + expected: false, + }, + { + name: "network selectors unchanged", + oldObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + NetworkSelectors: []types.NetworkSelector{ + { + NetworkSelectionType: types.ClusterUserDefinedNetworks, + ClusterUserDefinedNetworkSelector: &types.ClusterUserDefinedNetworkSelector{ + NetworkSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + }, + }, + }, + }, + newObj: &networkconnectv1.ClusterNetworkConnect{ + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + NetworkSelectors: []types.NetworkSelector{ + { + NetworkSelectionType: types.ClusterUserDefinedNetworks, + ClusterUserDefinedNetworkSelector: &types.ClusterUserDefinedNetworkSelector{ + NetworkSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + }, + }, + }, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := cncNeedsUpdate(tt.oldObj, tt.newObj) + assert.Equal(t, tt.expected, result) + }) + } +} + +// Test for nodeNeedsUpdate function +func TestNodeNeedsUpdate(t *testing.T) { + tests := []struct { + name string + oldObj *corev1.Node + newObj *corev1.Node + expected bool + }{ + { + name: "create event (oldObj nil)", + oldObj: nil, + newObj: &corev1.Node{}, + expected: true, + }, + { + name: "delete event (newObj nil)", + oldObj: &corev1.Node{}, + newObj: nil, + expected: true, + }, + { + name: "no changes", + oldObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{}, + }, + }, + newObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + { + name: "zone annotation changed", + oldObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"k8s.ovn.org/zone-name": "zone1"}, + }, + }, + newObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"k8s.ovn.org/zone-name": "zone2"}, + }, + }, + expected: true, + }, + { + name: "node subnet annotation changed", + oldObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"k8s.ovn.org/node-subnets": `{"default":"10.244.0.0/24"}`}, + }, + }, + newObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"k8s.ovn.org/node-subnets": `{"default":"10.244.1.0/24"}`}, + }, + }, + expected: true, + }, + { + name: "node ID annotation changed", + oldObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"k8s.ovn.org/node-id": "1"}, + }, + }, + newObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"k8s.ovn.org/node-id": "2"}, + }, + }, + expected: false, + }, + { + name: "node ID annotation update during add time", + oldObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + newObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"k8s.ovn.org/node-id": "2"}, + }, + }, + expected: true, + }, + { + name: "irrelevant annotation changed - should not trigger update", + oldObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"some-other-annotation": "value1"}, + }, + }, + newObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{"some-other-annotation": "value2"}, + }, + }, + expected: false, + }, + { + name: "relevant annotations unchanged", + oldObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + "k8s.ovn.org/zone-name": "zone1", + "k8s.ovn.org/node-subnets": `{"default":"10.244.0.0/24"}`, + "k8s.ovn.org/node-id": "1", + }, + }, + }, + newObj: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + "k8s.ovn.org/zone-name": "zone1", + "k8s.ovn.org/node-subnets": `{"default":"10.244.0.0/24"}`, + "k8s.ovn.org/node-id": "1", + }, + }, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := nodeNeedsUpdate(tt.oldObj, tt.newObj) + assert.Equal(t, tt.expected, result) + }) + } +} + +// TestController_reconcileNode tests that reconcileNode requeues all CNCs +func TestController_reconcileNode(t *testing.T) { + g := gomega.NewWithT(t) + + err := config.PrepareTestConfig() + g.Expect(err).ToNot(gomega.HaveOccurred()) + + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableNetworkConnect = true + + fakeClientset := util.GetOVNClientset().GetOVNKubeControllerClientset() + + // Create test CNCs + cnc1 := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{Name: "cnc1"}, + } + cnc2 := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{Name: "cnc2"}, + } + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc1, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc2, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Create test node + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + "k8s.ovn.org/zone-name": "zone1", + "k8s.ovn.org/node-subnets": `{"default":"10.244.0.0/24"}`, + }, + }, + } + _, err = fakeClientset.KubeClient.CoreV1().Nodes().Create( + context.Background(), node, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClientset) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + err = wf.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer wf.Shutdown() + + // Wait for informer caches to sync + syncCtx, syncCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer syncCancel() + synced := cache.WaitForCacheSync( + syncCtx.Done(), + wf.ClusterNetworkConnectInformer().Informer().HasSynced, + wf.NodeCoreInformer().Informer().HasSynced, + ) + g.Expect(synced).To(gomega.BeTrue(), "informer caches should sync") + + // Track reconciled CNCs + reconciledCNCs := sets.New[string]() + reconciledMutex := sync.Mutex{} + + // Create controller with listers from watch factory + c := &Controller{ + cncLister: wf.ClusterNetworkConnectInformer().Lister(), + nodeLister: wf.NodeCoreInformer().Lister(), + } + + // Create CNC controller with custom reconcile function that tracks calls + cncCfg := &controllerutil.ControllerConfig[networkconnectv1.ClusterNetworkConnect]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Informer: wf.ClusterNetworkConnectInformer().Informer(), + Lister: wf.ClusterNetworkConnectInformer().Lister().List, + Reconcile: func(key string) error { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + reconciledCNCs.Insert(key) + return nil + }, + ObjNeedsUpdate: cncNeedsUpdate, + Threadiness: 1, + } + c.cncController = controllerutil.NewController("test-cnc-controller", cncCfg) + + err = controllerutil.Start(c.cncController) + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer controllerutil.Stop(c.cncController) + + // Wait for initial sync, then clear recorded reconciliations + g.Eventually(func() int { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + return reconciledCNCs.Len() + }).Should(gomega.BeNumerically(">=", 2)) + reconciledMutex.Lock() + reconciledCNCs = sets.New[string]() + reconciledMutex.Unlock() + + // Call reconcileNode + err = c.reconcileNode("node1") + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Verify all CNCs were requeued + g.Eventually(func() []string { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + return reconciledCNCs.UnsortedList() + }).Should(gomega.ConsistOf("cnc1", "cnc2")) +} + +// TestController_syncNAD tests that syncNAD requeues CNCs matching the NAD network ID. +func TestController_syncNAD(t *testing.T) { + g := gomega.NewWithT(t) + err := config.PrepareTestConfig() + g.Expect(err).ToNot(gomega.HaveOccurred()) + + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableNetworkConnect = true + + fakeClientset := util.GetOVNClientset().GetOVNKubeControllerClientset() + + networkID := 7 + ownerKey := fmt.Sprintf("layer3_%d", networkID) + cnc := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cnc1", + Annotations: map[string]string{ + "k8s.ovn.org/network-connect-subnet": fmt.Sprintf("{\"%s\":{\"ipv4\":\"192.168.0.0/24\"}}", ownerKey), + }, + }, + } + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + nadConfig := `{"cniVersion":"0.4.0","name":"net1","type":"ovn-k8s-cni-overlay","topology":"layer3","role":"primary","netAttachDefName":"ns1/nad1"}` + nad := &nettypes.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "nad1", + Annotations: map[string]string{ + "k8s.ovn.org/network-id": strconv.Itoa(networkID), + }, + }, + Spec: nettypes.NetworkAttachmentDefinitionSpec{ + Config: nadConfig, + }, + } + _, err = fakeClientset.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(nad.Namespace).Create( + context.Background(), nad, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + nadKey := util.GetNADName(nad.Namespace, nad.Name) + netInfo, err := util.ParseNADInfo(nad) + g.Expect(err).ToNot(gomega.HaveOccurred()) + mutableNetInfo := util.NewMutableNetInfo(netInfo) + mutableNetInfo.AddNADs(nadKey) + mutableNetInfo.SetNetworkID(networkID) + + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClientset) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + err = wf.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer wf.Shutdown() + + // Wait for informer caches to sync + syncCtx, syncCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer syncCancel() + synced := cache.WaitForCacheSync( + syncCtx.Done(), + wf.ClusterNetworkConnectInformer().Informer().HasSynced, + wf.NADInformer().Informer().HasSynced, + ) + g.Expect(synced).To(gomega.BeTrue(), "informer caches should sync") + + // Track reconciled CNCs + reconciledCNCs := sets.New[string]() + reconciledMutex := sync.Mutex{} + + // Create controller with listers from watch factory + c := &Controller{ + cncLister: wf.ClusterNetworkConnectInformer().Lister(), + nadLister: wf.NADInformer().Lister(), + networkManager: &networkmanager.FakeNetworkManager{ + NADNetworks: map[string]util.NetInfo{ + nadKey: mutableNetInfo, + }, + }, + } + + // Create CNC controller with custom reconcile function that tracks calls + cncCfg := &controllerutil.ControllerConfig[networkconnectv1.ClusterNetworkConnect]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Informer: wf.ClusterNetworkConnectInformer().Informer(), + Lister: wf.ClusterNetworkConnectInformer().Lister().List, + Reconcile: func(key string) error { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + reconciledCNCs.Insert(key) + return nil + }, + ObjNeedsUpdate: cncNeedsUpdate, + Threadiness: 1, + } + c.cncController = controllerutil.NewController("test-cnc-controller", cncCfg) + + err = controllerutil.Start(c.cncController) + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer controllerutil.Stop(c.cncController) + + // Wait for initial sync, then clear recorded reconciliations + g.Eventually(func() int { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + return reconciledCNCs.Len() + }).Should(gomega.BeNumerically(">=", 1)) + reconciledMutex.Lock() + reconciledCNCs = sets.New[string]() + reconciledMutex.Unlock() + + err = c.syncNAD("ns1/nad1") + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Verify CNC was requeued + g.Eventually(func() []string { + reconciledMutex.Lock() + defer reconciledMutex.Unlock() + return reconciledCNCs.UnsortedList() + }).Should(gomega.ConsistOf("cnc1")) +} diff --git a/go-controller/pkg/ovn/controller/networkconnect/controller_suite_test.go b/go-controller/pkg/ovn/controller/networkconnect/controller_suite_test.go new file mode 100644 index 0000000000..8459de0ffa --- /dev/null +++ b/go-controller/pkg/ovn/controller/networkconnect/controller_suite_test.go @@ -0,0 +1,13 @@ +package networkconnect + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestNetworkConnectController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "OVNKube NetworkConnect Controller Suite") +} diff --git a/go-controller/pkg/ovn/controller/networkconnect/controller_test.go b/go-controller/pkg/ovn/controller/networkconnect/controller_test.go new file mode 100644 index 0000000000..f683a38e23 --- /dev/null +++ b/go-controller/pkg/ovn/controller/networkconnect/controller_test.go @@ -0,0 +1,2918 @@ +package networkconnect + +import ( + "context" + "fmt" + "net" + "strconv" + "strings" + "time" + + cnitypes "github.com/containernetworking/cni/pkg/types" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + libovsdbclient "github.com/ovn-kubernetes/libovsdb/client" + + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// ============================================================================= +// Test Helper Types and Functions +// ============================================================================= + +const ( + // Annotation keys used in tests (matching unexported constants in util package) + ovnNodeSubnetsAnnotation = "k8s.ovn.org/node-subnets" + ovnNetworkConnectSubnetAnnotation = "k8s.ovn.org/network-connect-subnet" +) + +// IP mode test configurations +var ipModes = []struct { + name string + v4 bool + v6 bool +}{ + {"IPv4", true, false}, + {"IPv6", false, true}, + {"DualStack", true, true}, +} + +// testNetwork represents a network configuration for testing +type testNetwork struct { + name string + id int + topologyType string + subnets []string // Pod subnets for the network (optional, defaults based on IP mode) +} + +// DefaultSubnets returns subnets based on IP mode and topology type +// For Layer3: format is "cidr/hostSubnetLength" e.g., "10.128.0.0/14/23" +// For Layer2: format is just "cidr" e.g., "10.200.0.0/16" +// When explicit subnets are provided, filters them based on IP mode +func (n testNetwork) DefaultSubnets() []string { + if len(n.subnets) > 0 { + // Filter explicit subnets by IP mode + var filtered []string + for _, subnet := range n.subnets { + // Check if subnet is IPv6 (contains ":") + isIPv6 := strings.Contains(subnet, ":") + if isIPv6 && config.IPv6Mode { + filtered = append(filtered, subnet) + } else if !isIPv6 && config.IPv4Mode { + filtered = append(filtered, subnet) + } + } + return filtered + } + var subnets []string + if n.topologyType == ovntypes.Layer3Topology { + if config.IPv4Mode { + subnets = append(subnets, "10.128.0.0/14/23") + } + if config.IPv6Mode { + subnets = append(subnets, "fd00:10:128::/48/64") + } + } else { // Layer2 + if config.IPv4Mode { + subnets = append(subnets, "10.200.0.0/16") + } + if config.IPv6Mode { + subnets = append(subnets, "fd00:10:200::/48") + } + } + return subnets +} + +// RouterName returns the OVN router name for this network +func (n testNetwork) RouterName() string { + prefix := util.GetUserDefinedNetworkPrefix(n.name) + if n.topologyType == ovntypes.Layer2Topology { + return prefix + ovntypes.TransitRouter + } + return prefix + ovntypes.OVNClusterRouter +} + +// testNode represents a node configuration for testing +type testNode struct { + name string + id int + zone string + nodeSubnets map[string]subnetPair // networkName -> subnet pair (v4, v6) +} + +// setupTestConfig initializes the test config with the given IP mode +func setupTestConfig(v4Enabled, v6Enabled bool) { + Expect(config.PrepareTestConfig()).NotTo(HaveOccurred()) + config.IPv4Mode = v4Enabled + config.IPv6Mode = v6Enabled + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableNetworkConnect = true +} + +// createTestNode creates a test Node object +// nodeSubnets is a map of network names to subnet pairs (v4, v6) +// If id is 0, node ID annotation is not set (to test nodes without ID allocation) +// If zone is empty, zone annotation is not set +func createTestNode(n testNode) *corev1.Node { + annotations := map[string]string{} + // Only set zone annotation if zone is not empty + if n.zone != "" { + annotations[util.OvnNodeZoneName] = n.zone + } + // Only set node ID annotation if id > 0 (0 means no node ID) + if n.id > 0 { + annotations[util.OvnNodeID] = strconv.Itoa(n.id) + } + // Add node subnet annotations based on IP mode + if len(n.nodeSubnets) > 0 { + annotations[ovnNodeSubnetsAnnotation] = buildNodeSubnetAnnotation(n.nodeSubnets) + } + annotations[util.OvnNodeChassisID] = chassisIDForNode(n.name) + + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: n.name, + Annotations: annotations, + }, + } +} + +// defaultConnectSubnets returns the default connect subnets based on the current IP mode +func defaultConnectSubnets() []networkconnectv1.ConnectSubnet { + var subnets []networkconnectv1.ConnectSubnet + if config.IPv4Mode { + subnets = append(subnets, networkconnectv1.ConnectSubnet{CIDR: "192.168.0.0/16", NetworkPrefix: 24}) + } + if config.IPv6Mode { + subnets = append(subnets, networkconnectv1.ConnectSubnet{CIDR: "fd00:192:168::/48", NetworkPrefix: 64}) + } + return subnets +} + +// subnetPair holds v4 and v6 subnet strings for a network owner +type subnetPair struct{ v4, v6 string } + +// buildConnectSubnetAnnotation builds the CNC subnet annotation JSON based on current IP mode +// Pass a map of owner keys (e.g., "layer3_1") to subnet pairs +// The function uses config.IPv4Mode/IPv6Mode to decide what to include +func buildConnectSubnetAnnotation(owners map[string]subnetPair) string { + if len(owners) == 0 { + return "" + } + + result := "{" + first := true + for owner, subnets := range owners { + includeV4 := config.IPv4Mode && subnets.v4 != "" + includeV6 := config.IPv6Mode && subnets.v6 != "" + + if !includeV4 && !includeV6 { + continue + } + + if !first { + result += "," + } + first = false + + if includeV4 && includeV6 { + result += fmt.Sprintf(`"%s":{"ipv4":"%s","ipv6":"%s"}`, owner, subnets.v4, subnets.v6) + } else if includeV4 { + result += fmt.Sprintf(`"%s":{"ipv4":"%s"}`, owner, subnets.v4) + } else { + result += fmt.Sprintf(`"%s":{"ipv6":"%s"}`, owner, subnets.v6) + } + } + result += "}" + return result +} + +// buildNodeSubnetAnnotation builds the node subnet annotation JSON based on current IP mode +// Pass a map of network names to subnet pairs +// The function uses config.IPv4Mode/IPv6Mode to decide what to include +func buildNodeSubnetAnnotation(networks map[string]subnetPair) string { + if len(networks) == 0 { + return "" + } + + result := "{" + first := true + for netName, subnets := range networks { + includeV4 := config.IPv4Mode && subnets.v4 != "" + includeV6 := config.IPv6Mode && subnets.v6 != "" + + if !includeV4 && !includeV6 { + continue + } + + if !first { + result += "," + } + first = false + + if includeV4 && includeV6 { + result += fmt.Sprintf(`"%s":["%s","%s"]`, netName, subnets.v4, subnets.v6) + } else if includeV4 { + result += fmt.Sprintf(`"%s":"%s"`, netName, subnets.v4) + } else { + result += fmt.Sprintf(`"%s":"%s"`, netName, subnets.v6) + } + } + result += "}" + return result +} + +// createTestCNC creates a test CNC object with proper annotations +func createTestCNC(name string, tunnelID int, connectSubnets []networkconnectv1.ConnectSubnet, subnetAnnotation string) *networkconnectv1.ClusterNetworkConnect { + cnc := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: make(map[string]string), + }, + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + ConnectSubnets: connectSubnets, + Connectivity: []networkconnectv1.ConnectivityType{ + networkconnectv1.PodNetwork, + }, + }, + } + if tunnelID > 0 { + cnc.Annotations[util.OvnConnectRouterTunnelKeyAnnotation] = strconv.Itoa(tunnelID) + } + if subnetAnnotation != "" { + cnc.Annotations[ovnNetworkConnectSubnetAnnotation] = subnetAnnotation + } + return cnc +} + +// createInitialDBWithRouters creates initial DB data with network routers +func createInitialDBWithRouters(networks []testNetwork) []libovsdbtest.TestData { + var data []libovsdbtest.TestData + for _, net := range networks { + routerName := net.RouterName() + data = append(data, &nbdb.LogicalRouter{ + UUID: routerName + "-uuid", + Name: routerName, + }) + } + return data +} + +// createNetInfo creates a real NetInfo for testing +func createNetInfo(net testNetwork) (util.NetInfo, error) { + // Use DefaultSubnets which handles IP mode + subnets := net.DefaultSubnets() + subnetsStr := strings.Join(subnets, ",") + + netConf := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: net.name}, + Topology: net.topologyType, + Role: ovntypes.NetworkRoleSecondary, + Subnets: subnetsStr, + } + netInfo, err := util.NewNetInfo(netConf) + if err != nil { + return nil, err + } + // Set the network ID + mutableNetInfo := util.NewMutableNetInfo(netInfo) + mutableNetInfo.SetNetworkID(net.id) + return mutableNetInfo, nil +} + +// verifyConnectRouter checks that a connect router exists with the expected properties +// Returns error if router doesn't exist or has wrong properties (for use in Eventually) +func verifyConnectRouter(nbClient libovsdbclient.Client, cncName string, tunnelID int) error { + routerName := getConnectRouterName(cncName) + router, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: routerName}) + if err != nil { + return err + } + if router.Options == nil || router.Options["requested-tnl-key"] != strconv.Itoa(tunnelID) { + return fmt.Errorf("connect router %s has wrong tunnel key, expected %d", routerName, tunnelID) + } + if router.ExternalIDs == nil || router.ExternalIDs[libovsdbops.ObjectNameKey.String()] != cncName { + return fmt.Errorf("connect router %s has wrong external ID, expected %s", routerName, cncName) + } + return nil +} + +// verifyRouterPorts checks that expected ports exist on a router +// Returns error for use in Eventually +func verifyRouterPortsCount(nbClient libovsdbclient.Client, routerName string, expectedPortCount int) error { + router, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: routerName}) + if err != nil { + return err + } + if len(router.Ports) != expectedPortCount { + return fmt.Errorf("expected %d ports on %s, got %d", expectedPortCount, routerName, len(router.Ports)) + } + return nil +} + +// verifyRouterPort verifies a port exists on any router with expected properties +// Returns error for use in Eventually +func verifyRouterPort(nbClient libovsdbclient.Client, portName, expectedCNCName string, expectedNetworks []string) error { + port, err := libovsdbops.GetLogicalRouterPort(nbClient, &nbdb.LogicalRouterPort{Name: portName}) + if err != nil { + return fmt.Errorf("port %s not found: %v", portName, err) + } + + // Verify external IDs + if port.ExternalIDs == nil || port.ExternalIDs[libovsdbops.ObjectNameKey.String()] != expectedCNCName { + return fmt.Errorf("port %s has wrong CNC name in external IDs, expected %s, got %v", portName, expectedCNCName, port.ExternalIDs) + } + + // Verify networks (IPs) if provided + if len(expectedNetworks) > 0 { + if len(port.Networks) != len(expectedNetworks) { + return fmt.Errorf("port %s networks count mismatch, expected %d, got %d", portName, len(expectedNetworks), len(port.Networks)) + } + for _, expected := range expectedNetworks { + found := false + for _, actual := range port.Networks { + if actual == expected { + found = true + break + } + } + if !found { + return fmt.Errorf("port %s missing expected network %s, got %v", portName, expected, port.Networks) + } + } + } + + // Verify MAC is set + if port.MAC == "" { + return fmt.Errorf("port %s has no MAC address", portName) + } + + return nil +} + +// verifyRouterPortIsRemote verifies a port exists and has requested-chassis option set (remote node port) +// Also verifies requested-tnl-key if expectedTunnelKey > 0 +// Returns error for use in Eventually +func verifyRouterPortIsRemote(nbClient libovsdbclient.Client, portName string, expectedTunnelKey int) error { + port, err := libovsdbops.GetLogicalRouterPort(nbClient, &nbdb.LogicalRouterPort{Name: portName}) + if err != nil { + return fmt.Errorf("port %s not found: %v", portName, err) + } + + // Remote ports have requested-chassis option set + if port.Options == nil || port.Options["requested-chassis"] == "" { + return fmt.Errorf("port %s is not a remote port (missing requested-chassis option), options: %v", portName, port.Options) + } + + // Verify tunnel key if expected + if expectedTunnelKey > 0 { + if port.Options["requested-tnl-key"] != strconv.Itoa(expectedTunnelKey) { + return fmt.Errorf("port %s has wrong tunnel key, expected %d, got %s", portName, expectedTunnelKey, port.Options["requested-tnl-key"]) + } + } + + return nil +} + +// verifyRouterPortIsLocal verifies a port exists and does NOT have requested-chassis option set (local node port) +// Also verifies requested-tnl-key if expectedTunnelKey > 0 +// Also verifies Peer field is set to expectedPeerPortName if not empty +// Returns error for use in Eventually +func verifyRouterPortIsLocal(nbClient libovsdbclient.Client, portName string, expectedTunnelKey int, expectedPeerPortName string) error { + port, err := libovsdbops.GetLogicalRouterPort(nbClient, &nbdb.LogicalRouterPort{Name: portName}) + if err != nil { + return fmt.Errorf("port %s not found: %v", portName, err) + } + + // Local ports should NOT have requested-chassis option set + if port.Options != nil && port.Options["requested-chassis"] != "" { + return fmt.Errorf("port %s is not a local port (has requested-chassis option), options: %v", portName, port.Options) + } + + // Verify tunnel key if expected + if expectedTunnelKey > 0 { + if port.Options == nil || port.Options["requested-tnl-key"] != strconv.Itoa(expectedTunnelKey) { + return fmt.Errorf("port %s has wrong tunnel key, expected %d, options: %v", portName, expectedTunnelKey, port.Options) + } + } + + // Verify Peer field for local ports + if expectedPeerPortName != "" { + if port.Peer == nil || *port.Peer != expectedPeerPortName { + var actualPeer string + if port.Peer != nil { + actualPeer = *port.Peer + } + return fmt.Errorf("port %s has wrong peer, expected %s, got %s", portName, expectedPeerPortName, actualPeer) + } + } + + return nil +} + +// getExpectedPortNetworks returns expected networks for connect and network router ports +// Pass allocated subnets (/24 for v4, /64 for v6) and node ID - function calculates P2P subnets +func getExpectedPortNetworks(v4AllocatedSubnet, v6AllocatedSubnet *net.IPNet, nodeID int) (connectRouterNetworks, networkRouterNetworks []string) { + var allocatedSubnets []*net.IPNet + if config.IPv4Mode && v4AllocatedSubnet != nil { + allocatedSubnets = append(allocatedSubnets, v4AllocatedSubnet) + } + if config.IPv6Mode && v6AllocatedSubnet != nil { + allocatedSubnets = append(allocatedSubnets, v6AllocatedSubnet) + } + + // Calculate P2P subnets for this node ID (same logic as controller) + portPairInfo, err := GetP2PAddresses(allocatedSubnets, nodeID) + if err != nil { + return nil, nil + } + + for _, ip := range portPairInfo.connectPortIPs { + connectRouterNetworks = append(connectRouterNetworks, ip.String()) + } + for _, ip := range portPairInfo.networkPortIPs { + networkRouterNetworks = append(networkRouterNetworks, ip.String()) + } + return connectRouterNetworks, networkRouterNetworks +} + +// getExpectedPortTunnelKey calculates the expected tunnel key for a port using the same +// logic as production code via getNetworkIndexAndMaxNodes. +// For Layer3: tunnelKey = networkIndex * maxNodes + nodeID + 1 +// For Layer2: tunnelKey = networkIndex * maxNodes + subIndex + 1 (subIndex from getLayer2SubIndex) +// +// Parameters: +// - allocatedSubnetV4/V6: the allocated subnet for this network from the annotation (e.g., "192.168.0.0/24") +// - topologyType: ovntypes.Layer3Topology or ovntypes.Layer2Topology +// - nodeID: node ID for Layer3, ignored for Layer2 +func getExpectedPortTunnelKey(allocatedSubnetV4, allocatedSubnetV6 string, topologyType string, nodeID int) int { + // Parse allocated subnets based on current IP mode + var subnets []*net.IPNet + if config.IPv4Mode && allocatedSubnetV4 != "" { + subnets = append(subnets, ovntest.MustParseIPNet(allocatedSubnetV4)) + } + if config.IPv6Mode && allocatedSubnetV6 != "" { + subnets = append(subnets, ovntest.MustParseIPNet(allocatedSubnetV6)) + } + + tunnelKey, err := GetTunnelKey(defaultConnectSubnets(), subnets, topologyType, nodeID) + if err != nil { + panic(err) // test helper should not fail with default connect subnets + } + return tunnelKey +} + +// extractNexthops extracts nexthop IPs (without mask) from network strings +// Uses ovntest.MustParseIPNet to parse CIDR strings like "192.168.0.3/31" +func extractNexthops(networks []string) (nexthopV4, nexthopV6 string) { + for _, net := range networks { + ipNet := ovntest.MustParseIPNet(net) + if ipNet.IP.To4() != nil { + nexthopV4 = ipNet.IP.String() + } else { + nexthopV6 = ipNet.IP.String() + } + } + return +} + +// verifyRouterPolicy verifies routing policies exist on a router with expected properties +// Returns error for use in Eventually +func verifyRouterPolicy(nbClient libovsdbclient.Client, routerName, cncName string, srcNetworkID, dstNetworkID, expectedCount int) error { + policies, err := libovsdbops.FindALogicalRouterPoliciesWithPredicate(nbClient, routerName, + func(policy *nbdb.LogicalRouterPolicy) bool { + return policy.ExternalIDs != nil && + policy.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + policy.ExternalIDs[libovsdbops.SourceNetworkIDKey.String()] == strconv.Itoa(srcNetworkID) && + policy.ExternalIDs[libovsdbops.DestinationNetworkIDKey.String()] == strconv.Itoa(dstNetworkID) + }) + if err != nil { + return fmt.Errorf("failed to get policies from %s: %v", routerName, err) + } + + if len(policies) != expectedCount { + return fmt.Errorf("expected %d policies on %s for CNC %s srcNetworkID %d dstNetworkID %d, got %d", + expectedCount, routerName, cncName, srcNetworkID, dstNetworkID, len(policies)) + } + return nil +} + +// verifyRouterPolicyCount verifies the count of policies on a router for a given CNC +// Returns error for use in Eventually +func verifyRouterPolicyCount(nbClient libovsdbclient.Client, routerName, cncName string, + srcNetworkID, dstNetworkID int, expectedCount int) error { + policies, err := libovsdbops.FindALogicalRouterPoliciesWithPredicate(nbClient, routerName, + func(policy *nbdb.LogicalRouterPolicy) bool { + return policy.ExternalIDs != nil && + policy.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + policy.ExternalIDs[libovsdbops.SourceNetworkIDKey.String()] == strconv.Itoa(srcNetworkID) && + policy.ExternalIDs[libovsdbops.DestinationNetworkIDKey.String()] == strconv.Itoa(dstNetworkID) + }) + if err != nil { + return fmt.Errorf("failed to get policies from %s: %v", routerName, err) + } + + if len(policies) != expectedCount { + return fmt.Errorf("expected %d policies on %s for CNC %s, got %d", + expectedCount, routerName, cncName, len(policies)) + } + return nil +} + +// verifyRouterStaticRoutes verifies static routes exist on a router with expected properties +// Returns error for use in Eventually +// networkID is the network's ID from the NAD annotation +// nodeID is the node's ID (0 for Layer2 networks) +// Pass v4/v6 prefix and nexthop pairs - function checks based on config.IPv4Mode/IPv6Mode +func verifyRouterStaticRoutes(nbClient libovsdbclient.Client, routerName, cncName string, networkID int, + nodeID int, prefixV4, prefixV6, nexthopV4, nexthopV6 string) error { + routes, err := libovsdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(nbClient, &nbdb.LogicalRouter{Name: routerName}, + func(route *nbdb.LogicalRouterStaticRoute) bool { + return route.ExternalIDs != nil && + route.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + route.ExternalIDs[libovsdbops.NetworkIDKey.String()] == strconv.Itoa(networkID) && + route.ExternalIDs[libovsdbops.NodeIDKey.String()] == strconv.Itoa(nodeID) + }) + if err != nil { + return fmt.Errorf("failed to get static routes from %s: %v", routerName, err) + } + + // Calculate expected count based on IP mode + expectedCount := 0 + if config.IPv4Mode { + expectedCount++ + } + if config.IPv6Mode { + expectedCount++ + } + + if len(routes) != expectedCount { + return fmt.Errorf("expected %d static routes on %s for CNC %s, networkID %d, node %d, got %d", + expectedCount, routerName, cncName, networkID, nodeID, len(routes)) + } + + // Verify v4 route if IPv4Mode is enabled + if config.IPv4Mode { + found := false + for _, route := range routes { + if route.IPPrefix == prefixV4 { + found = true + if route.Nexthop != nexthopV4 { + return fmt.Errorf("v4 route %s on %s has wrong Nexthop, expected %s, got %s", + prefixV4, routerName, nexthopV4, route.Nexthop) + } + if route.ExternalIDs[libovsdbops.IPFamilyKey.String()] != "v4" { + return fmt.Errorf("v4 route %s on %s has wrong IPFamilyKey, expected v4, got %s", + prefixV4, routerName, route.ExternalIDs[libovsdbops.IPFamilyKey.String()]) + } + break + } + } + if !found { + return fmt.Errorf("v4 route with IPPrefix %s not found on %s", prefixV4, routerName) + } + } + + // Verify v6 route if IPv6Mode is enabled + if config.IPv6Mode { + found := false + for _, route := range routes { + if route.IPPrefix == prefixV6 { + found = true + if route.Nexthop != nexthopV6 { + return fmt.Errorf("v6 route %s on %s has wrong Nexthop, expected %s, got %s", + prefixV6, routerName, nexthopV6, route.Nexthop) + } + if route.ExternalIDs[libovsdbops.IPFamilyKey.String()] != "v6" { + return fmt.Errorf("v6 route %s on %s has wrong IPFamilyKey, expected v6, got %s", + prefixV6, routerName, route.ExternalIDs[libovsdbops.IPFamilyKey.String()]) + } + break + } + } + if !found { + return fmt.Errorf("v6 route with IPPrefix %s not found on %s", prefixV6, routerName) + } + } + + return nil +} + +// verifyRouterStaticRoutesCount verifies the count of static routes on a router for a given CNC +// Returns error for use in Eventually +func verifyRouterStaticRoutesCount(nbClient libovsdbclient.Client, routerName, cncName string, + networkID, nodeID int, expectedCount int) error { + routes, err := libovsdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(nbClient, &nbdb.LogicalRouter{Name: routerName}, + func(route *nbdb.LogicalRouterStaticRoute) bool { + return route.ExternalIDs != nil && + route.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + route.ExternalIDs[libovsdbops.NetworkIDKey.String()] == strconv.Itoa(networkID) && + route.ExternalIDs[libovsdbops.NodeIDKey.String()] == strconv.Itoa(nodeID) + }) + if err != nil { + return fmt.Errorf("failed to get static routes from %s: %v", routerName, err) + } + + if len(routes) != expectedCount { + return fmt.Errorf("expected %d static routes on %s for CNC %s, got %d", + expectedCount, routerName, cncName, len(routes)) + } + return nil +} + +// ============================================================================= +// Integration Tests for Network Connect Controller +// ============================================================================= + +var _ = Describe("OVNKube Network Connect Controller Integration Tests", func() { + for _, ipMode := range ipModes { + + Context("["+ipMode.name+"]", func() { + var ( + nbClient libovsdbclient.Client + testCtx *libovsdbtest.Context + fakeClientset *util.OVNKubeControllerClientset + wf *factory.WatchFactory + fakeNM *networkmanager.FakeNetworkManager + controller *Controller + zoneName string + ) + + // start initializes and starts the controller with the given initial state + start := func(initialDB []libovsdbtest.TestData, nodes []testNode, networks map[string]testNetwork) { + var err error + + // Always add COPP to initialDB (required for router creation) + // Appending to nil is safe in Go - it creates a new slice + initialDB = append(initialDB, &nbdb.Copp{ + UUID: "copp-uuid", + Name: ovntypes.DefaultCOPPName, + }) + + // Create libovsdb test harness + nbClient, testCtx, err = libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: initialDB, + }, nil) + Expect(err).NotTo(HaveOccurred()) + + // Create fake clientset with nodes + var nodeObjs []runtime.Object + for _, n := range nodes { + nodeObjs = append(nodeObjs, createTestNode(n)) + } + fakeClientset = util.GetOVNClientset(nodeObjs...).GetOVNKubeControllerClientset() + + // Create watch factory + wf, err = factory.NewOVNKubeControllerWatchFactory(fakeClientset) + Expect(err).NotTo(HaveOccurred()) + + err = wf.Start() + Expect(err).NotTo(HaveOccurred()) + + // Create FakeNetworkManager with networks + fakeNM = &networkmanager.FakeNetworkManager{ + PrimaryNetworks: make(map[string]util.NetInfo), + } + for name, net := range networks { + netInfo, err := createNetInfo(net) + Expect(err).NotTo(HaveOccurred()) + fakeNM.PrimaryNetworks[name] = netInfo + } + + // Create and start controller + controller = NewController(zoneName, nbClient, wf, fakeNM.Interface()) + + err = controller.Start() + Expect(err).NotTo(HaveOccurred()) + } + + cleanup := func() { + if controller != nil { + controller.Stop() + } + if testCtx != nil { + testCtx.Cleanup() + } + if wf != nil { + wf.Shutdown() + } + } + + BeforeEach(func() { + setupTestConfig(ipMode.v4, ipMode.v6) + zoneName = "node1" // Default zone name + }) + + AfterEach(func() { + cleanup() + }) + + // ============================================================================= + // Context: CNC Lifecycle Tests + // ============================================================================= + Context("CNC Lifecycle", func() { + + It("should create connect router when CNC is created", func() { + // Setup - no networks needed for just connect router creation + start(nil, nil, nil) + + // Create CNC with tunnel ID - no subnet annotation needed + cnc := createTestCNC("test-cnc", 100, defaultConnectSubnets(), "") + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for connect router to be created and verify properties + Eventually(func() error { + return verifyConnectRouter(nbClient, "test-cnc", 100) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should not create connect router without tunnel ID annotation", func() { + // Setup - no networks needed + start(nil, nil, nil) + + // Create CNC without tunnel ID (tunnelID=0 means no annotation) + cnc := createTestCNC("no-tunnel-cnc", 0, defaultConnectSubnets(), "") + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait a bit and verify no router was created + Consistently(func() error { + routerName := getConnectRouterName("no-tunnel-cnc") + _, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: routerName}) + return err + }).WithTimeout(2 * time.Second).Should(HaveOccurred()) + }) + + It("should create connect router when tunnel ID annotation is added later", func() { + // Setup - no networks needed for just connect router creation + start(nil, nil, nil) + + // Create CNC without tunnel ID + cnc := createTestCNC("delayed-tunnel-cnc", 0, defaultConnectSubnets(), "") + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify no router created initially + Consistently(func() error { + routerName := getConnectRouterName("delayed-tunnel-cnc") + _, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: routerName}) + return err + }).WithTimeout(1 * time.Second).Should(HaveOccurred()) + + // Now update CNC to add tunnel ID annotation + cnc.Annotations[util.OvnConnectRouterTunnelKeyAnnotation] = "150" + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), cnc, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify router is now created + Eventually(func() error { + return verifyConnectRouter(nbClient, "delayed-tunnel-cnc", 150) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should not update connect router tunnel ID once set", func() { + // Setup - no networks needed for just connect router update + start(nil, nil, nil) + + // Create CNC with initial tunnel ID + cnc := createTestCNC("update-tunnel-cnc", 100, defaultConnectSubnets(), "") + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify router is created with initial tunnel ID + Eventually(func() error { + return verifyConnectRouter(nbClient, "update-tunnel-cnc", 100) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Update CNC to change tunnel ID annotation + cnc.Annotations[util.OvnConnectRouterTunnelKeyAnnotation] = "250" + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), cnc, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify router still has the original tunnel ID (tunnel ID is immutable once set) + Consistently(func() error { + return verifyConnectRouter(nbClient, "update-tunnel-cnc", 100) + }).WithTimeout(2 * time.Second).Should(Succeed()) + }) + + It("should delete connect router when CNC is deleted", func() { + // Setup - no networks needed for just connect router lifecycle + start(nil, nil, nil) + + // Create CNC + cnc := createTestCNC("delete-me-cnc", 300, defaultConnectSubnets(), "") + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for connect router to be created + Eventually(func() error { + return verifyConnectRouter(nbClient, "delete-me-cnc", 300) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Delete CNC + err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Delete( + context.Background(), "delete-me-cnc", metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for connect router to be deleted + Eventually(func() error { + routerName := getConnectRouterName("delete-me-cnc") + _, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: routerName}) + if err != nil { + return nil // Router deleted, which is what we want + } + return errRouterStillExists + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should create ports and routes when CNC has only one allocated network", func() { + // Setup with Layer3 network + networks := []testNetwork{ + {name: "red-network", id: 1, topologyType: ovntypes.Layer3Topology}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "red-network": {"10.128.1.0/24", "fd00:10:128:1::/64"}, + }}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"red-network": networks[0]}) + + // Create CNC with allocated network + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + cnc := createTestCNC("ports-cnc", 200, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Get expected port IPs based on allocated connect subnet and node ID + // Pass the allocated subnet (/24, /64) and node ID - function calculates P2P subnet + connectNets, networkNets := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 1, // node1 has ID 1 + ) + + // Verify connect router + Eventually(func() error { + return verifyConnectRouter(nbClient, "ports-cnc", 200) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port count on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("ports-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port count on network router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify ports are local (node is in same zone as controller) + connectPortName := getConnectRouterToNetworkRouterPortName("ports-cnc", networks[0].name, "node1") + networkPortName := getNetworkRouterToConnectRouterPortName(networks[0].name, "node1", "ports-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, connectPortName, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 1), networkPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, networkPortName, 0, connectPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port on connect router (crtor-{cnc}-{network}-{node}) + // Connect router port uses first IP of P2P subnet + Eventually(func() error { + return verifyRouterPort(nbClient, getConnectRouterToNetworkRouterPortName( + "ports-cnc", networks[0].name, "node1"), "ports-cnc", connectNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port on network router (rtocr-{network}-{node}-{cnc}) + // Network router port uses second IP of P2P subnet + Eventually(func() error { + return verifyRouterPort(nbClient, getNetworkRouterToConnectRouterPortName( + networks[0].name, "node1", "ports-cnc"), "ports-cnc", networkNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no routing policies exist on network router since + // there is only 1 network - so nothing to connect to + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), + "ports-cnc", networks[0].id, networks[0].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify static routes on connect router + // IPPrefix = node's subnet, Nexthop = network router port IP + nexthopV4, nexthopV6 := extractNexthops(networkNets) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("ports-cnc"), + "ports-cnc", 1, 1, + "10.128.1.0/24", "fd00:10:128:1::/64", nexthopV4, nexthopV6) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should not create ports, policies and routes when CNC has no subnet annotation", func() { + // Setup with Layer3 network + networks := []testNetwork{ + {name: "red-network", id: 1, topologyType: ovntypes.Layer3Topology}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "red-network": {"10.128.1.0/24", "fd00:10:128:1::/64"}, + }}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"red-network": networks[0]}) + + cnc := createTestCNC("ports-cnc", 200, defaultConnectSubnets(), "") + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify connect router is created + Eventually(func() error { + return verifyConnectRouter(nbClient, "ports-cnc", 200) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no ports on connect router (no subnet annotation = no allocated networks) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("ports-cnc"), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no ports on network router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), + "ports-cnc", networks[0].id, networks[0].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no static routes on connect router + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("ports-cnc"), + "ports-cnc", 1, 1, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should create ports, policies and routes when CNC has more than one allocated network", func() { + // Setup with two Layer3 networks + networks := []testNetwork{ + {name: "red-network", id: 1, topologyType: ovntypes.Layer3Topology}, + {name: "blue-network", id: 2, topologyType: ovntypes.Layer3Topology}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "red-network": {"10.128.1.0/24", "fd00:10:128:1::/64"}, + "blue-network": {"10.129.1.0/24", "fd00:10:129:1::/64"}, + }}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"red-network": networks[0], "blue-network": networks[1]}) + + // Allocated connect subnets for each network + type networkTestData struct { + connectSubnetV4 string + connectSubnetV6 string + nodeSubnetV4 string + nodeSubnetV6 string + networkOwner string + } + networkData := []networkTestData{ + {"192.168.0.0/24", "fd00:192:168::/64", "10.128.1.0/24", "fd00:10:128:1::/64", "layer3_1"}, + {"192.168.1.0/24", "fd00:192:168:1::/64", "10.129.1.0/24", "fd00:10:129:1::/64", "layer3_2"}, + } + + // Create CNC with both networks allocated + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + "layer3_2": {"192.168.1.0/24", "fd00:192:168:1::/64"}, + }) + cnc := createTestCNC("ports-cnc", 200, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify connect router is created + Eventually(func() error { + return verifyConnectRouter(nbClient, "ports-cnc", 200) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port count on connect router (1 port per network = 2 total) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("ports-cnc"), len(networks)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify each network's ports, policies, and routes + for i, net := range networks { + data := networkData[i] + + // Verify port count on this network's router (1 port for the node) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify ports are local (node is in same zone as controller) + connectPortName := getConnectRouterToNetworkRouterPortName("ports-cnc", net.name, "node1") + networkPortName := getNetworkRouterToConnectRouterPortName(net.name, "node1", "ports-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, connectPortName, getExpectedPortTunnelKey(data.connectSubnetV4, data.connectSubnetV6, ovntypes.Layer3Topology, 1), networkPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, networkPortName, 0, connectPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Get expected port IPs based on allocated connect subnet and node ID + connectNets, networkNets := getExpectedPortNetworks( + ovntest.MustParseIPNet(data.connectSubnetV4), + ovntest.MustParseIPNet(data.connectSubnetV6), + 1, // node1 has ID 1 + ) + + // Verify port on connect router (crtor-{cnc}-{network}-{node}) + Eventually(func() error { + return verifyRouterPort(nbClient, getConnectRouterToNetworkRouterPortName( + "ports-cnc", net.name, "node1"), "ports-cnc", connectNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port on network router (rtocr-{network}-{node}-{cnc}) + Eventually(func() error { + return verifyRouterPort(nbClient, getNetworkRouterToConnectRouterPortName( + net.name, "node1", "ports-cnc"), "ports-cnc", networkNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routing policies on this network's router + // Policies point to the OTHER network (1 per IP family) + otherNetworkID := networks[1-i].id // 0->1, 1->0 + expectedPolicyCount := 0 + if config.IPv4Mode { + expectedPolicyCount++ + } + if config.IPv6Mode { + expectedPolicyCount++ + } + Eventually(func() error { + return verifyRouterPolicy(nbClient, net.RouterName(), + "ports-cnc", net.id, otherNetworkID, expectedPolicyCount) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify static routes on connect router for this network + // IPPrefix = node's subnet, Nexthop = network router port IP + nexthopV4, nexthopV6 := extractNexthops(networkNets) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("ports-cnc"), + "ports-cnc", net.id, 1, + data.nodeSubnetV4, data.nodeSubnetV6, nexthopV4, nexthopV6) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + }) + + It("should delete all ports, policies and routes when CNC with two networks is deleted", func() { + // Setup with two Layer3 networks + networks := []testNetwork{ + {name: "red-network", id: 1, topologyType: ovntypes.Layer3Topology}, + {name: "blue-network", id: 2, topologyType: ovntypes.Layer3Topology}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "red-network": {"10.128.1.0/24", "fd00:10:128:1::/64"}, + "blue-network": {"10.129.1.0/24", "fd00:10:129:1::/64"}, + }}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"red-network": networks[0], "blue-network": networks[1]}) + + // Create CNC with both networks allocated + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + "layer3_2": {"192.168.1.0/24", "fd00:192:168:1::/64"}, + }) + cnc := createTestCNC("delete-cnc", 300, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for connect router, ports, and routes to be created + Eventually(func() error { + return verifyConnectRouter(nbClient, "delete-cnc", 300) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("delete-cnc"), len(networks)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Calculate expected counts per IP family + expectedPerIPFamily := 0 + if config.IPv4Mode { + expectedPerIPFamily++ + } + if config.IPv6Mode { + expectedPerIPFamily++ + } + + // Verify policies exist for each network before deletion + // Each network's router has policies pointing to the OTHER network + for i, net := range networks { + otherNetworkID := networks[1-i].id + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, net.RouterName(), + "delete-cnc", net.id, otherNetworkID, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Verify static routes exist for each network before deletion + for _, net := range networks { + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("delete-cnc"), + "delete-cnc", net.id, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Delete the CNC + err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Delete( + context.Background(), "delete-cnc", metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify connect router is deleted + Eventually(func() error { + routerName := getConnectRouterName("delete-cnc") + _, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: routerName}) + if err != nil { + return nil // Router deleted, which is what we want + } + return errRouterStillExists + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify all ports are deleted from each network router + for _, net := range networks { + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Verify all policies are deleted from each network router + for i, net := range networks { + otherNetworkID := networks[1-i].id + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, net.RouterName(), + "delete-cnc", net.id, otherNetworkID, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Note: Static routes are on the connect router which is already deleted + }) + }) + + // ============================================================================= + // Context: Network Changes + // ============================================================================= + Context("Network Changes", func() { + + It("should add ports, policies and routes when new network is connected via annotation update", func() { + // Setup with two networks (provide both v4 and v6 subnets) + networks := []testNetwork{ + {name: "net1", id: 1, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + {name: "net2", id: 2, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.132.0.0/14/23", "fd00:10:132::/48/64"}}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "net1": {"10.128.1.0/24", "fd00:10:128:1::/64"}, "net2": {"10.129.1.0/24", "fd00:10:129:1::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"net1": networks[0], "net2": networks[1]}) + + // Create CNC with only first network + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + cnc := createTestCNC("update-cnc", 400, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify connect router exists + Eventually(func() error { + return verifyConnectRouter(nbClient, "update-cnc", 400) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port on connect router for first network + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("update-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port on first network's router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify static routes for first network (no policies since only 1 network) + expectedRouteCount := 0 + if config.IPv4Mode { + expectedRouteCount++ + } + if config.IPv6Mode { + expectedRouteCount++ + } + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("update-cnc"), + "update-cnc", 1, 1, expectedRouteCount) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no policies since only 1 network (nothing to connect to) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), + "update-cnc", networks[0].id, networks[1].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Update CNC to add second network + updatedCNC, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "update-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + "layer3_2": {"192.168.1.0/24", "fd00:192:168:1::/64"}, + }) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify port count on connect router (2 ports now) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("update-cnc"), 2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port on second network's router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[1].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify static routes for second network + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("update-cnc"), + "update-cnc", 2, 1, expectedRouteCount) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Now with 2 networks, policies should exist on both network routers + // Each network's router has policies pointing to the other network + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), + "update-cnc", networks[0].id, networks[1].id, expectedRouteCount) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[1].RouterName(), + "update-cnc", networks[1].id, networks[0].id, expectedRouteCount) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should remove ports, policies and routes when network is disconnected via annotation update", func() { + // Setup with two networks + networks := []testNetwork{ + {name: "remove-net1", id: 1, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + {name: "remove-net2", id: 2, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.132.0.0/14/23", "fd00:10:132::/48/64"}}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "remove-net1": {"10.128.1.0/24", "fd00:10:128:1::/64"}, "remove-net2": {"10.129.1.0/24", "fd00:10:129:1::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"remove-net1": networks[0], "remove-net2": networks[1]}) + + // Calculate expected count per IP family + expectedPerIPFamily := 0 + if config.IPv4Mode { + expectedPerIPFamily++ + } + if config.IPv6Mode { + expectedPerIPFamily++ + } + + // Create CNC with both networks (2 networks) + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + "layer3_2": {"192.168.1.0/24", "fd00:192:168:1::/64"}, + }) + cnc := createTestCNC("remove-net-cnc", 500, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // ========== VERIFY 2 NETWORKS ========== + // Verify connect router + Eventually(func() error { + return verifyConnectRouter(nbClient, "remove-net-cnc", 500) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify ports on connect router (2) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("remove-net-cnc"), 2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify each network has ports, policies, routes + for i, net := range networks { + otherNetworkID := networks[1-i].id + + // Port on network router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Policies pointing to other network + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, net.RouterName(), + "remove-net-cnc", net.id, otherNetworkID, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Static routes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("remove-net-cnc"), + "remove-net-cnc", net.id, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // ========== UPDATE TO 1 NETWORK ========== + updatedCNC, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "remove-net-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify ports on connect router (1 now) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("remove-net-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify first network still has port + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify second network has no ports + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[1].RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no policies on first network (only 1 network now, nothing to connect to) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), + "remove-net-cnc", networks[0].id, networks[1].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify first network still has routes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("remove-net-cnc"), + "remove-net-cnc", 1, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify second network has no routes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("remove-net-cnc"), + "remove-net-cnc", 2, 1, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // ========== UPDATE TO 0 NETWORKS ========== + updatedCNC, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "remove-net-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = "" + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify no ports on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("remove-net-cnc"), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no ports on either network router + for _, net := range networks { + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Verify no routes on connect router + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("remove-net-cnc"), + "remove-net-cnc", 1, 1, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should handle Layer2 networks with 0 -> 1 -> 2 -> 1 -> 0 network transitions", func() { + // Setup with two Layer2 networks + networks := []testNetwork{ + {name: "l2-net1", id: 1, topologyType: ovntypes.Layer2Topology, subnets: []string{"10.200.0.0/16", "fd00:10:200::/48"}}, + {name: "l2-net2", id: 2, topologyType: ovntypes.Layer2Topology, subnets: []string{"10.201.0.0/16", "fd00:10:201::/48"}}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1"}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"l2-net1": networks[0], "l2-net2": networks[1]}) + + // Calculate expected count per IP family + expectedPerIPFamily := 0 + if config.IPv4Mode { + expectedPerIPFamily++ + } + if config.IPv6Mode { + expectedPerIPFamily++ + } + + // ========== START WITH 0 NETWORKS ========== + // Create CNC with no networks (empty annotation) + cnc := createTestCNC("l2-cnc", 850, defaultConnectSubnets(), "") + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify connect router exists but has no ports + Eventually(func() error { + return verifyConnectRouter(nbClient, "l2-cnc", 850) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("l2-cnc"), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no ports on network routers + for _, net := range networks { + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Verify no routes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("l2-cnc"), + "l2-cnc", 1, 0, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("l2-cnc"), + "l2-cnc", 2, 0, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no policies on network routers + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), + "l2-cnc", networks[0].id, networks[1].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[1].RouterName(), + "l2-cnc", networks[1].id, networks[0].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // ========== ADD NETWORK 1 (0 -> 1) ========== + updatedCNC, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "l2-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer2_1": {"192.168.0.0/31", "fd00:192:168::/127"}, + }) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify 1 port on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("l2-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify 1 port on network1 router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port is local (node1 is in same zone as controller) + // Layer2 uses subIndex=0 for tunnel key, and no per-node ports (empty nodeName) + l2ConnectPort1 := getConnectRouterToNetworkRouterPortName("l2-cnc", "l2-net1", "") + l2NetworkPort1 := getNetworkRouterToConnectRouterPortName("l2-net1", "", "l2-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, l2ConnectPort1, getExpectedPortTunnelKey("192.168.0.0/31", "fd00:192:168::/127", ovntypes.Layer2Topology, 0), l2NetworkPort1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, l2NetworkPort1, 0, l2ConnectPort1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes for network1 (Layer2 uses nodeID=0) + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("l2-cnc"), + "l2-cnc", 1, 0, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // No policies with only 1 network + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), + "l2-cnc", networks[0].id, networks[1].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // ========== ADD NETWORK 2 (1 -> 2) ========== + updatedCNC, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "l2-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer2_1": {"192.168.0.0/31", "fd00:192:168::/127"}, + "layer2_2": {"192.168.1.0/31", "fd00:192:168:1::/127"}, + }) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify 2 ports on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("l2-cnc"), 2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify each network + l2Subnets := []subnetPair{ + {"192.168.0.0/31", "fd00:192:168::/127"}, + {"192.168.1.0/31", "fd00:192:168:1::/127"}, + } + for i, net := range networks { + otherNetworkID := networks[1-i].id + + // Port on network router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port is local (node1 in same zone as controller) + // Layer2 uses empty nodeName (no per-node ports) + connectPort := getConnectRouterToNetworkRouterPortName("l2-cnc", net.name, "") + networkPort := getNetworkRouterToConnectRouterPortName(net.name, "", "l2-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, connectPort, getExpectedPortTunnelKey(l2Subnets[i].v4, l2Subnets[i].v6, ovntypes.Layer2Topology, 0), networkPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, networkPort, 0, connectPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Policies pointing to other network + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, net.RouterName(), + "l2-cnc", net.id, otherNetworkID, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Static routes (Layer2 uses nodeID=0) + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("l2-cnc"), + "l2-cnc", net.id, 0, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // ========== REMOVE NETWORK 1 (2 -> 1) ========== + updatedCNC, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "l2-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer2_2": {"192.168.1.0/31", "fd00:192:168:1::/127"}, + }) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify 1 port on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("l2-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify network1 has no ports + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify network2 still has port + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[1].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify network2 port is local (Layer2: empty nodeName) + l2ConnectPort2 := getConnectRouterToNetworkRouterPortName("l2-cnc", "l2-net2", "") + l2NetworkPort2 := getNetworkRouterToConnectRouterPortName("l2-net2", "", "l2-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, l2ConnectPort2, getExpectedPortTunnelKey("192.168.1.0/31", "fd00:192:168:1::/127", ovntypes.Layer2Topology, 0), l2NetworkPort2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, l2NetworkPort2, 0, l2ConnectPort2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no policies on network2 (only 1 network now) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[1].RouterName(), + "l2-cnc", networks[1].id, networks[0].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify network1 has no routes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("l2-cnc"), + "l2-cnc", 1, 0, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify network2 still has routes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("l2-cnc"), + "l2-cnc", 2, 0, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // ========== REMOVE NETWORK 2 (1 -> 0) ========== + updatedCNC, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "l2-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = "" + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify no ports on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("l2-cnc"), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no ports on either network router + for _, net := range networks { + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Verify no policies on network routers (both networks pointing to each other should be gone) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), + "l2-cnc", networks[0].id, networks[1].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[1].RouterName(), + "l2-cnc", networks[1].id, networks[0].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no routes on connect router + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("l2-cnc"), + "l2-cnc", 2, 0, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should handle mixed Layer2 and Layer3 networks with 0 -> 1 -> 2 -> 1 -> 0 transitions", func() { + // Setup with one Layer2 and one Layer3 network (provide both v4 and v6 subnets) + l2Net := testNetwork{name: "mixed-l2", id: 1, topologyType: ovntypes.Layer2Topology, + subnets: []string{"10.200.0.0/16", "fd00:10:200::/48"}} + l3Net := testNetwork{name: "mixed-l3", id: 2, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}} + networks := []testNetwork{l2Net, l3Net} + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "mixed-l3": {"10.128.1.0/24", "fd00:10:128:1::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"mixed-l2": l2Net, "mixed-l3": l3Net}) + + // Calculate expected count per IP family + expectedPerIPFamily := 0 + if config.IPv4Mode { + expectedPerIPFamily++ + } + if config.IPv6Mode { + expectedPerIPFamily++ + } + + // ========== START WITH 0 NETWORKS ========== + cnc := createTestCNC("mixed-cnc", 900, defaultConnectSubnets(), "") + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify connect router exists but has no ports + Eventually(func() error { + return verifyConnectRouter(nbClient, "mixed-cnc", 900) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("mixed-cnc"), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no ports on network routers + for _, net := range networks { + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Verify no routes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("mixed-cnc"), + "mixed-cnc", 1, 0, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("mixed-cnc"), + "mixed-cnc", 2, 1, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // ========== ADD LAYER2 NETWORK (0 -> 1) ========== + updatedCNC, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "mixed-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer2_1": {"192.168.0.0/31", "fd00:192:168::/127"}, + }) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify 1 port on connect router (Layer2 has 1 port) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("mixed-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer2 network router has port + Eventually(func() error { + return verifyRouterPortsCount(nbClient, l2Net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer2 port is local (node1 in same zone as controller) + // Layer2: empty nodeName (no per-node ports) + mixedL2ConnectPort := getConnectRouterToNetworkRouterPortName("mixed-cnc", l2Net.name, "") + mixedL2NetworkPort := getNetworkRouterToConnectRouterPortName(l2Net.name, "", "mixed-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, mixedL2ConnectPort, getExpectedPortTunnelKey("192.168.0.0/31", "fd00:192:168::/127", ovntypes.Layer2Topology, 0), mixedL2NetworkPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, mixedL2NetworkPort, 0, mixedL2ConnectPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer3 network router has no port yet + Eventually(func() error { + return verifyRouterPortsCount(nbClient, l3Net.RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes for Layer2 (nodeID=0) + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("mixed-cnc"), + "mixed-cnc", 1, 0, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // No policies with only 1 network + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, l2Net.RouterName(), + "mixed-cnc", l2Net.id, l3Net.id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // ========== ADD LAYER3 NETWORK (1 -> 2) ========== + updatedCNC, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "mixed-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer2_1": {"192.168.0.0/31", "fd00:192:168::/127"}, + "layer3_2": {"192.168.1.0/24", "fd00:192:168:1::/64"}, + }) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify 2 ports on connect router (1 for Layer2, 1 for Layer3 node1) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("mixed-cnc"), 2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify both network routers have ports + Eventually(func() error { + return verifyRouterPortsCount(nbClient, l2Net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortsCount(nbClient, l3Net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer2 port is local + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, mixedL2ConnectPort, getExpectedPortTunnelKey("192.168.0.0/31", "fd00:192:168::/127", ovntypes.Layer2Topology, 0), mixedL2NetworkPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer3 port is local (node1 id=1) + mixedL3ConnectPort := getConnectRouterToNetworkRouterPortName("mixed-cnc", l3Net.name, "node1") + mixedL3NetworkPort := getNetworkRouterToConnectRouterPortName(l3Net.name, "node1", "mixed-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, mixedL3ConnectPort, getExpectedPortTunnelKey("192.168.1.0/24", "fd00:192:168:1::/64", ovntypes.Layer3Topology, 1), mixedL3NetworkPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, mixedL3NetworkPort, 0, mixedL3ConnectPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes for both networks + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("mixed-cnc"), + "mixed-cnc", 1, 0, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("mixed-cnc"), + "mixed-cnc", 2, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify policies on both routers pointing to each other + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, l2Net.RouterName(), + "mixed-cnc", l2Net.id, l3Net.id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, l3Net.RouterName(), + "mixed-cnc", l3Net.id, l2Net.id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // ========== REMOVE LAYER2 NETWORK (2 -> 1) ========== + updatedCNC, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "mixed-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_2": {"192.168.1.0/24", "fd00:192:168:1::/64"}, + }) + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify 1 port on connect router (only Layer3 now) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("mixed-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer2 has no ports + Eventually(func() error { + return verifyRouterPortsCount(nbClient, l2Net.RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer3 still has port + Eventually(func() error { + return verifyRouterPortsCount(nbClient, l3Net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer3 port is local (node1 id=1) + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, mixedL3ConnectPort, getExpectedPortTunnelKey("192.168.1.0/24", "fd00:192:168:1::/64", ovntypes.Layer3Topology, 1), mixedL3NetworkPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, mixedL3NetworkPort, 0, mixedL3ConnectPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer2 routes removed + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("mixed-cnc"), + "mixed-cnc", 1, 0, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer3 routes still exist + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("mixed-cnc"), + "mixed-cnc", 2, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no policies on Layer3 (only 1 network now) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, l3Net.RouterName(), + "mixed-cnc", l3Net.id, l2Net.id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // ========== REMOVE LAYER3 NETWORK (1 -> 0) ========== + updatedCNC, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Get( + context.Background(), "mixed-cnc", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedCNC.Annotations[ovnNetworkConnectSubnetAnnotation] = "" + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Update( + context.Background(), updatedCNC, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify no ports on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("mixed-cnc"), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify no ports on either network router + for _, net := range networks { + Eventually(func() error { + return verifyRouterPortsCount(nbClient, net.RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + + // Verify no routes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("mixed-cnc"), + "mixed-cnc", 2, 1, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + }) + + // ============================================================================= + // Context: Node Events + // ============================================================================= + Context("Node Events", func() { + + It("should add ports and routes when a new node is added", func() { + // Setup with one node initially + networks := []testNetwork{ + {name: "node-add-net", id: 1, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "node-add-net": {"10.128.1.0/24", "fd00:10:128:1::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"node-add-net": networks[0]}) + + // Create CNC + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + cnc := createTestCNC("node-add-cnc", 1600, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify initial state: 1 port on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("node-add-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify initial port is local (node1 is in same zone as controller) + connectPortName := getConnectRouterToNetworkRouterPortName("node-add-cnc", "node-add-net", "node1") + networkPortName := getNetworkRouterToConnectRouterPortName("node-add-net", "node1", "node-add-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, connectPortName, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 1), networkPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details for node1 + connectNets1, networkNets1 := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 1, // node1 has ID 1 + ) + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName, "node-add-cnc", connectNets1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, networkPortName, "node-add-cnc", networkNets1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify initial routes for node1 + // IPPrefix = node's subnet, Nexthop = network router port IP + nexthopV4_1, nexthopV6_1 := extractNexthops(networkNets1) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("node-add-cnc"), + "node-add-cnc", 1, 1, + "10.128.1.0/24", "fd00:10:128:1::/64", nexthopV4_1, nexthopV6_1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Add a new node + newNode := createTestNode(testNode{ + name: "node2", id: 2, zone: "node2", + nodeSubnets: map[string]subnetPair{"node-add-net": {"10.128.2.0/24", "fd00:10:128:2::/64"}}, + }) + _, err = fakeClientset.KubeClient.CoreV1().Nodes().Create( + context.Background(), newNode, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify new port added (2 total now) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("node-add-cnc"), 2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify node2's port is remote (node2 is in different zone "node2") + connectPortName2 := getConnectRouterToNetworkRouterPortName("node-add-cnc", "node-add-net", "node2") + Eventually(func() error { + return verifyRouterPortIsRemote(nbClient, connectPortName2, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 2)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details for node2 (remote - only connect router port) + connectNets2, networkNets2 := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 2, // node2 has ID 2 + ) + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName2, "node-add-cnc", connectNets2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes for both nodes + // node1 routes verified above, now verify node2 routes + nexthopV4_2, nexthopV6_2 := extractNexthops(networkNets2) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("node-add-cnc"), + "node-add-cnc", 1, 2, + "10.128.2.0/24", "fd00:10:128:2::/64", nexthopV4_2, nexthopV6_2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should remove ports and routes when a node is deleted", func() { + // Setup with two nodes initially + networks := []testNetwork{ + {name: "node-del-net", id: 1, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "node-del-net": {"10.128.1.0/24", "fd00:10:128:1::/64"}}}, + {name: "node2", id: 2, zone: "node2", nodeSubnets: map[string]subnetPair{ + "node-del-net": {"10.128.2.0/24", "fd00:10:128:2::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"node-del-net": networks[0]}) + + // Create CNC + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + cnc := createTestCNC("node-del-cnc", 1700, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify initial state: 2 ports on connect router + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("node-del-cnc"), 2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details for node1 (local) + connectPortName1 := getConnectRouterToNetworkRouterPortName("node-del-cnc", "node-del-net", "node1") + networkPortName1 := getNetworkRouterToConnectRouterPortName("node-del-net", "node1", "node-del-cnc") + connectNets1, networkNets1 := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 1, // node1 has ID 1 + ) + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, connectPortName1, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 1), networkPortName1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName1, "node-del-cnc", connectNets1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, networkPortName1, "node-del-cnc", networkNets1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details for node2 (remote) + connectPortName2 := getConnectRouterToNetworkRouterPortName("node-del-cnc", "node-del-net", "node2") + connectNets2, networkNets2 := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 2, // node2 has ID 2 + ) + Eventually(func() error { + return verifyRouterPortIsRemote(nbClient, connectPortName2, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 2)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName2, "node-del-cnc", connectNets2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes for both nodes with full details + nexthopV4_1, nexthopV6_1 := extractNexthops(networkNets1) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("node-del-cnc"), + "node-del-cnc", 1, 1, + "10.128.1.0/24", "fd00:10:128:1::/64", nexthopV4_1, nexthopV6_1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + nexthopV4_2, nexthopV6_2 := extractNexthops(networkNets2) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("node-del-cnc"), + "node-del-cnc", 1, 2, + "10.128.2.0/24", "fd00:10:128:2::/64", nexthopV4_2, nexthopV6_2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Delete node2 + err = fakeClientset.KubeClient.CoreV1().Nodes().Delete( + context.Background(), "node2", metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify port removed (1 remaining) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("node-del-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify node1 port still exists with correct details + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName1, "node-del-cnc", connectNets1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes for node2 are removed + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("node-del-cnc"), + "node-del-cnc", 1, 2, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes for node1 still exist with full details + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("node-del-cnc"), + "node-del-cnc", 1, 1, + "10.128.1.0/24", "fd00:10:128:1::/64", nexthopV4_1, nexthopV6_1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should not create ports or routes until node ID is allocated", func() { + // Setup with network but node WITHOUT node ID annotation + networks := []testNetwork{ + {name: "nodeid-wait-net", id: 1, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + } + // Create node without node ID (id: 0 means no annotation) + nodes := []testNode{ + {name: "node1", id: 0, zone: "node1", nodeSubnets: map[string]subnetPair{ + "nodeid-wait-net": {"10.128.1.0/24", "fd00:10:128:1::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"nodeid-wait-net": networks[0]}) + + // Create CNC + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + cnc := createTestCNC("nodeid-wait-cnc", 1750, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify connect router is created but has no ports (node has no ID) + Eventually(func() error { + return verifyConnectRouter(nbClient, "nodeid-wait-cnc", 1750) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Consistently(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("nodeid-wait-cnc"), 0) + }).WithTimeout(2 * time.Second).Should(Succeed()) + + // Verify network router also has no connect ports + Consistently(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 0) + }).WithTimeout(2 * time.Second).Should(Succeed()) + + // Verify no static routes either + Consistently(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("nodeid-wait-cnc"), + "nodeid-wait-cnc", networks[0].id, 0, 0) + }).WithTimeout(2 * time.Second).Should(Succeed()) + + // Now add node ID annotation + node, err := fakeClientset.KubeClient.CoreV1().Nodes().Get( + context.Background(), "node1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + node.Annotations[util.OvnNodeID] = "1" + _, err = fakeClientset.KubeClient.CoreV1().Nodes().Update( + context.Background(), node, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Now ports and routes should be created + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("nodeid-wait-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify network router also has the connect port + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify ports are local (node is in same zone as controller) + connectPortName := getConnectRouterToNetworkRouterPortName("nodeid-wait-cnc", "nodeid-wait-net", "node1") + networkPortName := getNetworkRouterToConnectRouterPortName("nodeid-wait-net", "node1", "nodeid-wait-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, connectPortName, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 1), networkPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, networkPortName, 0, connectPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details + connectNets, networkNets := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 1, // node1 has ID 1 + ) + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName, "nodeid-wait-cnc", connectNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, networkPortName, "nodeid-wait-cnc", networkNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify static routes with full details + nexthopV4, nexthopV6 := extractNexthops(networkNets) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("nodeid-wait-cnc"), + "nodeid-wait-cnc", 1, 1, + "10.128.1.0/24", "fd00:10:128:1::/64", nexthopV4, nexthopV6) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should reconcile when node zone annotation is updated", func() { + // Setup with node in local zone + // The controller considers a node "local" when util.GetNodeZone(node) == c.zone + // So we set controller's zoneName to match the node's zone annotation + zoneName = "node1" + networks := []testNetwork{ + {name: "zone-update-net", id: 1, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "zone-update-net": {"10.128.1.0/24", "fd00:10:128:1::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"zone-update-net": networks[0]}) + + // Create CNC + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + cnc := createTestCNC("zone-update-cnc", 1800, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify initial port (local zone node) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("zone-update-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify ports are local with peer set + connectPortName := getConnectRouterToNetworkRouterPortName("zone-update-cnc", "zone-update-net", "node1") + networkPortName := getNetworkRouterToConnectRouterPortName("zone-update-net", "node1", "zone-update-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, connectPortName, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 1), networkPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, networkPortName, 0, connectPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details + connectNets, networkNets := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 1, // node1 has ID 1 + ) + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName, "zone-update-cnc", connectNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, networkPortName, "zone-update-cnc", networkNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify initial static routes with full details + nexthopV4, nexthopV6 := extractNexthops(networkNets) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("zone-update-cnc"), + "zone-update-cnc", 1, 1, + "10.128.1.0/24", "fd00:10:128:1::/64", nexthopV4, nexthopV6) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Update node's zone annotation to a different zone + node, err := fakeClientset.KubeClient.CoreV1().Nodes().Get( + context.Background(), "node1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + node.Annotations[util.OvnNodeZoneName] = "zone2" + _, err = fakeClientset.KubeClient.CoreV1().Nodes().Update( + context.Background(), node, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Controller should reconcile - port and routes should still exist (now treated as remote zone) + Consistently(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("zone-update-cnc"), 1) + }).WithTimeout(2 * time.Second).Should(Succeed()) + + // After zone update, connect router port should be remote (no peer, has requested-chassis) + Eventually(func() error { + return verifyRouterPortIsRemote(nbClient, connectPortName, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 1)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Network router port is deleted for remote nodes - only connect router side port exists + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes still exist after zone change + Consistently(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("zone-update-cnc"), + "zone-update-cnc", 1, 1, + "10.128.1.0/24", "fd00:10:128:1::/64", nexthopV4, nexthopV6) + }).WithTimeout(2 * time.Second).Should(Succeed()) + }) + + It("should handle multiple zones", func() { + // Controller in zone "local-node" + zoneName = "local-node" + networks := []testNetwork{ + {name: "zone-net", id: 1, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + } + nodes := []testNode{ + // local-node has zone annotation "local-node" which matches c.zone + {name: "local-node", id: 1, zone: "local-node", nodeSubnets: map[string]subnetPair{ + "zone-net": {"10.128.1.0/24", "fd00:10:128:1::/64"}}}, + // remote-node is in different zone + {name: "remote-node-1", id: 2, zone: "remote-zone", nodeSubnets: map[string]subnetPair{ + "zone-net": {"10.128.2.0/24", "fd00:10:128:2::/64"}}}, + {name: "remote-node-2", id: 3, zone: "remote-zone", nodeSubnets: map[string]subnetPair{ + "zone-net": {"10.128.3.0/24", "fd00:10:128:3::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"zone-net": networks[0]}) + + expectedPerIPFamily := 0 + if config.IPv4Mode { + expectedPerIPFamily++ + } + if config.IPv6Mode { + expectedPerIPFamily++ + } + + // Create CNC with IP mode-aware subnets + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + cnc := createTestCNC("zone-cnc", 1500, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify 3 ports on connect router (one for each node: 1 local + 2 remote) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("zone-cnc"), 3) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify local-node port is local (with peer to network router) + localConnectPort := getConnectRouterToNetworkRouterPortName("zone-cnc", "zone-net", "local-node") + localNetworkPort := getNetworkRouterToConnectRouterPortName("zone-net", "local-node", "zone-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, localConnectPort, 0, localNetworkPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details for local-node + localConnectNets, localNetworkNets := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 1, // local-node has ID 1 + ) + Eventually(func() error { + return verifyRouterPort(nbClient, localConnectPort, "zone-cnc", localConnectNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, localNetworkPort, "zone-cnc", localNetworkNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify network router has port for local node only (remote nodes don't get network router ports) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, localNetworkPort, 0, localConnectPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify remote-node-1 port is remote (no peer, has requested-chassis) + remoteConnectPort1 := getConnectRouterToNetworkRouterPortName("zone-cnc", "zone-net", "remote-node-1") + Eventually(func() error { + return verifyRouterPortIsRemote(nbClient, remoteConnectPort1, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 2)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details for remote-node-1 + remoteConnectNets1, _ := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 2, // remote-node-1 has ID 2 + ) + Eventually(func() error { + return verifyRouterPort(nbClient, remoteConnectPort1, "zone-cnc", remoteConnectNets1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify remote-node-2 port is remote (no peer, has requested-chassis) + remoteConnectPort2 := getConnectRouterToNetworkRouterPortName("zone-cnc", "zone-net", "remote-node-2") + Eventually(func() error { + return verifyRouterPortIsRemote(nbClient, remoteConnectPort2, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 3)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify port details for remote-node-2 + remoteConnectNets2, _ := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 3, // remote-node-2 has ID 3 + ) + Eventually(func() error { + return verifyRouterPort(nbClient, remoteConnectPort2, "zone-cnc", remoteConnectNets2) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify static routes on connect router for all 3 nodes + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("zone-cnc"), + "zone-cnc", 1, 1, expectedPerIPFamily) // local-node routes + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("zone-cnc"), + "zone-cnc", 1, 2, expectedPerIPFamily) // remote-node-1 routes + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("zone-cnc"), + "zone-cnc", networks[0].id, 3, expectedPerIPFamily) // remote-node-2 routes + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should update routes when node subnet annotation is updated", func() { + // Setup + networks := []testNetwork{ + {name: "subnet-update-net", id: 1, topologyType: ovntypes.Layer3Topology, + subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "subnet-update-net": {"10.128.1.0/24", "fd00:10:128:1::/64"}}}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"subnet-update-net": networks[0]}) + + // Create CNC + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + }) + cnc := createTestCNC("subnet-update-cnc", 1900, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify initial port + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("subnet-update-cnc"), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify initial port details + connectPortName := getConnectRouterToNetworkRouterPortName("subnet-update-cnc", "subnet-update-net", "node1") + networkPortName := getNetworkRouterToConnectRouterPortName("subnet-update-net", "node1", "subnet-update-cnc") + connectNets, networkNets := getExpectedPortNetworks( + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/64"), + 1, // node1 has ID 1 + ) + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, connectPortName, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 1), networkPortName) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName, "subnet-update-cnc", connectNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPort(nbClient, networkPortName, "subnet-update-cnc", networkNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify initial routes with full details + nexthopV4, nexthopV6 := extractNexthops(networkNets) + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("subnet-update-cnc"), + "subnet-update-cnc", 1, 1, + "10.128.1.0/24", "fd00:10:128:1::/64", nexthopV4, nexthopV6) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Update node subnet annotation + node, err := fakeClientset.KubeClient.CoreV1().Nodes().Get( + context.Background(), "node1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + node.Annotations[ovnNodeSubnetsAnnotation] = buildNodeSubnetAnnotation(map[string]subnetPair{ + "subnet-update-net": {"10.128.10.0/24", "fd00:10:128:10::/64"}, + }) + _, err = fakeClientset.KubeClient.CoreV1().Nodes().Update( + context.Background(), node, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Controller should react and update routes + // Port count stays the same + Consistently(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("subnet-update-cnc"), 1) + }).WithTimeout(2 * time.Second).Should(Succeed()) + + // Port details should remain unchanged + Eventually(func() error { + return verifyRouterPort(nbClient, connectPortName, "subnet-update-cnc", connectNets) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Routes should still exist with updated destination prefix + Eventually(func() error { + return verifyRouterStaticRoutes(nbClient, getConnectRouterName("subnet-update-cnc"), + "subnet-update-cnc", 1, 1, + "10.128.10.0/24", "fd00:10:128:10::/64", nexthopV4, nexthopV6) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + + It("should create ports for all nodes when multiple networks are connected", func() { + // Setup with 1 Layer3 and 1 Layer2 network and 2 nodes (1 local, 1 remote) + zoneName = "node1" + l3Net := testNetwork{name: "net1", id: 1, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}} + l2Net := testNetwork{name: "net2", id: 2, topologyType: ovntypes.Layer2Topology, subnets: []string{"10.132.0.0/16", "fd00:10:132::/48"}} + networks := []testNetwork{l3Net, l2Net} + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "net1": {"10.128.1.0/24", "fd00:10:128:1::/64"}, + }}, + {name: "node2", id: 2, zone: "node2", nodeSubnets: map[string]subnetPair{ + "net1": {"10.128.2.0/24", "fd00:10:128:2::/64"}, + }}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{"net1": l3Net, "net2": l2Net}) + + // Create CNC with Layer3 and Layer2 networks + // Network index is based on subnet position within connect subnet range (192.168.0.0/16): + // - 192.168.0.0/24 → index 0 (Layer3) + // - 192.168.1.0/31 → index 1 (Layer2) + subnetAnnotation := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + "layer2_2": {"192.168.1.0/31", "fd00:192:168:1::/127"}, + }) + cnc := createTestCNC("multi-cnc", 1000, defaultConnectSubnets(), subnetAnnotation) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Verify 3 ports on connect router (Layer3: 2 nodes = 2 ports, Layer2: 1 port) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("multi-cnc"), 3) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer3 network router has 1 port (local node only) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, l3Net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer2 network router has 1 port + Eventually(func() error { + return verifyRouterPortsCount(nbClient, l2Net.RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer3 local node port (node1, id=1) - should be local with peer + l3ConnectPort := getConnectRouterToNetworkRouterPortName("multi-cnc", l3Net.name, "node1") + l3NetworkPort := getNetworkRouterToConnectRouterPortName(l3Net.name, "node1", "multi-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, l3ConnectPort, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 1), l3NetworkPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, l3NetworkPort, 0, l3ConnectPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer3 remote node port (node2, id=2) - should be remote with requested-chassis + l3RemoteConnectPort := getConnectRouterToNetworkRouterPortName("multi-cnc", l3Net.name, "node2") + Eventually(func() error { + return verifyRouterPortIsRemote(nbClient, l3RemoteConnectPort, getExpectedPortTunnelKey("192.168.0.0/24", "fd00:192:168::/64", ovntypes.Layer3Topology, 2)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify Layer2 port (single port, no node name) + l2ConnectPort := getConnectRouterToNetworkRouterPortName("multi-cnc", l2Net.name, "") + l2NetworkPort := getNetworkRouterToConnectRouterPortName(l2Net.name, "", "multi-cnc") + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, l2ConnectPort, getExpectedPortTunnelKey("192.168.1.0/31", "fd00:192:168:1::/127", ovntypes.Layer2Topology, 0), l2NetworkPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyRouterPortIsLocal(nbClient, l2NetworkPort, 0, l2ConnectPort) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify static routes + expectedPerIPFamily := 0 + if config.IPv4Mode { + expectedPerIPFamily++ + } + if config.IPv6Mode { + expectedPerIPFamily++ + } + + // Layer3 routes: one per node + for _, node := range nodes { + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc"), + "multi-cnc", l3Net.id, node.id, expectedPerIPFamily) + }).WithTimeout(5*time.Second).Should(Succeed(), "routes for %s node %s (id=%d)", l3Net.name, node.name, node.id) + } + + // Layer2 routes: single route (nodeID=0) + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc"), + "multi-cnc", 2, 0, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify policies on Layer3 network router pointing to Layer2 network + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, l3Net.RouterName(), + "multi-cnc", l3Net.id, l2Net.id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify policies on Layer2 network router pointing to Layer3 network + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, l2Net.RouterName(), + "multi-cnc", l2Net.id, l3Net.id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + }) + + // ============================================================================= + // Context: Multiple CNCs + // ============================================================================= + Context("Multiple CNCs", func() { + + It("should handle multiple CNCs independently", func() { + // Setup: 4 networks (2 per CNC), 2 nodes + // CNC1 connects net1 and net2, CNC2 connects net3 and net4 + zoneName = "node1" + networks := []testNetwork{ + {name: "net1", id: 1, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.128.0.0/14/23", "fd00:10:128::/48/64"}}, + {name: "net2", id: 2, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.132.0.0/14/23", "fd00:10:132::/48/64"}}, + {name: "net3", id: 3, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.136.0.0/14/23", "fd00:10:136::/48/64"}}, + {name: "net4", id: 4, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.140.0.0/14/23", "fd00:10:140::/48/64"}}, + } + nodes := []testNode{ + {name: "node1", id: 1, zone: "node1", nodeSubnets: map[string]subnetPair{ + "net1": {"10.128.1.0/24", "fd00:10:128:1::/64"}, + "net2": {"10.132.1.0/24", "fd00:10:132:1::/64"}, + "net3": {"10.136.1.0/24", "fd00:10:136:1::/64"}, + "net4": {"10.140.1.0/24", "fd00:10:140:1::/64"}, + }}, + {name: "node2", id: 2, zone: "node2", nodeSubnets: map[string]subnetPair{ + "net1": {"10.128.2.0/24", "fd00:10:128:2::/64"}, + "net2": {"10.132.2.0/24", "fd00:10:132:2::/64"}, + "net3": {"10.136.2.0/24", "fd00:10:136:2::/64"}, + "net4": {"10.140.2.0/24", "fd00:10:140:2::/64"}, + }}, + } + initialDB := createInitialDBWithRouters(networks) + start(initialDB, nodes, map[string]testNetwork{ + "net1": networks[0], "net2": networks[1], + "net3": networks[2], "net4": networks[3], + }) + + expectedPerIPFamily := 0 + if config.IPv4Mode { + expectedPerIPFamily++ + } + if config.IPv6Mode { + expectedPerIPFamily++ + } + + // Create first CNC connecting net1 and net2 + subnetAnnotation1 := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_1": {"192.168.0.0/24", "fd00:192:168::/64"}, + "layer3_2": {"192.168.1.0/24", "fd00:192:168:1::/64"}, + }) + cnc1 := createTestCNC("multi-cnc1", 600, defaultConnectSubnets(), subnetAnnotation1) + + _, err := fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc1, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Create second CNC connecting net3 and net4 + subnetAnnotation2 := buildConnectSubnetAnnotation(map[string]subnetPair{ + "layer3_3": {"192.168.2.0/24", "fd00:192:168:2::/64"}, + "layer3_4": {"192.168.3.0/24", "fd00:192:168:3::/64"}, + }) + cnc2 := createTestCNC("multi-cnc2", 700, defaultConnectSubnets(), subnetAnnotation2) + + _, err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Create( + context.Background(), cnc2, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for both connect routers + Eventually(func() error { + return verifyConnectRouter(nbClient, "multi-cnc1", 600) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + Eventually(func() error { + return verifyConnectRouter(nbClient, "multi-cnc2", 700) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify ports on CNC1's connect router + // 2 networks x 2 nodes = 4 ports, but node2 is remote so only 2 local + 2 remote = 4 + // Local node (node1) ports for net1 and net2 + remote node (node2) ports for net1 and net2 + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("multi-cnc1"), 4) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify ports on CNC2's connect router (same: 4 ports) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("multi-cnc2"), 4) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify ports on network routers (only local node1 gets ports) + // net1 and net2 connected to CNC1 + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[1].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // net3 and net4 connected to CNC2 + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[2].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[3].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes on CNC1's connect router + // Routes for net1: node1 (local) and node2 (remote) subnets + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc1"), + "multi-cnc1", 1, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc1"), + "multi-cnc1", 1, 2, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + // Routes for net2 + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc1"), + "multi-cnc1", 2, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc1"), + "multi-cnc1", 2, 2, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routes on CNC2's connect router + // Routes for net3 + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc2"), + "multi-cnc2", 3, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc2"), + "multi-cnc2", 3, 2, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + // Routes for net4 + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc2"), + "multi-cnc2", 4, 1, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterStaticRoutesCount(nbClient, getConnectRouterName("multi-cnc2"), + "multi-cnc2", 4, 2, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify routing policies on network routers + // net1's router has policy for traffic to net2 (srcOwner=layer3_1, dstNetworkID=2) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), "multi-cnc1", networks[0].id, networks[1].id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + // net2's router has policy for traffic to net1 (srcOwner=layer3_2, dstNetworkID=1) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[1].RouterName(), "multi-cnc1", networks[1].id, networks[0].id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + // net3's router has policy for traffic to net4 (srcOwner=layer3_3, dstNetworkID=4) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[2].RouterName(), "multi-cnc2", networks[2].id, networks[3].id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + // net4's router has policy for traffic to net3 (srcOwner=layer3_4, dstNetworkID=3) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[3].RouterName(), "multi-cnc2", networks[3].id, networks[2].id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Delete first CNC + err = fakeClientset.NetworkConnectClient.K8sV1().ClusterNetworkConnects().Delete( + context.Background(), "multi-cnc1", metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for first router to be deleted + Eventually(func() error { + routerName := getConnectRouterName("multi-cnc1") + _, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: routerName}) + if err != nil { + return nil // Deleted + } + return errRouterStillExists + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify CNC1's ports are removed from network routers + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[0].RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[1].RouterName(), 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Verify CNC1's policies are removed from network routers + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[0].RouterName(), "multi-cnc1", networks[0].id, networks[1].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[1].RouterName(), "multi-cnc1", networks[1].id, networks[0].id, 0) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // Second CNC should still be fully functional + Expect(verifyConnectRouter(nbClient, "multi-cnc2", 700)).To(Succeed()) + + // CNC2's ports should still exist + Eventually(func() error { + return verifyRouterPortsCount(nbClient, getConnectRouterName("multi-cnc2"), 4) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[2].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPortsCount(nbClient, networks[3].RouterName(), 1) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // CNC2's policies should still exist + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[2].RouterName(), "multi-cnc2", networks[2].id, networks[3].id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + Eventually(func() error { + return verifyRouterPolicyCount(nbClient, networks[3].RouterName(), "multi-cnc2", networks[3].id, networks[2].id, expectedPerIPFamily) + }).WithTimeout(5 * time.Second).Should(Succeed()) + }) + }) + + }) // end Context for ipMode + } +}) + +// Sentinel error for router existence check +var errRouterStillExists = &routerStillExistsError{} + +type routerStillExistsError struct{} + +func (e *routerStillExistsError) Error() string { + return "router still exists" +} diff --git a/go-controller/pkg/ovn/controller/networkconnect/static_subnet_and_tunnel_key_generator.go b/go-controller/pkg/ovn/controller/networkconnect/static_subnet_and_tunnel_key_generator.go new file mode 100644 index 0000000000..96055f2292 --- /dev/null +++ b/go-controller/pkg/ovn/controller/networkconnect/static_subnet_and_tunnel_key_generator.go @@ -0,0 +1,198 @@ +package networkconnect + +import ( + "fmt" + "math/big" + "net" + + utilnet "k8s.io/utils/net" + + networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/generator/ip" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" +) + +// Helper functions + +// findMatchingConnectSubnet returns the networkPrefix and connectCIDR for the given IP family. +// It finds the matching connect subnet based on whether the subnet is IPv4 or IPv6. +func getNetworkPrefixAndConnectCIDR(connectSubnets []networkconnectv1.ConnectSubnet, subnet *net.IPNet) (networkPrefix int, connectCIDR *net.IPNet, err error) { + isIPv4 := subnet.IP.To4() != nil + + for _, cs := range connectSubnets { + _, cidr, err := net.ParseCIDR(string(cs.CIDR)) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse connect subnet %s: %v", cs.CIDR, err) + } + if (cidr.IP.To4() != nil) == isIPv4 { + return int(cs.NetworkPrefix), cidr, nil + } + } + return 0, nil, fmt.Errorf("no connect subnet found for IP family IPv4=%v", isIPv4) +} + +// getTotalBits returns 32 for IPv4 or 128 for IPv6. +func getTotalBits(ip net.IP) int { + if ip.To4() != nil { + return 32 + } + return 128 +} + +// getSubnetIndexInParent calculates the index of a child subnet within a parent subnet. +// For example, finding the index of a /24 subnet within a /16 block, or a /31 within a /24. +// The index is calculated as: (childIP - parentIP) >> (totalBits - childPrefixLen) +func getSubnetIndexInParent(parentBaseIP net.IP, childSubnet *net.IPNet) int { + parentInt := utilnet.BigForIP(parentBaseIP) + childInt := utilnet.BigForIP(childSubnet.IP) + + offset := new(big.Int).Sub(childInt, parentInt) + + childOnes, totalBits := childSubnet.Mask.Size() + shift := totalBits - childOnes + + if shift > 0 { + offset.Rsh(offset, uint(shift)) + } + + return int(offset.Int64()) +} + +// getNetworkIndexAndMaxNodes calculates the network index and maxNodes from the CNC's connect subnets. +// This is used for deterministic tunnel key allocation per the OKEP. +// +// Algorithm Overview (from OKEP): +// +// 1. Calculate maxNodes: maxNodes = 2^(bits - NetworkPrefix) +// - For IPv4 with NetworkPrefix=24: maxNodes = 2^(32-24) = 256 +// - For IPv6 with NetworkPrefix=96: maxNodes = 2^(128-96) = 4 billion (capped at 5000 as claimed by Kubernetes) +// +// 2. Calculate networkIndex: Based on the subnet's position in the connectSubnet range +// - For a connect CIDR 192.168.0.0/16 with /24 prefix, subnet 192.168.5.0/24 has networkIndex=5 +// +// 3. Tunnel key allocation (done by caller): +// - Layer3 networks: tunnelKey = networkIndex * maxNodes + nodeID + 1 +// - Layer2 networks: tunnelKey = networkIndex * maxNodes + subIndex + 1 +// +// Example with NetworkPrefix=24 (maxNodes=256): +// +// | Network | Subnet | Type | Index | Tunnel Key Range | +// |-----------|----------------|--------|-------|------------------| +// | network1 | 192.168.0.0/24 | Layer3 | 0 | [1, 256] | +// | network2 | 192.168.1.0/24 | Layer3 | 1 | [257, 512] | +// | network40 | 192.168.4.0/31 | Layer2 | 4 | [1025] | +func getNetworkIndexAndMaxNodes(subnet *net.IPNet, networkPrefix int, connectCIDR *net.IPNet) (networkIndex, maxNodes int, err error) { + totalBits := getTotalBits(subnet.IP) + + // Calculate maxNodes based on network prefix + // maxNodes = 2^(TotalBits - networkPrefix) + maxNodes = 1 << (totalBits - networkPrefix) + if maxNodes > 5000 { // limit max as claimed by Kubernetes + maxNodes = 5000 + } + + // Calculate network index from the subnet's position within the connect subnet range + connectOnes, _ := connectCIDR.Mask.Size() + + // Validate configuration (CRD CEL validation should prevent this, but check for defense in depth) + shift := totalBits - networkPrefix + if shift <= 0 || networkPrefix <= connectOnes { + return 0, 0, fmt.Errorf("invalid configuration: networkPrefix (%d) must be greater than connect CIDR prefix (%d) and less than %d", + networkPrefix, connectOnes, totalBits) + } + + // Create a temporary IPNet with /networkPrefix mask to use with getSubnetIndexInParent + networkPrefixSubnet := &net.IPNet{ + IP: subnet.IP, + Mask: net.CIDRMask(networkPrefix, totalBits), + } + networkIndex = getSubnetIndexInParent(connectCIDR.IP, networkPrefixSubnet) + + return networkIndex, maxNodes, nil +} + +// getLayer2SubIndex returns the index for a Layer2 /31 (or /127) subnet within its /networkPrefix block. +func getLayer2SubIndex(subnet *net.IPNet, networkPrefix int) int { + // Calculate the base IP of the /networkPrefix block this /31 belongs to + totalBits := getTotalBits(subnet.IP) + blockBaseIP := subnet.IP.Mask(net.CIDRMask(networkPrefix, totalBits)) + return getSubnetIndexInParent(blockBaseIP, subnet) +} + +// GetTunnelKey calculates the tunnel key for a network based on its topology type. +// For Layer3: tunnelKey = networkIndex * maxNodes + nodeID + 1 +// For Layer2: tunnelKey = networkIndex * maxNodes + subIndex + 1 (where subIndex is derived from subnet) +// The +1 ensures tunnel keys are always > 0 (0 is reserved/invalid). +func GetTunnelKey(connectSubnets []networkconnectv1.ConnectSubnet, allocatedSubnets []*net.IPNet, topologyType string, nodeID int) (int, error) { + if len(allocatedSubnets) == 0 { + return 0, fmt.Errorf("no allocated subnets provided") + } + subnet := allocatedSubnets[0] + + // Get connect subnet info once for use by both calculations. + // In dual-stack, the first subnet is IPv4 (added first during annotation parsing). + // The tunnel key calculation is consistent across IP families because CNC has CEL validation + // ensuring (32 - IPv4NetworkPrefix) == (128 - IPv6NetworkPrefix), so maxNodes is the same. + networkPrefix, connectCIDR, err := getNetworkPrefixAndConnectCIDR(connectSubnets, subnet) + if err != nil { + return 0, err + } + + networkIndex, maxNodes, err := getNetworkIndexAndMaxNodes(subnet, networkPrefix, connectCIDR) + if err != nil { + return 0, fmt.Errorf("failed to get network index and max nodes: %v", err) + } + + if topologyType == ovntypes.Layer2Topology { + subIndex := getLayer2SubIndex(subnet, networkPrefix) + return networkIndex*maxNodes + subIndex + 1, nil + } + // Layer3 + return networkIndex*maxNodes + nodeID + 1, nil +} + +// connectPortPairInfo contains the IP addresses for the connect port and the network port +// and the corresponding nodeID (layer3 and 0 for layer2), tunnelKey +type connectPortPairInfo struct { + connectPortIPs []*net.IPNet + networkPortIPs []*net.IPNet +} + +// GetP2PAddresses calculates /31 (IPv4) or /127 (IPv6) point-to-point addresses for a node. +// It takes the allocated subnets and nodeID, and returns both IPs of the P2P subnet. +// The first IP is typically used for the router side, the second for the network side. +func GetP2PAddresses(subnets []*net.IPNet, nodeID int) (*connectPortPairInfo, error) { + portPairInfo := &connectPortPairInfo{ + connectPortIPs: make([]*net.IPNet, 0), + networkPortIPs: make([]*net.IPNet, 0), + } + for _, subnet := range subnets { + generator, err := ip.NewIPGenerator(subnet.String()) + if err != nil { + return nil, fmt.Errorf("failed to create IP generator: %v", err) + } + // Use GenerateIPPair to get two IPs forming a /31 or /127 subnet + p2pSubnet, _, err := generator.GenerateIPPair(nodeID) + if err != nil { + return nil, fmt.Errorf("failed to generate P2P subnet for node ID %d: %v", nodeID, err) + } + + // First IP of the P2P subnet + firstIPNet := &net.IPNet{ + IP: p2pSubnet.IP, + Mask: p2pSubnet.Mask, + } + portPairInfo.connectPortIPs = append(portPairInfo.connectPortIPs, firstIPNet) + + // Second IP (increment the last byte) + secondIP := make(net.IP, len(p2pSubnet.IP)) + copy(secondIP, p2pSubnet.IP) + secondIP[len(secondIP)-1]++ + secondIPNet := &net.IPNet{ + IP: secondIP, + Mask: p2pSubnet.Mask, + } + portPairInfo.networkPortIPs = append(portPairInfo.networkPortIPs, secondIPNet) + } + return portPairInfo, nil +} diff --git a/go-controller/pkg/ovn/controller/networkconnect/static_subnet_and_tunnel_key_generator_test.go b/go-controller/pkg/ovn/controller/networkconnect/static_subnet_and_tunnel_key_generator_test.go new file mode 100644 index 0000000000..afa7fadce8 --- /dev/null +++ b/go-controller/pkg/ovn/controller/networkconnect/static_subnet_and_tunnel_key_generator_test.go @@ -0,0 +1,568 @@ +package networkconnect + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" +) + +func TestGetLayer2SubIndex(t *testing.T) { + tests := []struct { + name string + subnet *net.IPNet + networkPrefix int + expected int + }{ + { + name: "IPv4 /31 at start of /24 block", + subnet: ovntest.MustParseIPNet("192.168.0.0/31"), + networkPrefix: 24, + expected: 0, + }, + { + name: "IPv4 /31 at offset 2 in /24 block", + subnet: ovntest.MustParseIPNet("192.168.0.2/31"), + networkPrefix: 24, + expected: 1, + }, + { + name: "IPv4 /31 at offset 10 in /24 block", + subnet: ovntest.MustParseIPNet("192.168.0.10/31"), + networkPrefix: 24, + expected: 5, + }, + { + name: "IPv6 /127 at start of /120 block", + subnet: ovntest.MustParseIPNet("fd00::0/127"), + networkPrefix: 120, + expected: 0, + }, + { + name: "IPv6 /127 at offset 4 in /120 block", + subnet: ovntest.MustParseIPNet("fd00::4/127"), + networkPrefix: 120, + expected: 2, + }, + // Test cases for networkPrefix spanning multiple octets + { + name: "IPv4 /31 in second octet of /20 block", + subnet: ovntest.MustParseIPNet("10.0.17.0/31"), // 10.0.16.0/20 block, offset 256 + networkPrefix: 20, + expected: 128, // (256 IPs in 10.0.16.x) / 2 + }, + { + name: "IPv4 /31 at end of /20 block", + subnet: ovntest.MustParseIPNet("10.0.31.254/31"), // 10.0.16.0/20 block, last /31 + networkPrefix: 20, + expected: 2047, // (4096 IPs - 2) / 2 + }, + { + name: "IPv6 /127 within /112 block", + subnet: ovntest.MustParseIPNet("fd00::1:2/127"), // within fd00::1:0/112 block, offset 2 + networkPrefix: 112, + expected: 1, // 2 / 2 = 1 + }, + { + name: "IPv6 /127 spanning octets - large offset", + subnet: ovntest.MustParseIPNet("fd00::100:0/127"), // within fd00::/72 block, offset 0x01000000 + networkPrefix: 72, + expected: 8388608, // 16777216 / 2 + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := getLayer2SubIndex(tt.subnet, tt.networkPrefix) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGetP2PAddresses(t *testing.T) { + tests := []struct { + name string + subnets []*net.IPNet + nodeID int + expectedFirst []string + expectedSecond []string + }{ + { + name: "IPv4 node ID 0", + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + }, + nodeID: 0, + expectedFirst: []string{"192.168.0.0/31"}, + expectedSecond: []string{"192.168.0.1/31"}, + }, + { + name: "IPv4 node ID 1", + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + }, + nodeID: 1, + expectedFirst: []string{"192.168.0.2/31"}, + expectedSecond: []string{"192.168.0.3/31"}, + }, + { + name: "IPv4 node ID 5", + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + }, + nodeID: 5, + expectedFirst: []string{"192.168.0.10/31"}, + expectedSecond: []string{"192.168.0.11/31"}, + }, + { + name: "IPv6 node ID 0", + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("fd00::/64"), + }, + nodeID: 0, + expectedFirst: []string{"fd00::/127"}, + expectedSecond: []string{"fd00::1/127"}, + }, + { + name: "IPv6 node ID 1", + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("fd00::/64"), + }, + nodeID: 1, + expectedFirst: []string{"fd00::2/127"}, + expectedSecond: []string{"fd00::3/127"}, + }, + { + name: "dual stack node ID 0", + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00::/64"), + }, + nodeID: 0, + expectedFirst: []string{"192.168.0.0/31", "fd00::/127"}, + expectedSecond: []string{"192.168.0.1/31", "fd00::1/127"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + portPairInfo, err := GetP2PAddresses(tt.subnets, tt.nodeID) + require.NoError(t, err) + require.Len(t, portPairInfo.connectPortIPs, len(tt.expectedFirst)) + require.Len(t, portPairInfo.networkPortIPs, len(tt.expectedSecond)) + + for i, expected := range tt.expectedFirst { + assert.Equal(t, expected, portPairInfo.connectPortIPs[i].String()) + } + for i, expected := range tt.expectedSecond { + assert.Equal(t, expected, portPairInfo.networkPortIPs[i].String()) + } + }) + } +} + +func TestGetNetworkIndexAndMaxNodes(t *testing.T) { + tests := []struct { + name string + connectSubnets []networkconnectv1.ConnectSubnet + subnet *net.IPNet + expectedNetworkIndex int + expectedMaxNodes int + expectedErr string + }{ + // IPv4 with /20 networkPrefix (4096 IPs per network) + { + name: "IPv4 /12 CIDR with /20 prefix, first network", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "172.16.0.0/12", NetworkPrefix: 20}, + }, + subnet: ovntest.MustParseIPNet("172.16.0.0/20"), + expectedNetworkIndex: 0, + expectedMaxNodes: 4096, // 2^(32-20) = 4096 + }, + { + name: "IPv4 /12 CIDR with /20 prefix, network at 172.20.0.0", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "172.16.0.0/12", NetworkPrefix: 20}, + }, + subnet: ovntest.MustParseIPNet("172.20.0.0/20"), + expectedNetworkIndex: 64, // (172.20.0.0 - 172.16.0.0) / 4096 = 262144 / 4096 = 64 + expectedMaxNodes: 4096, + }, + // IPv4 with /22 networkPrefix (1024 IPs per network) + { + name: "IPv4 /16 CIDR with /22 prefix, network index 7", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.100.0.0/16", NetworkPrefix: 22}, + }, + subnet: ovntest.MustParseIPNet("10.100.28.0/22"), + expectedNetworkIndex: 7, // 28 / 4 = 7 (each /22 spans 4 in third octet) + expectedMaxNodes: 1024, + }, + // IPv4 with /26 networkPrefix (64 IPs per network) + { + name: "IPv4 /20 CIDR with /26 prefix, network spanning octets", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.0.16.0/20", NetworkPrefix: 26}, + }, + subnet: ovntest.MustParseIPNet("10.0.17.128/26"), + expectedNetworkIndex: 6, // offset 384 / 64 = 6 + expectedMaxNodes: 64, + }, + // IPv4 with /28 networkPrefix (16 IPs per network) + { + name: "IPv4 /24 CIDR with /28 prefix, last network", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.100.0/24", NetworkPrefix: 28}, + }, + subnet: ovntest.MustParseIPNet("192.168.100.240/28"), + expectedNetworkIndex: 15, // 240 / 16 = 15 + expectedMaxNodes: 16, + }, + // IPv4 with /18 networkPrefix (16384 IPs, capped at 5000) + { + name: "IPv4 /8 CIDR with /18 prefix, large network", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.0.0.0/8", NetworkPrefix: 18}, + }, + subnet: ovntest.MustParseIPNet("10.4.64.0/18"), + expectedNetworkIndex: 17, // (10.4.64.0 - 10.0.0.0) / 16384 = 279552 / 16384 = 17 + expectedMaxNodes: 5000, + }, + // IPv6 with /116 networkPrefix (4096 IPs per network) + { + name: "IPv6 /108 CIDR with /116 prefix, network index 3", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "fd00:abcd::/108", NetworkPrefix: 116}, + }, + subnet: ovntest.MustParseIPNet("fd00:abcd::3000/116"), + expectedNetworkIndex: 3, // 0x3000 / 0x1000 = 3 + expectedMaxNodes: 4096, + }, + // IPv6 with /124 networkPrefix (16 IPs per network) + { + name: "IPv6 /120 CIDR with /124 prefix, network index 10", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "2001:db8:cafe::/120", NetworkPrefix: 124}, + }, + subnet: ovntest.MustParseIPNet("2001:db8:cafe::a0/124"), + expectedNetworkIndex: 10, // 0xa0 / 0x10 = 10 + expectedMaxNodes: 16, + }, + // IPv6 with /104 networkPrefix (16M IPs, capped at 5000) + { + name: "IPv6 /96 CIDR with /104 prefix, capped maxNodes", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "fd12:3456::/96", NetworkPrefix: 104}, + }, + subnet: ovntest.MustParseIPNet("fd12:3456::500:0/104"), + expectedNetworkIndex: 5, // 0x05000000 >> 24 = 5 + expectedMaxNodes: 5000, + }, + // DualStack with /20 and /116 (matching host bits: 32-20 = 128-116 = 12) + // Note: function only uses the first (IPv4) subnet for calculation + { + name: "DualStack /16+/112 with /20+/116 prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.50.0.0/16", NetworkPrefix: 20}, + {CIDR: "fd00:50::/112", NetworkPrefix: 116}, + }, + subnet: ovntest.MustParseIPNet("10.50.48.0/20"), + expectedNetworkIndex: 3, // IPv4: 48 / 16 = 3 + expectedMaxNodes: 4096, + }, + // DualStack with /22 and /118 (matching host bits: 32-22 = 128-118 = 10) + { + name: "DualStack /14+/110 with /22+/118 prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "172.16.0.0/14", NetworkPrefix: 22}, + {CIDR: "fd00:172::/110", NetworkPrefix: 118}, + }, + subnet: ovntest.MustParseIPNet("172.18.8.0/22"), + expectedNetworkIndex: 130, // IPv4: (172.18.8.0 - 172.16.0.0) / 1024 = 133120 / 1024 = 130 + expectedMaxNodes: 1024, + }, + // Error cases (validation errors - getNetworkPrefixAndConnectCIDR errors are tested via TestGetTunnelKey) + { + name: "error: networkPrefix equal to connect CIDR prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/24", NetworkPrefix: 24}, // same as CIDR prefix - invalid + }, + subnet: ovntest.MustParseIPNet("192.168.0.0/24"), + expectedErr: "invalid configuration: networkPrefix (24) must be greater than connect CIDR prefix (24)", + }, + { + name: "error: networkPrefix smaller than connect CIDR prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/24", NetworkPrefix: 16}, // smaller than CIDR prefix - invalid + }, + subnet: ovntest.MustParseIPNet("192.168.0.0/16"), + expectedErr: "invalid configuration: networkPrefix (16) must be greater than connect CIDR prefix (24)", + }, + // Note: In practice, this case is prevented by CRD CEL validation which enforces networkPrefix < 32 for IPv4. + // This test is for defense-in-depth validation in the code. + { + name: "error: shift is zero (IPv4 networkPrefix equals totalBits)", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/16", NetworkPrefix: 32}, // shift = 32 - 32 = 0 + }, + subnet: ovntest.MustParseIPNet("192.168.0.1/32"), + expectedErr: "invalid configuration: networkPrefix (32) must be greater than connect CIDR prefix (16) and less than 32", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // First get networkPrefix and connectCIDR from connectSubnets + networkPrefix, connectCIDR, err := getNetworkPrefixAndConnectCIDR(tt.connectSubnets, tt.subnet) + if err != nil { + // This shouldn't happen for valid test cases; errors from getNetworkPrefixAndConnectCIDR + // are tested via TestGetTunnelKey + t.Fatalf("unexpected error from getNetworkPrefixAndConnectCIDR: %v", err) + } + + networkIndex, maxNodes, err := getNetworkIndexAndMaxNodes(tt.subnet, networkPrefix, connectCIDR) + if tt.expectedErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedErr) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedNetworkIndex, networkIndex, "networkIndex mismatch") + assert.Equal(t, tt.expectedMaxNodes, maxNodes, "maxNodes mismatch") + } + }) + } +} + +func TestGetTunnelKey(t *testing.T) { + tests := []struct { + name string + connectSubnets []networkconnectv1.ConnectSubnet + allocatedSubnets []*net.IPNet + topologyType string + nodeID int + expectedTunnelKey int + expectedErr string + }{ + // Layer3 with /20 networkPrefix (4096 maxNodes) + { + name: "Layer3 IPv4 /12 CIDR with /20 prefix, network 5, node 100", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "172.16.0.0/12", NetworkPrefix: 20}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("172.16.80.0/20")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 100, + expectedTunnelKey: 5*4096 + 100 + 1, // networkIndex=5 (80/16=5), maxNodes=4096, nodeID=100 -> 20581 + }, + // Layer3 with /22 networkPrefix (1024 maxNodes) + { + name: "Layer3 IPv4 /16 CIDR with /22 prefix, network 12, node 500", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.100.0.0/16", NetworkPrefix: 22}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("10.100.48.0/22")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 500, + expectedTunnelKey: 12*1024 + 500 + 1, // networkIndex=12 (48/4=12), maxNodes=1024 -> 12789 + }, + // Layer3 with /26 networkPrefix (64 maxNodes) + { + name: "Layer3 IPv4 /20 CIDR with /26 prefix, network spanning octets", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.0.16.0/20", NetworkPrefix: 26}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("10.0.19.64/26")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 30, + expectedTunnelKey: 13*64 + 30 + 1, // networkIndex=13 (offset 832/64=13), maxNodes=64 -> 863 + }, + // Layer3 with /28 networkPrefix (16 maxNodes) + { + name: "Layer3 IPv4 /24 CIDR with /28 prefix, small network", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.100.0/24", NetworkPrefix: 28}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("192.168.100.176/28")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 7, + expectedTunnelKey: 11*16 + 7 + 1, // networkIndex=11 (176/16=11), maxNodes=16 -> 184 + }, + // Layer3 IPv6 with /116 networkPrefix (4096 maxNodes) + { + name: "Layer3 IPv6 /108 CIDR with /116 prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "fd00:abcd::/108", NetworkPrefix: 116}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("fd00:abcd::7000/116")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 200, + expectedTunnelKey: 7*4096 + 200 + 1, // networkIndex=7, maxNodes=4096 -> 28873 + }, + // Layer3 IPv6 with /124 networkPrefix (16 maxNodes) + { + name: "Layer3 IPv6 /120 CIDR with /124 prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "2001:db8:cafe::/120", NetworkPrefix: 124}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("2001:db8:cafe::b0/124")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 5, + expectedTunnelKey: 11*16 + 5 + 1, // networkIndex=11 (0xb0/0x10=11), maxNodes=16 -> 182 + }, + // Layer3 DualStack with /20+/116 (matching host bits) + { + name: "Layer3 DualStack /16+/112 with /20+/116 prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.50.0.0/16", NetworkPrefix: 20}, + {CIDR: "fd00:50::/112", NetworkPrefix: 116}, + }, + allocatedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("10.50.64.0/20"), + ovntest.MustParseIPNet("fd00:50::4000/116"), + }, + topologyType: ovntypes.Layer3Topology, + nodeID: 1000, + expectedTunnelKey: 4*4096 + 1000 + 1, // networkIndex=4, maxNodes=4096 -> 17385 + }, + // Layer2 with /20 networkPrefix (4096 maxNodes), /31 spanning octets + { + name: "Layer2 IPv4 /12 CIDR with /20 prefix, /31 in second octet", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "172.16.0.0/12", NetworkPrefix: 20}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("172.16.17.128/31")}, + topologyType: ovntypes.Layer2Topology, + nodeID: 0, + expectedTunnelKey: 1*4096 + 192 + 1, // networkIndex=1 (172.16.16.0/20 block), subIndex=384/2=192 -> 4289 + }, + // Layer2 with /22 networkPrefix (1024 maxNodes) + { + name: "Layer2 IPv4 /16 CIDR with /22 prefix, /31 at high offset", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.100.0.0/16", NetworkPrefix: 22}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("10.100.51.254/31")}, + topologyType: ovntypes.Layer2Topology, + nodeID: 0, + expectedTunnelKey: 12*1024 + 511 + 1, // networkIndex=12 (48-51 = block 12), subIndex=(3*256+254)/2=511 -> 12800 + }, + // Layer2 with /26 networkPrefix (64 maxNodes) + { + name: "Layer2 IPv4 /20 CIDR with /26 prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.0.16.0/20", NetworkPrefix: 26}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("10.0.20.62/31")}, + topologyType: ovntypes.Layer2Topology, + nodeID: 0, + expectedTunnelKey: 16*64 + 31 + 1, // networkIndex=16 (offset 1024/64), subIndex=62/2=31 -> 1056 + }, + // Layer2 IPv6 with /116 networkPrefix (4096 maxNodes) + { + name: "Layer2 IPv6 /108 CIDR with /116 prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "fd00:abcd::/108", NetworkPrefix: 116}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("fd00:abcd::5800/127")}, + topologyType: ovntypes.Layer2Topology, + nodeID: 0, + expectedTunnelKey: 5*4096 + 1024 + 1, // networkIndex=5, subIndex=0x800/2=1024 -> 21505 + }, + // Layer2 DualStack with /20+/116 + { + name: "Layer2 DualStack /16+/112 with /20+/116 prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.50.0.0/16", NetworkPrefix: 20}, + {CIDR: "fd00:50::/112", NetworkPrefix: 116}, + }, + allocatedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("10.50.96.100/31"), + ovntest.MustParseIPNet("fd00:50::6064/127"), + }, + topologyType: ovntypes.Layer2Topology, + nodeID: 0, + expectedTunnelKey: 6*4096 + 50 + 1, // networkIndex=6, subIndex=100/2=50 -> 24627 + }, + // Large maxNodes (capped at 5000) + { + name: "Layer3 with /18 prefix, maxNodes capped", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.0.0.0/8", NetworkPrefix: 18}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("10.4.64.0/18")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 2500, + expectedTunnelKey: 17*5000 + 2500 + 1, // networkIndex=17, maxNodes=5000 (capped) -> 87501 + }, + { + name: "Layer2 with /18 prefix, maxNodes capped, high subIndex", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "10.0.0.0/8", NetworkPrefix: 18}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("10.8.127.254/31")}, + topologyType: ovntypes.Layer2Topology, + nodeID: 0, + expectedTunnelKey: 33*5000 + 8191 + 1, // networkIndex=33 (10.8.64.0/18 block), subIndex=16382/2=8191 -> 173192 + }, + // Error cases + { + name: "error: invalid CIDR", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "invalid", NetworkPrefix: 24}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("192.168.0.0/24")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 1, + expectedErr: "failed to parse connect subnet", + }, + { + name: "error: no matching connect subnet for Layer3 IPv4", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "fd00::/112", NetworkPrefix: 120}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("192.168.0.0/24")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 1, + expectedErr: "no connect subnet found for IP family", + }, + { + name: "error: no matching connect subnet for Layer2 IPv6", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/16", NetworkPrefix: 24}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("fd00::0/127")}, + topologyType: ovntypes.Layer2Topology, + nodeID: 0, + expectedErr: "no connect subnet found for IP family", + }, + { + name: "error: networkPrefix equal to connect CIDR prefix", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/24", NetworkPrefix: 24}, + }, + allocatedSubnets: []*net.IPNet{ovntest.MustParseIPNet("192.168.0.0/24")}, + topologyType: ovntypes.Layer3Topology, + nodeID: 1, + expectedErr: "invalid configuration: networkPrefix", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tunnelKey, err := GetTunnelKey(tt.connectSubnets, tt.allocatedSubnets, tt.topologyType, tt.nodeID) + if tt.expectedErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedErr) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedTunnelKey, tunnelKey, "tunnelKey mismatch") + } + }) + } +} diff --git a/go-controller/pkg/ovn/controller/networkconnect/topology.go b/go-controller/pkg/ovn/controller/networkconnect/topology.go new file mode 100644 index 0000000000..471fc0d93b --- /dev/null +++ b/go-controller/pkg/ovn/controller/networkconnect/topology.go @@ -0,0 +1,972 @@ +package networkconnect + +import ( + "fmt" + "net" + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" + + "github.com/ovn-kubernetes/libovsdb/ovsdb" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" + libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + utilerrors "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" +) + +// getConnectRouterName returns the connect router name for a CNC. +func getConnectRouterName(cncName string) string { + return ovntypes.ConnectRouterPrefix + cncName +} + +// getConnectRouterToNetworkRouterPortName returns the name of the port on the connect router +// that connects to the network router. For Layer3, includes the node name. +func getConnectRouterToNetworkRouterPortName(cncName, networkName, nodeName string) string { + if nodeName == "" { + // Layer2: no per-node ports + return ovntypes.ConnectRouterToRouterPrefix + cncName + "_" + networkName + } + // Layer3: per-node ports + return ovntypes.ConnectRouterToRouterPrefix + cncName + "_" + networkName + "_" + nodeName +} + +// getNetworkRouterToConnectRouterPortName returns the name of the port on the network router +// that connects to the connect router. For Layer3, includes the node name. +func getNetworkRouterToConnectRouterPortName(networkName, nodeName, cncName string) string { + if nodeName == "" { + // Layer2: no per-node ports + return ovntypes.RouterToConnectRouterPrefix + networkName + "_" + cncName + } + // Layer3: per-node ports + return ovntypes.RouterToConnectRouterPrefix + networkName + "_" + nodeName + "_" + cncName +} + +// ensureConnectRouter creates or updates the connect router for a CNC. +func (c *Controller) ensureConnectRouter(cnc *networkconnectv1.ClusterNetworkConnect, tunnelID int) error { + routerName := getConnectRouterName(cnc.Name) + // The default COPP is used for all routers in all networks. + // Since the default COPP is created in SetupMaster() which is + // called before the network connect controller is initialized (run() method), + // we can safely fetch and use the default COPP here. + copp, err := libovsdbops.GetCOPP(c.nbClient, &nbdb.Copp{Name: ovntypes.DefaultCOPPName}) + if err != nil { + return fmt.Errorf("unable to create router control plane protection: %w", err) + } + router := &nbdb.LogicalRouter{ + Name: routerName, + ExternalIDs: map[string]string{ + libovsdbops.ObjectNameKey.String(): cnc.Name, + libovsdbops.OwnerControllerKey.String(): controllerName, + libovsdbops.OwnerTypeKey.String(): libovsdbops.ClusterNetworkConnectOwnerType, + }, + Options: map[string]string{ + // Set the tunnel key for the connect router + "requested-tnl-key": strconv.Itoa(tunnelID), + }, + Copp: &copp.UUID, + } + + // Create or update the router + err = libovsdbops.CreateOrUpdateLogicalRouter(c.nbClient, router, &router.ExternalIDs, &router.Options, &router.Copp) + if err != nil { + return fmt.Errorf("failed to create/update connect router %s for CNC %s: %v", routerName, cnc.Name, err) + } + + klog.V(4).Infof("Ensured connect router %s with tunnel ID %d", routerName, tunnelID) + return nil +} + +// deleteConnectRouter deletes the connect router for a CNC. +func (c *Controller) deleteConnectRouter(cncName string) error { + routerName := getConnectRouterName(cncName) + + router := &nbdb.LogicalRouter{Name: routerName} + err := libovsdbops.DeleteLogicalRouter(c.nbClient, router) + if err != nil { + return fmt.Errorf("failed to delete connect router %s: %v", routerName, err) + } + + klog.V(4).Infof("Deleted connect router %s", routerName) + return nil +} + +// syncNetworkConnections syncs all network connections for a CNC. +// STEP2: Create the patch ports connecting network router's to the connect router +// using IPs from the network subnet CNC annotation. +// STEP3: If PodNetworkConnect is enabled, create the logical router policies on network router's +// to steer traffic to the connect router for other connected networks. +// STEP4: If PodNetworkConnect is enabled, add static routes to connect router towards +// each of the connected networks. +func (c *Controller) syncNetworkConnections(cnc *networkconnectv1.ClusterNetworkConnect, allocatedSubnets map[string][]*net.IPNet) error { + cncName := cnc.Name + cncState, exists := c.cncCache[cncName] + if !exists || cncState == nil { + return fmt.Errorf("CNC %s not found in cache", cncName) + } + + // Get all nodes - the connect-router needs static routes to ALL node subnets + allNodes, err := c.nodeLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("failed to list nodes: %v", err) + } + // Build set of current node IDs for comparison. (used for deleting ports for nodes that no longer exist) + currentNodeIDs := sets.New[string]() + var localNode *corev1.Node + for _, node := range allNodes { + if util.GetNodeZone(node) == c.zone { + // we don't support multiple local nodes per zone for this feature + localNode = node + } + nodeID, err := util.GetNodeID(node) + if err != nil { + continue + } + currentNodeIDs.Insert(strconv.Itoa(nodeID)) + } + + desiredNetworks := sets.New[string]() + for owner := range allocatedSubnets { + desiredNetworks.Insert(owner) + } + networksToDelete := cncState.connectedNetworks.Difference(desiredNetworks) + networksToCreate := desiredNetworks.Difference(cncState.connectedNetworks) + + klog.V(5).Infof("CNC %s: desiredNetworks=%v, connectedNetworks=%v, networksToCreate=%v, networksToDelete=%v", + cncName, desiredNetworks.UnsortedList(), cncState.connectedNetworks.UnsortedList(), + networksToCreate.UnsortedList(), networksToDelete.UnsortedList()) + + var errs []error + + // Ensure ports, routing policies and static routes for ALL desired networks. + // All operations are idempotent (CreateOrUpdate), so we reconcile them on every sync. + // This handles: + // - New networks: creates ports, policies, and static routes + // - New nodes added to existing networks: creates ports and static routes + // - Existing networks needing policies to newly added networks + // - Node annotation changes (new subnets becoming available) + // Each network is transacted separately to keep transaction sizes bounded. + for owner, subnets := range allocatedSubnets { + isNewNetwork := networksToCreate.Has(owner) + _, networkID, err := util.ParseNetworkOwner(owner) + if err != nil { + klog.Warningf("Failed to parse owner key %s: %v", owner, err) + continue + } + + // Find the network info for this owner + netInfo := c.networkManager.GetNetworkByID(networkID) + if netInfo == nil { + klog.V(4).Infof("Network with ID %d not found, skipping", networkID) + continue + } + localActive := localNode != nil && c.networkManager.NodeHasNetwork(localNode.Name, netInfo.GetNetworkName()) + + // Check if the network router exists before trying to create ports on it. + // The network might be registered in the network manager but not yet created in OVN NB. + // If the router doesn't exist, skip this network and retry later. + if localActive { + networkRouterName := netInfo.GetNetworkScopedClusterRouterName() + _, err = libovsdbops.GetLogicalRouter(c.nbClient, &nbdb.LogicalRouter{Name: networkRouterName}) + if err != nil { + klog.V(4).Infof("Network router %s for network %s does not exist yet, will retry: %v", networkRouterName, netInfo.GetNetworkName(), err) + errs = append(errs, fmt.Errorf("network router %s for network %s does not exist yet: %w", networkRouterName, netInfo.GetNetworkName(), err)) + continue + } + } + + klog.V(5).Infof("CNC %s: ensuring ports, policies and routes for network %s (new=%v)", cncName, netInfo.GetNetworkName(), isNewNetwork) + + // Build ops per network to keep transaction sizes bounded + var createOps []ovsdb.Operation + + // Create/update ports connecting the connect router and network router + // Local node: full port pair with peer; Remote nodes: connect-router port only + // This is idempotent - existing ports are unchanged, new node ports are created + createOps, err = c.ensureConnectPortsOps(createOps, cnc, netInfo, subnets, allNodes, localActive) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to ensure connect ports for network %s: %w", cncName, netInfo.GetNetworkName(), err)) + continue + } + + // Ensure routing policies on the network router (local-active networks only) + if localActive { + createOps, err = c.ensureRoutingPoliciesOps(createOps, cncName, netInfo, allocatedSubnets, localNode) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to ensure routing policies for network %s: %w", cncName, netInfo.GetNetworkName(), err)) + continue + } + } + + // Ensure static routes on the connect router. For Layer2, skip when the local network is inactive. + if netInfo.TopologyType() != ovntypes.Layer2Topology || localActive { + createOps, err = c.ensureStaticRoutesOps(createOps, cnc, netInfo, subnets, allNodes) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to ensure static routes for network %s: %w", cncName, netInfo.GetNetworkName(), err)) + continue + } + } + + // Transact per network to keep transaction sizes bounded + if len(createOps) > 0 { + if _, err := libovsdbops.TransactAndCheck(c.nbClient, createOps); err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to execute create operations for network %s: %w", cncName, netInfo.GetNetworkName(), err)) + continue + } + klog.Infof("CNC %s: executed %d create operations for network %s", cncName, len(createOps), netInfo.GetNetworkName()) + } + + // Update cache after successful transact for this network + if isNewNetwork { + cncState.connectedNetworks.Insert(owner) + } + } + + connectRouterName := getConnectRouterName(cncName) + + // Cleanup ports and routes for nodes that no longer exist. - transact separately + var nodeDeleteOps []ovsdb.Operation + nodeDeleteOps, err = libovsdbops.DeleteLogicalRouterPortWithPredicateOps(c.nbClient, nodeDeleteOps, connectRouterName, + func(item *nbdb.LogicalRouterPort) bool { + // Only delete ports owned by this CNC + if item.ExternalIDs[libovsdbops.ObjectNameKey.String()] != cncName { + return false + } + nodeIDStr := item.ExternalIDs[libovsdbops.NodeIDKey.String()] + // nodeID 0 is used for Layer2 networks which don't have per-node ports + if nodeIDStr == "" || nodeIDStr == "0" { + return false + } + // Delete if nodeID is not in current nodes + return !currentNodeIDs.Has(nodeIDStr) + }) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to cleanup ports for deleted nodes: %w", cncName, err)) + } + + // Delete static routes for nodes that no longer exist + nodeDeleteOps, err = libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicateOps(c.nbClient, nodeDeleteOps, connectRouterName, + func(item *nbdb.LogicalRouterStaticRoute) bool { + // Only delete routes owned by this CNC + if item.ExternalIDs[libovsdbops.ObjectNameKey.String()] != cncName { + return false + } + nodeIDStr := item.ExternalIDs[libovsdbops.NodeIDKey.String()] + // nodeID 0 is used for Layer2 networks which don't have per-node routes + if nodeIDStr == "" || nodeIDStr == "0" { + return false + } + // Delete if nodeID is not in current nodes + return !currentNodeIDs.Has(nodeIDStr) + }) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to cleanup routes for deleted nodes: %w", cncName, err)) + } + + if len(nodeDeleteOps) > 0 { + if _, err := libovsdbops.TransactAndCheck(c.nbClient, nodeDeleteOps); err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to execute node cleanup operations: %w", cncName, err)) + } else { + klog.Infof("CNC %s: executed %d node cleanup operations", cncName, len(nodeDeleteOps)) + } + } + + // Cleanup networks that are no longer connected - transact per network + for owner := range networksToDelete { + klog.V(5).Infof("CNC %s: cleaning up network owner=%s", cncName, owner) + _, networkID, err := util.ParseNetworkOwner(owner) + if err != nil { + klog.Warningf("Failed to parse owner key %s: %v", owner, err) + continue + } + + // Find all ports matching this CNC and network ID (across all routers) + // This allows cleanup even if the network has been deleted from the network manager + ports, err := libovsdbops.FindLogicalRouterPortWithPredicate(c.nbClient, func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + item.ExternalIDs[libovsdbops.NetworkIDKey.String()] == strconv.Itoa(networkID) + }) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to find ports for network %s: %w", cncName, owner, err)) + continue + } + + // Collect unique router names from ports + routerNames := sets.New[string]() + for _, port := range ports { + routerName := port.ExternalIDs[libovsdbops.RouterNameKey.String()] + if routerName == "" { + klog.Warningf("Port %s missing router name in ExternalIDs, skipping", port.Name) + continue + } + routerNames.Insert(routerName) + } + + // Build delete ops for this network + var deleteOps []ovsdb.Operation + for routerName := range routerNames { + deleteOps, err = libovsdbops.DeleteLogicalRouterPortWithPredicateOps(c.nbClient, deleteOps, routerName, + func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + item.ExternalIDs[libovsdbops.NetworkIDKey.String()] == strconv.Itoa(networkID) && + item.ExternalIDs[libovsdbops.RouterNameKey.String()] == routerName + }) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to delete network router ports for network %s: %w", cncName, owner, err)) + continue + } + } + + // Find all routing policies owned by this CNC that need to be deleted + // This includes: + // 1. Policies on the disconnected network's router (routing FROM this network TO others) + // 2. Policies on other networks' routers that reference this deleted network (routing TO this network) + allPolicies, err := libovsdbops.FindLogicalRouterPoliciesWithPredicate(c.nbClient, func(item *nbdb.LogicalRouterPolicy) bool { + // Find all policies owned by this CNC + if item.ExternalIDs[libovsdbops.ObjectNameKey.String()] != cncName { + return false + } + // Match policies that either: + // - Are FROM this network (SourceNetworkIDKey == networkID), OR + // - Reference this network as destination (DestinationNetworkIDKey == networkID) + policySourceNetworkID := item.ExternalIDs[libovsdbops.SourceNetworkIDKey.String()] + policyDestinationNetworkID := item.ExternalIDs[libovsdbops.DestinationNetworkIDKey.String()] + return policySourceNetworkID == strconv.Itoa(networkID) || policyDestinationNetworkID == strconv.Itoa(networkID) + }) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to find routing policies for network %s: %w", cncName, owner, err)) + continue + } + + // Group policies by router name and delete them + policiesByRouter := make(map[string][]*nbdb.LogicalRouterPolicy) + for _, policy := range allPolicies { + routerName := policy.ExternalIDs[libovsdbops.RouterNameKey.String()] + if routerName == "" { + klog.Warningf("Policy %s missing router name in ExternalIDs, skipping", policy.UUID) + continue + } + policiesByRouter[routerName] = append(policiesByRouter[routerName], policy) + } + + // Delete policies from each router + for routerName, routerPolicies := range policiesByRouter { + deleteOps, err = libovsdbops.DeleteLogicalRouterPoliciesOps(c.nbClient, deleteOps, routerName, routerPolicies...) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to delete routing policies from router %s for network %s: %w", cncName, routerName, owner, err)) + // Don't continue here - we still want to try other cleanups + } + } + + // Delete static routes from the connect router for this network + // Note: Static routes don't have RouterNameKey in ExternalIDs, but we're deleting from connectRouterName + // so matching by ObjectNameKey and NetworkIDKey is sufficient + deleteOps, err = libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicateOps(c.nbClient, deleteOps, connectRouterName, + func(item *nbdb.LogicalRouterStaticRoute) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + item.ExternalIDs[libovsdbops.NetworkIDKey.String()] == strconv.Itoa(networkID) + }) + if err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to delete static routes for network %s: %w", cncName, owner, err)) + continue + } + + // Transact per network to keep transaction sizes bounded + if len(deleteOps) > 0 { + if _, err := libovsdbops.TransactAndCheck(c.nbClient, deleteOps); err != nil { + errs = append(errs, fmt.Errorf("CNC %s: failed to execute delete operations for network %s: %w", cncName, owner, err)) + continue + } + klog.Infof("CNC %s: executed %d delete operations for network %s", cncName, len(deleteOps), owner) + } + + // Update cache after successful transact for this network + cncState.connectedNetworks.Delete(owner) + } + + return utilerrors.Join(errs...) +} + +// cleanupNetworkConnections removes all network connections for a CNC. +// This is called when a CNC is being deleted. +// 1. First delete network router ports from the network routers for this CNC +// 2. Then delete routing policies on the network routers for this CNC +func (c *Controller) cleanupNetworkConnections(cncName string) error { + var ops []ovsdb.Operation + + // Find all ports owned by this CNC (across all routers and networks) + // This allows cleanup even if networks have been deleted from the network manager + allPorts, err := libovsdbops.FindLogicalRouterPortWithPredicate(c.nbClient, func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName + }) + if err != nil { + return fmt.Errorf("failed to find ports for CNC %s: %w", cncName, err) + } + + // Collect unique router names from ports + routerNames := sets.New[string]() + + for _, port := range allPorts { + routerName := port.ExternalIDs[libovsdbops.RouterNameKey.String()] + if routerName == "" { + klog.Warningf("Port %s missing router name in ExternalIDs, skipping", port.Name) + continue + } + routerNames.Insert(routerName) + } + + // Delete all ports for this CNC + // All deletions happen in a single transaction, so order doesn't matter + // OVN handles peer references gracefully when both ports are deleted atomically + for routerName := range routerNames { + if routerName == getConnectRouterName(cncName) { + // the whole connect router will be deleted when the CNC is deleted + // so no need to delete the ports on the connect router + continue + } + ops, err = libovsdbops.DeleteLogicalRouterPortWithPredicateOps(c.nbClient, ops, routerName, + func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + item.ExternalIDs[libovsdbops.RouterNameKey.String()] == routerName + }) + if err != nil { + return fmt.Errorf("failed to delete router ports for router %s: %w", routerName, err) + } + } + + // Find all routing policies owned by this CNC + allPolicies, err := libovsdbops.FindLogicalRouterPoliciesWithPredicate(c.nbClient, func(item *nbdb.LogicalRouterPolicy) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName + }) + if err != nil { + return fmt.Errorf("failed to find routing policies for CNC %s: %w", cncName, err) + } + + // Group policies by router name and delete them + policiesByRouter := make(map[string][]*nbdb.LogicalRouterPolicy) + for _, policy := range allPolicies { + routerName := policy.ExternalIDs[libovsdbops.RouterNameKey.String()] + if routerName == "" { + klog.Warningf("Policy %s missing router name in ExternalIDs, skipping", policy.UUID) + continue + } + policiesByRouter[routerName] = append(policiesByRouter[routerName], policy) + } + + // Delete policies from each router + for routerName, routerPolicies := range policiesByRouter { + ops, err = libovsdbops.DeleteLogicalRouterPoliciesOps(c.nbClient, ops, routerName, routerPolicies...) + if err != nil { + return fmt.Errorf("failed to delete routing policies from router %s: %w", routerName, err) + } + } + + // Execute all delete operations + if len(ops) > 0 { + if _, err := libovsdbops.TransactAndCheck(c.nbClient, ops); err != nil { + return fmt.Errorf("failed to execute cleanup operations for CNC %s: %w", cncName, err) + } + klog.Infof("CNC %s: executed %d cleanup operations", cncName, len(ops)) + } + + return nil +} + +// ensureConnectPortsOps returns ops to create the ports connecting the connect router and network router. +// For Layer3: +// - Local node: creates full port pair (connect-router ↔ network-router) with peer relationship +// - Remote nodes: creates only the connect-router side port (with tunnel key, no peer) +// +// For Layer2: creates a single port pair (transit router is distributed) +func (c *Controller) ensureConnectPortsOps(ops []ovsdb.Operation, cnc *networkconnectv1.ClusterNetworkConnect, netInfo util.NetInfo, + subnets []*net.IPNet, nodes []*corev1.Node, localActive bool) ([]ovsdb.Operation, error) { + cncName := cnc.Name + networkName := netInfo.GetNetworkName() + connectRouterName := getConnectRouterName(cncName) + networkRouterName := netInfo.GetNetworkScopedClusterRouterName() + networkID := netInfo.GetNetworkID() + + // Validate subnets are allocated for tunnel key calculation + if len(subnets) == 0 { + return nil, fmt.Errorf("no subnets allocated for network %s", networkName) + } + + if netInfo.TopologyType() == ovntypes.Layer3Topology { + // For Layer3 networks, create ports for all nodes + for _, node := range nodes { + nodeID, err := util.GetNodeID(node) + if err != nil { + // node update event will trigger the reconciliation again. + klog.V(4).Infof("Node %s does not have node ID, skipping: %v", node.Name, err) + continue + } + + // Calculate the /31 subnet for this node from the allocated subnet + portPairInfo, err := GetP2PAddresses(subnets, nodeID) + if err != nil { + return nil, fmt.Errorf("failed to calculate P2P IP addresses for node %s: %v", node.Name, err) + } + + // Calculate tunnel key using the unified function + tunnelKey, err := GetTunnelKey(cnc.Spec.ConnectSubnets, subnets, ovntypes.Layer3Topology, nodeID) + if err != nil { + return nil, fmt.Errorf("failed to calculate tunnel key for node %s: %v", node.Name, err) + } + + connectPortName := getConnectRouterToNetworkRouterPortName(cncName, networkName, node.Name) + networkPortName := getNetworkRouterToConnectRouterPortName(networkName, node.Name, cncName) + + isLocalNode := util.GetNodeZone(node) == c.zone + + if isLocalNode { + if !localActive { + ops, err = libovsdbops.DeleteLogicalRouterPortWithPredicateOps(c.nbClient, ops, connectRouterName, + func(item *nbdb.LogicalRouterPort) bool { + return item.Name == connectPortName + }) + if err != nil { + return nil, fmt.Errorf("failed to delete connect router port ops %s: %v", connectPortName, err) + } + ops, err = libovsdbops.DeleteLogicalRouterPortWithPredicateOps(c.nbClient, ops, networkRouterName, + func(item *nbdb.LogicalRouterPort) bool { + return item.Name == networkPortName + }) + if err != nil { + return nil, fmt.Errorf("failed to delete network router port ops %s: %v", networkPortName, err) + } + continue + } + // Local node: create both ports with peer relationship + ops, err = c.createRouterPortOps(ops, connectRouterName, connectPortName, portPairInfo.connectPortIPs, + networkPortName, cncName, networkID, nodeID, tunnelKey, "") + if err != nil { + return nil, fmt.Errorf("failed to create connect router port ops %s: %v", connectPortName, err) + } + ops, err = c.createRouterPortOps(ops, networkRouterName, networkPortName, portPairInfo.networkPortIPs, + connectPortName, cncName, networkID, nodeID, 0, "") + if err != nil { + return nil, fmt.Errorf("failed to create network router port ops %s: %v", networkPortName, err) + } + } else { + // Remote node: create only the connect-router side port with requested-chassis set + // This makes the port type: remote in SB, enabling cross-zone tunneling + chassisID, err := util.ParseNodeChassisIDAnnotation(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + return nil, ovntypes.NewSuppressedError(err) + } + return nil, fmt.Errorf("failed to parse node chassis-id for node %s: %w", node.Name, err) + } + ops, err = c.createRouterPortOps(ops, connectRouterName, connectPortName, portPairInfo.connectPortIPs, + "", cncName, networkID, nodeID, tunnelKey, chassisID) + if err != nil { + return nil, fmt.Errorf("failed to create remote connect router port ops %s: %v", connectPortName, err) + } + // Delete the network router port if it exists (cleanup from when node was local) + ops, err = libovsdbops.DeleteLogicalRouterPortWithPredicateOps(c.nbClient, ops, networkRouterName, + func(item *nbdb.LogicalRouterPort) bool { + return item.Name == networkPortName + }) + if err != nil { + return nil, fmt.Errorf("failed to delete network router port ops %s: %v", networkPortName, err) + } + } + } + } + if netInfo.TopologyType() == ovntypes.Layer2Topology { + // For Layer2 networks, create a single port pair to the transit router + portPairInfo, err := GetP2PAddresses(subnets, 0) + if err != nil { + return nil, fmt.Errorf("failed to calculate P2P IP addresses for Layer2 network %s: %v", networkName, err) + } + + // Calculate tunnel key using the unified function (nodeID=0 for Layer2) + tunnelKey, err := GetTunnelKey(cnc.Spec.ConnectSubnets, subnets, ovntypes.Layer2Topology, 0) + if err != nil { + return nil, fmt.Errorf("failed to calculate tunnel key for Layer2 network %s: %v", networkName, err) + } + + connectPortName := getConnectRouterToNetworkRouterPortName(cncName, networkName, "") + networkPortName := getNetworkRouterToConnectRouterPortName(networkName, "", cncName) + + if !localActive { + ops, err = libovsdbops.DeleteLogicalRouterPortWithPredicateOps(c.nbClient, ops, connectRouterName, + func(item *nbdb.LogicalRouterPort) bool { + return item.Name == connectPortName + }) + if err != nil { + return nil, fmt.Errorf("failed to delete connect router port ops %s: %v", connectPortName, err) + } + ops, err = libovsdbops.DeleteLogicalRouterPortWithPredicateOps(c.nbClient, ops, networkRouterName, + func(item *nbdb.LogicalRouterPort) bool { + return item.Name == networkPortName + }) + if err != nil { + return nil, fmt.Errorf("failed to delete network router port ops %s: %v", networkPortName, err) + } + return ops, nil + } + + // Create the port on the connect router (with peer set) + ops, err = c.createRouterPortOps(ops, connectRouterName, connectPortName, portPairInfo.connectPortIPs, + networkPortName, cncName, networkID, 0, tunnelKey, "") + if err != nil { + return nil, fmt.Errorf("failed to create connect router port ops %s: %v", connectPortName, err) + } + + // Create the peer port on the transit router (with peer set) + ops, err = c.createRouterPortOps(ops, networkRouterName, networkPortName, portPairInfo.networkPortIPs, + connectPortName, cncName, networkID, 0, 0, "") + if err != nil { + return nil, fmt.Errorf("failed to create network router port ops %s: %v", networkPortName, err) + } + } + + return ops, nil +} + +// createRouterPortOps returns ops to create a logical router port with peer and tunnel key set. +// If remoteChassisName is provided, the port is configured as a remote port (type: remote in SB). +func (c *Controller) createRouterPortOps(ops []ovsdb.Operation, routerName, portName string, ipNets []*net.IPNet, peerPortName string, + cncName string, networkID, nodeID, tunnelKey int, remoteChassisName string) ([]ovsdb.Operation, error) { + if len(ipNets) == 0 { + return nil, fmt.Errorf("no IPNets provided for router port %s", portName) + } + + dbIndexes := libovsdbops.NewDbObjectIDs(libovsdbops.LogicalRouterPortClusterNetworkConnect, controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.NodeIDKey: strconv.Itoa(nodeID), + libovsdbops.NetworkIDKey: strconv.Itoa(networkID), + libovsdbops.ObjectNameKey: cncName, + libovsdbops.RouterNameKey: routerName, + }) + + port := &nbdb.LogicalRouterPort{ + Name: portName, + MAC: util.IPAddrToHWAddr(ipNets[0].IP).String(), + Networks: util.IPNetsToStringSlice(ipNets), + ExternalIDs: dbIndexes.GetExternalIDs(), + } + if peerPortName != "" { + port.Peer = &peerPortName + } + + options := map[string]string{} + if tunnelKey != 0 { + options[libovsdbops.RequestedTnlKey] = strconv.Itoa(tunnelKey) + } + if remoteChassisName != "" { + options[libovsdbops.RequestedChassis] = remoteChassisName + } + if len(options) > 0 { + port.Options = options + } + + router := &nbdb.LogicalRouter{Name: routerName} + var err error + ops, err = libovsdbops.CreateOrUpdateLogicalRouterPortOps(c.nbClient, ops, router, port, nil) + if err != nil { + return nil, fmt.Errorf("failed to create port ops %s on router %s: %v", portName, routerName, err) + } + + klog.V(5).Infof("Created/updated router port ops %s on %s with peer %s and tunnel key %d, options %v", portName, routerName, peerPortName, tunnelKey, options) + return ops, nil +} + +// ensureRoutingPoliciesOps returns ops to create routing policies on the network router to steer traffic to connected networks. +// For Layer3: creates policy for the local node only (each zone handles its own node) +// For Layer2: creates a single policy (transit router is distributed) +func (c *Controller) ensureRoutingPoliciesOps(ops []ovsdb.Operation, cncName string, srcNetwork util.NetInfo, + allocatedSubnets map[string][]*net.IPNet, localNode *corev1.Node) ([]ovsdb.Operation, error) { + networkRouterName := srcNetwork.GetNetworkScopedClusterRouterName() + + // Get the source network's subnets to build the inport match + srcSubnets := srcNetwork.Subnets() + if len(srcSubnets) == 0 { + return nil, fmt.Errorf("source network %s has no subnets", srcNetwork.GetNetworkName()) + } + + // Get the source network's connect subnets - these determine the nexthop for routing policies + // The nexthop is the connect-router's port IP that connects to the source network + srcOwnerKey := util.ComputeNetworkOwner(srcNetwork.TopologyType(), srcNetwork.GetNetworkID()) + srcConnectSubnets, found := allocatedSubnets[srcOwnerKey] + if !found || len(srcConnectSubnets) == 0 { + return nil, fmt.Errorf("source network %s connect subnets not found in allocated subnets", srcNetwork.GetNetworkName()) + } + + // Calculate inport and nexthop once - these are constant for the source network + // The nexthop is the connect-router's port that connects to the source network. + // Traffic flow: srcNetwork router -> connect-router (via srcConnectSubnets) -> dstNetwork + var inportName string + var nexthops []net.IP + + if srcNetwork.TopologyType() == ovntypes.Layer3Topology { + // For Layer3, create policy for the local node + // If there's no local node (node moved to different zone), skip policy creation. + // The controller in the node's zone will handle its policies. + if localNode == nil { + klog.Infof("No local node found for zone %s, skipping routing policy "+ + "creation for Layer3 network %s (node moved to different zone)", c.zone, srcNetwork.GetNetworkName()) + return ops, nil + } + nodeID, err := util.GetNodeID(localNode) + if err != nil { + return nil, fmt.Errorf("local node %s does not have node ID: %v", localNode.Name, err) + } + + inportName = srcNetwork.GetNetworkScopedRouterToSwitchPortName(localNode.Name) + + portPairInfo, err := GetP2PAddresses(srcConnectSubnets, nodeID) + if err != nil { + return nil, fmt.Errorf("failed to calculate P2P IP addresses for node %s: %v", localNode.Name, err) + } + nexthops = util.IPNetsToIPs(portPairInfo.connectPortIPs) + } else if srcNetwork.TopologyType() == ovntypes.Layer2Topology { + // For Layer2, create a single policy (nodeName ignored for Layer2 switch) + inportName = srcNetwork.GetNetworkScopedRouterToSwitchPortName("") + // For Layer2, srcConnectSubnets is already a /31 (IPv4) or /127 (IPv6) subnet. + // Using nodeID=0 extracts the first and second IPs from this existing P2P subnet. + // The first IP is the connect router port IP. + portPairInfo, err := GetP2PAddresses(srcConnectSubnets, 0) + if err != nil { + return nil, fmt.Errorf("failed to calculate P2P IP addresses for Layer2 network %s: %v", srcNetwork.GetNetworkName(), err) + } + nexthops = util.IPNetsToIPs(portPairInfo.connectPortIPs) + } + + // For each other connected network, add a routing policy. + // Note: We iterate allocatedSubnets again here (it's also iterated by the caller) because + // this creates the full mesh of policies. The outer loop in syncNetworkConnections selects + // the SOURCE network (where policies are created), while this inner loop finds all + // DESTINATION networks (what the policies route to). This is O(N²) which is intentional + // for a full mesh connectivity between N networks. + // This is typically fine since number of networks that are expected to be connected by a CNC is small, eg. 10. + for owner := range allocatedSubnets { + _, dstNetworkID, err := util.ParseNetworkOwner(owner) + if err != nil { + continue + } + + // Skip if this is the same network + if dstNetworkID == srcNetwork.GetNetworkID() { + continue + } + + // Find destination network info + dstNetwork := c.networkManager.GetNetworkByID(dstNetworkID) + if dstNetwork == nil { + klog.V(4).Infof("Destination network %d not found, skipping policy", dstNetworkID) + continue + } + + // Get destination network's pod subnets + dstPodSubnets := dstNetwork.Subnets() + + // Create policies for each destination subnet + ops, err = c.createRoutingPoliciesOps(ops, dstNetworkID, networkRouterName, inportName, dstPodSubnets, + srcNetwork.GetNetworkID(), nexthops, cncName) + if err != nil { + return nil, err + } + } + klog.V(5).Infof("Created/updated routing policies ops on %s: %s -> %s", networkRouterName, inportName, nexthops) + + return ops, nil +} + +// createRoutingPoliciesOps returns ops to create logical router policies. +func (c *Controller) createRoutingPoliciesOps(ops []ovsdb.Operation, dstNetworkID int, routerName, inportName string, + dstSubnets []config.CIDRNetworkEntry, srcNetworkID int, nexthops []net.IP, cncName string) ([]ovsdb.Operation, error) { + for _, dstSubnet := range dstSubnets { + // Determine IP version and get appropriate nexthop + var nexthop string + for _, nh := range nexthops { + isIPv4Subnet := utilnet.IsIPv4(dstSubnet.CIDR.IP) + isIPv4Nexthop := utilnet.IsIPv4(nh) + if isIPv4Subnet == isIPv4Nexthop { + nexthop = nh.String() + break + } + } + if nexthop == "" { + continue + } + + // Build the match string + ipVersion := "ip4" + ipFamily := "v4" + if utilnet.IsIPv6(dstSubnet.CIDR.IP) { + ipVersion = "ip6" + ipFamily = "v6" + } + match := fmt.Sprintf(`inport == "%s" && %s.dst == %s`, inportName, ipVersion, dstSubnet.CIDR.String()) + + dbIndexes := libovsdbops.NewDbObjectIDs(libovsdbops.LogicalRouterPolicyClusterNetworkConnect, controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.DestinationNetworkIDKey: strconv.Itoa(dstNetworkID), + libovsdbops.SourceNetworkIDKey: strconv.Itoa(srcNetworkID), + libovsdbops.IPFamilyKey: ipFamily, + libovsdbops.ObjectNameKey: cncName, + libovsdbops.RouterNameKey: routerName, + }) + policy := &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.NetworkConnectPolicyPriority, + Match: match, + Action: nbdb.LogicalRouterPolicyActionReroute, + Nexthops: []string{nexthop}, + ExternalIDs: dbIndexes.GetExternalIDs(), + } + + var err error + ops, err = libovsdbops.CreateOrUpdateLogicalRouterPolicyWithPredicateOps(c.nbClient, ops, routerName, policy, + libovsdbops.GetPredicate[*nbdb.LogicalRouterPolicy](dbIndexes, nil)) + if err != nil { + return nil, fmt.Errorf("failed to create routing policy ops on %s: %v", routerName, err) + } + + klog.V(5).Infof("Created/updated routing policy ops on %s: %s -> %s", routerName, match, nexthop) + } + + return ops, nil +} + +// ensureStaticRoutesOps returns ops to create static routes on the connect router for reaching network subnets. +func (c *Controller) ensureStaticRoutesOps(ops []ovsdb.Operation, cnc *networkconnectv1.ClusterNetworkConnect, + netInfo util.NetInfo, subnets []*net.IPNet, nodes []*corev1.Node) ([]ovsdb.Operation, error) { + cncName := cnc.Name + networkName := netInfo.GetNetworkName() + connectRouterName := getConnectRouterName(cncName) + + networkID := netInfo.GetNetworkID() + + // Get the network's pod subnets + podSubnets := netInfo.Subnets() + + if netInfo.TopologyType() == ovntypes.Layer3Topology { + // For Layer3, create routes to each node's subnet slice + for _, node := range nodes { + nodeID, err := util.GetNodeID(node) + if err != nil { + continue + } + + // Get the node's subnet from the network + nodeSubnets, err := c.getNodeSubnet(netInfo, node.Name) + if err != nil { + klog.V(4).Infof("Could not get node subnet for %s on network %s: %v", node.Name, networkName, err) + continue + } + + // Calculate nexthop (second IP of the P2P subnet on network router side) + portPairInfo, err := GetP2PAddresses(subnets, nodeID) + if err != nil { + return nil, fmt.Errorf("failed to calculate P2P IP addresses for node %s and Layer3 network %s: %v", node.Name, networkName, err) + } + nexthops := util.IPNetsToIPs(portPairInfo.networkPortIPs) + + // Create route for this node's subnets + ops, err = c.createStaticRoutesOps(ops, networkID, connectRouterName, nodeSubnets, nexthops, cncName, nodeID) + if err != nil { + return nil, fmt.Errorf("failed to create static route ops for node %s: %v", node.Name, err) + } + } + } + if netInfo.TopologyType() == ovntypes.Layer2Topology { + // For Layer2, create a single route to the network's subnets + portPairInfo, err := GetP2PAddresses(subnets, 0) + if err != nil { + return nil, fmt.Errorf("failed to calculate P2P IP addresses for Layer2 network %s: %v", networkName, err) + } + nexthops := util.IPNetsToIPs(portPairInfo.networkPortIPs) + + var podSubnetIPNets []*net.IPNet + for _, entry := range podSubnets { + podSubnetIPNets = append(podSubnetIPNets, entry.CIDR) + } + + ops, err = c.createStaticRoutesOps(ops, networkID, connectRouterName, podSubnetIPNets, nexthops, cncName, 0) + if err != nil { + return nil, fmt.Errorf("failed to create static route ops for Layer2 network %s: %v", networkName, err) + } + } + + return ops, nil +} + +// createStaticRoutesOps returns ops to create logical router static routes. +func (c *Controller) createStaticRoutesOps(ops []ovsdb.Operation, networkID int, routerName string, dstSubnets []*net.IPNet, + nexthops []net.IP, cncName string, nodeID int) ([]ovsdb.Operation, error) { + for _, dstSubnet := range dstSubnets { + // Find matching nexthop (same IP family) + var nexthop string + isIPv4Subnet := utilnet.IsIPv4(dstSubnet.IP) + for _, nh := range nexthops { + isIPv4Nexthop := utilnet.IsIPv4(nh) + if isIPv4Subnet == isIPv4Nexthop { + nexthop = nh.String() + break + } + } + if nexthop == "" { + continue + } + + ipFamily := "v4" + if !isIPv4Subnet { + ipFamily = "v6" + } + + dbIndexes := libovsdbops.NewDbObjectIDs(libovsdbops.LogicalRouterStaticRouteClusterNetworkConnect, controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.NetworkIDKey: strconv.Itoa(networkID), + libovsdbops.NodeIDKey: strconv.Itoa(nodeID), + libovsdbops.IPFamilyKey: ipFamily, + libovsdbops.ObjectNameKey: cncName, // CNC name + }) + route := &nbdb.LogicalRouterStaticRoute{ + IPPrefix: dstSubnet.String(), + Nexthop: nexthop, + ExternalIDs: dbIndexes.GetExternalIDs(), + } + + var err error + // Don't limit fields to update - when node subnets change, IPPrefix and Nexthop need to be updated too + ops, err = libovsdbops.CreateOrUpdateLogicalRouterStaticRoutesWithPredicateOps(c.nbClient, ops, routerName, route, + libovsdbops.GetPredicate[*nbdb.LogicalRouterStaticRoute](dbIndexes, nil)) + if err != nil { + return nil, fmt.Errorf("failed to create static route ops on %s: %v", routerName, err) + } + + klog.V(5).Infof("Created/updated static route ops on %s: %s via %s", routerName, dstSubnet.String(), nexthop) + } + + return ops, nil +} + +// getNodeSubnet gets the subnet allocated to a specific node for a network. +func (c *Controller) getNodeSubnet(netInfo util.NetInfo, nodeName string) ([]*net.IPNet, error) { + // For Layer3 networks, each node gets a subnet slice + // Get node info to find its allocated subnet + node, err := c.nodeLister.Get(nodeName) + if err != nil { + return nil, fmt.Errorf("failed to get node %s: %v", nodeName, err) + } + + // Parse the subnet for this network + nodeSubnets, err := util.ParseNodeHostSubnetAnnotation(node, netInfo.GetNetworkName()) + if err != nil { + if util.IsAnnotationNotSetError(err) { + // we must continue setting up the next network, the node update event will trigger the reconciliation again. + return nil, nil + } + return nil, fmt.Errorf("failed to parse node subnet for network %s: %v", netInfo.GetNetworkName(), err) + } + return nodeSubnets, nil +} diff --git a/go-controller/pkg/ovn/controller/networkconnect/topology_test.go b/go-controller/pkg/ovn/controller/networkconnect/topology_test.go new file mode 100644 index 0000000000..b182f4099e --- /dev/null +++ b/go-controller/pkg/ovn/controller/networkconnect/topology_test.go @@ -0,0 +1,2164 @@ +package networkconnect + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/mocks/multinetwork" +) + +func chassisIDForNode(nodeName string) string { + return uuid.NewSHA1(uuid.NameSpaceOID, []byte(nodeName)).String() +} + +type testNetworkManager struct { + networkmanager.FakeNetworkManager + nodeHas map[string]bool +} + +func (t *testNetworkManager) NodeHasNetwork(_ string, networkName string) bool { + return t.nodeHas[networkName] +} + +func TestGetConnectRouterName(t *testing.T) { + tests := []struct { + name string + cncName string + expected string + }{ + { + name: "simple cnc name", + cncName: "my-cnc", + expected: "connect_router_my-cnc", + }, + { + name: "cnc name with dashes", + cncName: "my-complex-cnc-name", + expected: "connect_router_my-complex-cnc-name", + }, + { + name: "short cnc name", + cncName: "a", + expected: "connect_router_a", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := getConnectRouterName(tt.cncName) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestEnsureConnectRouter(t *testing.T) { + tests := []struct { + name string + cncName string + tunnelID int + initialDB []libovsdbtest.TestData + expectedRouters []*nbdb.LogicalRouter + }{ + { + name: "create new connect router", + cncName: "test-cnc", + tunnelID: 100, + initialDB: []libovsdbtest.TestData{ + &nbdb.Copp{ + UUID: "copp-uuid", + Name: ovntypes.DefaultCOPPName, + }, + }, + expectedRouters: []*nbdb.LogicalRouter{ + { + Name: "connect_router_test-cnc", + Options: map[string]string{ + "requested-tnl-key": "100", + }, + }, + }, + }, + { + name: "update existing connect router", + cncName: "existing-cnc", + tunnelID: 200, + initialDB: []libovsdbtest.TestData{ + &nbdb.Copp{ + UUID: "copp-uuid", + Name: ovntypes.DefaultCOPPName, + }, + &nbdb.LogicalRouter{ + UUID: "existing-router-uuid", + Name: "connect_router_existing-cnc", + ExternalIDs: map[string]string{ + libovsdbops.OwnerControllerKey.String(): controllerName, + libovsdbops.OwnerTypeKey.String(): string(libovsdbops.ClusterNetworkConnectOwnerType), + libovsdbops.ObjectNameKey.String(): "existing-cnc", + }, + Options: map[string]string{ + "requested-tnl-key": "100", // old tunnel ID + }, + }, + }, + expectedRouters: []*nbdb.LogicalRouter{ + { + Name: "connect_router_existing-cnc", + Options: map[string]string{ + "requested-tnl-key": "200", // updated tunnel ID + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + c := &Controller{ + nbClient: nbClient, + } + + cnc := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.cncName, + }, + } + + err = c.ensureConnectRouter(cnc, tt.tunnelID) + require.NoError(t, err) + + // Get the COPP UUID for verification + copp, err := libovsdbops.GetCOPP(nbClient, &nbdb.Copp{Name: ovntypes.DefaultCOPPName}) + require.NoError(t, err, "expected default COPP to exist") + + // Verify the router was created/updated correctly + for _, expectedRouter := range tt.expectedRouters { + router, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: expectedRouter.Name}) + require.NoError(t, err) + assert.Equal(t, expectedRouter.Name, router.Name) + assert.Equal(t, expectedRouter.Options, router.Options) + // Check that required external IDs are present (there may be additional IDs like k8s.ovn.org/id) + assert.Equal(t, controllerName, router.ExternalIDs[libovsdbops.OwnerControllerKey.String()]) + assert.Equal(t, string(libovsdbops.ClusterNetworkConnectOwnerType), router.ExternalIDs[libovsdbops.OwnerTypeKey.String()]) + assert.Equal(t, tt.cncName, router.ExternalIDs[libovsdbops.ObjectNameKey.String()]) + // Verify COPP is set on the router + require.NotNil(t, router.Copp, "expected COPP to be set on router") + assert.Equal(t, copp.UUID, *router.Copp, "COPP UUID mismatch") + } + }) + } +} + +func TestDeleteConnectRouter(t *testing.T) { + tests := []struct { + name string + cncName string + initialDB []libovsdbtest.TestData + expectRouterGone bool + expectedRemaining []string // router names that should still exist + }{ + { + name: "delete existing connect router", + cncName: "test-cnc", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect_router_test-cnc", + }, + }, + expectRouterGone: true, + expectedRemaining: []string{}, + }, + { + name: "delete non-existent router (should not error)", + cncName: "non-existent", + initialDB: []libovsdbtest.TestData{}, + expectRouterGone: true, + expectedRemaining: []string{}, + }, + { + name: "delete one router, keep others", + cncName: "delete-me", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-1-uuid", + Name: "connect_router_delete-me", + }, + &nbdb.LogicalRouter{ + UUID: "router-2-uuid", + Name: "connect_router_keep-me", + }, + }, + expectRouterGone: true, + expectedRemaining: []string{"connect_router_keep-me"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + c := &Controller{ + nbClient: nbClient, + } + + err = c.deleteConnectRouter(tt.cncName) + require.NoError(t, err) + + // Verify the router was deleted + deletedRouterName := getConnectRouterName(tt.cncName) + _, err = libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: deletedRouterName}) + if tt.expectRouterGone { + require.Error(t, err, "expected router %s to be deleted", deletedRouterName) + } + + // Verify remaining routers + for _, routerName := range tt.expectedRemaining { + _, err = libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: routerName}) + assert.NoError(t, err, "expected router %s to still exist", routerName) + } + }) + } +} + +func TestGetConnectRouterToNetworkRouterPortName(t *testing.T) { + tests := []struct { + name string + cncName string + networkName string + nodeName string + expected string + }{ + { + name: "Layer2 network (no node)", + cncName: "my-cnc", + networkName: "blue-network", + nodeName: "", + expected: ovntypes.ConnectRouterToRouterPrefix + "my-cnc_blue-network", + }, + { + name: "Layer3 network with node", + cncName: "my-cnc", + networkName: "red-network", + nodeName: "node-1", + expected: ovntypes.ConnectRouterToRouterPrefix + "my-cnc_red-network_node-1", + }, + { + name: "Layer3 network with different node", + cncName: "test-cnc", + networkName: "network-a", + nodeName: "worker-node-2", + expected: ovntypes.ConnectRouterToRouterPrefix + "test-cnc_network-a_worker-node-2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := getConnectRouterToNetworkRouterPortName(tt.cncName, tt.networkName, tt.nodeName) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGetNetworkRouterToConnectRouterPortName(t *testing.T) { + tests := []struct { + name string + networkName string + nodeName string + cncName string + expected string + }{ + { + name: "Layer2 network (no node)", + networkName: "blue-network", + nodeName: "", + cncName: "my-cnc", + expected: ovntypes.RouterToConnectRouterPrefix + "blue-network_my-cnc", + }, + { + name: "Layer3 network with node", + networkName: "red-network", + nodeName: "node-1", + cncName: "my-cnc", + expected: ovntypes.RouterToConnectRouterPrefix + "red-network_node-1_my-cnc", + }, + { + name: "Layer3 network with different node", + networkName: "network-a", + nodeName: "worker-node-2", + cncName: "test-cnc", + expected: ovntypes.RouterToConnectRouterPrefix + "network-a_worker-node-2_test-cnc", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := getNetworkRouterToConnectRouterPortName(tt.networkName, tt.nodeName, tt.cncName) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestCreateRouterPortOps(t *testing.T) { + tests := []struct { + name string + routerName string + portName string + ipNets []*net.IPNet + peerPortName string + cncName string + networkID int + nodeID int + tunnelKey int + remoteChassisName string + initialDB []libovsdbtest.TestData + expectError bool + }{ + { + name: "create local port with peer", + routerName: "connect_router_test", + portName: "crtor-test_network_node1", + ipNets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/31"), + }, + peerPortName: "rtocr-network_node1_test", + cncName: "test-cnc", + networkID: 1, + nodeID: 1, + tunnelKey: 100, + remoteChassisName: "", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect_router_test", + }, + }, + expectError: false, + }, + { + name: "create remote port with requested-chassis", + routerName: "connect_router_test", + portName: "crtor-test_network_node2", + ipNets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.2/31"), + }, + peerPortName: "", + cncName: "test-cnc", + networkID: 1, + nodeID: 2, + tunnelKey: 101, + remoteChassisName: chassisIDForNode("node2"), + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect_router_test", + }, + }, + expectError: false, + }, + { + name: "create dual-stack port", + routerName: "connect_router_test", + portName: "crtor-test_network_node1", + ipNets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/31"), + ovntest.MustParseIPNet("fd00::0/127"), + }, + peerPortName: "rtocr-network_node1_test", + cncName: "test-cnc", + networkID: 1, + nodeID: 1, + tunnelKey: 100, + remoteChassisName: "", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect_router_test", + }, + }, + expectError: false, + }, + { + name: "create port on network router (rtocr)", + routerName: "test-network_ovn_cluster_router", + portName: "rtocr-test_network_node1_test-cnc", + ipNets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.1/31"), + }, + peerPortName: "crtor-test_cnc_test_network_node1", + cncName: "test-cnc", + networkID: 1, + nodeID: 1, + tunnelKey: 0, // network router ports don't have tunnel keys + remoteChassisName: "", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "network-router-uuid", + Name: "test-network_ovn_cluster_router", + }, + }, + expectError: false, + }, + { + name: "error when no IPNets provided", + routerName: "connect_router_test", + portName: "test-port", + ipNets: []*net.IPNet{}, + peerPortName: "", + cncName: "test-cnc", + networkID: 1, + nodeID: 1, + tunnelKey: 0, + remoteChassisName: "", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect_router_test", + }, + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + c := &Controller{ + nbClient: nbClient, + } + + ops, err := c.createRouterPortOps(nil, tt.routerName, tt.portName, tt.ipNets, tt.peerPortName, tt.cncName, tt.networkID, tt.nodeID, tt.tunnelKey, tt.remoteChassisName) + + if tt.expectError { + assert.Error(t, err) + return + } + + require.NoError(t, err) + require.NotEmpty(t, ops) + + // Execute ops + _, err = libovsdbops.TransactAndCheck(nbClient, ops) + require.NoError(t, err) + + // Verify the port was created with correct fields + router, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: tt.routerName}) + require.NoError(t, err) + require.NotEmpty(t, router.Ports, "expected router to have ports") + + // Get the port and verify its fields + port, err := libovsdbops.GetLogicalRouterPort(nbClient, &nbdb.LogicalRouterPort{Name: tt.portName}) + require.NoError(t, err) + + // Verify port name + assert.Equal(t, tt.portName, port.Name) + + // Verify ExternalIDs + assert.Equal(t, tt.cncName, port.ExternalIDs[libovsdbops.ObjectNameKey.String()], "ObjectNameKey mismatch") + assert.Equal(t, strconv.Itoa(tt.networkID), port.ExternalIDs[libovsdbops.NetworkIDKey.String()], "NetworkIDKey mismatch") + assert.Equal(t, strconv.Itoa(tt.nodeID), port.ExternalIDs[libovsdbops.NodeIDKey.String()], "NodeIDKey mismatch") + assert.Equal(t, tt.routerName, port.ExternalIDs[libovsdbops.RouterNameKey.String()], "RouterNameKey mismatch") + + // Verify Networks (IP addresses) + expectedNetworks := make([]string, len(tt.ipNets)) + for i, ipNet := range tt.ipNets { + expectedNetworks[i] = ipNet.String() + } + assert.ElementsMatch(t, expectedNetworks, port.Networks, "Networks mismatch") + + // Verify peer port name + if tt.peerPortName != "" { + require.NotNil(t, port.Peer) + assert.Equal(t, tt.peerPortName, *port.Peer, "Peer port name mismatch") + } else { + assert.Nil(t, port.Peer, "Expected no peer port") + } + + // Verify Options (tunnel key, requested-chassis) + if tt.tunnelKey != 0 { + assert.Equal(t, strconv.Itoa(tt.tunnelKey), port.Options[libovsdbops.RequestedTnlKey], "Tunnel key mismatch") + } + if tt.remoteChassisName != "" { + assert.Equal(t, tt.remoteChassisName, port.Options[libovsdbops.RequestedChassis], "Requested chassis mismatch") + } + }) + } +} + +func TestEnsureConnectPortsOps(t *testing.T) { + tests := []struct { + name string + cncName string + zone string // controller's zone (local node name) + connectSubnets []networkconnectv1.ConnectSubnet + networkName string + networkID int + topologyType string + subnets []*net.IPNet + nodes []*corev1.Node + initialDB []libovsdbtest.TestData + expectError bool + expectedConnectPorts []string // port names on connect router + expectedNetworkPorts []string // port names on network router + }{ + { + name: "Layer3 with local node - creates both ports with peer", + cncName: "test-cnc", + zone: "node1", // local node + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/16", NetworkPrefix: 24}, + }, + networkName: "test-network", + networkID: 1, + topologyType: ovntypes.Layer3Topology, + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + }, + nodes: []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + "k8s.ovn.org/node-id": "1", + util.OvnNodeChassisID: chassisIDForNode("node1"), + util.OvnNodeZoneName: "node1", // local zone + }, + }, + }, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_test-network"}, + }, + expectError: false, + expectedConnectPorts: []string{ovntypes.ConnectRouterToRouterPrefix + "test-cnc_test-network_node1"}, + expectedNetworkPorts: []string{ovntypes.RouterToConnectRouterPrefix + "test-network_node1_test-cnc"}, + }, + { + name: "Layer3 with remote node - creates only connect router port with requested-chassis", + cncName: "test-cnc", + zone: "node1", // local node is node1, but we're creating for node2 + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/16", NetworkPrefix: 24}, + }, + networkName: "test-network", + networkID: 1, + topologyType: ovntypes.Layer3Topology, + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + }, + nodes: []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", // remote node + Annotations: map[string]string{ + "k8s.ovn.org/node-id": "2", + util.OvnNodeChassisID: chassisIDForNode("node2"), + util.OvnNodeZoneName: "node2", // different zone + }, + }, + }, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_test-network"}, + }, + expectError: false, + expectedConnectPorts: []string{ovntypes.ConnectRouterToRouterPrefix + "test-cnc_test-network_node2"}, + expectedNetworkPorts: []string{}, // no network router port for remote nodes + }, + { + name: "Layer3 with 2 nodes (1 local + 1 remote) - creates correct ports for each", + cncName: "test-cnc", + zone: "node1", // node1 is local, node2 is remote + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/16", NetworkPrefix: 24}, + }, + networkName: "test-network", + networkID: 1, + topologyType: ovntypes.Layer3Topology, + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + }, + nodes: []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", // local node + Annotations: map[string]string{ + "k8s.ovn.org/node-id": "1", + util.OvnNodeChassisID: chassisIDForNode("node1"), + util.OvnNodeZoneName: "node1", // local zone + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", // remote node + Annotations: map[string]string{ + "k8s.ovn.org/node-id": "2", + util.OvnNodeChassisID: chassisIDForNode("node2"), + util.OvnNodeZoneName: "node2", // different zone + }, + }, + }, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_test-network"}, + }, + expectError: false, + expectedConnectPorts: []string{ + ovntypes.ConnectRouterToRouterPrefix + "test-cnc_test-network_node1", + ovntypes.ConnectRouterToRouterPrefix + "test-cnc_test-network_node2", + }, + expectedNetworkPorts: []string{ + ovntypes.RouterToConnectRouterPrefix + "test-network_node1_test-cnc", // only local node + }, + }, + { + name: "Layer2 - creates port pair on connect and transit router", + cncName: "test-cnc", + zone: "node1", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/16", NetworkPrefix: 24}, + }, + networkName: "test-l2-network", + networkID: 2, + topologyType: ovntypes.Layer2Topology, + subnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/31"), // /31 for L2 + }, + nodes: []*corev1.Node{}, // Layer2 doesn't need nodes + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + &nbdb.LogicalRouter{UUID: "transit-router-uuid", Name: "transit_test-l2-network"}, + }, + expectError: false, + expectedConnectPorts: []string{ovntypes.ConnectRouterToRouterPrefix + "test-cnc_test-l2-network"}, + expectedNetworkPorts: []string{ovntypes.RouterToConnectRouterPrefix + "test-l2-network_test-cnc"}, + }, + { + name: "error when no subnets provided", + cncName: "test-cnc", + zone: "node1", + connectSubnets: []networkconnectv1.ConnectSubnet{ + {CIDR: "192.168.0.0/16", NetworkPrefix: 24}, + }, + networkName: "test-network", + networkID: 1, + topologyType: ovntypes.Layer3Topology, + subnets: []*net.IPNet{}, // empty + nodes: []*corev1.Node{}, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + c := &Controller{ + nbClient: nbClient, + zone: tt.zone, + } + + cnc := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{Name: tt.cncName}, + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + ConnectSubnets: tt.connectSubnets, + }, + } + + // Create a mock NetInfo + netInfo := &mocks.NetInfo{} + netInfo.On("GetNetworkName").Return(tt.networkName) + netInfo.On("GetNetworkID").Return(tt.networkID) + netInfo.On("TopologyType").Return(tt.topologyType) + if tt.topologyType == ovntypes.Layer2Topology { + netInfo.On("GetNetworkScopedClusterRouterName").Return("transit_" + tt.networkName) + } else { + netInfo.On("GetNetworkScopedClusterRouterName").Return("cluster-router_" + tt.networkName) + } + + ops, err := c.ensureConnectPortsOps(nil, cnc, netInfo, tt.subnets, tt.nodes, true) + + if tt.expectError { + assert.Error(t, err) + return + } + require.NoError(t, err) + + // Execute ops + if len(ops) > 0 { + _, err = libovsdbops.TransactAndCheck(nbClient, ops) + require.NoError(t, err) + } + + // Verify connect router ports + connectRouterName := getConnectRouterName(tt.cncName) + connectRouter, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: connectRouterName}) + require.NoError(t, err) + + for _, expectedPortName := range tt.expectedConnectPorts { + port, err := libovsdbops.GetLogicalRouterPort(nbClient, &nbdb.LogicalRouterPort{Name: expectedPortName}) + require.NoError(t, err, "expected connect router port %s to exist", expectedPortName) + assert.Equal(t, tt.cncName, port.ExternalIDs[libovsdbops.ObjectNameKey.String()]) + } + assert.Len(t, connectRouter.Ports, len(tt.expectedConnectPorts), "connect router port count mismatch") + + // Verify network router ports + for _, expectedPortName := range tt.expectedNetworkPorts { + port, err := libovsdbops.GetLogicalRouterPort(nbClient, &nbdb.LogicalRouterPort{Name: expectedPortName}) + require.NoError(t, err, "expected network router port %s to exist", expectedPortName) + assert.Equal(t, tt.cncName, port.ExternalIDs[libovsdbops.ObjectNameKey.String()]) + } + }) + } +} + +func TestCleanupNetworkConnections(t *testing.T) { + tests := []struct { + name string + cncName string + initialDB []libovsdbtest.TestData + expectError bool + expectedConnectPortCount int // connect router ports should remain (not deleted) + expectedNetworkPortCount int // network router ports should be deleted + networkRouterName string // name of network router to check (if any) + }{ + { + name: "cleanup single network connection", + cncName: "test-cnc", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "connect-router-uuid", + Name: "connect_router_test-cnc", + Ports: []string{"port-1-uuid"}, + }, + &nbdb.LogicalRouterPort{ + UUID: "port-1-uuid", + Name: "crtor-test-cnc-net1-node1", + ExternalIDs: map[string]string{ + libovsdbops.ObjectNameKey.String(): "test-cnc", + libovsdbops.NetworkIDKey.String(): "1", + libovsdbops.RouterNameKey.String(): "connect_router_test-cnc", + }, + }, + &nbdb.LogicalRouter{ + UUID: "network-router-uuid", + Name: "network-router-1", + Ports: []string{"port-2-uuid"}, + }, + &nbdb.LogicalRouterPort{ + UUID: "port-2-uuid", + Name: "rtocr-net1-node1-test-cnc", + ExternalIDs: map[string]string{ + libovsdbops.ObjectNameKey.String(): "test-cnc", + libovsdbops.NetworkIDKey.String(): "1", + libovsdbops.RouterNameKey.String(): "network-router-1", + }, + }, + }, + expectError: false, + expectedConnectPortCount: 1, // connect router ports are NOT deleted + expectedNetworkPortCount: 0, // network router ports ARE deleted + networkRouterName: "network-router-1", + }, + { + name: "cleanup remote port (no peer on network router)", + cncName: "test-cnc", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "connect-router-uuid", + Name: "connect_router_test-cnc", + Ports: []string{"port-1-uuid", "port-2-uuid"}, + }, + &nbdb.LogicalRouterPort{ + UUID: "port-1-uuid", + Name: "crtor-test-cnc-net1-node1", + ExternalIDs: map[string]string{ + libovsdbops.ObjectNameKey.String(): "test-cnc", + libovsdbops.NetworkIDKey.String(): "1", + libovsdbops.NodeIDKey.String(): "1", + libovsdbops.RouterNameKey.String(): "connect_router_test-cnc", + }, + // Local port has peer set + Peer: func() *string { s := "rtocr-net1-node1-test-cnc"; return &s }(), + }, + &nbdb.LogicalRouterPort{ + UUID: "port-2-uuid", + Name: "crtor-test-cnc-net1-node2", + ExternalIDs: map[string]string{ + libovsdbops.ObjectNameKey.String(): "test-cnc", + libovsdbops.NetworkIDKey.String(): "1", + libovsdbops.NodeIDKey.String(): "2", + libovsdbops.RouterNameKey.String(): "connect_router_test-cnc", + }, + Options: map[string]string{ + libovsdbops.RequestedChassis: chassisIDForNode("node2"), + }, + // Remote port has no peer + }, + }, + expectError: false, + expectedConnectPortCount: 2, // both local and remote connect router ports are NOT deleted + expectedNetworkPortCount: 0, // no network router ports in this test + }, + { + name: "cleanup with no connected networks", + cncName: "empty-cnc", + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "connect-router-uuid", + Name: "connect_router_empty-cnc", + }, + }, + expectError: false, + expectedConnectPortCount: 0, // no ports to begin with + expectedNetworkPortCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + c := &Controller{ + nbClient: nbClient, + } + + err = c.cleanupNetworkConnections(tt.cncName) + if tt.expectError { + assert.Error(t, err) + return + } + require.NoError(t, err) + + // Verify connect router ports are NOT deleted (they remain because the router will be deleted separately) + connectRouterName := getConnectRouterName(tt.cncName) + connectRouter, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: connectRouterName}) + if err == nil { + assert.Len(t, connectRouter.Ports, tt.expectedConnectPortCount, "connect router ports should remain") + } + + // Verify network router ports ARE deleted + if tt.networkRouterName != "" { + networkRouter, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: tt.networkRouterName}) + if err == nil { + assert.Len(t, networkRouter.Ports, tt.expectedNetworkPortCount, "network router ports should be deleted") + } + } + }) + } +} + +func TestSyncNetworkConnectionsInactiveNetwork(t *testing.T) { + g := gomega.NewWithT(t) + + err := config.PrepareTestConfig() + g.Expect(err).ToNot(gomega.HaveOccurred()) + + fakeClientset := util.GetOVNClientset().GetOVNKubeControllerClientset() + + nodeSubnets := map[string]string{ + "netA": "10.0.0.0/24", + "netB": "10.1.0.0/24", + } + subnetsBytes, err := json.Marshal(nodeSubnets) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Seed a local node with per-network subnets and required annotations. + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + util.OvnNodeZoneName: "zone1", + util.OvnNodeID: "1", + util.OvnNodeChassisID: chassisIDForNode("node1"), + "k8s.ovn.org/node-subnets": string(subnetsBytes), + }, + }, + } + _, err = fakeClientset.KubeClient.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // Start node informer so getNodeSubnet can resolve node annotations. + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClientset) + g.Expect(err).ToNot(gomega.HaveOccurred()) + err = wf.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer wf.Shutdown() + + syncCtx, syncCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer syncCancel() + synced := cache.WaitForCacheSync(syncCtx.Done(), wf.NodeCoreInformer().Informer().HasSynced) + g.Expect(synced).To(gomega.BeTrue(), "informer caches should sync") + + cncName := "cnc1" + // Mock netinfos for two connected networks. + networkA := &mocks.NetInfo{} + networkA.On("GetNetworkName").Return("netA") + networkA.On("GetNetworkID").Return(1) + networkA.On("TopologyType").Return(ovntypes.Layer3Topology) + networkA.On("GetNetworkScopedClusterRouterName").Return("cluster-router_netA") + networkA.On("GetNetworkScopedRouterToSwitchPortName", "node1").Return("rtos-netA-node1") + networkA.On("Subnets").Return([]config.CIDRNetworkEntry{ + {CIDR: ovntest.MustParseIPNet("10.0.0.0/16")}, + }) + + networkB := &mocks.NetInfo{} + networkB.On("GetNetworkName").Return("netB") + networkB.On("GetNetworkID").Return(2) + networkB.On("TopologyType").Return(ovntypes.Layer3Topology) + networkB.On("GetNetworkScopedClusterRouterName").Return("cluster-router_netB") + networkB.On("GetNetworkScopedRouterToSwitchPortName", "node1").Return("rtos-netB-node1") + networkB.On("Subnets").Return([]config.CIDRNetworkEntry{ + {CIDR: ovntest.MustParseIPNet("10.1.0.0/16")}, + }) + + // netA is active locally, netB is inactive initially. + networks := map[string]util.NetInfo{ + "netA": networkA, + "netB": networkB, + } + nm := &testNetworkManager{ + FakeNetworkManager: networkmanager.FakeNetworkManager{ + PrimaryNetworks: networks, + }, + nodeHas: map[string]bool{ + "netA": true, + "netB": false, + }, + } + + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{Name: getConnectRouterName(cncName)}, + &nbdb.LogicalRouter{Name: "cluster-router_netA"}, + &nbdb.LogicalRouter{Name: "cluster-router_netB"}, + }, + }, nil) + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer cleanup.Cleanup() + + // Controller with connect router and both network routers. + c := &Controller{ + nbClient: nbClient, + zone: "zone1", + nodeLister: wf.NodeCoreInformer().Lister(), + networkManager: nm, + cncCache: map[string]*networkConnectState{ + cncName: { + name: cncName, + connectedNetworks: sets.New[string](), + }, + }, + } + + cnc := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{Name: cncName}, + Spec: networkconnectv1.ClusterNetworkConnectSpec{ + ConnectSubnets: []networkconnectv1.ConnectSubnet{ + { + CIDR: "192.168.0.0/16", + NetworkPrefix: 24, + }, + }, + }, + } + allocatedSubnets := map[string][]*net.IPNet{ + util.ComputeNetworkOwner(ovntypes.Layer3Topology, 1): {ovntest.MustParseIPNet("192.168.0.0/24")}, + util.ComputeNetworkOwner(ovntypes.Layer3Topology, 2): {ovntest.MustParseIPNet("192.168.1.0/24")}, + } + + // First sync: netB inactive, so only remote/static programming is expected. + err = c.syncNetworkConnections(cnc, allocatedSubnets) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + policies, err := libovsdbops.FindLogicalRouterPoliciesWithPredicate(nbClient, func(item *nbdb.LogicalRouterPolicy) bool { + return item.ExternalIDs[libovsdbops.SourceNetworkIDKey.String()] == "1" && + item.ExternalIDs[libovsdbops.DestinationNetworkIDKey.String()] == "2" + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(policies).ToNot(gomega.BeEmpty()) + + // Static routes for netB should be programmed on the connect router. + routes, err := libovsdbops.FindLogicalRouterStaticRoutesWithPredicate(nbClient, func(item *nbdb.LogicalRouterStaticRoute) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == cncName && + item.ExternalIDs[libovsdbops.NetworkIDKey.String()] == "2" + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(routes).ToNot(gomega.BeEmpty()) + portPairInfo, err := GetP2PAddresses(allocatedSubnets[util.ComputeNetworkOwner(ovntypes.Layer3Topology, 2)], 1) + g.Expect(err).ToNot(gomega.HaveOccurred()) + expectedNexthops := util.IPNetsToIPs(portPairInfo.networkPortIPs) + g.Expect(expectedNexthops).ToNot(gomega.BeEmpty()) + + foundRoute := false + for _, route := range routes { + if route.IPPrefix == "10.1.0.0/24" { + g.Expect(route.Nexthop).To(gomega.Equal(expectedNexthops[0].String())) + foundRoute = true + break + } + } + g.Expect(foundRoute).To(gomega.BeTrue()) + + // No local router port should be created for inactive netB. + ports, err := libovsdbops.FindLogicalRouterPortWithPredicate(nbClient, func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[libovsdbops.NetworkIDKey.String()] == "2" && + item.ExternalIDs[libovsdbops.RouterNameKey.String()] == "cluster-router_netB" + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(ports).To(gomega.BeEmpty()) + + // Activate netB and re-sync: local router ports and policies should now be created. + nm.nodeHas["netB"] = true + err = c.syncNetworkConnections(cnc, allocatedSubnets) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + ports, err = libovsdbops.FindLogicalRouterPortWithPredicate(nbClient, func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[libovsdbops.NetworkIDKey.String()] == "2" && + item.ExternalIDs[libovsdbops.RouterNameKey.String()] == "cluster-router_netB" + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(ports).ToNot(gomega.BeEmpty()) + + policies, err = libovsdbops.FindLogicalRouterPoliciesWithPredicate(nbClient, func(item *nbdb.LogicalRouterPolicy) bool { + return item.ExternalIDs[libovsdbops.SourceNetworkIDKey.String()] == "2" && + item.ExternalIDs[libovsdbops.DestinationNetworkIDKey.String()] == "1" + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(policies).ToNot(gomega.BeEmpty()) +} + +func TestCreateRoutingPoliciesOps(t *testing.T) { + type expectedPolicy struct { + match string + nexthop string + ipFamily string // "v4" or "v6" + } + + tests := []struct { + name string + dstNetworkID int + routerName string + inportName string + nexthops []net.IP + cncName string + srcNetworkID int + dstSubnets []string // CIDR strings + initialDB []libovsdbtest.TestData + expectError bool + expectedPolicies []expectedPolicy + }{ + { + name: "create single IPv4 routing policy", + dstNetworkID: 2, + routerName: "network-router-1", + inportName: "switch-to-router-port", + nexthops: []net.IP{ + net.ParseIP("192.168.0.0"), + }, + cncName: "test-cnc", + srcNetworkID: 1, + dstSubnets: []string{"10.200.0.0/24"}, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "network-router-1", + }, + }, + expectError: false, + expectedPolicies: []expectedPolicy{ + { + match: `inport == "switch-to-router-port" && ip4.dst == 10.200.0.0/24`, + nexthop: "192.168.0.0", + ipFamily: "v4", + }, + }, + }, + { + name: "create dual-stack routing policies", + dstNetworkID: 2, + routerName: "network-router-1", + inportName: "switch-to-router-port", + nexthops: []net.IP{ + net.ParseIP("192.168.0.0"), + net.ParseIP("fd00::"), + }, + cncName: "test-cnc", + srcNetworkID: 1, + dstSubnets: []string{"10.200.0.0/24", "fd00:10:200::/64"}, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "network-router-1", + }, + }, + expectError: false, + expectedPolicies: []expectedPolicy{ + { + match: `inport == "switch-to-router-port" && ip4.dst == 10.200.0.0/24`, + nexthop: "192.168.0.0", + ipFamily: "v4", + }, + { + match: `inport == "switch-to-router-port" && ip6.dst == fd00:10:200::/64`, + nexthop: "fd00::", + ipFamily: "v6", + }, + }, + }, + { + name: "skip when no matching nexthop for IP family", + dstNetworkID: 2, + routerName: "network-router-1", + inportName: "switch-to-router-port", + nexthops: []net.IP{ + net.ParseIP("fd00::"), // IPv6 only + }, + cncName: "test-cnc", + srcNetworkID: 1, + dstSubnets: []string{"10.200.0.0/24"}, // IPv4 destination + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "network-router-1", + }, + }, + expectError: false, + expectedPolicies: nil, // no policies created due to family mismatch + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + c := &Controller{ + nbClient: nbClient, + } + + // Convert string subnets to config.CIDRNetworkEntry + var dstSubnets []config.CIDRNetworkEntry + for _, s := range tt.dstSubnets { + dstSubnets = append(dstSubnets, config.CIDRNetworkEntry{CIDR: ovntest.MustParseIPNet(s)}) + } + + ops, err := c.createRoutingPoliciesOps(nil, tt.dstNetworkID, tt.routerName, tt.inportName, dstSubnets, + tt.srcNetworkID, tt.nexthops, tt.cncName) + + if tt.expectError { + assert.Error(t, err) + return + } + + require.NoError(t, err) + + if len(tt.expectedPolicies) == 0 { + assert.Empty(t, ops) + return + } + + require.NotEmpty(t, ops) + + // Execute ops + _, err = libovsdbops.TransactAndCheck(nbClient, ops) + require.NoError(t, err) + + // Verify the policies were created + router, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: tt.routerName}) + require.NoError(t, err) + assert.Len(t, router.Policies, len(tt.expectedPolicies)) + + // Fetch all policies attached to the router and verify their fields + policies, err := libovsdbops.FindLogicalRouterPoliciesWithPredicate(nbClient, func(item *nbdb.LogicalRouterPolicy) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == tt.cncName + }) + require.NoError(t, err) + assert.Len(t, policies, len(tt.expectedPolicies)) + + for _, expected := range tt.expectedPolicies { + // Find the policy with matching IP family + var found *nbdb.LogicalRouterPolicy + for _, p := range policies { + if p.ExternalIDs[libovsdbops.IPFamilyKey.String()] == expected.ipFamily { + found = p + break + } + } + require.NotNilf(t, found, "expected policy with IP family %s not found", expected.ipFamily) + + // Verify policy fields + assert.Equal(t, ovntypes.NetworkConnectPolicyPriority, found.Priority, "Priority mismatch for IP family %s", expected.ipFamily) + assert.Equal(t, expected.match, found.Match, "Match mismatch for IP family %s", expected.ipFamily) + assert.Equal(t, nbdb.LogicalRouterPolicyActionReroute, found.Action, "Action mismatch for IP family %s", expected.ipFamily) + assert.Equal(t, []string{expected.nexthop}, found.Nexthops, "Nexthops mismatch for IP family %s", expected.ipFamily) + + // Verify ExternalIDs + assert.Equal(t, strconv.Itoa(tt.dstNetworkID), found.ExternalIDs[libovsdbops.DestinationNetworkIDKey.String()], "DestinationNetworkIDKey mismatch") + assert.Equal(t, strconv.Itoa(tt.srcNetworkID), found.ExternalIDs[libovsdbops.SourceNetworkIDKey.String()], "SourceNetworkIDKey mismatch") + assert.Equal(t, tt.cncName, found.ExternalIDs[libovsdbops.ObjectNameKey.String()], "ObjectNameKey mismatch") + assert.Equal(t, expected.ipFamily, found.ExternalIDs[libovsdbops.IPFamilyKey.String()], "IPFamilyKey mismatch") + } + }) + } +} + +func TestEnsureRoutingPoliciesOps(t *testing.T) { + tests := []struct { + name string + cncName string + zone string + srcNetworkName string + srcNetworkID int + srcTopologyType string + srcSubnets []string // CIDR strings for source network + allocatedSubnets map[string][]*net.IPNet + dstNetworks []struct { // destination networks + name string + id int + topologyType string + subnets []string + } + initialDB []libovsdbtest.TestData + expectError bool + expectedPolicies int + }{ + { + name: "Layer3 source with single destination network", + cncName: "test-cnc", + zone: "node1", + srcNetworkName: "src-network", + srcNetworkID: 1, + srcTopologyType: ovntypes.Layer3Topology, + srcSubnets: []string{"10.128.0.0/24"}, + allocatedSubnets: map[string][]*net.IPNet{ + "layer3_1": {ovntest.MustParseIPNet("192.168.0.0/24")}, // source network's connect subnet (large enough for P2P) + "layer3_2": {ovntest.MustParseIPNet("192.168.1.0/24")}, // destination network's connect subnet + }, + dstNetworks: []struct { + name string + id int + topologyType string + subnets []string + }{ + {name: "dst-network", id: 2, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.129.0.0/24"}}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_src-network"}, + }, + expectError: false, + expectedPolicies: 1, // one policy for dst-network's subnet + }, + { + name: "Layer3 source with multiple destination networks", + cncName: "test-cnc", + zone: "node1", + srcNetworkName: "src-network", + srcNetworkID: 1, + srcTopologyType: ovntypes.Layer3Topology, + srcSubnets: []string{"10.128.0.0/24"}, + allocatedSubnets: map[string][]*net.IPNet{ + "layer3_1": {ovntest.MustParseIPNet("192.168.0.0/24")}, + "layer3_2": {ovntest.MustParseIPNet("192.168.1.0/24")}, + "layer3_3": {ovntest.MustParseIPNet("192.168.2.0/24")}, + }, + dstNetworks: []struct { + name string + id int + topologyType string + subnets []string + }{ + {name: "dst-network-1", id: 2, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.129.0.0/24"}}, + {name: "dst-network-2", id: 3, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.130.0.0/24"}}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_src-network"}, + }, + expectError: false, + expectedPolicies: 2, // one policy for each dst network + }, + { + name: "Layer2 source with destination network", + cncName: "test-cnc", + zone: "node1", + srcNetworkName: "src-l2-network", + srcNetworkID: 1, + srcTopologyType: ovntypes.Layer2Topology, + srcSubnets: []string{"10.128.0.0/24"}, + allocatedSubnets: map[string][]*net.IPNet{ + "layer2_1": {ovntest.MustParseIPNet("192.168.0.0/31")}, // Layer2 doesn't use P2P per node + "layer3_2": {ovntest.MustParseIPNet("192.168.1.0/24")}, + }, + dstNetworks: []struct { + name string + id int + topologyType string + subnets []string + }{ + {name: "dst-network", id: 2, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.129.0.0/24"}}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "transit_src-l2-network"}, + }, + expectError: false, + expectedPolicies: 1, + }, + { + name: "Layer2 only - both source and destination are Layer2", + cncName: "test-cnc", + zone: "node1", + srcNetworkName: "src-l2-network", + srcNetworkID: 1, + srcTopologyType: ovntypes.Layer2Topology, + srcSubnets: []string{"10.128.0.0/24"}, + allocatedSubnets: map[string][]*net.IPNet{ + "layer2_1": {ovntest.MustParseIPNet("192.168.0.0/31")}, + "layer2_2": {ovntest.MustParseIPNet("192.168.0.2/31")}, + }, + dstNetworks: []struct { + name string + id int + topologyType string + subnets []string + }{ + {name: "dst-l2-network", id: 2, topologyType: ovntypes.Layer2Topology, subnets: []string{"10.129.0.0/24"}}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "transit_src-l2-network"}, + }, + expectError: false, + expectedPolicies: 1, + }, + { + name: "Mixed Layer2 and Layer3 networks", + cncName: "test-cnc", + zone: "node1", + srcNetworkName: "src-l3-network", + srcNetworkID: 1, + srcTopologyType: ovntypes.Layer3Topology, + srcSubnets: []string{"10.128.0.0/24"}, + allocatedSubnets: map[string][]*net.IPNet{ + "layer3_1": {ovntest.MustParseIPNet("192.168.0.0/24")}, + "layer2_2": {ovntest.MustParseIPNet("192.168.1.0/31")}, + "layer3_3": {ovntest.MustParseIPNet("192.168.2.0/24")}, + }, + dstNetworks: []struct { + name string + id int + topologyType string + subnets []string + }{ + {name: "dst-l2-network", id: 2, topologyType: ovntypes.Layer2Topology, subnets: []string{"10.129.0.0/24"}}, + {name: "dst-l3-network", id: 3, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.130.0.0/24"}}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_src-l3-network"}, + }, + expectError: false, + expectedPolicies: 2, // one for each dst network + }, + { + name: "IPv6 single-stack allocated subnets", + cncName: "test-cnc", + zone: "node1", + srcNetworkName: "src-network", + srcNetworkID: 1, + srcTopologyType: ovntypes.Layer3Topology, + srcSubnets: []string{"fd00:10:128::/64"}, + allocatedSubnets: map[string][]*net.IPNet{ + "layer3_1": {ovntest.MustParseIPNet("fd00:192:168::/120")}, // IPv6 connect subnet + "layer3_2": {ovntest.MustParseIPNet("fd00:192:169::/120")}, + }, + dstNetworks: []struct { + name string + id int + topologyType string + subnets []string + }{ + {name: "dst-network", id: 2, topologyType: ovntypes.Layer3Topology, subnets: []string{"fd00:10:129::/64"}}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_src-network"}, + }, + expectError: false, + expectedPolicies: 1, + }, + { + name: "Dual-stack allocated subnets", + cncName: "test-cnc", + zone: "node1", + srcNetworkName: "src-network", + srcNetworkID: 1, + srcTopologyType: ovntypes.Layer3Topology, + srcSubnets: []string{"10.128.0.0/24", "fd00:10:128::/64"}, + allocatedSubnets: map[string][]*net.IPNet{ + "layer3_1": {ovntest.MustParseIPNet("192.168.0.0/24"), ovntest.MustParseIPNet("fd00:192:168::/120")}, + "layer3_2": {ovntest.MustParseIPNet("192.168.1.0/24"), ovntest.MustParseIPNet("fd00:192:169::/120")}, + }, + dstNetworks: []struct { + name string + id int + topologyType string + subnets []string + }{ + {name: "dst-network", id: 2, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.129.0.0/24", "fd00:10:129::/64"}}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_src-network"}, + }, + expectError: false, + expectedPolicies: 2, // one for IPv4, one for IPv6 + }, + { + name: "Dual-stack with mixed Layer2 and Layer3", + cncName: "test-cnc", + zone: "node1", + srcNetworkName: "src-l3-network", + srcNetworkID: 1, + srcTopologyType: ovntypes.Layer3Topology, + srcSubnets: []string{"10.128.0.0/24", "fd00:10:128::/64"}, + allocatedSubnets: map[string][]*net.IPNet{ + "layer3_1": {ovntest.MustParseIPNet("192.168.0.0/24"), ovntest.MustParseIPNet("fd00:192:168::/120")}, + "layer2_2": {ovntest.MustParseIPNet("192.168.1.0/31"), ovntest.MustParseIPNet("fd00:192:169::/127")}, + "layer3_3": {ovntest.MustParseIPNet("192.168.2.0/24"), ovntest.MustParseIPNet("fd00:192:170::/120")}, + }, + dstNetworks: []struct { + name string + id int + topologyType string + subnets []string + }{ + {name: "dst-l2-network", id: 2, topologyType: ovntypes.Layer2Topology, subnets: []string{"10.129.0.0/24", "fd00:10:129::/64"}}, + {name: "dst-l3-network", id: 3, topologyType: ovntypes.Layer3Topology, subnets: []string{"10.130.0.0/24", "fd00:10:130::/64"}}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "network-router-uuid", Name: "cluster-router_src-l3-network"}, + }, + expectError: false, + expectedPolicies: 4, // 2 dst networks x 2 IP families + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create fake clientset with multiple nodes + nodes := []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + "k8s.ovn.org/node-id": "1", + util.OvnNodeChassisID: chassisIDForNode("node1"), + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + Annotations: map[string]string{ + "k8s.ovn.org/node-id": "2", + util.OvnNodeChassisID: chassisIDForNode("node2"), + }, + }, + }, + } + fakeClientset := util.GetOVNClientset().GetOVNKubeControllerClientset() + for _, node := range nodes { + _, err := fakeClientset.KubeClient.CoreV1().Nodes().Create( + context.Background(), node, metav1.CreateOptions{}) + require.NoError(t, err) + } + defer func() { + for _, node := range nodes { + err := fakeClientset.KubeClient.CoreV1().Nodes().Delete(context.Background(), node.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + } + }() + + // Create watch factory + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClientset) + require.NoError(t, err) + err = wf.Start() + require.NoError(t, err) + defer wf.Shutdown() + + // Wait for cache sync + syncCtx, syncCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer syncCancel() + synced := cache.WaitForCacheSync(syncCtx.Done(), wf.NodeCoreInformer().Informer().HasSynced) + require.True(t, synced, "informer caches should sync") + + // Create NB test harness + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + // Create mock source network + srcNetwork := &mocks.NetInfo{} + srcNetwork.On("GetNetworkName").Return(tt.srcNetworkName) + srcNetwork.On("GetNetworkID").Return(tt.srcNetworkID) + srcNetwork.On("TopologyType").Return(tt.srcTopologyType) + if tt.srcTopologyType == ovntypes.Layer2Topology { + srcNetwork.On("GetNetworkScopedClusterRouterName").Return("transit_" + tt.srcNetworkName) + srcNetwork.On("GetNetworkScopedRouterToSwitchPortName", "").Return("trtos-" + tt.srcNetworkName) + } else { + srcNetwork.On("GetNetworkScopedClusterRouterName").Return("cluster-router_" + tt.srcNetworkName) + srcNetwork.On("GetNetworkScopedRouterToSwitchPortName", tt.zone).Return("rtos-" + tt.srcNetworkName + "-" + tt.zone) + } + var srcSubnets []config.CIDRNetworkEntry + for _, s := range tt.srcSubnets { + srcSubnets = append(srcSubnets, config.CIDRNetworkEntry{CIDR: ovntest.MustParseIPNet(s)}) + } + srcNetwork.On("Subnets").Return(srcSubnets) + + // Create mock destination networks and FakeNetworkManager + fakeNM := &networkmanager.FakeNetworkManager{ + PrimaryNetworks: make(map[string]util.NetInfo), + } + for _, dst := range tt.dstNetworks { + dstNet := &mocks.NetInfo{} + dstNet.On("GetNetworkName").Return(dst.name) + dstNet.On("GetNetworkID").Return(dst.id) + dstNet.On("TopologyType").Return(dst.topologyType) + var dstSubnets []config.CIDRNetworkEntry + for _, s := range dst.subnets { + dstSubnets = append(dstSubnets, config.CIDRNetworkEntry{CIDR: ovntest.MustParseIPNet(s)}) + } + dstNet.On("Subnets").Return(dstSubnets) + // Add to FakeNetworkManager (key is namespace but we use name for simplicity) + fakeNM.PrimaryNetworks[dst.name] = dstNet + } + + // Create controller + c := &Controller{ + nbClient: nbClient, + zone: tt.zone, + nodeLister: wf.NodeCoreInformer().Lister(), + networkManager: fakeNM, + } + + cnc := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{Name: tt.cncName}, + } + + ops, err := c.ensureRoutingPoliciesOps(nil, cnc.Name, srcNetwork, tt.allocatedSubnets, nodes[0]) + + if tt.expectError { + assert.Error(t, err) + return + } + + require.NoError(t, err) + + if tt.expectedPolicies == 0 { + assert.Empty(t, ops) + return + } + + require.NotEmpty(t, ops) + + // Execute ops + _, err = libovsdbops.TransactAndCheck(nbClient, ops) + require.NoError(t, err) + + // Verify policies were created + policies, err := libovsdbops.FindLogicalRouterPoliciesWithPredicate(nbClient, func(item *nbdb.LogicalRouterPolicy) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == tt.cncName + }) + require.NoError(t, err) + assert.Len(t, policies, tt.expectedPolicies) + + // Verify each policy has expected fields + for _, policy := range policies { + assert.Equal(t, ovntypes.NetworkConnectPolicyPriority, policy.Priority) + assert.Equal(t, nbdb.LogicalRouterPolicyActionReroute, policy.Action) + assert.NotEmpty(t, policy.Match) + assert.NotEmpty(t, policy.Nexthops) + assert.Equal(t, tt.cncName, policy.ExternalIDs[libovsdbops.ObjectNameKey.String()]) + } + }) + } +} + +func TestCreateStaticRoutesOps(t *testing.T) { + type expectedRoute struct { + ipPrefix string + nexthop string + ipFamily string // "v4" or "v6" + } + + tests := []struct { + name string + networkID int + routerName string + dstSubnets []*net.IPNet + nexthops []net.IP + cncName string + nodeID int + initialDB []libovsdbtest.TestData + expectError bool + expectedRoutes []expectedRoute + }{ + { + name: "create IPv4 static route", + networkID: 1, + routerName: "connect-router-test", + dstSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("10.128.0.0/24"), + }, + nexthops: []net.IP{ + net.ParseIP("192.168.0.1"), + }, + cncName: "test-cnc", + nodeID: 1, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect-router-test", + }, + }, + expectError: false, + expectedRoutes: []expectedRoute{ + {ipPrefix: "10.128.0.0/24", nexthop: "192.168.0.1", ipFamily: "v4"}, + }, + }, + { + name: "create dual-stack static routes", + networkID: 1, + routerName: "connect-router-test", + dstSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("10.128.0.0/24"), + ovntest.MustParseIPNet("fd00:10:128::/64"), + }, + nexthops: []net.IP{ + net.ParseIP("192.168.0.1"), + net.ParseIP("fd00::1"), + }, + cncName: "test-cnc", + nodeID: 1, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect-router-test", + }, + }, + expectError: false, + expectedRoutes: []expectedRoute{ + {ipPrefix: "10.128.0.0/24", nexthop: "192.168.0.1", ipFamily: "v4"}, + {ipPrefix: "fd00:10:128::/64", nexthop: "fd00::1", ipFamily: "v6"}, + }, + }, + { + name: "skip when no matching nexthop for IP family", + networkID: 1, + routerName: "connect-router-test", + dstSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("10.128.0.0/24"), + }, + nexthops: []net.IP{ + net.ParseIP("fd00::1"), // IPv6 nexthop for IPv4 destination + }, + cncName: "test-cnc", + nodeID: 1, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect-router-test", + }, + }, + expectError: false, + expectedRoutes: nil, // no routes created due to family mismatch + }, + { + name: "Layer2 route with nodeID 0", + networkID: 1, + routerName: "connect-router-test", + dstSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("10.200.0.0/24"), + }, + nexthops: []net.IP{ + net.ParseIP("192.168.100.1"), + }, + cncName: "test-cnc", + nodeID: 0, // Layer2 networks use nodeID 0 + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "router-uuid", + Name: "connect-router-test", + }, + }, + expectError: false, + expectedRoutes: []expectedRoute{ + {ipPrefix: "10.200.0.0/24", nexthop: "192.168.100.1", ipFamily: "v4"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + c := &Controller{ + nbClient: nbClient, + } + + ops, err := c.createStaticRoutesOps(nil, tt.networkID, tt.routerName, tt.dstSubnets, tt.nexthops, tt.cncName, tt.nodeID) + + if tt.expectError { + assert.Error(t, err) + return + } + + require.NoError(t, err) + + if len(tt.expectedRoutes) == 0 { + assert.Empty(t, ops) + return + } + + require.NotEmpty(t, ops) + + // Execute ops + _, err = libovsdbops.TransactAndCheck(nbClient, ops) + require.NoError(t, err) + + // Verify the routes were created + router, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: tt.routerName}) + require.NoError(t, err) + assert.Len(t, router.StaticRoutes, len(tt.expectedRoutes)) + + routes, err := libovsdbops.FindLogicalRouterStaticRoutesWithPredicate(nbClient, func(item *nbdb.LogicalRouterStaticRoute) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == tt.cncName + }) + require.NoError(t, err) + assert.Len(t, routes, len(tt.expectedRoutes)) + + for _, expected := range tt.expectedRoutes { + // Find the route with matching IP family + var found *nbdb.LogicalRouterStaticRoute + for _, r := range routes { + if r.ExternalIDs[libovsdbops.IPFamilyKey.String()] == expected.ipFamily { + found = r + break + } + } + require.NotNilf(t, found, "expected route with IP family %s not found", expected.ipFamily) + + // Verify route fields + assert.Equal(t, expected.ipPrefix, found.IPPrefix, "IPPrefix mismatch for IP family %s", expected.ipFamily) + assert.Equal(t, expected.nexthop, found.Nexthop, "Nexthop mismatch for IP family %s", expected.ipFamily) + + // Verify ExternalIDs + assert.Equal(t, tt.cncName, found.ExternalIDs[libovsdbops.ObjectNameKey.String()], "ObjectNameKey mismatch") + assert.Equal(t, strconv.Itoa(tt.networkID), found.ExternalIDs[libovsdbops.NetworkIDKey.String()], "NetworkIDKey mismatch") + assert.Equal(t, strconv.Itoa(tt.nodeID), found.ExternalIDs[libovsdbops.NodeIDKey.String()], "NodeIDKey mismatch") + assert.Equal(t, expected.ipFamily, found.ExternalIDs[libovsdbops.IPFamilyKey.String()], "IPFamilyKey mismatch") + } + }) + } +} + +func TestEnsureStaticRoutesOps(t *testing.T) { + tests := []struct { + name string + cncName string + zone string + networkName string + networkID int + topologyType string + podSubnets []string // network's pod subnets + connectSubnets []*net.IPNet + nodes []struct { + name string + id string + nodeSubnets map[string]string // networkName -> subnet annotation + } + initialDB []libovsdbtest.TestData + expectError bool + expectedRoutes int + }{ + { + name: "Layer3 with single node", + cncName: "test-cnc", + zone: "node1", + networkName: "test-network", + networkID: 1, + topologyType: ovntypes.Layer3Topology, + podSubnets: []string{"10.128.0.0/16"}, + connectSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + }, + nodes: []struct { + name string + id string + nodeSubnets map[string]string + }{ + { + name: "node1", + id: "1", + nodeSubnets: map[string]string{ + "test-network": "10.128.1.0/24", + }, + }, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + }, + expectError: false, + expectedRoutes: 1, + }, + { + name: "Layer3 with multiple nodes", + cncName: "test-cnc", + zone: "node1", + networkName: "test-network", + networkID: 1, + topologyType: ovntypes.Layer3Topology, + podSubnets: []string{"10.128.0.0/16"}, + connectSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + }, + nodes: []struct { + name string + id string + nodeSubnets map[string]string + }{ + { + name: "node1", + id: "1", + nodeSubnets: map[string]string{ + "test-network": "10.128.1.0/24", + }, + }, + { + name: "node2", + id: "2", + nodeSubnets: map[string]string{ + "test-network": "10.128.2.0/24", + }, + }, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + }, + expectError: false, + expectedRoutes: 2, // one route per node + }, + { + name: "Layer2 creates single route", + cncName: "test-cnc", + zone: "node1", + networkName: "test-l2-network", + networkID: 1, + topologyType: ovntypes.Layer2Topology, + podSubnets: []string{"10.200.0.0/24"}, + connectSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/31"), + }, + nodes: []struct { + name string + id string + nodeSubnets map[string]string + }{ + {name: "node1", id: "1", nodeSubnets: nil}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + }, + expectError: false, + expectedRoutes: 1, // single route for L2 + }, + { + name: "Dual-stack Layer3 with single node", + cncName: "test-cnc", + zone: "node1", + networkName: "test-network", + networkID: 1, + topologyType: ovntypes.Layer3Topology, + podSubnets: []string{"10.128.0.0/16", "fd00:10:128::/48"}, + connectSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/24"), + ovntest.MustParseIPNet("fd00:192:168::/120"), + }, + nodes: []struct { + name string + id string + nodeSubnets map[string]string + }{ + { + name: "node1", + id: "1", + nodeSubnets: map[string]string{ + // Dual-stack format: JSON array + "test-network": `["10.128.1.0/24", "fd00:10:128:1::/64"]`, + }, + }, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + }, + expectError: false, + expectedRoutes: 2, // one IPv4 + one IPv6 route + }, + { + name: "Dual-stack Layer2", + cncName: "test-cnc", + zone: "node1", + networkName: "test-l2-network", + networkID: 1, + topologyType: ovntypes.Layer2Topology, + podSubnets: []string{"10.200.0.0/24", "fd00:10:200::/64"}, + connectSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("192.168.0.0/31"), + ovntest.MustParseIPNet("fd00:192:168::/127"), + }, + nodes: []struct { + name string + id string + nodeSubnets map[string]string + }{ + {name: "node1", id: "1", nodeSubnets: nil}, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + }, + expectError: false, + expectedRoutes: 2, // one IPv4 + one IPv6 route for L2 + }, + { + name: "IPv6 single-stack Layer3", + cncName: "test-cnc", + zone: "node1", + networkName: "test-network", + networkID: 1, + topologyType: ovntypes.Layer3Topology, + podSubnets: []string{"fd00:10:128::/48"}, + connectSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("fd00:192:168::/120"), + }, + nodes: []struct { + name string + id string + nodeSubnets map[string]string + }{ + { + name: "node1", + id: "1", + nodeSubnets: map[string]string{ + "test-network": "fd00:10:128:1::/64", + }, + }, + }, + initialDB: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{UUID: "connect-router-uuid", Name: "connect_router_test-cnc"}, + }, + expectError: false, + expectedRoutes: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create fake clientset with nodes + fakeClientset := util.GetOVNClientset().GetOVNKubeControllerClientset() + var createdNodes []*corev1.Node + for _, n := range tt.nodes { + annotations := map[string]string{ + "k8s.ovn.org/node-id": n.id, + } + // Add node subnet annotations + for netName, subnet := range n.nodeSubnets { + // If subnet is already a JSON array (starts with [), use it directly + // Otherwise, wrap it as a single subnet string + if len(subnet) > 0 && subnet[0] == '[' { + annotations["k8s.ovn.org/node-subnets"] = fmt.Sprintf(`{"%s":%s}`, netName, subnet) + } else { + annotations["k8s.ovn.org/node-subnets"] = fmt.Sprintf(`{"%s":"%s"}`, netName, subnet) + } + } + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: n.name, + Annotations: annotations, + }, + } + _, err := fakeClientset.KubeClient.CoreV1().Nodes().Create( + context.Background(), node, metav1.CreateOptions{}) + require.NoError(t, err) + createdNodes = append(createdNodes, node) + } + defer func() { + for _, node := range createdNodes { + err := fakeClientset.KubeClient.CoreV1().Nodes().Delete(context.Background(), node.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + } + }() + + // Create watch factory + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClientset) + require.NoError(t, err) + err = wf.Start() + require.NoError(t, err) + defer wf.Shutdown() + + // Wait for cache sync + syncCtx, syncCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer syncCancel() + synced := cache.WaitForCacheSync(syncCtx.Done(), wf.NodeCoreInformer().Informer().HasSynced) + require.True(t, synced, "informer caches should sync") + + // Create NB test harness + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: tt.initialDB, + }, nil) + require.NoError(t, err) + defer cleanup.Cleanup() + + // Create mock network + netInfo := &mocks.NetInfo{} + netInfo.On("GetNetworkName").Return(tt.networkName) + netInfo.On("GetNetworkID").Return(tt.networkID) + netInfo.On("TopologyType").Return(tt.topologyType) + var podSubnets []config.CIDRNetworkEntry + for _, s := range tt.podSubnets { + podSubnets = append(podSubnets, config.CIDRNetworkEntry{CIDR: ovntest.MustParseIPNet(s)}) + } + netInfo.On("Subnets").Return(podSubnets) + + // Create controller + c := &Controller{ + nbClient: nbClient, + zone: tt.zone, + nodeLister: wf.NodeCoreInformer().Lister(), + } + + cnc := &networkconnectv1.ClusterNetworkConnect{ + ObjectMeta: metav1.ObjectMeta{Name: tt.cncName}, + } + + // Get nodes for the function call + var nodes []*corev1.Node + for _, n := range tt.nodes { + node, err := wf.NodeCoreInformer().Lister().Get(n.name) + require.NoError(t, err) + nodes = append(nodes, node) + } + + ops, err := c.ensureStaticRoutesOps(nil, cnc, netInfo, tt.connectSubnets, nodes) + + if tt.expectError { + assert.Error(t, err) + return + } + + require.NoError(t, err) + + if tt.expectedRoutes == 0 { + assert.Empty(t, ops) + return + } + + require.NotEmpty(t, ops) + + // Execute ops + _, err = libovsdbops.TransactAndCheck(nbClient, ops) + require.NoError(t, err) + + // Verify routes were created + router, err := libovsdbops.GetLogicalRouter(nbClient, &nbdb.LogicalRouter{Name: getConnectRouterName(tt.cncName)}) + require.NoError(t, err) + assert.Len(t, router.StaticRoutes, tt.expectedRoutes) + + routes, err := libovsdbops.FindLogicalRouterStaticRoutesWithPredicate(nbClient, func(item *nbdb.LogicalRouterStaticRoute) bool { + return item.ExternalIDs[libovsdbops.ObjectNameKey.String()] == tt.cncName + }) + require.NoError(t, err) + assert.Len(t, routes, tt.expectedRoutes) + + expectedNodeIDs := map[string]struct{}{} + if tt.topologyType == ovntypes.Layer2Topology { + expectedNodeIDs["0"] = struct{}{} + } else { + for _, node := range tt.nodes { + expectedNodeIDs[node.id] = struct{}{} + } + } + + for _, route := range routes { + _, ok := expectedNodeIDs[route.ExternalIDs[libovsdbops.NodeIDKey.String()]] + assert.True(t, ok, "unexpected NodeID %s", route.ExternalIDs[libovsdbops.NodeIDKey.String()]) + assert.Equal(t, tt.cncName, route.ExternalIDs[libovsdbops.ObjectNameKey.String()], "ObjectNameKey mismatch") + assert.Equal(t, strconv.Itoa(tt.networkID), route.ExternalIDs[libovsdbops.NetworkIDKey.String()], "NetworkIDKey mismatch") + expectedIPFamily := "v4" + if strings.Contains(route.IPPrefix, ":") { + expectedIPFamily = "v6" + } + assert.Equal(t, expectedIPFamily, route.ExternalIDs[libovsdbops.IPFamilyKey.String()], "IPFamilyKey mismatch") + } + }) + } +} diff --git a/go-controller/pkg/ovn/copp.go b/go-controller/pkg/ovn/copp.go index 39f2f092d4..8c96cbd9fa 100644 --- a/go-controller/pkg/ovn/copp.go +++ b/go-controller/pkg/ovn/copp.go @@ -21,9 +21,6 @@ const ( OVNRejectRateLimiter = "reject" OVNTCPRSTRateLimiter = "tcp-reset" OVNServiceMonitorLimiter = "svc-monitor" - - // Default COPP object name - defaultCOPPName = "ovnkube-default" ) var defaultProtocolNames = [...]string{ @@ -83,7 +80,7 @@ func EnsureDefaultCOPP(nbClient libovsdbclient.Client) (string, error) { } defaultCOPP := &nbdb.Copp{ - Name: defaultCOPPName, + Name: types.DefaultCOPPName, Meters: meterNames, } ops, err = libovsdbops.CreateOrUpdateCOPPsOps(nbClient, ops, defaultCOPP) diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index 455d2d1b21..5e850fef14 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -29,6 +29,7 @@ import ( apbroutecontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/apbroute" efcontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/egressfirewall" egresssvc "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/egressservice" + networkconnectcontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/networkconnect" svccontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/unidling" dnsnameresolver "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/dns_name_resolver" @@ -98,6 +99,9 @@ type DefaultNetworkController struct { // Controller used for programming OVN for Admin Network Policy anpController *anpcontroller.Controller + // Controller used for programming OVN for Network Connect + networkConnectController *networkconnectcontroller.Controller + // Controller used to handle the admin policy based external route resources apbExternalRouteController *apbroutecontroller.ExternalGatewayMasterController @@ -342,6 +346,9 @@ func (oc *DefaultNetworkController) Stop() { if oc.routeImportManager != nil { oc.routeImportManager.ForgetNetwork(oc.GetNetworkName()) } + if oc.networkConnectController != nil { + oc.networkConnectController.Stop() + } close(oc.stopChan) oc.cancelableCtx.Cancel() @@ -564,6 +571,16 @@ func (oc *DefaultNetworkController) run(_ context.Context) error { }() } + if util.IsNetworkConnectEnabled() { + err := oc.newNetworkConnectController() + if err != nil { + return fmt.Errorf("unable to create network connect controller, err: %w", err) + } + if err := oc.networkConnectController.Start(); err != nil { + return fmt.Errorf("unable to start network connect controller, err: %w", err) + } + } + end := time.Since(start) klog.Infof("Completing all the Watchers took %v", end) metrics.MetricOVNKubeControllerSyncDuration.WithLabelValues("all watchers").Set(end.Seconds()) @@ -747,14 +764,14 @@ func (h *defaultNetworkControllerEventHandler) AddResource(obj interface{}, from case factory.PodType: pod, ok := obj.(*corev1.Pod) if !ok { - return fmt.Errorf("could not cast %T object to *knet.Pod", obj) + return fmt.Errorf("could not cast %T object to *corev1.Pod", obj) } return h.oc.ensurePod(nil, pod, true) case factory.NodeType: node, ok := obj.(*corev1.Node) if !ok { - return fmt.Errorf("could not cast %T object to *kapi.Node", obj) + return fmt.Errorf("could not cast %T object to *corev1.Node", obj) } if config.HybridOverlay.Enabled { if util.NoHostSubnet(node) { @@ -878,7 +895,7 @@ func (h *defaultNetworkControllerEventHandler) AddResource(obj interface{}, from case factory.NamespaceType: ns, ok := obj.(*corev1.Namespace) if !ok { - return fmt.Errorf("could not cast %T object to *kapi.Namespace", obj) + return fmt.Errorf("could not cast %T object to *corev1.Namespace", obj) } return h.oc.AddNamespace(ns) @@ -902,11 +919,11 @@ func (h *defaultNetworkControllerEventHandler) UpdateResource(oldObj, newObj int case factory.NodeType: newNode, ok := newObj.(*corev1.Node) if !ok { - return fmt.Errorf("could not cast newObj of type %T to *kapi.Node", newObj) + return fmt.Errorf("could not cast newObj of type %T to *corev1.Node", newObj) } oldNode, ok := oldObj.(*corev1.Node) if !ok { - return fmt.Errorf("could not cast oldObj of type %T to *kapi.Node", oldObj) + return fmt.Errorf("could not cast oldObj of type %T to *corev1.Node", oldObj) } var switchToOvnNode bool if config.HybridOverlay.Enabled { @@ -1108,7 +1125,7 @@ func (h *defaultNetworkControllerEventHandler) DeleteResource(obj, cachedObj int case factory.NodeType: node, ok := obj.(*corev1.Node) if !ok { - return fmt.Errorf("could not cast obj of type %T to *knet.Node", obj) + return fmt.Errorf("could not cast obj of type %T to *corev1.Node", obj) } return h.oc.deleteNodeEvent(node) diff --git a/go-controller/pkg/ovn/egressfirewall_test.go b/go-controller/pkg/ovn/egressfirewall_test.go index 955fcaed82..79874f372d 100644 --- a/go-controller/pkg/ovn/egressfirewall_test.go +++ b/go-controller/pkg/ovn/egressfirewall_test.go @@ -861,6 +861,8 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }, }) + egressFirewall.ResourceVersion = "1" + egressFirewall1.ResourceVersion = "2" startOvn(dbSetup, []corev1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}, true) @@ -1098,6 +1100,8 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }, }) + egressFirewall.ResourceVersion = "1" + egressFirewall1.ResourceVersion = "2" startOvn(dbSetup, []corev1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}, true) diff --git a/go-controller/pkg/ovn/egressgw_test.go b/go-controller/pkg/ovn/egressgw_test.go index 9b6f4810eb..1e93c4fa6d 100644 --- a/go-controller/pkg/ovn/egressgw_test.go +++ b/go-controller/pkg/ovn/egressgw_test.go @@ -136,7 +136,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -172,7 +172,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -276,7 +276,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -312,7 +312,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -420,7 +420,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -466,7 +466,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -898,7 +898,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -969,7 +969,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1079,7 +1079,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1119,7 +1119,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1240,7 +1240,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1280,7 +1280,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1411,7 +1411,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1451,7 +1451,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1592,7 +1592,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1632,7 +1632,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1665,7 +1665,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1711,7 +1711,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1745,7 +1745,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1785,7 +1785,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1939,7 +1939,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1979,7 +1979,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2013,7 +2013,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2053,7 +2053,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2086,7 +2086,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2132,7 +2132,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2165,7 +2165,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2211,7 +2211,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2245,7 +2245,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2285,7 +2285,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2330,7 +2330,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2370,7 +2370,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - "requested-chassis": "node1", + "requested-chassis": chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2484,7 +2484,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2618,7 +2618,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2759,7 +2759,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2951,7 +2951,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { }, Name: "namespace1_myPod", Options: map[string]string{ - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), "iface-id-ver": "myPod", }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, @@ -3133,7 +3133,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { }, Name: "namespace1_myPod", Options: map[string]string{ - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), "iface-id-ver": "myPod", }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, @@ -3704,7 +3704,7 @@ func injectNode(fakeOvn *FakeOVN) { ObjectMeta: metav1.ObjectMeta{ Name: "node1", Annotations: map[string]string{"k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.2/24", "next-hop":"169.254.33.1"}}`, - "k8s.ovn.org/node-chassis-id": "79fdcfc4-6fe6-4cd3-8242-c0f85a4668ec", + "k8s.ovn.org/node-chassis-id": chassisIDForNode("node1"), "k8s.ovn.org/node-subnets": `{"default":"10.128.1.0/24"}`, }, }, diff --git a/go-controller/pkg/ovn/egressip.go b/go-controller/pkg/ovn/egressip.go index 3823804558..41f2e9a6af 100644 --- a/go-controller/pkg/ovn/egressip.go +++ b/go-controller/pkg/ovn/egressip.go @@ -790,20 +790,16 @@ func (e *EgressIPController) addPodEgressIPAssignments(ni util.NetInfo, name str return nil } var remainingAssignments, staleAssignments []egressipv1.EgressIPStatusItem - nadName := ni.GetNetworkName() - if ni.IsUserDefinedNetwork() { - nadNames := ni.GetNADs() - if len(nadNames) == 0 { - return fmt.Errorf("expected at least one NAD name for Namespace %s", pod.Namespace) - } - nadName = nadNames[0] // there should only be one active network + nadKey, err := e.getPodNADKeyForNetwork(ni, pod) + if err != nil { + return err } - podIPNets, err := e.getPodIPs(ni, pod, nadName) + podIPNets, err := e.getPodIPs(ni, pod, nadKey) if err != nil { return fmt.Errorf("failed to get pod %s/%s IPs: %v", pod.Namespace, pod.Name, err) } if len(podIPNets) == 0 { - return fmt.Errorf("failed to get pod ips for pod %s on network %s with NAD name %s", podKey, ni.GetNetworkName(), nadName) + return fmt.Errorf("failed to get pod ips for pod %s on network %s with NAD key %s", podKey, ni.GetNetworkName(), nadKey) } podIPs := make([]net.IP, 0, len(podIPNets)) for _, ipNet := range podIPNets { @@ -2071,15 +2067,6 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { egressLocalPods: map[string]sets.Set[string]{}, egressRemotePods: map[string]sets.Set[string]{}, } - nadName := types.DefaultNetworkName - if ni.IsUserDefinedNetwork() { - nadNames := ni.GetNADs() - if len(nadNames) == 0 { - klog.Errorf("Network %s: error build egress IP sync cache, expected at least one NAD name for Namespace %s", ni.GetNetworkName(), namespace.Name) - continue - } - nadName = nadNames[0] // there should only be one active network - } for _, pod := range pods { if !util.PodNeedsSNAT(pod) { continue @@ -2087,7 +2074,12 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { if egressLocalNodesCache.Len() == 0 && !e.isPodScheduledinLocalZone(pod) { continue // don't process anything on master's that have nothing to do with the pod } - podIPs, err := e.getPodIPs(ni, pod, nadName) + nadKey, err := e.getPodNADKeyForNetwork(ni, pod) + if err != nil { + klog.Errorf("Network %s: failed to resolve NAD for pod %s/%s: %v", ni.GetNetworkName(), pod.Namespace, pod.Name, err) + continue + } + podIPs, err := e.getPodIPs(ni, pod, nadKey) if err != nil { klog.Errorf("Network %s: error build egress IP sync cache, error while trying to get pod %s/%s IPs: %v", ni.GetNetworkName(), pod.Namespace, pod.Name, err) continue @@ -2410,17 +2402,13 @@ func (e *EgressIPController) addStandByEgressIPAssignment(ni util.NetInfo, podKe return nil } // get IPs - nadName := ni.GetNetworkName() - if ni.IsUserDefinedNetwork() { - nadNames := ni.GetNADs() - if len(nadNames) == 0 { - return fmt.Errorf("expected at least one NAD name for Namespace %s", pod.Namespace) - } - nadName = nadNames[0] // there should only be one active network + nadKey, err := e.getPodNADKeyForNetwork(ni, pod) + if err != nil { + return err } - podIPNets, err := e.getPodIPs(ni, pod, nadName) + podIPNets, err := e.getPodIPs(ni, pod, nadKey) if err != nil { - return fmt.Errorf("failed to get pod %s/%s IPs using nad name %q: %v", pod.Namespace, pod.Name, nadName, err) + return fmt.Errorf("failed to get pod %s/%s IPs using NAD key %q: %v", pod.Namespace, pod.Name, nadKey, err) } if len(podIPNets) == 0 { return fmt.Errorf("no IP(s) available for pod %s/%s on network %s", pod.Namespace, pod.Name, ni.GetNetworkName()) @@ -2698,7 +2686,7 @@ func (e *EgressIPController) addExternalGWPodSNATOps(ni util.NetInfo, ops []ovsd if err != nil { return nil, err } - podIPs, err := util.GetPodCIDRsWithFullMask(pod, &util.DefaultNetInfo{}) + podIPs, err := util.GetPodCIDRsWithFullMask(pod, &util.DefaultNetInfo{}, nil) if err != nil { return nil, err } @@ -3722,7 +3710,24 @@ func ensureDefaultNoRerouteNodePolicies(nbClient libovsdbclient.Client, addressS return nil } -func (e *EgressIPController) getPodIPs(ni util.NetInfo, pod *corev1.Pod, nadName string) ([]*net.IPNet, error) { +func (e *EgressIPController) getPodNADKeyForNetwork(ni util.NetInfo, pod *corev1.Pod) (string, error) { + if !ni.IsUserDefinedNetwork() { + return ni.GetNetworkName(), nil + } + nadKeys, err := util.PodNADKeys(pod, ni, e.networkManager.GetNetworkNameForNADKey) + if err != nil { + return "", err + } + if len(nadKeys) == 0 { + return "", fmt.Errorf("expected at least one NAD key for pod %s/%s on network %s", pod.Namespace, pod.Name, ni.GetNetworkName()) + } + if len(nadKeys) > 1 { + return "", fmt.Errorf("expected one NAD key for pod %s/%s on network %s, got %d", pod.Namespace, pod.Name, ni.GetNetworkName(), len(nadKeys)) + } + return nadKeys[0], nil +} + +func (e *EgressIPController) getPodIPs(ni util.NetInfo, pod *corev1.Pod, nadKey string) ([]*net.IPNet, error) { podIPs := make([]*net.IPNet, 0) getIPFromIPNetFn := func(podIPNets []*net.IPNet) []*net.IPNet { ipNetsCopy := make([]*net.IPNet, 0, len(podIPNets)) @@ -3739,7 +3744,7 @@ func (e *EgressIPController) getPodIPs(ni util.NetInfo, pod *corev1.Pod, nadName // addLogicalPort has finished successfully setting up networking for // the pod, so we can proceed with retrieving its IP and deleting the // external GW configuration created in addLogicalPort for the pod. - logicalPort, err := e.logicalPortCache.get(pod, nadName) + logicalPort, err := e.logicalPortCache.get(pod, nadKey) if err != nil { return nil, nil } @@ -3756,7 +3761,7 @@ func (e *EgressIPController) getPodIPs(ni util.NetInfo, pod *corev1.Pod, nadName podIPs = getIPFromIPNetFn(logicalPort.ips) } else { // means this is egress node's local master if ni.IsDefault() { - podIPNets, err := util.GetPodCIDRsWithFullMask(pod, ni) + podIPNets, err := util.GetPodCIDRsWithFullMask(pod, ni, nil) if err != nil { return nil, fmt.Errorf("failed to get pod %s/%s IP: %v", pod.Namespace, pod.Name, err) } @@ -3765,7 +3770,7 @@ func (e *EgressIPController) getPodIPs(ni util.NetInfo, pod *corev1.Pod, nadName } podIPs = getIPFromIPNetFn(podIPNets) } else if ni.IsUserDefinedNetwork() { - podIPNets := util.GetPodCIDRsWithFullMaskOfNetwork(pod, nadName) + podIPNets := util.GetPodCIDRsWithFullMaskOfNetwork(pod, nadKey) if len(podIPNets) == 0 { return nil, fmt.Errorf("failed to get pod %s/%s IPs", pod.Namespace, pod.Name) } diff --git a/go-controller/pkg/ovn/egressip_test.go b/go-controller/pkg/ovn/egressip_test.go index b618aa4bec..bd4402eea0 100644 --- a/go-controller/pkg/ovn/egressip_test.go +++ b/go-controller/pkg/ovn/egressip_test.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/urfave/cli/v2" @@ -7042,7 +7043,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" "namespace": egressPod1.Namespace, }, Options: map[string]string{ - libovsdbops.RequestedChassis: egressPod1.Spec.NodeName, + libovsdbops.RequestedChassis: node1.Annotations[util.OvnNodeChassisID], "iface-id-ver": egressPod1.Name, }, PortSecurity: []string{podAddr}, @@ -7395,7 +7396,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" "namespace": egressPod1.Namespace, }, Options: map[string]string{ - libovsdbops.RequestedChassis: egressPod1.Spec.NodeName, + libovsdbops.RequestedChassis: node1.Annotations[util.OvnNodeChassisID], "iface-id-ver": egressPod1.Name, }, PortSecurity: []string{podAddr}, @@ -7678,7 +7679,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Expect(err).NotTo(gomega.HaveOccurred()) ePod, err := fakeOvn.fakeClient.KubeClient.CoreV1().Pods(egressPod1.Namespace).Get(context.TODO(), egressPod1.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}) + egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) egressNetPodIP, _, err := net.ParseCIDR(egressPodPortInfo.ips[0].String()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -7816,7 +7817,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" "namespace": egressPod1.Namespace, }, Options: map[string]string{ - libovsdbops.RequestedChassis: egressPod1.Spec.NodeName, + libovsdbops.RequestedChassis: node1.Annotations[util.OvnNodeChassisID], "iface-id-ver": egressPod1.Name, }, PortSecurity: []string{podAddr}, @@ -9463,7 +9464,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Expect(err).NotTo(gomega.HaveOccurred()) ePod, err := fakeOvn.fakeClient.KubeClient.CoreV1().Pods(egressPod.Namespace).Get(context.TODO(), egressPod.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}) + egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) egressNetPodIP, _, err := net.ParseCIDR(egressPodPortInfo.ips[0].String()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -10435,7 +10436,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Expect(err).NotTo(gomega.HaveOccurred()) ePod, err := fakeOvn.fakeClient.KubeClient.CoreV1().Pods(egressPod.Namespace).Get(context.TODO(), egressPod.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}) + egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) egressNetPodIP, _, err := net.ParseCIDR(egressPodPortInfo.ips[0].String()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -10632,7 +10633,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Expect(err).NotTo(gomega.HaveOccurred()) ePod, err := fakeOvn.fakeClient.KubeClient.CoreV1().Pods(egressPod.Namespace).Get(context.TODO(), egressPod.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}) + egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) egressNetPodIP, _, err := net.ParseCIDR(egressPodPortInfo.ips[0].String()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -10959,7 +10960,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Expect(err).NotTo(gomega.HaveOccurred()) ePod, err := fakeOvn.fakeClient.KubeClient.CoreV1().Pods(egressPod.Namespace).Get(context.TODO(), egressPod.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}) + egressPodIP, err := util.GetPodIPsOfNetwork(ePod, &util.DefaultNetInfo{}, nil) index := 0 //ipv4 address at zero index gomega.Expect(err).NotTo(gomega.HaveOccurred()) egressNetPodIP, _, err := net.ParseCIDR(egressPodPortInfo.ips[0].String()) @@ -15681,10 +15682,20 @@ func getReRouteStaticRoute(clusterSubnet, nextHop string) *nbdb.LogicalRouterSta } func getNodeObj(nodeName string, annotations, labels map[string]string) corev1.Node { + nodeAnnotations := map[string]string{} + if annotations != nil { + nodeAnnotations = make(map[string]string, len(annotations)+1) + for k, v := range annotations { + nodeAnnotations[k] = v + } + } + if _, ok := nodeAnnotations[util.OvnNodeChassisID]; !ok { + nodeAnnotations[util.OvnNodeChassisID] = uuid.NewSHA1(uuid.NameSpaceOID, []byte(nodeName)).String() + } return corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: nodeName, - Annotations: annotations, + Annotations: nodeAnnotations, Labels: labels, }, Status: corev1.NodeStatus{ diff --git a/go-controller/pkg/ovn/egressip_udn_l2_test.go b/go-controller/pkg/ovn/egressip_udn_l2_test.go index 8fe85b972d..1b4d845800 100644 --- a/go-controller/pkg/ovn/egressip_udn_l2_test.go +++ b/go-controller/pkg/ovn/egressip_udn_l2_test.go @@ -132,6 +132,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) @@ -517,6 +518,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) @@ -1047,6 +1049,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) @@ -1552,6 +1555,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, nil) egressPodCDN := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDN := *newPodWithLabels(eipNamespace2, podName2, node1Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDN, util.GetNADName(eipNamespace2, nadName1), fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) nadNsName := util.GetNADName(eipNamespace2, nadName1) netconf := ovncnitypes.NetConf{ @@ -1929,6 +1933,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDN := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDN := *newPodWithLabels(eipNamespace2, podName2, node1Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDN, util.GetNADName(eipNamespace2, nadName1), fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) nadNsName := util.GetNADName(eipNamespace2, nadName1) netconf := ovncnitypes.NetConf{ @@ -2293,6 +2298,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, nil) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, nil) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) @@ -2676,6 +2682,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) diff --git a/go-controller/pkg/ovn/egressip_udn_l3_test.go b/go-controller/pkg/ovn/egressip_udn_l3_test.go index 424bc79385..a8a50a3724 100644 --- a/go-controller/pkg/ovn/egressip_udn_l3_test.go +++ b/go-controller/pkg/ovn/egressip_udn_l3_test.go @@ -41,26 +41,25 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol ) const ( - nadName1 = "nad1" - networkName1 = "network1" - networkName1_ = networkName1 + "_" - node1Name = "node1" - v4Net1 = "20.128.0.0/14" - v4Node1Net1 = "20.128.0.0/16" - v4Pod1IPNode1Net1 = "20.128.0.5" - podName3 = "egress-pod3" - v4Pod2IPNode1Net1 = "20.128.0.6" - v4Node1Tsp = "100.88.0.2" - node2Name = "node2" - v4Node2Net1 = "20.129.0.0/16" - v4Node2Tsp = "100.88.0.3" - podName4 = "egress-pod4" - v4Pod1IPNode2Net1 = "20.129.0.2" - v4Pod2IPNode2Net1 = "20.129.0.3" - eIP1Mark = 50000 - eIP2Mark = 50001 - userDefinedNetworkID = "2" - //tnlKey = zoneinterconnect.BaseTransitSwitchTunnelKey + userDefinedNetworkID + nadName1 = "nad1" + networkName1 = "network1" + networkName1_ = networkName1 + "_" + node1Name = "node1" + v4Net1 = "20.128.0.0/14" + v4Node1Net1 = "20.128.0.0/16" + v4Pod1IPNode1Net1 = "20.128.0.5" + podName3 = "egress-pod3" + v4Pod2IPNode1Net1 = "20.128.0.6" + v4Node1Tsp = "100.88.0.2" + node2Name = "node2" + v4Node2Net1 = "20.129.0.0/16" + v4Node2Tsp = "100.88.0.3" + podName4 = "egress-pod4" + v4Pod1IPNode2Net1 = "20.129.0.2" + v4Pod2IPNode2Net1 = "20.129.0.3" + eIP1Mark = 50000 + eIP2Mark = 50001 + // tnlKey = zoneinterconnect.BaseTransitSwitchTunnelKey + userDefinedNetworkID tnlKey = "16711685" ) @@ -137,6 +136,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) @@ -168,6 +168,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), util.OvnNodeID: "2", } + addL3GatewayConfig(node1Annotations, node1IPv4CIDR, "7e:57:f8:f0:3c:49") labels := map[string]string{ "k8s.ovn.org/egress-assignable": "", } @@ -180,6 +181,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), util.OvnNodeID: "3", } + addL3GatewayConfig(node2Annotations, node2IPv4CIDR, "7e:57:f8:f0:3c:50") node2 := getNodeObj(node2Name, node2Annotations, labels) eIP := egressipv1.EgressIP{ ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), @@ -261,7 +263,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{}, }, } @@ -473,7 +475,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, }, getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), @@ -512,6 +514,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) @@ -544,6 +547,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), util.OvnNodeID: "2", } + addL3GatewayConfig(node1Annotations, node1IPv4CIDR, "7e:57:f8:f0:3c:49") labels := map[string]string{ "k8s.ovn.org/egress-assignable": "", } @@ -556,6 +560,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), util.OvnNodeID: "3", } + addL3GatewayConfig(node2Annotations, node2IPv4CIDR, "7e:57:f8:f0:3c:50") node2 := getNodeObj(node2Name, node2Annotations, labels) twoNodeStatus := []egressipv1.EgressIPStatusItem{ { @@ -635,7 +640,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), }, } fakeOvn.startWithDBSetup( @@ -848,7 +853,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, }, getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), @@ -986,7 +991,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, }, getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), @@ -1031,6 +1036,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) @@ -1062,6 +1068,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), util.OvnNodeID: "2", } + addL3GatewayConfig(node1Annotations, node1IPv4CIDR, "7e:57:f8:f0:3c:49") labels := map[string]string{ "k8s.ovn.org/egress-assignable": "", } @@ -1074,7 +1081,30 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), util.OvnNodeID: "3", } + addL3GatewayConfig(node2Annotations, node2IPv4CIDR, "7e:57:f8:f0:3c:50") node2 := getNodeObj(node2Name, node2Annotations, labels) + gwConfig, err := util.ParseNodeL3GatewayAnnotation(&node1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + rtosPortName := "rtos-" + networkName1_ + node1Name + rtosPortUUID := rtosPortName + "-UUID" + rtosChassisName := rtosPortName + "-" + node1.Annotations[util.OvnNodeChassisID] + rtosChassisUUID := rtosChassisName + "-UUID" + rtosPort := &nbdb.LogicalRouterPort{ + UUID: rtosPortUUID, + Name: rtosPortName, + MAC: util.IPAddrToHWAddr(util.GetNodeGatewayIfAddr(node1UDNSubnet).IP).String(), + Networks: []string{util.GetNodeGatewayIfAddr(node1UDNSubnet).String()}, + Options: map[string]string{ + "gateway_mtu": fmt.Sprintf("%d", config.Default.MTU), + }, + GatewayChassis: []string{rtosChassisUUID}, + } + rtosGatewayChassis := &nbdb.GatewayChassis{ + UUID: rtosChassisUUID, + Name: rtosChassisName, + ChassisName: node1.Annotations[util.OvnNodeChassisID], + Priority: 1, + } twoNodeStatus := []egressipv1.EgressIPStatusItem{ { Node: node1Name, @@ -1153,7 +1183,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), }, } fakeOvn.startWithDBSetup( @@ -1387,17 +1417,20 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fmt.Sprintf("static-route-%s-%s-UUID", node2UDNLogicalRouterIPv4[0], v4Node2Tsp), }, Nat: []string{networkName1_ + node1Name + "-masqueradeNAT-UUID"}, - Ports: []string{netInfo.GetNetworkScopedName(ovntypes.RouterToTransitSwitchPrefix+node1.Name) + "-UUID"}, + Ports: []string{rtosPortUUID, netInfo.GetNetworkScopedName(ovntypes.RouterToTransitSwitchPrefix+node1.Name) + "-UUID"}, }, &nbdb.LogicalRouter{ UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), Ports: []string{ ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + networkName1_ + node1.Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: gwRouterExternalIDs(netInfo, *gwConfig), + Options: gwRouterOptions(*gwConfig), Policies: []string{getGWPktMarkLRPUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), getGWPktMarkLRPUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName())}, }, + rtosPort, + rtosGatewayChassis, &nbdb.LogicalSwitchPort{ UUID: "k8s-" + networkName1_ + node1Name + "-UUID", Name: "k8s-" + networkName1_ + node1Name, @@ -1430,7 +1463,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID", "stor-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, OtherConfig: map[string]string{ "exclude_ips": util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String(), @@ -1459,7 +1492,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol "node": node2.Name, }, Options: map[string]string{ - libovsdbops.RequestedChassis: node2.Name, + libovsdbops.RequestedChassis: node2.Annotations[util.OvnNodeChassisID], libovsdbops.RequestedTnlKey: node2.Annotations[util.OvnNodeID], }, Type: "remote", @@ -1489,7 +1522,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol udnEnabledSvcV4, } ginkgo.By("ensure expected equals actual") - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveDataIgnoringUUIDs(expectedDatabaseStateTwoEgressNodes)) ginkgo.By("delete EgressIP") err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Delete(context.TODO(), eIP.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1639,14 +1672,17 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fmt.Sprintf("static-route-%s-%s-UUID", node2UDNLogicalRouterIPv4[0], v4Node2Tsp), }, Nat: []string{networkName1_ + node1Name + "-masqueradeNAT-UUID"}, - Ports: []string{netInfo.GetNetworkScopedName(ovntypes.RouterToTransitSwitchPrefix+node1.Name) + "-UUID"}, + Ports: []string{rtosPortUUID, netInfo.GetNetworkScopedName(ovntypes.RouterToTransitSwitchPrefix+node1.Name) + "-UUID"}, }, &nbdb.LogicalRouter{ UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + networkName1_ + node1.Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: secConInfo.bnc.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: gwRouterExternalIDs(netInfo, *gwConfig), + Options: gwRouterOptions(*gwConfig), }, + rtosPort, + rtosGatewayChassis, &nbdb.LogicalSwitchPort{ UUID: "k8s-" + networkName1_ + node1Name + "-UUID", Name: "k8s-" + networkName1_ + node1Name, @@ -1679,7 +1715,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID", "stor-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, OtherConfig: map[string]string{ "exclude_ips": util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String(), @@ -1708,7 +1744,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol "node": node2.Name, }, Options: map[string]string{ - libovsdbops.RequestedChassis: node2.Name, + libovsdbops.RequestedChassis: node2.Annotations[util.OvnNodeChassisID], libovsdbops.RequestedTnlKey: node2.Annotations[util.OvnNodeID], }, Type: "remote", @@ -1738,7 +1774,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol udnEnabledSvcV4, } ginkgo.By("ensure expected equals actual") - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveDataIgnoringUUIDs(expectedDatabaseState)) return nil } err := app.Run([]string{app.Name}) @@ -1766,6 +1802,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, nil) egressPodCDN := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDN := *newPodWithLabels(eipNamespace2, podName2, node1Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDN, util.GetNADName(eipNamespace2, nadName1), fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) nadNsName := util.GetNADName(eipNamespace2, nadName1) netconf := ovncnitypes.NetConf{ @@ -1795,6 +1832,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), util.OvnNodeID: "2", } + addL3GatewayConfig(node1Annotations, node1IPv4CIDR, "7e:57:f8:f0:3c:49") labels := map[string]string{ "k8s.ovn.org/egress-assignable": "", } @@ -1807,6 +1845,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), util.OvnNodeID: "3", } + addL3GatewayConfig(node2Annotations, node2IPv4CIDR, "7e:57:f8:f0:3c:50") node2 := getNodeObj(node2Name, node2Annotations, labels) twoNodeStatus := []egressipv1.EgressIPStatusItem{ { @@ -1886,7 +1925,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), }, } fakeOvn.startWithDBSetup( @@ -2096,7 +2135,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, }, getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), @@ -2105,7 +2144,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol udnEnabledSvcV4, } ginkgo.By("ensure expected equals actual") - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveDataIgnoringUUIDs(expectedDatabaseStateTwoEgressNodes)) return nil } err := app.Run([]string{app.Name}) @@ -2133,6 +2172,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDN := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) egressPodUDN := *newPodWithLabels(eipNamespace2, podName2, node1Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDN, util.GetNADName(eipNamespace2, nadName1), fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) nadNsName := util.GetNADName(eipNamespace2, nadName1) netconf := ovncnitypes.NetConf{ @@ -2162,6 +2202,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), util.OvnNodeID: "2", } + addL3GatewayConfig(node1Annotations, node1IPv4CIDR, "7e:57:f8:f0:3c:49") labels := map[string]string{ "k8s.ovn.org/egress-assignable": "", } @@ -2174,6 +2215,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), util.OvnNodeID: "3", } + addL3GatewayConfig(node2Annotations, node2IPv4CIDR, "7e:57:f8:f0:3c:50") node2 := getNodeObj(node2Name, node2Annotations, labels) twoNodeStatus := []egressipv1.EgressIPStatusItem{ { @@ -2253,7 +2295,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), }, } fakeOvn.startWithDBSetup( @@ -2451,7 +2493,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, }, getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), @@ -2489,6 +2531,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol egressUDNNamespace := newUDNNamespaceWithLabels(eipNamespace2, egressPodLabel) egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, nil) egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, nil) + setPrimaryNetworkAnnot(&egressPodUDNLocal, nadName, fmt.Sprintf("%s%s", v4Pod1IPNode1Net1, util.GetIPFullMaskString(v4Pod1IPNode1Net1))) egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) @@ -2520,6 +2563,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), util.OvnNodeID: "2", } + addL3GatewayConfig(node1Annotations, node1IPv4CIDR, "7e:57:f8:f0:3c:49") labels := map[string]string{ "k8s.ovn.org/egress-assignable": "", } @@ -2532,7 +2576,30 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), util.OvnNodeID: "3", } + addL3GatewayConfig(node2Annotations, node2IPv4CIDR, "7e:57:f8:f0:3c:50") node2 := getNodeObj(node2Name, node2Annotations, labels) + gwConfig, err := util.ParseNodeL3GatewayAnnotation(&node1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + rtosPortName := "rtos-" + networkName1_ + node1Name + rtosPortUUID := rtosPortName + "-UUID" + rtosChassisName := rtosPortName + "-" + node1.Annotations[util.OvnNodeChassisID] + rtosChassisUUID := rtosChassisName + "-UUID" + rtosPort := &nbdb.LogicalRouterPort{ + UUID: rtosPortUUID, + Name: rtosPortName, + MAC: util.IPAddrToHWAddr(util.GetNodeGatewayIfAddr(node1UDNSubnet).IP).String(), + Networks: []string{util.GetNodeGatewayIfAddr(node1UDNSubnet).String()}, + Options: map[string]string{ + "gateway_mtu": fmt.Sprintf("%d", config.Default.MTU), + }, + GatewayChassis: []string{rtosChassisUUID}, + } + rtosGatewayChassis := &nbdb.GatewayChassis{ + UUID: rtosChassisUUID, + Name: rtosChassisName, + ChassisName: node1.Annotations[util.OvnNodeChassisID], + Priority: 1, + } twoNodeStatus := []egressipv1.EgressIPStatusItem{ { Node: node1Name, @@ -2611,7 +2678,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), }, } fakeOvn.startWithDBSetup( @@ -2853,16 +2920,19 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fmt.Sprintf("static-route-%s-%s-UUID", node2UDNLogicalRouterIPv4[0], v4Node2Tsp), }, Nat: []string{networkName1_ + node1Name + "-masqueradeNAT-UUID"}, - Ports: []string{netInfo.GetNetworkScopedName(ovntypes.RouterToTransitSwitchPrefix+node1.Name) + "-UUID"}, + Ports: []string{rtosPortUUID, netInfo.GetNetworkScopedName(ovntypes.RouterToTransitSwitchPrefix+node1.Name) + "-UUID"}, }, &nbdb.LogicalRouter{ UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + networkName1_ + node1.Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: gwRouterExternalIDs(netInfo, *gwConfig), + Options: gwRouterOptions(*gwConfig), Policies: []string{getGWPktMarkLRPUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), getGWPktMarkLRPUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName())}, }, + rtosPort, + rtosGatewayChassis, &nbdb.LogicalSwitchPort{ UUID: "k8s-" + networkName1_ + node1Name + "-UUID", Name: "k8s-" + networkName1_ + node1Name, @@ -2895,7 +2965,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", Name: netInfo.GetNetworkScopedSwitchName(node1.Name), Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID", "stor-" + networkName1_ + node1Name + "-UUID"}, - ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer3Topology}, + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(netInfo), QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, OtherConfig: map[string]string{ "exclude_ips": util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String(), @@ -2924,7 +2994,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol "node": node2.Name, }, Options: map[string]string{ - libovsdbops.RequestedChassis: node2.Name, + libovsdbops.RequestedChassis: node2.Annotations[util.OvnNodeChassisID], libovsdbops.RequestedTnlKey: node2.Annotations[util.OvnNodeID], }, Type: "remote", @@ -2964,6 +3034,14 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol }) }) +func addL3GatewayConfig(annotations map[string]string, nodeIPv4CIDR, mac string) { + annotations["k8s.ovn.org/l3-gateway-config"] = fmt.Sprintf( + `{"default":{"mode":"local","mac-address":%q, "ip-address":%q, "next-hop":"192.168.126.1"}}`, + mac, + nodeIPv4CIDR, + ) +} + // returns the address set with externalID "k8s.ovn.org/name": "egressip-served-pods"" func buildEgressIPServedPodsAddressSetsForController(ips []string, network, controller string) (*nbdb.AddressSet, *nbdb.AddressSet) { dbIDs := getEgressIPAddrSetDbIDs(EgressIPServedPodsAddrSetName, network, controller) diff --git a/go-controller/pkg/ovn/egressqos.go b/go-controller/pkg/ovn/egressqos.go index 605b127d03..5b70ab54f5 100644 --- a/go-controller/pkg/ovn/egressqos.go +++ b/go-controller/pkg/ovn/egressqos.go @@ -169,7 +169,7 @@ func (oc *DefaultNetworkController) createASForEgressQoSRule(podSelector metav1. for _, pod := range pods { // we don't handle HostNetworked or completed pods or not-scheduled pods or remote-zone pods if !util.PodWantsHostNetwork(pod) && !util.PodCompleted(pod) && util.PodScheduled(pod) && oc.isPodScheduledinLocalZone(pod) { - podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo()) + podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo(), nil) if err != nil && !errors.Is(err, util.ErrNoPodIPFound) { return nil, nil, err } @@ -755,7 +755,7 @@ func (oc *DefaultNetworkController) syncEgressQoSPod(key string) error { return nil } - podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo()) + podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo(), nil) if errors.Is(err, util.ErrNoPodIPFound) { return nil // reprocess it when it is updated with an IP } @@ -846,8 +846,8 @@ func (oc *DefaultNetworkController) onEgressQoSPodUpdate(oldObj, newObj interfac oldPodLabels := labels.Set(oldPod.Labels) newPodLabels := labels.Set(newPod.Labels) - oldPodIPs, _ := util.GetPodIPsOfNetwork(oldPod, oc.GetNetInfo()) - newPodIPs, _ := util.GetPodIPsOfNetwork(newPod, oc.GetNetInfo()) + oldPodIPs, _ := util.GetPodIPsOfNetwork(oldPod, oc.GetNetInfo(), nil) + newPodIPs, _ := util.GetPodIPsOfNetwork(newPod, oc.GetNetInfo(), nil) isOldPodLocal := oc.isPodScheduledinLocalZone(oldPod) isNewPodLocal := oc.isPodScheduledinLocalZone(newPod) oldPodCompleted := util.PodCompleted(oldPod) diff --git a/go-controller/pkg/ovn/egressservices_test.go b/go-controller/pkg/ovn/egressservices_test.go index 25412c13d5..612941a338 100644 --- a/go-controller/pkg/ovn/egressservices_test.go +++ b/go-controller/pkg/ovn/egressservices_test.go @@ -1700,6 +1700,7 @@ func nodeFor(name, ipv4, ipv6, v4subnet, v6subnet, transitIPv4, transitIPv6 stri "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", ipv4, ipv6), util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\",\"%s\"]", fmt.Sprintf("%s/24", ipv4), fmt.Sprintf("%s/64", ipv6)), "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\",\"%s\"]}", v4subnet, v6subnet), + util.OvnNodeChassisID: chassisIDForNode(name), // Used only with IC tests "k8s.ovn.org/zone-name": name, diff --git a/go-controller/pkg/ovn/external_gateway_apb_test.go b/go-controller/pkg/ovn/external_gateway_apb_test.go index b237174ae0..2e91281041 100644 --- a/go-controller/pkg/ovn/external_gateway_apb_test.go +++ b/go-controller/pkg/ovn/external_gateway_apb_test.go @@ -178,7 +178,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -214,7 +214,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -324,7 +324,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -360,7 +360,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -463,7 +463,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -499,7 +499,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -606,7 +606,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -652,7 +652,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -814,7 +814,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -896,7 +896,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1039,7 +1039,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:49:a1:93:cb fd00:10:244:2::3"}, }, @@ -1167,7 +1167,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1238,7 +1238,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1339,7 +1339,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1375,7 +1375,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1481,7 +1481,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1517,7 +1517,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1643,7 +1643,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1679,7 +1679,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1797,7 +1797,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1833,7 +1833,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1860,7 +1860,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1902,7 +1902,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -1992,7 +1992,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2112,7 +2112,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2243,7 +2243,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2285,7 +2285,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2341,7 +2341,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { Name: "namespace1_myPod", Options: map[string]string{ "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, }, @@ -2539,7 +2539,7 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { }, Name: "namespace1_myPod", Options: map[string]string{ - libovsdbops.RequestedChassis: "node1", + libovsdbops.RequestedChassis: chassisIDForNode("node1"), "iface-id-ver": "myPod", }, PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, diff --git a/go-controller/pkg/ovn/gateway.go b/go-controller/pkg/ovn/gateway.go index e51878926f..ddce0de5c7 100644 --- a/go-controller/pkg/ovn/gateway.go +++ b/go-controller/pkg/ovn/gateway.go @@ -34,16 +34,17 @@ import ( ) type GatewayManager struct { - nodeName string - clusterRouterName string - gwRouterName string - extSwitchName string - joinSwitchName string - coppUUID string - kube kube.InterfaceOVN - nbClient libovsdbclient.Client - netInfo util.NetInfo - watchFactory *factory.WatchFactory + nodeName string + clusterRouterName string + gwRouterName string + extSwitchName string + joinSwitchName string + coppUUID string + kube kube.InterfaceOVN + nbClient libovsdbclient.Client + netInfo util.NetInfo + watchFactory *factory.WatchFactory + getNetworkNameForNADKey func(nadKey string) string // Cluster wide Load_Balancer_Group UUID. // Includes all node switches and node gateway routers. clusterLoadBalancerGroupUUID string @@ -148,6 +149,12 @@ func WithLoadBalancerGroups(routerLBGroup, clusterLBGroup, switchLBGroup string) } } +func WithNetworkNameForNADKeyResolver(getNetworkNameForNADKey func(nadKey string) string) GatewayOption { + return func(manager *GatewayManager) { + manager.getNetworkNameForNADKey = getNetworkNameForNADKey + } +} + // cleanupStalePodSNATs removes pod SNATs against nodeIP for the given node if // the SNAT.logicalIP isn't an active podIP, or disableSNATMultipleGWs=false. // We don't have to worry about @@ -160,6 +167,9 @@ func WithLoadBalancerGroups(routerLBGroup, clusterLBGroup, switchLBGroup string) // pod->nodeSNATs which won't get cleared up unless explicitly deleted. // NOTE2: egressIP SNATs are synced in EIP controller. func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.IPNet, gwLRPIPs []net.IP) error { + if gw.netInfo.IsUserDefinedNetwork() && gw.getNetworkNameForNADKey == nil { + return fmt.Errorf("missing NAD resolver for network %q", gw.netInfo.GetNetworkName()) + } // collect all the pod IPs for which we should be doing the SNAT; // if DisableSNATMultipleGWs==false we consider all // the SNATs stale @@ -179,7 +189,7 @@ func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.I continue } if util.PodCompleted(&pod) { - collidingPod, err := findPodWithIPAddresses(gw.watchFactory, gw.netInfo, []net.IP{utilnet.ParseIPSloppy(pod.Status.PodIP)}, "") //even if a pod is completed we should still delete the nat if the ip is not in use anymore + collidingPod, err := findPodWithIPAddresses(gw.watchFactory, gw.netInfo, []net.IP{utilnet.ParseIPSloppy(pod.Status.PodIP)}, "", gw.getNetworkNameForNADKey) //even if a pod is completed we should still delete the nat if the ip is not in use anymore if err != nil { return fmt.Errorf("lookup for pods with same ip as %s %s failed: %w", pod.Namespace, pod.Name, err) } @@ -187,7 +197,7 @@ func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.I continue } } - podIPs, err := util.GetPodIPsOfNetwork(&pod, gw.netInfo) + podIPs, err := util.GetPodIPsOfNetwork(&pod, gw.netInfo, gw.getNetworkNameForNADKey) if err != nil && errors.Is(err, util.ErrNoPodIPFound) { // It is possible that the pod is scheduled during this time, but the LSP add or // IP Allocation has not happened and it is waiting for the WatchPods to start diff --git a/go-controller/pkg/ovn/hybrid.go b/go-controller/pkg/ovn/hybrid.go index 0164cc4076..d9a5610940 100644 --- a/go-controller/pkg/ovn/hybrid.go +++ b/go-controller/pkg/ovn/hybrid.go @@ -279,7 +279,7 @@ func (oc *DefaultNetworkController) setupHybridLRPolicySharedGw(nodeSubnets []*n }, &clusterRouterStaticRoutes.Nexthop); err != nil { return fmt.Errorf("failed to add policy route static '%s %s' for on %s , error: %w", clusterRouterStaticRoutes.IPPrefix, clusterRouterStaticRoutes.Nexthop, - oc.GetNetworkScopedGWRouterName(nodeName), err) + ovntypes.OVNClusterRouter, err) } klog.Infof("Created hybrid overlay logical route static route at cluster router for node %s", nodeName) diff --git a/go-controller/pkg/ovn/kubevirt_test.go b/go-controller/pkg/ovn/kubevirt_test.go index 5f2ce57b3f..9aad857d9c 100644 --- a/go-controller/pkg/ovn/kubevirt_test.go +++ b/go-controller/pkg/ovn/kubevirt_test.go @@ -748,7 +748,7 @@ var _ = Describe("OVN Kubevirt Operations", func() { "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf(`{"ipv4": %q, "ipv6": %q}`, nodeByName[node1].transitSwitchPortIPv4, nodeByName[node1].transitSwitchPortIPv6), "k8s.ovn.org/node-subnets": fmt.Sprintf(`{"default":[%q,%q]}`, nodeByName[node1].subnetIPv4, nodeByName[node1].subnetIPv6), "k8s.ovn.org/l3-gateway-config": fmt.Sprintf(`{"default": {"mode": "local", "mac-address":"7e:57:f8:f0:3c:51", "ip-addresses":[%q, %q]}}`, nodeByName[node1].addressIPv4, nodeByName[node1].addressIPv6), - "k8s.ovn.org/node-chassis-id": "1", + "k8s.ovn.org/node-chassis-id": chassisIDForNode(node1), util.OvnNodeID: nodeByName[node1].nodeID, }, }, @@ -760,7 +760,7 @@ var _ = Describe("OVN Kubevirt Operations", func() { "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf(`{"ipv4": %q, "ipv6": %q}`, nodeByName[node2].transitSwitchPortIPv4, nodeByName[node2].transitSwitchPortIPv6), "k8s.ovn.org/node-subnets": fmt.Sprintf(`{"default":[%q,%q]}`, nodeByName[node2].subnetIPv4, nodeByName[node2].subnetIPv6), "k8s.ovn.org/l3-gateway-config": fmt.Sprintf(`{"default": {"mode": "local", "mac-address":"7e:57:f8:f0:3c:52", "ip-addresses":[%q, %q]}}`, nodeByName[node2].addressIPv4, nodeByName[node2].addressIPv6), - "k8s.ovn.org/node-chassis-id": "2", + "k8s.ovn.org/node-chassis-id": chassisIDForNode(node2), util.OvnNodeID: nodeByName[node2].nodeID, }, }, @@ -772,7 +772,7 @@ var _ = Describe("OVN Kubevirt Operations", func() { "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf(`{"ipv4": %q, "ipv6": %q}`, nodeByName[node3].transitSwitchPortIPv4, nodeByName[node3].transitSwitchPortIPv6), "k8s.ovn.org/node-subnets": fmt.Sprintf(`{"default":[%q,%q]}`, nodeByName[node3].subnetIPv4, nodeByName[node3].subnetIPv6), "k8s.ovn.org/l3-gateway-config": fmt.Sprintf(`{"default": {"mode": "local", "mac-address":"7e:57:f8:f0:3c:53", "ip-addresses":[%q, %q]}}`, nodeByName[node3].addressIPv4, nodeByName[node3].addressIPv6), - "k8s.ovn.org/node-chassis-id": "3", + "k8s.ovn.org/node-chassis-id": chassisIDForNode(node3), util.OvnNodeID: nodeByName[node3].nodeID, }, }, diff --git a/go-controller/pkg/ovn/layer2_user_defined_network_controller.go b/go-controller/pkg/ovn/layer2_user_defined_network_controller.go index 8f95983ecb..63f4994cfa 100644 --- a/go-controller/pkg/ovn/layer2_user_defined_network_controller.go +++ b/go-controller/pkg/ovn/layer2_user_defined_network_controller.go @@ -138,6 +138,16 @@ func (h *layer2UserDefinedNetworkControllerEventHandler) AddResource(obj interfa } return h.oc.addUpdateLocalNodeEvent(node, nodeParams) } + if config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + if !h.oc.networkManager.NodeHasNetwork(node.Name, h.oc.GetNetworkName()) { + klog.V(5).Infof("Ignoring processing remote node: %s as it has no active NAD for network: %s", + node.Name, h.oc.GetNetworkName()) + // store sync IC failed for the node, so if on node update if the NAD is no longer filtered, we actually + // process it + h.oc.syncZoneICFailed.Store(node.Name, true) + return nil + } + } return h.oc.addUpdateRemoteNodeEvent(node, config.OVNKubernetesFeature.EnableInterconnect) default: return h.oc.AddUserDefinedNetworkResourceCommon(h.objType, obj) @@ -211,6 +221,14 @@ func (h *layer2UserDefinedNetworkControllerEventHandler) UpdateResource(oldObj, return h.oc.addUpdateLocalNodeEvent(newNode, nodeSyncsParam) } else { + if config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + if !h.oc.networkManager.NodeHasNetwork(newNode.Name, h.oc.GetNetworkName()) { + klog.V(5).Infof("Ignoring processing remote node: %s as it has no active NAD for network: %s", + newNode.Name, h.oc.GetNetworkName()) + h.oc.syncZoneICFailed.Store(newNode.Name, true) + return nil + } + } _, syncZoneIC := h.oc.syncZoneICFailed.Load(newNode.Name) _, oldNodeNoRouter := h.oc.remoteNodesNoRouter.Load(oldNode.Name) if oldNodeNoRouter && util.UDNLayer2NodeUsesTransitRouter(newNode) { @@ -325,7 +343,8 @@ func NewLayer2UserDefinedNetworkController( networkManager networkmanager.Interface, routeImportManager routeimport.Manager, portCache *PortCache, - eIPController *EgressIPController) (*Layer2UserDefinedNetworkController, error) { + eIPController *EgressIPController, +) (*Layer2UserDefinedNetworkController, error) { stopChan := make(chan struct{}) @@ -398,7 +417,12 @@ func NewLayer2UserDefinedNetworkController( if err != nil { return nil, fmt.Errorf("unable to create new service controller while creating new layer2 network controller: %w", err) } - oc.defaultGatewayReconciler = kubevirt.NewDefaultGatewayReconciler(oc.watchFactory, oc.GetNetInfo(), util.GetNetworkScopedK8sMgmtHostIntfName(uint(oc.GetNetworkID()))) + oc.defaultGatewayReconciler = kubevirt.NewDefaultGatewayReconciler( + oc.watchFactory, + oc.GetNetInfo(), + util.GetNetworkScopedK8sMgmtHostIntfName(uint(oc.GetNetworkID())), + oc.networkManager.GetNetworkNameForNADKey, + ) } if oc.allocatesPodAnnotation() { @@ -793,9 +817,19 @@ func (oc *Layer2UserDefinedNetworkController) addSwitchPortForRemoteNodeGR(node return fmt.Errorf("failed to fetch tunnelID annotation from the node %s for network %s, err: %w", node.Name, oc.GetNetworkName(), err) } + + chassisID, err := util.ParseNodeChassisIDAnnotation(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + // remote node may not have the annotation yet, suppress it + return types.NewSuppressedError(err) + } + return fmt.Errorf("failed to parse node chassis-id for node %s: %w", node.Name, err) + } + logicalSwitchPort.Options = map[string]string{ libovsdbops.RequestedTnlKey: strconv.Itoa(tunnelID), - libovsdbops.RequestedChassis: node.Name, + libovsdbops.RequestedChassis: chassisID, } sw := nbdb.LogicalSwitch{Name: oc.GetNetworkScopedSwitchName(types.OVNLayer2Switch)} err = libovsdbops.CreateOrUpdateLogicalSwitchPortsOnSwitch(oc.nbClient, &sw, &logicalSwitchPort) @@ -865,13 +899,23 @@ func (oc *Layer2UserDefinedNetworkController) addRouterSetupForRemoteNodeGR(node if err != nil { return nil } + + chassisID, err := util.ParseNodeChassisIDAnnotation(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + // remote node may not have the annotation yet, suppress it + return types.NewSuppressedError(err) + } + return fmt.Errorf("failed to parse node chassis-id for node %s: %w", node.Name, err) + } + transitPort := nbdb.LogicalRouterPort{ Name: types.TransitRouterToRouterPrefix + oc.GetNetworkScopedGWRouterName(node.Name), MAC: util.IPAddrToHWAddr(transitRouterInfo.transitRouterNets[0].IP).String(), Networks: util.IPNetsToStringSlice(transitRouterInfo.transitRouterNets), Options: map[string]string{ libovsdbops.RequestedTnlKey: getTransitRouterPortTunnelKey(transitRouterInfo.nodeID), - libovsdbops.RequestedChassis: node.Name, + libovsdbops.RequestedChassis: chassisID, }, ExternalIDs: map[string]string{ types.NetworkExternalID: oc.GetNetworkName(), @@ -922,6 +966,19 @@ func (oc *Layer2UserDefinedNetworkController) addTransitRouterRoutes(node *corev return nil } +func (oc *Layer2UserDefinedNetworkController) delPortForRemoteNodeGR(node *corev1.Node) error { + swName := oc.GetNetworkScopedSwitchName(types.OVNLayer2Switch) + sw := &nbdb.LogicalSwitch{Name: swName} + logicalSwitchPort := &nbdb.LogicalSwitchPort{ + Name: types.SwitchToRouterPrefix + oc.GetNetworkScopedSwitchName(types.OVNLayer2Switch) + "_" + node.Name, + } + if err := libovsdbops.DeleteLogicalSwitchPorts(oc.nbClient, sw, logicalSwitchPort); err != nil { + return fmt.Errorf("failed to delete remote GR switch port %q from layer 2 switch %q for the node %q, error: %w", + logicalSwitchPort.Name, swName, node.Name, err) + } + return nil +} + func (oc *Layer2UserDefinedNetworkController) cleanupRouterSetupForRemoteNodeGR(nodeName string) error { transitPort := &nbdb.LogicalRouterPort{ Name: types.TransitRouterToRouterPrefix + oc.GetNetworkScopedGWRouterName(nodeName), @@ -954,22 +1011,29 @@ func (oc *Layer2UserDefinedNetworkController) cleanupRouterSetupForRemoteNodeGR( } func (oc *Layer2UserDefinedNetworkController) deleteNodeEvent(node *corev1.Node) error { - // GatewayManager only exists for local nodes. - if err := oc.gatewayManagerForNode(node.Name).Cleanup(); err != nil { - return fmt.Errorf("failed to cleanup gateway on node %q: %w", node.Name, err) + if _, local := oc.localZoneNodes.Load(node.Name); local { + if util.IsNetworkSegmentationSupportEnabled() && oc.IsPrimaryNetwork() { + if err := oc.gatewayManagerForNode(node.Name).Cleanup(); err != nil { + return fmt.Errorf("failed to cleanup gateway on node %q: %w", node.Name, err) + } + oc.gatewayManagers.Delete(node.Name) + } + } else { + if config.Layer2UsesTransitRouter { + // this is a no-op for local nodes + if err := oc.cleanupRouterSetupForRemoteNodeGR(node.Name); err != nil { + return fmt.Errorf("failed to cleanup remote node %q gateway: %w", node.Name, err) + } + } else if config.OVNKubernetesFeature.EnableInterconnect { // Legacy check - pre-transit router + if err := oc.delPortForRemoteNodeGR(node); err != nil { + return fmt.Errorf("failed to cleanup remote zone node %s's remote LRP, %w", node.Name, err) + } + } } - oc.gatewayManagers.Delete(node.Name) oc.localZoneNodes.Delete(node.Name) oc.mgmtPortFailed.Delete(node.Name) oc.syncEIPNodeRerouteFailed.Delete(node.Name) - - if config.Layer2UsesTransitRouter { - // this is a no-op for local nodes - if err := oc.cleanupRouterSetupForRemoteNodeGR(node.Name); err != nil { - return fmt.Errorf("failed to cleanup remote node %q gateway: %w", node.Name, err) - } - oc.syncZoneICFailed.Delete(node.Name) - } + oc.syncZoneICFailed.Delete(node.Name) return nil } @@ -1110,6 +1174,9 @@ func (oc *Layer2UserDefinedNetworkController) gatewayOptions() []GatewayOption { oc.switchLoadBalancerGroupUUID, )) } + if resolver := oc.getNetworkNameForNADKeyFunc(); resolver != nil { + opts = append(opts, WithNetworkNameForNADKeyResolver(resolver)) + } return opts } @@ -1385,16 +1452,30 @@ func (oc *Layer2UserDefinedNetworkController) syncNodes(nodes []interface{}) err return err } foundNodeNames := sets.New[string]() - foundNodes := make([]*corev1.Node, len(nodes)) - for i, obj := range nodes { + activeNodes := make([]*corev1.Node, 0, len(nodes)) + dynamicUDN := config.OVNKubernetesFeature.EnableDynamicUDNAllocation + for _, obj := range nodes { node, ok := obj.(*corev1.Node) if !ok { return fmt.Errorf("spurious object in syncNodes: %v", obj) } + if oc.isLocalZoneNode(node) { + foundNodeNames.Insert(node.Name) + activeNodes = append(activeNodes, node) + continue + } + // Clean up remote nodes that went inactive + if dynamicUDN && !oc.nodeHasActiveNetwork(node.Name) { + if err := oc.deleteNodeEvent(node); err != nil { + return err + } + continue + } foundNodeNames.Insert(node.Name) - foundNodes[i] = node + activeNodes = append(activeNodes, node) } - oc.setRemoteNodesNoRouter(foundNodes) + // Transit Router cleanup + oc.setRemoteNodesNoTransitRouter(activeNodes) // Get the transit router. If it's not present - no cleanup to do tr := &nbdb.LogicalRouter{ Name: oc.GetNetworkScopedClusterRouterName(), @@ -1435,8 +1516,12 @@ func (oc *Layer2UserDefinedNetworkController) syncNodes(nodes []interface{}) err return nil } -// setRemoteNodesNoRouter finds remote nodes that do not use transit router. -func (oc *Layer2UserDefinedNetworkController) setRemoteNodesNoRouter(nodes []*corev1.Node) { +func (oc *Layer2UserDefinedNetworkController) nodeHasActiveNetwork(nodeName string) bool { + return oc.networkManager.NodeHasNetwork(nodeName, oc.GetNetworkName()) +} + +// setRemoteNodesNoTransitRouter finds remote nodes that do not use transit router. +func (oc *Layer2UserDefinedNetworkController) setRemoteNodesNoTransitRouter(nodes []*corev1.Node) { for _, node := range nodes { if oc.isLocalZoneNode(node) { continue @@ -1446,3 +1531,11 @@ func (oc *Layer2UserDefinedNetworkController) setRemoteNodesNoRouter(nodes []*co } } } + +// HandleNetworkRefChange marks the node for interconnect sync so a queued update does not skip it. +func (oc *Layer2UserDefinedNetworkController) HandleNetworkRefChange(nodeName string, active bool) { + if active { + oc.syncZoneICFailed.Store(nodeName, true) + } + oc.BaseNetworkController.HandleNetworkRefChange(nodeName, active) +} diff --git a/go-controller/pkg/ovn/layer2_user_defined_network_controller_test.go b/go-controller/pkg/ovn/layer2_user_defined_network_controller_test.go index 3a20b00985..7461784139 100644 --- a/go-controller/pkg/ovn/layer2_user_defined_network_controller_test.go +++ b/go-controller/pkg/ovn/layer2_user_defined_network_controller_test.go @@ -20,6 +20,7 @@ import ( ovnkcnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" testnm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" @@ -93,8 +94,15 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 2 network", func() { By(fmt.Sprintf("Creating a node named %q, with IP: %s", nodeName, nodeIPv4CIDR)) testNode, err := newNodeWithUserDefinedNetworks(nodeName, nodeIPv4CIDR) Expect(err).NotTo(HaveOccurred()) - - Expect(setupFakeOvnForLayer2Topology(fakeOvn, initialDB, netInfo, testNode, podInfo, pod)).To(Succeed()) + nodes := []corev1.Node{*testNode} + if config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + testNode2, err := newNodeWithUserDefinedNetworks("test-node2", "192.168.127.202/24", netInfo) + Expect(err).NotTo(HaveOccurred()) + testNode2.Annotations["k8s.ovn.org/zone-name"] = "blah" + By("adding an extra node that should be ignored by Dynamic UDN Allocation") + nodes = append(nodes, *testNode2) + } + Expect(setupFakeOvnForLayer2Topology(fakeOvn, initialDB, netInfo, nodes, podInfo, pod)).To(Succeed()) defer fakeOvn.networkManager.Stop() // for layer2 on interconnect, it is the cluster manager that @@ -127,7 +135,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 2 network", func() { fakeOvn, []testPod{podInfo}, expectationOptions..., - ).expectedLogicalSwitchesAndPorts(netInfo.isPrimary)...)) + ).expectedLogicalSwitchesAndPorts()...)) return nil } @@ -171,6 +179,14 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 2 network", func() { }), config.GatewayModeShared, ), + Entry("with dynamic UDN allocation, a remote node with no NAD is ignored", + dummyLayer2PrimaryUserDefinedNetwork("100.200.0.0/16"), + icClusterTestConfiguration(func(config *testConfiguration) { + config.configToOverride.EnableDynamicUDNAllocation = true + config.configToOverride.EnableNetworkSegmentation = true + }), + config.GatewayModeShared, + ), /** FIXME: tests do not support ipv6 yet Entry("pod on a IPv6 user defined primary network on an IC cluster with per-pod SNATs enabled", dummyPrimaryLayer2UserDefinedNetwork("2001:db8:abcd:0012::/64"), @@ -215,7 +231,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 2 network", func() { testNode, err := newNodeWithUserDefinedNetworks(nodeName, nodeIPv4CIDR) Expect(err).NotTo(HaveOccurred()) - Expect(setupFakeOvnForLayer2Topology(fakeOvn, initialDB, netInfo, testNode, sourcePodInfo, sourcePod, + Expect(setupFakeOvnForLayer2Topology(fakeOvn, initialDB, netInfo, []corev1.Node{*testNode}, sourcePodInfo, sourcePod, &ipamclaimsapi.IPAMClaimList{Items: []ipamclaimsapi.IPAMClaim{ipamClaim}}), ).To(Succeed()) defer fakeOvn.networkManager.Stop() @@ -250,7 +266,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 2 network", func() { fakeOvn, []testPod{sourcePodInfo}, expectationOptions..., - ).expectedLogicalSwitchesAndPorts(netInfo.isPrimary)...)) + ).expectedLogicalSwitchesAndPorts()...)) targetPodInfo := dummyL2TestPod(ns, netInfo, targetPodInfoIdx, userDefinedNetworkIdx) targetKvPod := newMultiHomedKubevirtPod( @@ -277,7 +293,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 2 network", func() { fakeOvn, testPods, expectationOptions..., - ).expectedLogicalSwitchesAndPortsWithLspEnabled(netInfo.isPrimary, expectedPodLspEnabled)...)) + ).expectedLogicalSwitchesAndPortsWithLspEnabled(expectedPodLspEnabled)...)) return nil } @@ -592,6 +608,240 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 2 network", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot use the last IP of the join subnet")) }) + + Describe("Dynamic UDN allocation with remote node", func() { + It("activates a remote node when a NAD becomes active and cleans it up when inactive", func() { + Expect(config.PrepareTestConfig()).To(Succeed()) + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + config.OVNKubernetesFeature.EnableInterconnect = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.Default.Zone = testICZone + config.Gateway.V4MasqueradeSubnet = "169.254.0.0/16" + + // Basic UDN setup + netInfo := dummyLayer2PrimaryUserDefinedNetwork("100.200.0.0/16") + n := newUDNNamespace(ns) + nad, err := newNetworkAttachmentDefinition(ns, nadName, *netInfo.netconf()) + Expect(err).NotTo(HaveOccurred()) + + // Local node and remote node with NAD + localNode, err := newNodeWithUserDefinedNetworks(nodeName, "192.168.126.202/24", netInfo) + Expect(err).NotTo(HaveOccurred()) + localNode.Annotations[util.OvnTransitSwitchPortAddr] = `{"ipv4":"100.88.0.3/16"}` + + remoteNode, err := newNodeWithUserDefinedNetworks("remoteNode", "192.168.127.202/24", netInfo) + Expect(err).NotTo(HaveOccurred()) + remoteNode.Annotations["k8s.ovn.org/zone-name"] = "other-zone" // force remote + remoteNode.Annotations[util.OvnTransitSwitchPortAddr] = `{"ipv4":"100.88.0.4/16"}` + + remotePod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "remote-pod", + Namespace: ns, + }, + Spec: corev1.PodSpec{ + NodeName: remoteNode.Name, + Containers: []corev1.Container{{Name: "c", Image: "scratch"}}, + }, + } + + localPod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-pod", + Namespace: ns, + }, + Spec: corev1.PodSpec{ + NodeName: localNode.Name, + Containers: []corev1.Container{{Name: "c", Image: "scratch"}}, + }, + } + + // Preload DB + fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{}, &corev1.NamespaceList{Items: []corev1.Namespace{*n}}, + &corev1.NodeList{Items: []corev1.Node{*localNode, *remoteNode}}, + &corev1.PodList{Items: []corev1.Pod{localPod}}, + &nadapi.NetworkAttachmentDefinitionList{Items: []nadapi.NetworkAttachmentDefinition{*nad}}) + + Expect(fakeOvn.networkManager.Start()).To(Succeed()) + defer fakeOvn.networkManager.Stop() + + userDefinedNetController, ok := fakeOvn.userDefinedNetworkControllers[userDefinedNetworkName] + Expect(ok).To(BeTrue()) + userDefinedNetController.bnc.ovnClusterLRPToJoinIfAddrs = dummyJoinIPs() + l2Controller, ok := fakeOvn.fullL2UDNControllers[netInfo.netName] + Expect(ok).To(BeTrue()) + mutableNetInfo := util.NewMutableNetInfo(l2Controller.GetNetInfo()) + mutableNetInfo.SetNetworkID(2) + err = util.ReconcileNetInfo(l2Controller.ReconcilableNetInfo, mutableNetInfo) + Expect(err).NotTo(HaveOccurred()) + err = l2Controller.init() + Expect(err).NotTo(HaveOccurred()) + Expect(userDefinedNetController.bnc.WatchNodes()).To(Succeed()) + + By("Remote node should not have a transit-router port before activation") + Consistently(func() bool { + p := func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[ovntypes.NodeExternalID] == remoteNode.Name && item.ExternalIDs[ovntypes.NetworkExternalID] == l2Controller.GetNetworkName() + } + ports, err := libovsdbops.FindLogicalRouterPortWithPredicate(fakeOvn.nbClient, p) + return err == nil && len(ports) > 0 + }).WithTimeout(500 * time.Millisecond).Should(BeFalse()) + + By("Creating a pod on the remote node should activate it") + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(ns).Create(context.TODO(), &remotePod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + return fakeOvn.networkManager.Interface().NodeHasNetwork(remoteNode.Name, netInfo.netName) + }).WithTimeout(3 * time.Second).Should(BeTrue()) + By("Triggering networkRefChange callback after updating remote node as active on NAD") + l2Controller.HandleNetworkRefChange(remoteNode.Name, true) + + By("Remote node should have a transit-router port created") + Eventually(func() bool { + p := func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[ovntypes.NodeExternalID] == remoteNode.Name && item.ExternalIDs[ovntypes.NetworkExternalID] == l2Controller.GetNetworkName() + } + ports, err := libovsdbops.FindLogicalRouterPortWithPredicate(fakeOvn.nbClient, p) + if err == nil && len(ports) > 0 { + return true + } + return false + }).WithTimeout(3 * time.Second).Should(BeTrue()) + + By("Deleting a pod on the remote node should set it as inactive") + err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(ns).Delete(context.TODO(), remotePod.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + return fakeOvn.networkManager.Interface().NodeHasNetwork(remoteNode.Name, netInfo.netName) + }).WithTimeout(3 * time.Second).Should(BeFalse()) + By("Triggering networkRefChange callback after updating remote node as inactive on NAD") + l2Controller.HandleNetworkRefChange(remoteNode.Name, false) + By("Remote node should not have a port on transit subnet") + Eventually(func() bool { + p := func(item *nbdb.LogicalRouterPort) bool { + return item.ExternalIDs[ovntypes.NodeExternalID] == remoteNode.Name && item.ExternalIDs[ovntypes.NetworkExternalID] == l2Controller.GetNetworkName() + } + ports, err := libovsdbops.FindLogicalRouterPortWithPredicate(fakeOvn.nbClient, p) + if err == nil && len(ports) > 0 { + return true + } + return false + }).WithTimeout(3 * time.Second).Should(BeFalse()) + + By("verifying that local node trtos and stotr ports still exist after remote node removal") + expectedLRP := &nbdb.LogicalRouterPort{ + Name: "trtos-isolatednet_ovn_layer2_switch", + MAC: "0a:58:64:c8:00:01", + GatewayChassis: []string{ + "00000000-0000-0000-0000-000000000000", + }, + Networks: []string{ + "100.200.0.1/16", + }, + Options: map[string]string{ + "gateway_mtu": "1400", + "requested-tnl-key": "1", + }, + } + + expectedLSP := &nbdb.LogicalSwitchPort{ + Name: "stotr-isolatednet_ovn_layer2_switch", + Type: "router", + Addresses: []string{"router"}, + Options: map[string]string{ + "router-port": "trtos-isolatednet_ovn_layer2_switch", + }, + ExternalIDs: map[string]string{ + "k8s.ovn.org/network": "isolatednet", + "k8s.ovn.org/topology": "layer2", + }, + } + + Eventually(fakeOvn.nbClient).WithTimeout(3 * time.Second).Should( + libovsdbtest.HaveDataSubset([]libovsdbtest.TestData{expectedLRP, expectedLSP}), + ) + }) + + It("does not filter pods from other namespaces of the same primary UDN", func() { + Expect(config.PrepareTestConfig()).To(Succeed()) + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + config.OVNKubernetesFeature.EnableInterconnect = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.Default.Zone = testICZone + + netInfo := dummyLayer2PrimaryUserDefinedNetwork("100.200.0.0/16") + nsA := "namespace-a" + nsB := "namespace-b" + nsAObj := newUDNNamespace(nsA) + nsBObj := newUDNNamespace(nsB) + + netInfoA := netInfo + netInfoA.nadName = namespacedName(nsA, nadName) + netInfoB := netInfo + netInfoB.nadName = namespacedName(nsB, nadName) + + nadA, err := newNetworkAttachmentDefinition(nsA, nadName, *netInfoA.netconf()) + Expect(err).NotTo(HaveOccurred()) + nadB, err := newNetworkAttachmentDefinition(nsB, nadName, *netInfoB.netconf()) + Expect(err).NotTo(HaveOccurred()) + + parsedNetInfoA, err := util.NewNetInfo(netInfoA.netconf()) + Expect(err).NotTo(HaveOccurred()) + mutableA := util.NewMutableNetInfo(parsedNetInfoA) + mutableA.SetNADs(namespacedName(nsA, nadName)) + + parsedNetInfoB, err := util.NewNetInfo(netInfoB.netconf()) + Expect(err).NotTo(HaveOccurred()) + mutableB := util.NewMutableNetInfo(parsedNetInfoB) + mutableB.SetNADs(namespacedName(nsB, nadName)) + + fakeOvn.networkManager = &testnm.FakeNetworkManager{ + PrimaryNetworks: map[string]util.NetInfo{ + nsA: mutableA, + nsB: mutableB, + }, + NADNetworks: map[string]util.NetInfo{ + namespacedName(nsA, nadName): mutableA, + namespacedName(nsB, nadName): mutableB, + }, + } + + localNode, err := newNodeWithUserDefinedNetworks(nodeName, "192.168.126.202/24", netInfo) + Expect(err).NotTo(HaveOccurred()) + + fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{}, + &corev1.NamespaceList{Items: []corev1.Namespace{*nsAObj, *nsBObj}}, + &corev1.NodeList{Items: []corev1.Node{*localNode}}, + &nadapi.NetworkAttachmentDefinitionList{Items: []nadapi.NetworkAttachmentDefinition{*nadA, *nadB}}, + ) + + Expect(fakeOvn.NewUserDefinedNetworkController(nadB)).To(Succeed()) + l2Controller, ok := fakeOvn.fullL2UDNControllers[netInfo.netName] + Expect(ok).To(BeTrue()) + mutableNetInfo := util.NewMutableNetInfo(l2Controller.GetNetInfo()) + mutableNetInfo.SetNADs(namespacedName(nsB, nadName)) + err = util.ReconcileNetInfo(l2Controller.ReconcilableNetInfo, mutableNetInfo) + Expect(err).NotTo(HaveOccurred()) + By("confirming the controller only tracks the local namespace NAD") + Expect(l2Controller.GetNetInfo().GetNADNamespaces()).To(ConsistOf(nsB)) + + remotePod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "remote-pod", + Namespace: nsA, + }, + Spec: corev1.PodSpec{ + NodeName: localNode.Name, + Containers: []corev1.Container{{Name: "c", Image: "scratch"}}, + }, + } + + By("ensuring the pod is not filtered out by the UDN controller") + Expect(l2Controller.FilterOutResource(factory.PodType, remotePod)).To(BeFalse()) + }) + }) }) func dummySecondaryLayer2UserDefinedNetwork(subnets string) userDefinedNetInfo { @@ -785,6 +1035,7 @@ func expectedLayer2EgressEntities(netInfo util.NetInfo, gwConfig util.L3GatewayC } if staleNode { staleNodeName := "stale-node" + staleNodeChassisID := chassisIDForNode("stale-node") // create remote router port remoteRouterName := fmt.Sprintf("GR_%s_%s", netInfo.GetNetworkName(), staleNodeName) remotePortName := fmt.Sprintf("%s%s", ovntypes.TransitRouterToRouterPrefix, remoteRouterName) @@ -799,7 +1050,7 @@ func expectedLayer2EgressEntities(netInfo util.NetInfo, gwConfig util.L3GatewayC MAC: util.IPAddrToHWAddr(remoteTRInfo.transitRouterNets[0].IP).String(), Options: map[string]string{ libovsdbops.RequestedTnlKey: "15", // as defined by getTransitRouterPortTunnelKey(nodeID) - libovsdbops.RequestedChassis: staleNodeName}, + libovsdbops.RequestedChassis: staleNodeChassisID}, ExternalIDs: externalIDs, } expectedEntities = append(expectedEntities, remotePort) @@ -834,7 +1085,7 @@ func dummyLayer2PrimaryUserDefinedNetwork(subnets string) userDefinedNetInfo { return secondaryNet } -func setupFakeOvnForLayer2Topology(fakeOvn *FakeOVN, initialDB libovsdbtest.TestSetup, netInfo userDefinedNetInfo, testNode *corev1.Node, podInfo testPod, pod *corev1.Pod, extraObjects ...runtime.Object) error { +func setupFakeOvnForLayer2Topology(fakeOvn *FakeOVN, initialDB libovsdbtest.TestSetup, netInfo userDefinedNetInfo, testNodes []corev1.Node, podInfo testPod, pod *corev1.Pod, extraObjects ...runtime.Object) error { By(fmt.Sprintf("creating a network attachment definition for network: %s", netInfo.netName)) nad, err := newNetworkAttachmentDefinition( ns, @@ -871,7 +1122,7 @@ func setupFakeOvnForLayer2Topology(fakeOvn *FakeOVN, initialDB libovsdbtest.Test *n, }, }, - &corev1.NodeList{Items: []corev1.Node{*testNode}}, + &corev1.NodeList{Items: testNodes}, &corev1.PodList{ Items: []corev1.Pod{ *pod, @@ -943,6 +1194,9 @@ func setupConfig(netInfo userDefinedNetInfo, testConfig testConfiguration, gatew // tests dont support dualstack yet config.IPv4Mode = false } + if config.OVNKubernetesFeature.EnableInterconnect { + config.Default.Zone = testICZone + } } func notReadyMigrationInfo() *liveMigrationInfo { diff --git a/go-controller/pkg/ovn/layer3_user_defined_network_controller.go b/go-controller/pkg/ovn/layer3_user_defined_network_controller.go index d93d827ce8..24ccf96a5f 100644 --- a/go-controller/pkg/ovn/layer3_user_defined_network_controller.go +++ b/go-controller/pkg/ovn/layer3_user_defined_network_controller.go @@ -140,6 +140,16 @@ func (h *Layer3UserDefinedNetworkControllerEventHandler) AddResource(obj interfa return err } } else { + if config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + if !h.oc.networkManager.NodeHasNetwork(node.Name, h.oc.GetNetworkName()) { + klog.V(5).Infof("Ignoring processing remote node: %s as it has no active NAD for network: %s", + node.Name, h.oc.GetNetworkName()) + // store sync IC failed for the node, so if on node update if the NAD is no longer filtered, we actually + // process it + h.oc.syncZoneICFailed.Store(node.Name, true) + return nil + } + } if err := h.oc.addUpdateRemoteNodeEvent(node, config.OVNKubernetesFeature.EnableInterconnect); err != nil { return err } @@ -211,6 +221,14 @@ func (h *Layer3UserDefinedNetworkControllerEventHandler) UpdateResource(oldObj, return h.oc.addUpdateLocalNodeEvent(newNode, nodeSyncsParam) } else { + if config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + if !h.oc.networkManager.NodeHasNetwork(newNode.Name, h.oc.GetNetworkName()) { + klog.V(5).Infof("Ignoring processing remote node: %s as it has no active NAD for network: %s", + newNode.Name, h.oc.GetNetworkName()) + h.oc.syncZoneICFailed.Store(newNode.Name, true) + return nil + } + } _, syncZoneIC := h.oc.syncZoneICFailed.Load(newNode.Name) // Check if the node moved from local zone to remote zone and if so syncZoneIC should be set to true. @@ -931,26 +949,35 @@ func (oc *Layer3UserDefinedNetworkController) deleteNodeEvent(node *corev1.Node) klog.V(5).Infof("Deleting Node %q for network %s. Removing the node from "+ "various caches", node.Name, oc.GetNetworkName()) - if err := oc.deleteNode(node.Name); err != nil { - return err - } - - if err := oc.gatewayManagerForNode(node.Name).Cleanup(); err != nil { - return fmt.Errorf("failed to cleanup gateway on node %q: %w", node.Name, err) - } - oc.gatewayManagers.Delete(node.Name) - oc.localZoneNodes.Delete(node.Name) - - oc.lsManager.DeleteSwitch(oc.GetNetworkScopedName(node.Name)) - oc.addNodeFailed.Delete(node.Name) - oc.mgmtPortFailed.Delete(node.Name) - oc.nodeClusterRouterPortFailed.Delete(node.Name) - if config.OVNKubernetesFeature.EnableInterconnect { - if err := oc.zoneICHandler.DeleteNode(node); err != nil { + if _, local := oc.localZoneNodes.Load(node.Name); local { + if err := oc.deleteNode(node.Name); err != nil { return err } - oc.syncZoneICFailed.Delete(node.Name) + + if gwObj, ok := oc.gatewayManagers.Load(node.Name); ok { + if gw, ok := gwObj.(*GatewayManager); ok { + if err := gw.Cleanup(); err != nil { + return fmt.Errorf("failed to cleanup gateway on node %q: %w", node.Name, err) + } + } else { + klog.Errorf("Failed to cleanup GW manager for network %q on node %s: could not retrieve GatewayManager", oc.GetNetworkName(), node.Name) + } + oc.gatewayManagers.Delete(node.Name) + } + oc.lsManager.DeleteSwitch(oc.GetNetworkScopedName(node.Name)) + oc.addNodeFailed.Delete(node.Name) + oc.mgmtPortFailed.Delete(node.Name) + oc.nodeClusterRouterPortFailed.Delete(node.Name) + oc.gatewaysFailed.Delete(node.Name) + } else { + if config.OVNKubernetesFeature.EnableInterconnect { + if err := oc.zoneICHandler.DeleteNode(node); err != nil { + return err + } + } } + oc.syncZoneICFailed.Delete(node.Name) + oc.localZoneNodes.Delete(node.Name) oc.syncEIPNodeRerouteFailed.Delete(node.Name) return nil } @@ -969,6 +996,11 @@ func (oc *Layer3UserDefinedNetworkController) deleteNode(nodeName string) error // do not want to delete. func (oc *Layer3UserDefinedNetworkController) syncNodes(nodes []interface{}) error { foundNodes := sets.New[string]() + activeNodes := nodes + dynamicUDN := config.OVNKubernetesFeature.EnableDynamicUDNAllocation + if dynamicUDN { + activeNodes = make([]interface{}, 0, len(nodes)) + } for _, tmp := range nodes { node, ok := tmp.(*corev1.Node) if !ok { @@ -982,6 +1014,20 @@ func (oc *Layer3UserDefinedNetworkController) syncNodes(nodes []interface{}) err if oc.isLocalZoneNode(node) { foundNodes.Insert(node.Name) oc.localZoneNodes.Store(node.Name, true) + if dynamicUDN { + activeNodes = append(activeNodes, node) + } + continue + } + if dynamicUDN { + if oc.nodeHasActiveNetwork(node.Name) { + foundNodes.Insert(node.Name) + activeNodes = append(activeNodes, node) + } else { + if err := oc.deleteNodeEvent(node); err != nil { + return err + } + } } } @@ -1002,14 +1048,18 @@ func (oc *Layer3UserDefinedNetworkController) syncNodes(nodes []interface{}) err } if config.OVNKubernetesFeature.EnableInterconnect { - if err := oc.zoneICHandler.SyncNodes(nodes); err != nil { - return fmt.Errorf("zoneICHandler failed to sync nodes: error: %w", err) + if err := oc.zoneICHandler.CleanupStaleNodes(activeNodes); err != nil { + return fmt.Errorf("zoneICHandler failed to cleanup stale nodes: error: %w", err) } } return nil } +func (oc *Layer3UserDefinedNetworkController) nodeHasActiveNetwork(nodeName string) bool { + return oc.networkManager.NodeHasNetwork(nodeName, oc.GetNetworkName()) +} + func (oc *Layer3UserDefinedNetworkController) gatherJoinSwitchIPs() error { // Allocate IPs for logical router port prefixed with // `GwRouterToJoinSwitchPrefix` for the network managed by this controller. @@ -1119,6 +1169,9 @@ func (oc *Layer3UserDefinedNetworkController) gatewayOptions() []GatewayOption { oc.switchLoadBalancerGroupUUID, )) } + if resolver := oc.getNetworkNameForNADKeyFunc(); resolver != nil { + opts = append(opts, WithNetworkNameForNADKeyResolver(resolver)) + } return opts } @@ -1150,3 +1203,11 @@ func (oc *Layer3UserDefinedNetworkController) StartServiceController(wg *sync.Wa } return nil } + +// HandleNetworkRefChange marks the node for interconnect sync so a queued update does not skip it. +func (oc *Layer3UserDefinedNetworkController) HandleNetworkRefChange(nodeName string, active bool) { + if active { + oc.syncZoneICFailed.Store(nodeName, true) + } + oc.BaseNetworkController.HandleNetworkRefChange(nodeName, active) +} diff --git a/go-controller/pkg/ovn/layer3_user_defined_network_controller_test.go b/go-controller/pkg/ovn/layer3_user_defined_network_controller_test.go index 8f5105c077..ed70df467f 100644 --- a/go-controller/pkg/ovn/layer3_user_defined_network_controller_test.go +++ b/go-controller/pkg/ovn/layer3_user_defined_network_controller_test.go @@ -19,6 +19,7 @@ import ( ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + userdefinednetworkv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" @@ -100,6 +101,9 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 3 network", func() { } } config.Gateway.Mode = gwMode + if config.OVNKubernetesFeature.EnableInterconnect { + config.Default.Zone = testICZone + } if knet.IsIPv6CIDRString(netInfo.clustersubnets) { config.IPv6Mode = true // tests dont support dualstack yet @@ -140,6 +144,14 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 3 network", func() { testNode, err := newNodeWithUserDefinedNetworks(nodeName, nodeIPv4CIDR, netInfo) Expect(err).NotTo(HaveOccurred()) networkPolicy := getMatchLabelsNetworkPolicy(denyPolicyName, ns, "", "", false, false) + nodes := []corev1.Node{*testNode} + if config.OVNKubernetesFeature.EnableDynamicUDNAllocation { + testNode2, err := newNodeWithUserDefinedNetworks("test-node2", "192.168.127.202/24", netInfo) + Expect(err).NotTo(HaveOccurred()) + testNode2.Annotations["k8s.ovn.org/zone-name"] = "blah" + By("adding an extra node that should be ignored by Dynamic UDN Allocation") + nodes = append(nodes, *testNode2) + } fakeOvn.startWithDBSetup( initialDB, &corev1.NamespaceList{ @@ -148,7 +160,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 3 network", func() { }, }, &corev1.NodeList{ - Items: []corev1.Node{*testNode}, + Items: nodes, }, &corev1.PodList{ Items: []corev1.Pod{ @@ -241,7 +253,7 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 3 network", func() { fakeOvn, []testPod{podInfo}, expectationOptions..., - ).expectedLogicalSwitchesAndPorts(netInfo.isPrimary)...))) + ).expectedLogicalSwitchesAndPorts()...))) return nil } @@ -287,6 +299,14 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 3 network", func() { }), config.GatewayModeShared, ), + Entry("with dynamic UDN allocation, a remote node with no NAD is ignored", + dummyPrimaryLayer3UserDefinedNetwork("192.168.0.0/16", "192.168.1.0/24"), + icClusterTestConfiguration(func(config *testConfiguration) { + config.configToOverride.EnableDynamicUDNAllocation = true + config.configToOverride.EnableNetworkSegmentation = true + }), + config.GatewayModeShared, + ), ) DescribeTable( @@ -439,6 +459,327 @@ var _ = Describe("OVN Multi-Homed pod operations for layer 3 network", func() { }), ), ) + Describe("Dynamic UDN allocation with remote node", func() { + It("activates a remote node when a NAD becomes active and cleans it up when inactive", func() { + Expect(config.PrepareTestConfig()).To(Succeed()) + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + config.OVNKubernetesFeature.EnableInterconnect = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.Default.Zone = testICZone + config.Gateway.V4MasqueradeSubnet = "169.254.0.0/16" + + // Basic UDN setup + netInfo := dummyPrimaryLayer3UserDefinedNetwork("192.168.0.0/16", "192.168.1.0/24") + n := newUDNNamespace(ns) + nad, err := newNetworkAttachmentDefinition(ns, nadName, *netInfo.netconf()) + Expect(err).NotTo(HaveOccurred()) + + // Local node and remote node with NAD + localNode, err := newNodeWithUserDefinedNetworks(nodeName, "192.168.126.202/24", netInfo) + Expect(err).NotTo(HaveOccurred()) + localNode.Annotations[util.OvnTransitSwitchPortAddr] = `{"ipv4":"100.88.0.3/16"}` + + remoteNode, err := newNodeWithUserDefinedNetworks("remoteNode", "192.168.127.202/24", netInfo) + Expect(err).NotTo(HaveOccurred()) + remoteNode.Annotations["k8s.ovn.org/zone-name"] = "other-zone" // force remote + remoteNode.Annotations[util.OvnTransitSwitchPortAddr] = `{"ipv4":"100.88.0.4/16"}` + + remotePod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "remote-pod", + Namespace: ns, + }, + Spec: corev1.PodSpec{ + NodeName: remoteNode.Name, + Containers: []corev1.Container{{Name: "c", Image: "scratch"}}, + }, + } + + localPod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-pod", + Namespace: ns, + }, + Spec: corev1.PodSpec{ + NodeName: localNode.Name, + Containers: []corev1.Container{{Name: "c", Image: "scratch"}}, + }, + } + + fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{}, &corev1.NamespaceList{Items: []corev1.Namespace{*n}}, + &corev1.NodeList{Items: []corev1.Node{*localNode, *remoteNode}}, + &corev1.PodList{Items: []corev1.Pod{localPod}}, + &nadapi.NetworkAttachmentDefinitionList{Items: []nadapi.NetworkAttachmentDefinition{*nad}}) + + Expect(fakeOvn.networkManager.Start()).To(Succeed()) + defer fakeOvn.networkManager.Stop() + + userDefinedNetController, ok := fakeOvn.userDefinedNetworkControllers[userDefinedNetworkName] + Expect(ok).To(BeTrue()) + userDefinedNetController.bnc.ovnClusterLRPToJoinIfAddrs = dummyJoinIPs() + l3Controller, ok := fakeOvn.fullL3UDNControllers[netInfo.netName] + Expect(ok).To(BeTrue()) + mutableNetInfo := util.NewMutableNetInfo(l3Controller.GetNetInfo()) + mutableNetInfo.SetNetworkID(2) + err = util.ReconcileNetInfo(l3Controller.ReconcilableNetInfo, mutableNetInfo) + Expect(err).NotTo(HaveOccurred()) + err = l3Controller.init() + Expect(err).NotTo(HaveOccurred()) + Expect(userDefinedNetController.bnc.WatchNodes()).To(Succeed()) + + By("Remote node should not have a port on transit subnet before activation") + Consistently(func() bool { + p := func(item *nbdb.LogicalSwitchPort) bool { + return item.ExternalIDs["node"] == remoteNode.Name + } + portList, err := libovsdbops.FindLogicalSwitchPortWithPredicate(fakeOvn.nbClient, p) + return err == nil && len(portList) > 0 + }).WithTimeout(500 * time.Millisecond).Should(BeFalse()) + + By("Creating a pod on the remote node should activate it") + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(ns).Create(context.TODO(), &remotePod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + return fakeOvn.networkManager.Interface().NodeHasNetwork(remoteNode.Name, netInfo.netName) + }).WithTimeout(3 * time.Second).Should(BeTrue()) + By("Triggering networkRefChange callback after updating remote node as active on NAD") + l3Controller.HandleNetworkRefChange(remoteNode.Name, true) + + By("Remote node should have a port created on transit subnet") + Eventually(func() bool { + p := func(item *nbdb.LogicalSwitchPort) bool { + return item.ExternalIDs["node"] == remoteNode.Name + } + portList, err := libovsdbops.FindLogicalSwitchPortWithPredicate(fakeOvn.nbClient, p) + if err == nil && len(portList) > 0 { + return true + } + return false + }).Should(BeTrue()) + + By("Deleting a pod on the remote node should set it as inactive") + err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(ns).Delete(context.TODO(), remotePod.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + return fakeOvn.networkManager.Interface().NodeHasNetwork(remoteNode.Name, netInfo.netName) + }).WithTimeout(3 * time.Second).Should(BeFalse()) + By("Triggering networkRefChange callback after updating remote node as inactive on NAD") + l3Controller.HandleNetworkRefChange(remoteNode.Name, false) + By("Remote node should not have a port on transit subnet") + Eventually(func() bool { + p := func(item *nbdb.LogicalSwitchPort) bool { + return item.ExternalIDs["node"] == remoteNode.Name + } + portList, err := libovsdbops.FindLogicalSwitchPortWithPredicate(fakeOvn.nbClient, p) + if err == nil && len(portList) > 0 { + return true + } + return false + }).WithTimeout(3 * time.Second).Should(BeFalse()) + + By("Verifying core gateway router and cluster router are intact after remote node removal") + hasPort := func(ports []string, target string) bool { + for _, port := range ports { + if port == target { + return true + } + } + return false + } + hasRouterPorts := func(routerName string, portNames ...string) bool { + routers, err := libovsdbops.FindLogicalRoutersWithPredicate(fakeOvn.nbClient, func(router *nbdb.LogicalRouter) bool { + return router.Name == routerName + }) + if err != nil || len(routers) == 0 { + return false + } + router := routers[0] + for _, portName := range portNames { + ports, err := libovsdbops.FindLogicalRouterPortWithPredicate(fakeOvn.nbClient, func(port *nbdb.LogicalRouterPort) bool { + return port.Name == portName + }) + if err != nil || len(ports) == 0 { + return false + } + if !hasPort(router.Ports, ports[0].UUID) { + return false + } + } + return true + } + hasSwitchPorts := func(switchName string, portNames ...string) bool { + switches, err := libovsdbops.FindLogicalSwitchesWithPredicate(fakeOvn.nbClient, func(sw *nbdb.LogicalSwitch) bool { + return sw.Name == switchName + }) + if err != nil || len(switches) == 0 { + return false + } + sw := switches[0] + for _, portName := range portNames { + ports, err := libovsdbops.FindLogicalSwitchPortWithPredicate(fakeOvn.nbClient, func(port *nbdb.LogicalSwitchPort) bool { + return port.Name == portName + }) + if err != nil || len(ports) == 0 { + return false + } + if !hasPort(sw.Ports, ports[0].UUID) { + return false + } + } + return true + } + Eventually(func() bool { + return hasRouterPorts( + "GR_isolatednet_test-node", + "rtoj-GR_isolatednet_test-node", + "rtoe-GR_isolatednet_test-node", + ) && + hasRouterPorts( + "isolatednet_ovn_cluster_router", + "rtoj-isolatednet_ovn_cluster_router", + "rtos-isolatednet_test-node", + "isolatednet_rtots-test-node", + ) && + hasSwitchPorts( + "isolatednet_join", + "jtor-GR_isolatednet_test-node", + "jtor-isolatednet_ovn_cluster_router", + ) && + hasSwitchPorts( + "isolatednet_transit_switch", + "isolatednet_tstor-test-node", + ) + }).WithTimeout(3 * time.Second).Should(BeTrue()) + }) + + It("activates a remote node when a CUDN NAD becomes active in another namespace", func() { + Expect(config.PrepareTestConfig()).To(Succeed()) + config.OVNKubernetesFeature.EnableDynamicUDNAllocation = true + config.OVNKubernetesFeature.EnableInterconnect = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.Default.Zone = testICZone + config.Gateway.V4MasqueradeSubnet = "169.254.0.0/16" + + const ( + cudnName = "cudn-shared" + nsA = "namespace-a" + nsB = "namespace-b" + nadAName = "cudn-nad-a" + nadBName = "cudn-nad-b" + remoteName = "remoteNode" + ) + + netName := util.GenerateCUDNNetworkName(cudnName) + netInfoA := userDefinedNetInfo{ + netName: netName, + nadName: namespacedName(nsA, nadAName), + topology: types.Layer3Topology, + clustersubnets: "192.168.0.0/16", + hostsubnets: "192.168.1.0/24", + isPrimary: true, + } + netInfoB := userDefinedNetInfo{ + netName: netName, + nadName: namespacedName(nsB, nadBName), + topology: types.Layer3Topology, + clustersubnets: "192.168.0.0/16", + hostsubnets: "192.168.2.0/24", + isPrimary: true, + } + + nsObjA := newUDNNamespace(nsA) + nsObjB := newUDNNamespace(nsB) + nadA, err := newNetworkAttachmentDefinition(nsA, nadAName, *netInfoA.netconf()) + Expect(err).NotTo(HaveOccurred()) + nadA.OwnerReferences = []metav1.OwnerReference{makeCUDNOwnerRef(cudnName)} + nadB, err := newNetworkAttachmentDefinition(nsB, nadBName, *netInfoB.netconf()) + Expect(err).NotTo(HaveOccurred()) + nadB.OwnerReferences = []metav1.OwnerReference{makeCUDNOwnerRef(cudnName)} + + localNode, err := newNodeWithUserDefinedNetworks(nodeName, "192.168.126.202/24", netInfoA) + Expect(err).NotTo(HaveOccurred()) + localNode.Annotations[util.OvnTransitSwitchPortAddr] = `{"ipv4":"100.88.0.3/16"}` + + remoteNode, err := newNodeWithUserDefinedNetworks(remoteName, "192.168.127.202/24", netInfoB) + Expect(err).NotTo(HaveOccurred()) + remoteNode.Annotations["k8s.ovn.org/zone-name"] = "other-zone" + remoteNode.Annotations[util.OvnTransitSwitchPortAddr] = `{"ipv4":"100.88.0.4/16"}` + + localPod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-pod", + Namespace: nsA, + }, + Spec: corev1.PodSpec{ + NodeName: localNode.Name, + Containers: []corev1.Container{{Name: "c", Image: "scratch"}}, + }, + } + + remotePod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "remote-pod", + Namespace: nsB, + }, + Spec: corev1.PodSpec{ + NodeName: remoteNode.Name, + Containers: []corev1.Container{{Name: "c", Image: "scratch"}}, + }, + } + + fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{}, &corev1.NamespaceList{Items: []corev1.Namespace{*nsObjA, *nsObjB}}, + &corev1.NodeList{Items: []corev1.Node{*localNode, *remoteNode}}, + &corev1.PodList{Items: []corev1.Pod{localPod}}, + &nadapi.NetworkAttachmentDefinitionList{Items: []nadapi.NetworkAttachmentDefinition{*nadA, *nadB}}, + ) + + Expect(fakeOvn.networkManager.Start()).To(Succeed()) + defer fakeOvn.networkManager.Stop() + + userDefinedNetController, ok := fakeOvn.userDefinedNetworkControllers[netName] + Expect(ok).To(BeTrue()) + userDefinedNetController.bnc.ovnClusterLRPToJoinIfAddrs = dummyJoinIPs() + l3Controller, ok := fakeOvn.fullL3UDNControllers[netName] + Expect(ok).To(BeTrue()) + + mutableNetInfo := util.NewMutableNetInfo(l3Controller.GetNetInfo()) + mutableNetInfo.SetNetworkID(2) + err = util.ReconcileNetInfo(l3Controller.ReconcilableNetInfo, mutableNetInfo) + Expect(err).NotTo(HaveOccurred()) + Expect(l3Controller.init()).To(Succeed()) + Expect(userDefinedNetController.bnc.WatchNodes()).To(Succeed()) + + By("Remote node should not have a port on transit subnet before activation") + Consistently(func() bool { + p := func(item *nbdb.LogicalSwitchPort) bool { + return item.ExternalIDs["node"] == remoteNode.Name + } + portList, err := libovsdbops.FindLogicalSwitchPortWithPredicate(fakeOvn.nbClient, p) + return err == nil && len(portList) > 0 + }).WithTimeout(500 * time.Millisecond).Should(BeFalse()) + + By("Creating a pod on the remote node in another namespace should activate it") + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(nsB).Create(context.TODO(), &remotePod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + return fakeOvn.networkManager.Interface().NodeHasNetwork(remoteNode.Name, netName) + }).WithTimeout(3 * time.Second).Should(BeTrue()) + By("Triggering networkRefChange callback after updating remote node as active on NAD") + l3Controller.HandleNetworkRefChange(remoteNode.Name, true) + + By("Remote node should have a port created on transit subnet") + Eventually(func() bool { + p := func(item *nbdb.LogicalSwitchPort) bool { + return item.ExternalIDs["node"] == remoteNode.Name + } + portList, err := libovsdbops.FindLogicalSwitchPortWithPredicate(fakeOvn.nbClient, p) + return err == nil && len(portList) > 0 + }).Should(BeTrue()) + }) + }) + }) func newPodWithPrimaryUDN( @@ -490,8 +831,14 @@ func newPodWithPrimaryUDN( func namespacedName(ns, name string) string { return fmt.Sprintf("%s/%s", ns, name) } -func (sni *userDefinedNetInfo) getNetworkRole() string { - return util.GetUserDefinedNetworkRole(sni.isPrimary) +func makeCUDNOwnerRef(name string) metav1.OwnerReference { + controller := true + return metav1.OwnerReference{ + APIVersion: userdefinednetworkv1.SchemeGroupVersion.String(), + Kind: "ClusterUserDefinedNetwork", + Name: name, + Controller: &controller, + } } func getNetworkRole(netInfo util.NetInfo) string { @@ -504,10 +851,7 @@ func (sni *userDefinedNetInfo) setupOVNDependencies(dbData *libovsdbtest.TestSet return err } - externalIDs := map[string]string{ - types.NetworkExternalID: sni.netName, - types.NetworkRoleExternalID: sni.getNetworkRole(), - } + externalIDs := util.GenerateExternalIDsForSwitchOrRouter(netInfo) switch sni.topology { case types.Layer2Topology: dbData.NBData = append(dbData.NBData, &nbdb.LogicalSwitch{ @@ -644,18 +988,24 @@ func newNodeWithUserDefinedNetworks(nodeName string, nodeIPv4CIDR string, netInf nextHopIP := util.GetNodeGatewayIfAddr(nodeCIDR).IP nodeCIDR.IP = nodeIP + zone := types.OvnDefaultZone + if config.OVNKubernetesFeature.EnableInterconnect { + zone = testICZone + } + return &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: nodeName, Annotations: map[string]string{ - "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", nodeIPv4CIDR, ""), - "k8s.ovn.org/node-subnets": parsedNodeSubnets, - util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", nodeIPv4CIDR), - "k8s.ovn.org/zone-name": "global", - "k8s.ovn.org/l3-gateway-config": fmt.Sprintf("{\"default\":{\"mode\":\"shared\",\"bridge-id\":\"breth0\",\"interface-id\":\"breth0_ovn-worker\",\"mac-address\":%q,\"ip-addresses\":[%[2]q],\"ip-address\":%[2]q,\"next-hops\":[%[3]q],\"next-hop\":%[3]q,\"node-port-enable\":\"true\",\"vlan-id\":\"0\"}}", util.IPAddrToHWAddr(nodeIP), nodeCIDR, nextHopIP), - util.OvnNodeChassisID: "abdcef", - "k8s.ovn.org/network-ids": fmt.Sprintf("{\"default\":\"0\",\"isolatednet\":\"%s\"}", userDefinedNetworkID), - util.OvnNodeID: "4", + util.Layer2TopologyVersion: util.TransitRouterTopoVersion, + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", nodeIPv4CIDR, ""), + "k8s.ovn.org/node-subnets": parsedNodeSubnets, + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", nodeIPv4CIDR), + "k8s.ovn.org/zone-name": zone, + "k8s.ovn.org/l3-gateway-config": fmt.Sprintf("{\"default\":{\"mode\":\"shared\",\"bridge-id\":\"breth0\",\"interface-id\":\"breth0_ovn-worker\",\"mac-address\":%q,\"ip-addresses\":[%[2]q],\"ip-address\":%[2]q,\"next-hops\":[%[3]q],\"next-hop\":%[3]q,\"node-port-enable\":\"true\",\"vlan-id\":\"0\"}}", util.IPAddrToHWAddr(nodeIP), nodeCIDR, nextHopIP), + util.OvnNodeChassisID: chassisIDForNode(nodeName), + "k8s.ovn.org/network-ids": fmt.Sprintf("{\"default\":\"0\",\"isolatednet\":\"%s\"}", userDefinedNetworkID), + util.OvnNodeID: "4", "k8s.ovn.org/udn-layer2-node-gateway-router-lrp-tunnel-ids": "{\"isolatednet\":\"25\"}", }, Labels: map[string]string{ diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 3fe803660e..d969e73d17 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -276,17 +276,17 @@ func (oc *DefaultNetworkController) syncNodesPeriodic() { return } - localZoneNodeNames := make([]string, 0, len(kNodes)) - remoteZoneNodeNames := make([]string, 0, len(kNodes)) + localZoneNodes := make([]*corev1.Node, 0, len(kNodes)) + remoteZoneNodes := make([]*corev1.Node, 0, len(kNodes)) for i := range kNodes { if oc.isLocalZoneNode(kNodes[i]) { - localZoneNodeNames = append(localZoneNodeNames, kNodes[i].Name) + localZoneNodes = append(localZoneNodes, kNodes[i]) } else { - remoteZoneNodeNames = append(remoteZoneNodeNames, kNodes[i].Name) + remoteZoneNodes = append(remoteZoneNodes, kNodes[i]) } } - if err := oc.syncChassis(localZoneNodeNames, remoteZoneNodeNames); err != nil { + if err := oc.syncChassis(localZoneNodes, remoteZoneNodes); err != nil { klog.Errorf("Failed to sync chassis: error: %v", err) } } @@ -297,8 +297,8 @@ func (oc *DefaultNetworkController) syncNodesPeriodic() { // do not want to delete. func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { foundNodes := sets.New[string]() - localZoneNodeNames := make([]string, 0, len(kNodes)) - remoteZoneKNodeNames := make([]string, 0, len(kNodes)) + localZoneNodes := make([]*corev1.Node, 0, len(kNodes)) + remoteZoneNodes := make([]*corev1.Node, 0, len(kNodes)) for _, tmp := range kNodes { node, ok := tmp.(*corev1.Node) if !ok { @@ -313,9 +313,9 @@ func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { if oc.isLocalZoneNode(node) { foundNodes.Insert(node.Name) oc.localZoneNodes.Store(node.Name, true) - localZoneNodeNames = append(localZoneNodeNames, node.Name) + localZoneNodes = append(localZoneNodes, node) } else { - remoteZoneKNodeNames = append(remoteZoneKNodeNames, node.Name) + remoteZoneNodes = append(remoteZoneNodes, node) } } @@ -359,7 +359,7 @@ func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { if ok { return false } - nodeName := strings.TrimPrefix(item.Name, types.GWRouterPrefix) + nodeName := util.GetWorkerFromGatewayRouter(item.Name) if nodeName != item.Name && len(nodeName) > 0 && !foundNodes.Has(nodeName) { staleSwitches.Insert(nodeName) return true @@ -378,17 +378,28 @@ func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { } } - if err := oc.syncChassis(localZoneNodeNames, remoteZoneKNodeNames); err != nil { + if err := oc.syncChassis(localZoneNodes, remoteZoneNodes); err != nil { return fmt.Errorf("failed to sync chassis: error: %v", err) } if config.OVNKubernetesFeature.EnableInterconnect { + // Chassis cleanup should happen regardless of transport mode to cleanup + // any stale remote chassis entries (e.g., from overlay->no-overlay migration) if err := oc.zoneChassisHandler.SyncNodes(kNodes); err != nil { return fmt.Errorf("zoneChassisHandler failed to sync nodes: error: %w", err) } - if err := oc.zoneICHandler.SyncNodes(kNodes); err != nil { - return fmt.Errorf("zoneICHandler failed to sync nodes: error: %w", err) + // Interconnect resource sync depends on transport mode: + // - For overlay: ensure transit switch exists and cleanup stale resources + // - For no-overlay: cleanup all interconnect resources (nodes and transit switch) + if oc.Transport() == types.NetworkTransportNoOverlay { + if err := oc.zoneICHandler.Cleanup(); err != nil { + return fmt.Errorf("zoneICHandler failed to cleanup: error: %w", err) + } + } else { + if err := oc.zoneICHandler.CleanupStaleNodes(kNodes); err != nil { + return fmt.Errorf("zoneICHandler failed to cleanup stale nodes: error: %w", err) + } } } @@ -397,7 +408,7 @@ func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { // Cleanup stale chassis and chassis template variables with no // corresponding nodes. -func (oc *DefaultNetworkController) syncChassis(localZoneNodeNames, remoteZoneNodeNames []string) error { +func (oc *DefaultNetworkController) syncChassis(localZoneNodes, remoteZoneNodes []*corev1.Node) error { chassisList, err := libovsdbops.ListChassis(oc.sbClient) if err != nil { return fmt.Errorf("failed to get chassis list: error: %v", err) @@ -418,10 +429,8 @@ func (oc *DefaultNetworkController) syncChassis(localZoneNodeNames, remoteZoneNo } } - chassisHostNameMap := map[string]*sbdb.Chassis{} chassisNameMap := map[string]*sbdb.Chassis{} for _, chassis := range chassisList { - chassisHostNameMap[chassis.Hostname] = chassis chassisNameMap[chassis.Name] = chassis } @@ -443,26 +452,33 @@ func (oc *DefaultNetworkController) syncChassis(localZoneNodeNames, remoteZoneNo // Delete existing nodes from the chassis map. // Also delete existing templateVars from the template map. - for _, nodeName := range localZoneNodeNames { - if chassis, ok := chassisHostNameMap[nodeName]; ok { - delete(chassisNameMap, chassis.Name) - delete(chassisHostNameMap, chassis.Hostname) - delete(templateChassisMap, chassis.Name) + for _, node := range localZoneNodes { + chassisID, err := util.ParseNodeChassisIDAnnotation(node) + if err != nil { + klog.Warningf("Unable to parse local node %s chassis-id annotation. Chassis may be removed during sync", + node.Name) + continue } + delete(chassisNameMap, chassisID) + delete(templateChassisMap, chassisID) } // Delete existing remote zone nodes from the chassis map, but not from the templateVars // as we need to cleanup chassisTemplateVars for the remote zone nodes - for _, nodeName := range remoteZoneNodeNames { - if chassis, ok := chassisHostNameMap[nodeName]; ok { - delete(chassisNameMap, chassis.Name) - delete(chassisHostNameMap, chassis.Hostname) + for _, node := range remoteZoneNodes { + chassisID, err := util.ParseNodeChassisIDAnnotation(node) + if err != nil { + klog.Warningf("Unable to parse remote node %s chassis-id annotation. Chassis may be removed during sync", + node.Name) + continue } + delete(chassisNameMap, chassisID) } - staleChassis := make([]*sbdb.Chassis, 0, len(chassisHostNameMap)) - for _, chassis := range chassisNameMap { + staleChassis := make([]*sbdb.Chassis, 0, len(chassisNameMap)) + for name, chassis := range chassisNameMap { staleChassis = append(staleChassis, chassis) + klog.Infof("Removing stale chassis with ID/Name: %s, hostname: %s", name, chassis.Hostname) } staleChassisTemplateVars := make([]*nbdb.ChassisTemplateVar, 0, len(templateChassisMap)) @@ -471,11 +487,11 @@ func (oc *DefaultNetworkController) syncChassis(localZoneNodeNames, remoteZoneNo } if err := libovsdbops.DeleteChassis(oc.sbClient, staleChassis...); err != nil { - return fmt.Errorf("failed Deleting chassis %v error: %v", chassisHostNameMap, err) + return fmt.Errorf("failed Deleting chassis %#v error: %v", chassisNameMap, err) } if err := libovsdbops.DeleteChassisTemplateVar(oc.nbClient, staleChassisTemplateVars...); err != nil { - return fmt.Errorf("failed Deleting chassis template vars %v error: %v", chassisHostNameMap, err) + return fmt.Errorf("failed Deleting chassis template vars %#v error: %v", staleChassisTemplateVars, err) } return nil @@ -638,21 +654,33 @@ func (oc *DefaultNetworkController) addUpdateLocalNodeEvent(node *corev1.Node, n } if nSyncs.syncZoneIC && config.OVNKubernetesFeature.EnableInterconnect { - // Call zone chassis handler's AddLocalZoneNode function to mark + // Always call zone chassis handler's AddLocalZoneNode function to mark // this node's chassis record in Southbound db as a local zone chassis. - // This is required when a node moves from a remote zone to local zone + // This is required even when the default network uses no-overlay transport, + // because user-defined networks may still use overlay transport and require + // the chassis entries for their transit switch connectivity. + chassisFailed := false if err := oc.zoneChassisHandler.AddLocalZoneNode(node); err != nil { errs = append(errs, err) oc.syncZoneICFailed.Store(node.Name, true) - } else { + chassisFailed = true + } + + // For no-overlay transport, the default network's interconnect resources are not needed. + // The transit switch and its resources are cleaned up during sync, so we only need + // to create IC resources for overlay transport. + if oc.Transport() != types.NetworkTransportNoOverlay { // Call zone IC handler's AddLocalZoneNode function to create // interconnect resources in the OVN Northbound db for this local zone node. if err := oc.zoneICHandler.AddLocalZoneNode(node); err != nil { errs = append(errs, err) oc.syncZoneICFailed.Store(node.Name, true) - } else { + } else if !chassisFailed { oc.syncZoneICFailed.Delete(node.Name) } + } else if !chassisFailed { + // In no-overlay mode, if chassis handler succeeded, clear the failed state + oc.syncZoneICFailed.Delete(node.Name) } } @@ -680,25 +708,34 @@ func (oc *DefaultNetworkController) addUpdateRemoteNodeEvent(node *corev1.Node, var err error if syncZoneIC && config.OVNKubernetesFeature.EnableInterconnect { - // Call zone chassis handler's AddRemoteZoneNode function to creates - // the remote chassis for the remote zone node in the SB DB or mark - // the entry as remote if it was local chassis earlier + // Always create remote chassis entry with geneve encapsulation. + // This is needed even when the default network uses no-overlay transport, + // because user-defined networks may still use overlay transport and require + // the remote chassis entries for their transit switch connectivity. if err = oc.zoneChassisHandler.AddRemoteZoneNode(node); err != nil { err = fmt.Errorf("adding or updating remote node chassis %s failed, err - %w", node.Name, err) oc.syncZoneICFailed.Store(node.Name, true) return err } - // Call zone IC handler's AddRemoteZoneNode function to create - // interconnect resources in the OVN NBDB for this remote zone node. - // Also, create the remote port binding in SBDB - if err = oc.zoneICHandler.AddRemoteZoneNode(node); err != nil { - err = fmt.Errorf("adding or updating remote node IC resources %s failed, err - %w", node.Name, err) - oc.syncZoneICFailed.Store(node.Name, true) + // For no-overlay transport, the default network's interconnect resources are not needed. + // The transit switch and its resources are cleaned up during sync, so we only need + // to create IC resources for overlay transport. + if oc.Transport() != types.NetworkTransportNoOverlay { + // Call zone IC handler's AddRemoteZoneNode function to create + // interconnect resources in the OVN NBDB for this remote zone node. + // Also, create the remote port binding in SBDB + if err = oc.zoneICHandler.AddRemoteZoneNode(node); err != nil { + err = fmt.Errorf("adding or updating remote node IC resources %s failed, err - %w", node.Name, err) + oc.syncZoneICFailed.Store(node.Name, true) + } else { + oc.syncZoneICFailed.Delete(node.Name) + } + klog.V(5).Infof("Creating Interconnect resources for remote node %q on network %q took: %s", node.Name, oc.GetNetworkName(), time.Since(start)) } else { + // In no-overlay mode, if chassis handler succeeded, clear the failed state oc.syncZoneICFailed.Delete(node.Name) } - klog.V(5).Infof("Creating Interconnect resources for remote node %q on network %q took: %s", node.Name, oc.GetNetworkName(), time.Since(start)) } return err } diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index 8de1687ee3..f040f1a8bb 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -1731,7 +1731,7 @@ var _ = ginkgo.Describe("Default network controller operations", func() { Name: "newNode", Annotations: map[string]string{ "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\", \"fd02:0:0:2::2895/64\"]}", newNodeSubnet), - "k8s.ovn.org/node-chassis-id": "2", + "k8s.ovn.org/node-chassis-id": chassisIDForNode("newNode"), util.OvnNodeID: "2", }, }, @@ -1793,7 +1793,7 @@ var _ = ginkgo.Describe("Default network controller operations", func() { Name: "newNode", Annotations: map[string]string{ "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\"]}", newNodeIpv4Subnet), - "k8s.ovn.org/node-chassis-id": "2", + "k8s.ovn.org/node-chassis-id": chassisIDForNode("newNode"), "k8s.ovn.org/node-gateway-router-lrp-ifaddr": "{\"ipv4\":\"100.64.0.2/16\"}", }, }, @@ -1906,7 +1906,7 @@ var _ = ginkgo.Describe("Default network controller operations", func() { newNodeSubnet := "10.1.2.0/24" transitSwitchSubnet := "100.88.0.3/16" testNode.Annotations["k8s.ovn.org/node-subnets"] = fmt.Sprintf("{\"default\":[\"%s\"]}", newNodeSubnet) - testNode.Annotations["k8s.ovn.org/node-chassis-id"] = "2" + testNode.Annotations["k8s.ovn.org/node-chassis-id"] = chassisIDForNode(testNode.Name) testNode.Annotations["k8s.ovn.org/node-transit-switch-port-ifaddr"] = fmt.Sprintf("{\"ipv4\":\"%s\"}", transitSwitchSubnet) testNode.Annotations["k8s.ovn.org/zone-name"] = "foo" updatedNode, err := fakeOvn.fakeClient.KubeClient.CoreV1().Nodes().Create(context.TODO(), &testNode, metav1.CreateOptions{}) @@ -2135,15 +2135,15 @@ func TestController_syncNodes(t *testing.T) { { name: "removes stale chassis and chassis private", initialSBDB: []libovsdbtest.TestData{ - &sbdb.Chassis{Name: "chassis-node1", Hostname: node1Name}, - &sbdb.ChassisPrivate{Name: "chassis-node1"}, - &sbdb.Chassis{Name: "chassis-node2", Hostname: nodeRmName}, - &sbdb.ChassisPrivate{Name: "chassis-node2"}, - &sbdb.ChassisPrivate{Name: "chassis-node3"}, + &sbdb.Chassis{Name: chassisIDForNode(node1Name), Hostname: node1Name}, + &sbdb.ChassisPrivate{Name: chassisIDForNode(node1Name)}, + &sbdb.Chassis{Name: chassisIDForNode(nodeRmName), Hostname: nodeRmName}, + &sbdb.ChassisPrivate{Name: chassisIDForNode(nodeRmName)}, + &sbdb.ChassisPrivate{Name: chassisIDForNode("node3")}, }, expectedSBDB: []libovsdbtest.TestData{ - &sbdb.Chassis{Name: "chassis-node1", Hostname: node1Name}, - &sbdb.ChassisPrivate{Name: "chassis-node1"}, + &sbdb.Chassis{Name: chassisIDForNode(node1Name), Hostname: node1Name}, + &sbdb.ChassisPrivate{Name: chassisIDForNode(node1Name)}, }, }, } @@ -2159,6 +2159,9 @@ func TestController_syncNodes(t *testing.T) { testNode := corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node1", + Annotations: map[string]string{ + "k8s.ovn.org/node-chassis-id": chassisIDForNode(node1Name), + }, }, } @@ -2243,20 +2246,20 @@ func TestController_deleteStaleNodeChassis(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node1", Annotations: map[string]string{ - "k8s.ovn.org/node-chassis-id": "chassis-node1-dpu", + "k8s.ovn.org/node-chassis-id": chassisIDForNode("node1-dpu"), }, }, }, name: "removes stale chassis when ovn running on DPU", initialSBDB: []libovsdbtest.TestData{ - &sbdb.Chassis{Name: "chassis-node1-dpu", Hostname: "node1"}, - &sbdb.ChassisPrivate{Name: "chassis-node1-dpu"}, - &sbdb.Chassis{Name: "chassis-node1", Hostname: "node1"}, - &sbdb.ChassisPrivate{Name: "chassis-node1"}, + &sbdb.Chassis{Name: chassisIDForNode("node1-dpu"), Hostname: "node1"}, + &sbdb.ChassisPrivate{Name: chassisIDForNode("node1-dpu")}, + &sbdb.Chassis{Name: chassisIDForNode("node1"), Hostname: "node1"}, + &sbdb.ChassisPrivate{Name: chassisIDForNode("node1")}, }, expectedSBDB: []libovsdbtest.TestData{ - &sbdb.Chassis{Name: "chassis-node1-dpu", Hostname: "node1"}, - &sbdb.ChassisPrivate{Name: "chassis-node1-dpu"}, + &sbdb.Chassis{Name: chassisIDForNode("node1-dpu"), Hostname: "node1"}, + &sbdb.ChassisPrivate{Name: chassisIDForNode("node1-dpu")}, }, }, } diff --git a/go-controller/pkg/ovn/multicast_test.go b/go-controller/pkg/ovn/multicast_test.go index d40cc618e4..b951fc0957 100644 --- a/go-controller/pkg/ovn/multicast_test.go +++ b/go-controller/pkg/ovn/multicast_test.go @@ -264,7 +264,7 @@ func newNodeWithNad(nad *nadapi.NetworkAttachmentDefinition, networkName, networ n.Annotations["k8s.ovn.org/node-subnets"] = fmt.Sprintf("{\"default\":\"192.168.126.202/24\", \"%s\":\"192.168.127.202/24\"}", networkName) n.Annotations["k8s.ovn.org/network-ids"] = fmt.Sprintf("{\"default\":\"0\",\"%s\":\"%s\"}", networkName, networkID) n.Annotations["k8s.ovn.org/node-mgmt-port-mac-addresses"] = fmt.Sprintf("{\"default\":\"96:8f:e8:25:a2:e5\",\"%s\":\"d6:bc:85:32:30:fb\"}", networkName) - n.Annotations["k8s.ovn.org/node-chassis-id"] = "abdcef" + n.Annotations["k8s.ovn.org/node-chassis-id"] = chassisIDForNode(n.Name) n.Annotations["k8s.ovn.org/l3-gateway-config"] = "{\"default\":{\"mac-address\":\"52:54:00:e2:ed:d0\",\"ip-addresses\":[\"10.1.1.10/24\"],\"ip-address\":\"10.1.1.10/24\",\"next-hops\":[\"10.1.1.1\"],\"next-hop\":\"10.1.1.1\"}}" n.Annotations[util.OvnNodeID] = "4" } @@ -761,7 +761,11 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { ports = append(ports, tPod.portUUID) } expectedData := getMulticastPolicyExpectedData(netInfo, namespace1.Name, ports) - expectedData = append(expectedData, getExpectedPodsAndSwitches(bnc.GetNetInfo(), tPods, []string{nodeName})...) + nadKey := "" + if nad != nil { + nadKey = util.GetNADName(nad.Namespace, nad.Name) + } + expectedData = append(expectedData, getExpectedPodsAndSwitches(bnc.GetNetInfo(), tPods, []string{nodeName}, nadKey)...) Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) asf.ExpectAddressSetWithAddresses(namespace1.Name, tPodIPs) return nil @@ -840,7 +844,11 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { Expect(acl.Name).To(BeNil()) Expect(acl.ExternalIDs[libovsdbops.ObjectNameKey.String()]).To(Equal(longNameSpace2Name)) } - expectedData = append(expectedData, getExpectedPodsAndSwitches(bnc.GetNetInfo(), []testPod{}, []string{node.Name})...) + nadKey := "" + if nad != nil { + nadKey = util.GetNADName(nad.Namespace, nad.Name) + } + expectedData = append(expectedData, getExpectedPodsAndSwitches(bnc.GetNetInfo(), []testPod{}, []string{node.Name}, nadKey)...) // Enable multicast in the namespace. updateMulticast(fakeOvn, ns1, true) updateMulticast(fakeOvn, ns2, true) @@ -929,7 +937,11 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { // Check pods were added asf.EventuallyExpectAddressSetWithAddresses(namespace1.Name, tPodIPs) expectedDataWithPods := getMulticastPolicyExpectedData(netInfo, namespace1.Name, ports) - expectedDataWithPods = append(expectedDataWithPods, getExpectedPodsAndSwitches(bnc, tPods, []string{nodeName})...) + nadKey := "" + if nad != nil { + nadKey = util.GetNADName(nad.Namespace, nad.Name) + } + expectedDataWithPods = append(expectedDataWithPods, getExpectedPodsAndSwitches(bnc.GetNetInfo(), tPods, []string{nodeName}, nadKey)...) Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedDataWithPods...)) // Delete the pod from the namespace. diff --git a/go-controller/pkg/ovn/multihoming_test.go b/go-controller/pkg/ovn/multihoming_test.go index e41593dd77..3743df4e03 100644 --- a/go-controller/pkg/ovn/multihoming_test.go +++ b/go-controller/pkg/ovn/multihoming_test.go @@ -118,11 +118,11 @@ func withClusterPortGroup() option { } } -func (em *userDefinedNetworkExpectationMachine) expectedLogicalSwitchesAndPorts(isPrimary bool) []libovsdbtest.TestData { - return em.expectedLogicalSwitchesAndPortsWithLspEnabled(isPrimary, nil) +func (em *userDefinedNetworkExpectationMachine) expectedLogicalSwitchesAndPorts() []libovsdbtest.TestData { + return em.expectedLogicalSwitchesAndPortsWithLspEnabled(nil) } -func (em *userDefinedNetworkExpectationMachine) expectedLogicalSwitchesAndPortsWithLspEnabled(isPrimary bool, expectedPodLspEnabled map[string]*bool) []libovsdbtest.TestData { +func (em *userDefinedNetworkExpectationMachine) expectedLogicalSwitchesAndPortsWithLspEnabled(expectedPodLspEnabled map[string]*bool) []libovsdbtest.TestData { data := []libovsdbtest.TestData{} for _, ocInfo := range em.fakeOvn.userDefinedNetworkControllers { nodeslsps := make(map[string][]string) @@ -260,10 +260,8 @@ func (em *userDefinedNetworkExpectationMachine) expectedLogicalSwitchesAndPortsW UUID: switchName + "-UUID", Name: switchName, Ports: nodeslsps[switchName], - ExternalIDs: map[string]string{ - ovntypes.NetworkExternalID: ocInfo.bnc.GetNetworkName(), - ovntypes.NetworkRoleExternalID: util.GetUserDefinedNetworkRole(isPrimary), - }, + + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(ocInfo.bnc), OtherConfig: otherConfig, ACLs: acls[switchName], } @@ -331,7 +329,7 @@ func newExpectedSwitchPort(lspUUID string, portName string, podAddr string, pod ovntypes.TopologyExternalID: netInfo.TopologyType(), }, Options: map[string]string{ - libovsdbops.RequestedChassis: pod.nodeName, + libovsdbops.RequestedChassis: requestedChassisForPod(pod), "iface-id-ver": pod.podName, }, PortSecurity: []string{podAddr}, diff --git a/go-controller/pkg/ovn/multipolicy_test.go b/go-controller/pkg/ovn/multipolicy_test.go index c3ba0f7d33..0d6ea4b2d3 100644 --- a/go-controller/pkg/ovn/multipolicy_test.go +++ b/go-controller/pkg/ovn/multipolicy_test.go @@ -152,7 +152,7 @@ func getExpectedDataPodsAndSwitchesForUserDefinedNetwork(fakeOvn *FakeOVN, pods ovntypes.TopologyExternalID: ocInfo.bnc.TopologyType(), }, Options: map[string]string{ - libovsdbops.RequestedChassis: pod.nodeName, + libovsdbops.RequestedChassis: requestedChassisForPod(pod), "iface-id-ver": pod.podName, }, diff --git a/go-controller/pkg/ovn/namespace.go b/go-controller/pkg/ovn/namespace.go index f09eea1b77..5ff3fa748e 100644 --- a/go-controller/pkg/ovn/namespace.go +++ b/go-controller/pkg/ovn/namespace.go @@ -170,7 +170,7 @@ func (oc *DefaultNetworkController) updateNamespace(old, newer *corev1.Namespace if util.PodWantsHostNetwork(pod) { continue } - podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo()) + podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo(), nil) if err != nil { errors = append(errors, fmt.Errorf("unable to get pod %q IPs for SNAT rule removal err (%v)", logicalPort, err)) } diff --git a/go-controller/pkg/ovn/network_segmentation_test.go b/go-controller/pkg/ovn/network_segmentation_test.go index cfcc0f7e83..97e48ec1b9 100644 --- a/go-controller/pkg/ovn/network_segmentation_test.go +++ b/go-controller/pkg/ovn/network_segmentation_test.go @@ -85,7 +85,7 @@ var _ = ginkgo.Describe("OVN Pod Operations with network segmentation", func() { }, Options: map[string]string{ // check requested-chassis will be updated to correct t1.nodeName value - libovsdbops.RequestedChassis: t1.nodeName, + libovsdbops.RequestedChassis: requestedChassisForPod(t1), // check old value for iface-id-ver will be updated to pod.UID "iface-id-ver": "wrong_value", }, diff --git a/go-controller/pkg/ovn/ovn.go b/go-controller/pkg/ovn/ovn.go index f427c5b177..db844753c5 100644 --- a/go-controller/pkg/ovn/ovn.go +++ b/go-controller/pkg/ovn/ovn.go @@ -25,6 +25,7 @@ import ( addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" anpcontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/admin_network_policy" egresssvc_zone "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/egressservice" + networkconnectcontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/networkconnect" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -216,7 +217,7 @@ func (oc *DefaultNetworkController) ensureLocalZonePod(oldPod, pod *corev1.Pod, func (oc *DefaultNetworkController) ensureRemotePodIP(oldPod, pod *corev1.Pod, addPort bool) error { if (addPort || (oldPod != nil && len(pod.Status.PodIPs) != len(oldPod.Status.PodIPs))) && !util.PodWantsHostNetwork(pod) { - podIfAddrs, err := util.GetPodCIDRsWithFullMask(pod, oc.GetNetInfo()) + podIfAddrs, err := util.GetPodCIDRsWithFullMask(pod, oc.GetNetInfo(), nil) if err != nil { // not finding pod IPs on a remote pod is common until the other node wires the pod, suppress it return fmt.Errorf("failed to obtain IPs to add remote pod %s/%s: %w", @@ -256,7 +257,7 @@ func (oc *DefaultNetworkController) ensureRemoteZonePod(oldPod, pod *corev1.Pod, } } if kubevirt.IsPodLiveMigratable(pod) { - return kubevirt.EnsureRemoteZonePodAddressesToNodeRoute(oc.watchFactory, oc.nbClient, pod, ovntypes.DefaultNetworkName) + return kubevirt.EnsureRemoteZonePodAddressesToNodeRoute(oc.watchFactory, oc.nbClient, pod) } return nil } @@ -347,7 +348,7 @@ func (oc *DefaultNetworkController) removeRemoteZonePod(pod *corev1.Pod) error { } if allVMPodsAreCompleted { - ips, err := util.GetPodCIDRsWithFullMask(pod, oc.GetNetInfo()) + ips, err := util.GetPodCIDRsWithFullMask(pod, oc.GetNetInfo(), nil) if err != nil && !errors.Is(err, util.ErrNoPodIPFound) { return fmt.Errorf("failed to get pod ips for the pod %s/%s: %w", pod.Namespace, pod.Name, err) } @@ -516,3 +517,13 @@ func (oc *DefaultNetworkController) newANPController() error { ) return err } + +func (oc *DefaultNetworkController) newNetworkConnectController() error { + oc.networkConnectController = networkconnectcontroller.NewController( + oc.zone, + oc.nbClient, + oc.watchFactory, + oc.networkManager, + ) + return nil +} diff --git a/go-controller/pkg/ovn/ovn_test.go b/go-controller/pkg/ovn/ovn_test.go index eca14ae0d8..52bbbf5956 100644 --- a/go-controller/pkg/ovn/ovn_test.go +++ b/go-controller/pkg/ovn/ovn_test.go @@ -12,7 +12,6 @@ import ( mnpfake "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/client/clientset/versioned/fake" nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" fakenadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" - "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ocpnetworkapiv1alpha1 "github.com/openshift/api/network/v1alpha1" ocpnetworkfake "github.com/openshift/client-go/network/clientset/versioned/fake" @@ -40,6 +39,7 @@ import ( egressservice "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1" egressservicefake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/clientset/versioned/fake" udnclientfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/fake" + vtepfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" @@ -64,6 +64,7 @@ const ( fakeUUIDv6 = "8a86f6d8-7972-4253-b0bd-ddbef66e9304" fakePgUUID = "bf02f460-5058-4689-8fcb-d31a1e484ed2" ovnClusterPortGroupUUID = fakePgUUID + testICZone = "test" ) type userDefinedNetworkControllerInfo struct { @@ -93,6 +94,7 @@ type FakeOVN struct { // information map of all UDN controllers userDefinedNetworkControllers map[string]userDefinedNetworkControllerInfo fullL2UDNControllers map[string]*Layer2UserDefinedNetworkController + fullL3UDNControllers map[string]*Layer3UserDefinedNetworkController } // NOTE: the FakeAddressSetFactory is no longer needed and should no longer be used. starting to phase out FakeAddressSetFactory @@ -110,6 +112,7 @@ func NewFakeOVN(useFakeAddressSet bool) *FakeOVN { userDefinedNetworkControllers: map[string]userDefinedNetworkControllerInfo{}, fullL2UDNControllers: map[string]*Layer2UserDefinedNetworkController{}, + fullL3UDNControllers: map[string]*Layer3UserDefinedNetworkController{}, } } @@ -177,6 +180,7 @@ func (o *FakeOVN) start(objects ...runtime.Object) { IPAMClaimsClient: fakeipamclaimclient.NewSimpleClientset(ipamClaimObjects...), NetworkAttchDefClient: nadClient, UserDefinedNetworkClient: udnclientfake.NewSimpleClientset(), + VTEPClient: vtepfake.NewSimpleClientset(), } o.init(nads) } @@ -194,6 +198,9 @@ func (o *FakeOVN) shutdown() { o.egressQoSWg.Wait() o.egressSVCWg.Wait() o.anpWg.Wait() + if o.networkManager != nil { + o.networkManager.Stop() + } o.nbsbCleanup.Cleanup() for _, ocInfo := range o.userDefinedNetworkControllers { close(ocInfo.bnc.stopChan) @@ -221,7 +228,7 @@ func (o *FakeOVN) init(nadList []nettypes.NetworkAttachmentDefinition) { if o.networkManager == nil { o.networkManager = networkmanager.Default() if config.OVNKubernetesFeature.EnableMultiNetwork { - o.networkManager, err = networkmanager.NewForZone("test", &networkmanager.FakeControllerManager{}, o.watcher) + o.networkManager, err = networkmanager.NewForZone(config.Default.Zone, &networkmanager.FakeControllerManager{}, o.watcher) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } } @@ -274,6 +281,9 @@ func (o *FakeOVN) init(nadList []nettypes.NetworkAttachmentDefinition) { err = o.watcher.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = o.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = o.eIPController.SyncLocalNodeZonesCache() gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "syncing Nodes OVN zones status must succeed to support EgressIP") @@ -281,9 +291,11 @@ func (o *FakeOVN) init(nadList []nettypes.NetworkAttachmentDefinition) { if err == nil { for _, node := range existingNodes { o.controller.localZoneNodes.Store(node.Name, true) - for _, udnController := range o.userDefinedNetworkControllers { - if udnController.bnc.localZoneNodes != nil { - udnController.bnc.localZoneNodes.Store(node.Name, true) + if util.GetNodeZone(node) == types.OvnDefaultZone || util.GetNodeZone(node) == config.Default.Zone { + for _, udnController := range o.userDefinedNetworkControllers { + if udnController.bnc.localZoneNodes != nil { + udnController.bnc.localZoneNodes.Store(node.Name, true) + } } } } @@ -515,7 +527,7 @@ func (o *FakeOVN) NewUserDefinedNetworkController(netattachdef *nettypes.Network } netName := nInfo.GetNetworkName() topoType := nInfo.TopologyType() - ocInfo, ok = o.userDefinedNetworkControllers[netName] + _, ok = o.userDefinedNetworkControllers[netName] if !ok { nbZoneFailed := false // Try to get the NBZone. If there is an error, create NB_Global record. @@ -524,7 +536,11 @@ func (o *FakeOVN) NewUserDefinedNetworkController(netattachdef *nettypes.Network _, err := libovsdbutil.GetNBZone(o.nbClient) if err != nil { nbZoneFailed = true - err = createTestNBGlobal(o.nbClient, "global") + zone := types.OvnDefaultZone + if config.OVNKubernetesFeature.EnableInterconnect && config.Default.Zone != "" { + zone = config.Default.Zone + } + err = createTestNBGlobal(o.nbClient, zone) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -552,16 +568,20 @@ func (o *FakeOVN) NewUserDefinedNetworkController(netattachdef *nettypes.Network asf := addressset.NewFakeAddressSetFactory(getNetworkControllerName(netName)) + mutableNetInfo := util.NewMutableNetInfo(nInfo) + mutableNetInfo.AddNADs(nadName) + switch topoType { case types.Layer3Topology: - l3Controller, err := NewLayer3UserDefinedNetworkController(cnci, nInfo, o.networkManager.Interface(), nil, o.eIPController, o.portCache) + l3Controller, err := NewLayer3UserDefinedNetworkController(cnci, mutableNetInfo, o.networkManager.Interface(), nil, o.eIPController, o.portCache) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if o.asf != nil { // use fake asf only when enabled l3Controller.addressSetFactory = asf } userDefinedNetworkController = &l3Controller.BaseUserDefinedNetworkController + o.fullL3UDNControllers[netName] = l3Controller case types.Layer2Topology: - l2Controller, err := NewLayer2UserDefinedNetworkController(cnci, nInfo, o.networkManager.Interface(), nil, o.portCache, o.eIPController) + l2Controller, err := NewLayer2UserDefinedNetworkController(cnci, mutableNetInfo, o.networkManager.Interface(), nil, o.portCache, o.eIPController) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if o.asf != nil { // use fake asf only when enabled l2Controller.addressSetFactory = asf @@ -569,7 +589,7 @@ func (o *FakeOVN) NewUserDefinedNetworkController(netattachdef *nettypes.Network userDefinedNetworkController = &l2Controller.BaseUserDefinedNetworkController o.fullL2UDNControllers[netName] = l2Controller case types.LocalnetTopology: - localnetController := NewLocalnetUserDefinedNetworkController(cnci, nInfo, o.networkManager.Interface()) + localnetController := NewLocalnetUserDefinedNetworkController(cnci, mutableNetInfo, o.networkManager.Interface()) if o.asf != nil { // use fake asf only when enabled localnetController.addressSetFactory = asf } @@ -586,15 +606,8 @@ func (o *FakeOVN) NewUserDefinedNetworkController(netattachdef *nettypes.Network err = deleteTestNBGlobal(o.nbClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - } else { - userDefinedNetworkController = ocInfo.bnc } - ginkgo.By(fmt.Sprintf("OVN test init: add NAD %s to user-defined network controller of %s network %s", nadName, topoType, netName)) - mutableNetInfo := util.NewMutableNetInfo(userDefinedNetworkController.GetNetInfo()) - mutableNetInfo.AddNADs(nadName) - err = util.ReconcileNetInfo(userDefinedNetworkController.ReconcilableNetInfo, mutableNetInfo) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) return nil } diff --git a/go-controller/pkg/ovn/pod_selector_address_set.go b/go-controller/pkg/ovn/pod_selector_address_set.go index 1eaf6f6b76..2cd90a5678 100644 --- a/go-controller/pkg/ovn/pod_selector_address_set.go +++ b/go-controller/pkg/ovn/pod_selector_address_set.go @@ -171,15 +171,16 @@ func (psas *PodSelectorAddressSet) init(bnc *BaseNetworkController) error { } ipv4Mode, ipv6Mode := bnc.IPMode() psas.handlerResources = &PodSelectorAddrSetHandlerInfo{ - addressSet: as, - key: psas.key, - podSelector: psas.podSelector, - namespaceSelector: psas.namespaceSelector, - namespace: psas.namespace, - netInfo: bnc.GetNetInfo(), - ipv4Mode: ipv4Mode, - ipv6Mode: ipv6Mode, - stopChan: psas.cancelableContext.Done(), + addressSet: as, + key: psas.key, + podSelector: psas.podSelector, + namespaceSelector: psas.namespaceSelector, + namespace: psas.namespace, + netInfo: bnc.GetNetInfo(), + getNetworkNameForNADKey: bnc.getNetworkNameForNADKeyFunc(), + ipv4Mode: ipv4Mode, + ipv6Mode: ipv6Mode, + stopChan: psas.cancelableContext.Done(), } } @@ -316,9 +317,10 @@ type PodSelectorAddrSetHandlerInfo struct { // namespace is used when namespaceSelector is nil to set static namespace namespace string - netInfo util.NetInfo - ipv4Mode bool - ipv6Mode bool + netInfo util.NetInfo + getNetworkNameForNADKey func(nadKey string) string + ipv4Mode bool + ipv6Mode bool stopChan <-chan struct{} } @@ -368,7 +370,7 @@ func (handlerInfo *PodSelectorAddrSetHandlerInfo) addPods(pods ...*corev1.Pod) e } ips := make([]net.IP, 0, len(pods)*podIPFactor) for _, pod := range pods { - podIPs, err := util.GetPodIPsOfNetwork(pod, handlerInfo.netInfo) + podIPs, err := util.GetPodIPsOfNetwork(pod, handlerInfo.netInfo, handlerInfo.getNetworkNameForNADKey) if err != nil { // not finding pod IPs on a remote pod is common until the other node wires the pod, suppress it return ovntypes.NewSuppressedError(err) @@ -380,7 +382,7 @@ func (handlerInfo *PodSelectorAddrSetHandlerInfo) addPods(pods ...*corev1.Pod) e // must be called with PodSelectorAddrSetHandlerInfo read lock func (handlerInfo *PodSelectorAddrSetHandlerInfo) deletePod(pod *corev1.Pod) error { - ips, err := util.GetPodIPsOfNetwork(pod, handlerInfo.netInfo) + ips, err := util.GetPodIPsOfNetwork(pod, handlerInfo.netInfo, handlerInfo.getNetworkNameForNADKey) if err != nil { // if pod ips can't be fetched on delete, we don't expect that information about ips will ever be updated, // therefore just log the error and return. @@ -466,7 +468,7 @@ func (bnc *BaseNetworkController) podSelectorPodNeedsDelete(pod *corev1.Pod, pod if !util.PodCompleted(pod) { return "", nil } - ips, err := util.GetPodIPsOfNetwork(pod, bnc.GetNetInfo()) + ips, err := util.GetPodIPsOfNetwork(pod, bnc.GetNetInfo(), bnc.getNetworkNameForNADKeyFunc()) if err != nil { // if pod has no IP, nothing to do klog.Warningf("Failed to get IPs of pod %s/%s during address_set pod selector removal: %v", @@ -481,7 +483,7 @@ func (bnc *BaseNetworkController) podSelectorPodNeedsDelete(pod *corev1.Pod, pod } // completed pod be deleted a long time ago, check if there is a new pod with that same ip - collidingPod, err := findPodWithIPAddresses(bnc.watchFactory, bnc.GetNetInfo(), ips, nodeName) + collidingPod, err := findPodWithIPAddresses(bnc.watchFactory, bnc.GetNetInfo(), ips, nodeName, bnc.getNetworkNameForNADKeyFunc()) if err != nil { return "", fmt.Errorf("lookup for pods with the same IPs [%s] failed: %w", util.JoinIPs(ips, " "), err) } diff --git a/go-controller/pkg/ovn/pods.go b/go-controller/pkg/ovn/pods.go index ebb2ed8433..e877cb9af6 100644 --- a/go-controller/pkg/ovn/pods.go +++ b/go-controller/pkg/ovn/pods.go @@ -245,7 +245,7 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *corev1.Pod) (err error) return nil } - _, networkMap, err := util.GetPodNADToNetworkMapping(pod, oc.GetNetInfo()) + _, networkMap, err := util.GetDefaultPodNADToNetworkMapping(pod) if err != nil { // multus won't add this Pod if this fails, should never happen return fmt.Errorf("error getting default-network's network-attachment for pod %s/%s: %v", pod.Namespace, pod.Name, err) @@ -272,8 +272,8 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *corev1.Pod) (err error) pod.Namespace, pod.Name, time.Since(start), libovsdbExecuteTime) }() - nadName := types.DefaultNetworkName - ops, lsp, podAnnotation, newlyCreatedPort, err = oc.addLogicalPortToNetwork(pod, nadName, network, nil) + nadKey := types.DefaultNetworkName + ops, lsp, podAnnotation, newlyCreatedPort, err = oc.addLogicalPortToNetwork(pod, nadKey, network, nil) if err != nil { return err } @@ -406,8 +406,8 @@ func (oc *DefaultNetworkController) allocateSyncPodsIPs(pod *corev1.Pod) (string } func (oc *DefaultNetworkController) allocateSyncMigratablePodIPsOnZone(vms map[ktypes.NamespacedName]bool, pod *corev1.Pod) (map[ktypes.NamespacedName]bool, string, *util.PodAnnotation, error) { - allocatePodIPsOnSwitchWrapFn := func(liveMigratablePod *corev1.Pod, liveMigratablePodAnnotation *util.PodAnnotation, switchName, nadName string) (string, error) { - return oc.allocatePodIPsOnSwitch(liveMigratablePod, liveMigratablePodAnnotation, switchName, nadName) + allocatePodIPsOnSwitchWrapFn := func(liveMigratablePod *corev1.Pod, liveMigratablePodAnnotation *util.PodAnnotation, switchName, nadKey string) (string, error) { + return oc.allocatePodIPsOnSwitch(liveMigratablePod, liveMigratablePodAnnotation, switchName, nadKey) } vmKey, expectedLogicalPortName, podAnnotation, err := kubevirt.AllocateSyncMigratablePodIPsOnZone(oc.watchFactory, oc.lsManager, oc.GetNetworkName(), pod, allocatePodIPsOnSwitchWrapFn) if err != nil { diff --git a/go-controller/pkg/ovn/pods_test.go b/go-controller/pkg/ovn/pods_test.go index 76383189a5..437e15bee3 100644 --- a/go-controller/pkg/ovn/pods_test.go +++ b/go-controller/pkg/ovn/pods_test.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/urfave/cli/v2" @@ -142,6 +143,7 @@ func newNode(nodeName, nodeIPv4CIDR string) *corev1.Node { "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", nodeIPv4CIDR, ""), "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\"}", v4Node1Subnet), util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", nodeIPv4CIDR), + util.OvnNodeChassisID: chassisIDForNode(nodeName), "k8s.ovn.org/zone-name": "global", }, Labels: map[string]string{ @@ -167,6 +169,7 @@ func newNodeGlobalZoneNotEgressableV4Only(nodeName, nodeIPv4 string) *corev1.Nod "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", nodeIPv4, ""), "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\"}", v4Node1Subnet), util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", nodeIPv4), + util.OvnNodeChassisID: chassisIDForNode(nodeName), "k8s.ovn.org/zone-name": "global", }, }, @@ -189,6 +192,7 @@ func newNodeGlobalZoneNotEgressableV6Only(nodeName, nodeIPv6 string) *corev1.Nod "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", "", nodeIPv6), "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\"}", v6Node1Subnet), util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", nodeIPv6), + util.OvnNodeChassisID: chassisIDForNode(nodeName), "k8s.ovn.org/zone-name": "global", }, }, @@ -210,19 +214,20 @@ func newNodeGlobalZoneNotEgressableV6Only(nodeName, nodeIPv6 string) *corev1.Nod } type testPod struct { - portUUID string - nodeName string - nodeSubnet string - nodeMgtIP string - nodeGWIP string - podName string - podIP string - podMAC string - namespace string - portName string - routes []util.PodRoute - noIfaceIdVer bool - networkRole string + portUUID string + nodeName string + nodeChassisID string + nodeSubnet string + nodeMgtIP string + nodeGWIP string + podName string + podIP string + podMAC string + namespace string + portName string + routes []util.PodRoute + noIfaceIdVer bool + networkRole string udnPodInfos map[string]*udnPodInfo } @@ -245,21 +250,40 @@ type portInfo struct { prefixLen int } +func chassisIDForNode(nodeName string) string { + return uuid.NewSHA1(uuid.NameSpaceOID, []byte(nodeName)).String() +} + +func requestedChassisForPod(pod testPod) string { + if pod.nodeChassisID != "" { + return pod.nodeChassisID + } + if pod.nodeName == "" { + return "" + } + return chassisIDForNode(pod.nodeName) +} + func newTPod(nodeName, nodeSubnet, nodeMgtIP, nodeGWIP, podName, podIPs, podMAC, namespace string) testPod { portName := util.GetLogicalPortName(namespace, podName) + nodeChassisID := "" + if nodeName != "" { + nodeChassisID = chassisIDForNode(nodeName) + } to := testPod{ - portUUID: portName + "-UUID", - nodeSubnet: nodeSubnet, - nodeMgtIP: nodeMgtIP, - nodeGWIP: nodeGWIP, - podIP: podIPs, - podMAC: podMAC, - portName: portName, - nodeName: nodeName, - podName: podName, - namespace: namespace, - udnPodInfos: map[string]*udnPodInfo{}, - networkRole: ovntypes.NetworkRolePrimary, // all tests here run with network-segmentation disabled by default by default + portUUID: portName + "-UUID", + nodeSubnet: nodeSubnet, + nodeMgtIP: nodeMgtIP, + nodeGWIP: nodeGWIP, + podIP: podIPs, + podMAC: podMAC, + portName: portName, + nodeName: nodeName, + nodeChassisID: nodeChassisID, + podName: podName, + namespace: namespace, + udnPodInfos: map[string]*udnPodInfo{}, + networkRole: ovntypes.NetworkRolePrimary, // all tests here run with network-segmentation disabled by default by default } var routeSources []*net.IPNet @@ -442,15 +466,18 @@ func getDefaultNetExpectedPodsAndSwitches(pods []testPod, nodes []string) []libo return getDefaultNetExpectedDataPodsSwitchesPortGroup(pods, nodes, "") } -func getExpectedPodsAndSwitches(netInfo util.NetInfo, pods []testPod, nodes []string) []libovsdbtest.TestData { - return getExpectedDataPodsSwitchesPortGroup(netInfo, pods, nodes, "") +func getExpectedPodsAndSwitches(netInfo util.NetInfo, pods []testPod, nodes []string, nadKey string) []libovsdbtest.TestData { + return getExpectedDataPodsSwitchesPortGroup(netInfo, pods, nodes, "", nadKey) } func getDefaultNetExpectedDataPodsSwitchesPortGroup(pods []testPod, nodes []string, namespacedPortGroup string) []libovsdbtest.TestData { - return getExpectedDataPodsSwitchesPortGroup(&util.DefaultNetInfo{}, pods, nodes, namespacedPortGroup) + return getExpectedDataPodsSwitchesPortGroup(&util.DefaultNetInfo{}, pods, nodes, namespacedPortGroup, "") } -func getExpectedDataPodsSwitchesPortGroup(netInfo util.NetInfo, pods []testPod, nodes []string, namespacedPortGroup string) []libovsdbtest.TestData { +func getExpectedDataPodsSwitchesPortGroup(netInfo util.NetInfo, pods []testPod, nodes []string, namespacedPortGroup string, nadKey string) []libovsdbtest.TestData { + if !netInfo.IsDefault() && nadKey == "" { + panic("missing NAD key for non-default network") + } nodeslsps := make(map[string][]string) var logicalSwitchPorts []*nbdb.LogicalSwitchPort for _, pod := range pods { @@ -458,7 +485,7 @@ func getExpectedDataPodsSwitchesPortGroup(netInfo util.NetInfo, pods []testPod, if netInfo.IsDefault() { portName = util.GetLogicalPortName(pod.namespace, pod.podName) } else { - portName = util.GetUserDefinedNetworkLogicalPortName(pod.namespace, pod.podName, netInfo.GetNADs()[0]) + portName = util.GetUserDefinedNetworkLogicalPortName(pod.namespace, pod.podName, nadKey) } var lspUUID string if len(pod.portUUID) == 0 { @@ -476,7 +503,7 @@ func getExpectedDataPodsSwitchesPortGroup(netInfo util.NetInfo, pods []testPod, "namespace": pod.namespace, }, Options: map[string]string{ - libovsdbops.RequestedChassis: pod.nodeName, + libovsdbops.RequestedChassis: requestedChassisForPod(pod), "iface-id-ver": pod.podName, }, PortSecurity: []string{podAddr}, @@ -486,7 +513,7 @@ func getExpectedDataPodsSwitchesPortGroup(netInfo util.NetInfo, pods []testPod, } if !netInfo.IsDefault() { lsp.ExternalIDs["k8s.ovn.org/network"] = netInfo.GetNetworkName() - lsp.ExternalIDs["k8s.ovn.org/nad"] = netInfo.GetNADs()[0] + lsp.ExternalIDs["k8s.ovn.org/nad"] = nadKey lsp.ExternalIDs["k8s.ovn.org/topology"] = netInfo.TopologyType() } logicalSwitchPorts = append(logicalSwitchPorts, lsp) @@ -1251,7 +1278,7 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) - ginkgo.It("correctly stops retrying adding a pod after failing n times", func() { + ginkgo.It("doesn't stop retrying adding a pod after failing n times", func() { app.Action = func(*cli.Context) error { namespace1 := *newNamespace("namespace1") podTest := newTPod( @@ -1336,12 +1363,13 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { gomega.BeNumerically("==", retry.MaxFailedAttempts), // failedAttempts should reach the max ) - // restore nbdb, trigger a retry and verify that the retry entry gets deleted - // because it reached retry.MaxFailedAttempts and the corresponding pod has NOT been added to OVN + // restore nbdb, trigger a retry and verify that the pod is added connCtx, cancel := context.WithTimeout(context.Background(), config.Default.OVSDBTxnTimeout) defer cancel() resetNBClient(connCtx, fakeOvn.controller.nbClient) + // reset backoff for immediate retry + retry.SetRetryObjWithNoBackoff(key, fakeOvn.controller.retryPods) fakeOvn.controller.retryPods.RequestRetryObjs() // check that pod is in API server pod, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(podTest.namespace).Get( @@ -1352,9 +1380,9 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { // check that the retry cache no longer has the entry retry.CheckRetryObjectEventually(key, false, fakeOvn.controller.retryPods) - // check that pod doesn't appear in OVN + // check that pod is configured in OVN gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData( - getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{"node1"})...)) + getDefaultNetExpectedPodsAndSwitches([]testPod{podTest}, []string{"node1"})...)) return nil } @@ -1363,7 +1391,7 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) - ginkgo.It("correctly stops retrying deleting a pod after failing n times", func() { + ginkgo.It("doesn't stop retrying deleting a pod after failing n times", func() { app.Action = func(*cli.Context) error { namespace1 := *newNamespace("namespace1") podTest := newTPod( @@ -1449,12 +1477,13 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { gomega.BeNumerically("==", retry.MaxFailedAttempts), // failedAttempts should be the max ) - // restore nbdb and verify that the retry entry gets deleted because it reached - // retry.MaxFailedAttempts and the corresponding pod has NOT been deleted from OVN + // restore nbdb and verify that the pod is deleted connCtx, cancel := context.WithTimeout(context.Background(), config.Default.OVSDBTxnTimeout) defer cancel() resetNBClient(connCtx, fakeOvn.controller.nbClient) + // reset backoff for immediate retry + retry.SetRetryObjWithNoBackoff(key, fakeOvn.controller.retryPods) fakeOvn.controller.retryPods.RequestRetryObjs() // check that the pod is not in API server @@ -1465,8 +1494,9 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { // check that the retry cache no longer has the entry retry.CheckRetryObjectEventually(key, false, fakeOvn.controller.retryPods) - // check that the pod is still in OVN - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedData...)) + // check that the pod is deleted in OVN + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData( + getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{"node1"})...)) return nil } @@ -2024,7 +2054,7 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { }, Options: map[string]string{ // check requested-chassis will be updated to correct t1.nodeName value - libovsdbops.RequestedChassis: t2.nodeName, + libovsdbops.RequestedChassis: requestedChassisForPod(t2), // check old value for iface-id-ver will be updated to pod.UID "iface-id-ver": "wrong_value", }, @@ -2039,7 +2069,7 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { "namespace": t2.namespace, }, Options: map[string]string{ - libovsdbops.RequestedChassis: t2.nodeName, + libovsdbops.RequestedChassis: requestedChassisForPod(t2), //"iface-id-ver": is empty to check that it won't be set on update }, PortSecurity: []string{fmt.Sprintf("%s %s", t2.podMAC, t2.podIP)}, @@ -2054,7 +2084,7 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { }, Options: map[string]string{ // check requested-chassis will be updated to correct t1.nodeName value - libovsdbops.RequestedChassis: t3.nodeName, + libovsdbops.RequestedChassis: requestedChassisForPod(t3), // check old value for iface-id-ver will be updated to pod.UID "iface-id-ver": "wrong_value", }, @@ -2224,7 +2254,7 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { }, Options: map[string]string{ // check requested-chassis will be updated to correct t1.nodeName value - libovsdbops.RequestedChassis: t1.nodeName, + libovsdbops.RequestedChassis: requestedChassisForPod(t1), // check old value for iface-id-ver will be updated to pod.UID "iface-id-ver": "wrong_value", }, diff --git a/go-controller/pkg/ovn/port_cache.go b/go-controller/pkg/ovn/port_cache.go index 9dbee646e3..56b3d6160d 100644 --- a/go-controller/pkg/ovn/port_cache.go +++ b/go-controller/pkg/ovn/port_cache.go @@ -18,7 +18,7 @@ type PortCache struct { stopChan <-chan struct{} // cache of logical port info (lpInfo). The first key is podName, in the form of - // podNamespace/podName; the second key is NAD name associated with specific port info + // podNamespace/podName; the second key is the NAD key associated with specific port info cache map[string]map[string]*lpInfo } @@ -40,24 +40,25 @@ func NewPortCache(stopChan <-chan struct{}) *PortCache { } } -func (c *PortCache) get(pod *corev1.Pod, nadName string) (*lpInfo, error) { +func (c *PortCache) get(pod *corev1.Pod, nadKey string) (*lpInfo, error) { var logicalPort string podName := fmt.Sprintf("%s/%s", pod.Namespace, pod.Name) - if nadName == types.DefaultNetworkName { + if nadKey == types.DefaultNetworkName { logicalPort = util.GetLogicalPortName(pod.Namespace, pod.Name) } else { - logicalPort = util.GetUserDefinedNetworkLogicalPortName(pod.Namespace, pod.Name, nadName) + logicalPort = util.GetUserDefinedNetworkLogicalPortName(pod.Namespace, pod.Name, nadKey) } c.RLock() defer c.RUnlock() if infoMap, ok := c.cache[podName]; ok { - if info, ok := infoMap[nadName]; ok { + if info, ok := infoMap[nadKey]; ok { x := *info return &x, nil } } - return nil, fmt.Errorf("logical port %s for pod %s not found in cache", podName, logicalPort) + return nil, fmt.Errorf("logical port %s (NAD key %s) for pod %s not found in cache", + logicalPort, nadKey, podName) } func (c *PortCache) getAll(pod *corev1.Pod) (map[string]*lpInfo, error) { @@ -75,14 +76,14 @@ func (c *PortCache) getAll(pod *corev1.Pod) (map[string]*lpInfo, error) { return nil, fmt.Errorf("logical port cache for pod %s not found", podName) } -func (c *PortCache) add(pod *corev1.Pod, logicalSwitch, nadName, uuid string, mac net.HardwareAddr, ips []*net.IPNet) *lpInfo { +func (c *PortCache) add(pod *corev1.Pod, logicalSwitch, nadKey, uuid string, mac net.HardwareAddr, ips []*net.IPNet) *lpInfo { var logicalPort string podName := fmt.Sprintf("%s/%s", pod.Namespace, pod.Name) - if nadName == types.DefaultNetworkName { + if nadKey == types.DefaultNetworkName { logicalPort = util.GetLogicalPortName(pod.Namespace, pod.Name) } else { - logicalPort = util.GetUserDefinedNetworkLogicalPortName(pod.Namespace, pod.Name, nadName) + logicalPort = util.GetUserDefinedNetworkLogicalPortName(pod.Namespace, pod.Name, nadKey) } c.Lock() defer c.Unlock() @@ -97,22 +98,22 @@ func (c *PortCache) add(pod *corev1.Pod, logicalSwitch, nadName, uuid string, ma logicalPort, portInfo, portInfo.ips, portInfo.mac) m, ok := c.cache[podName] if ok { - m[nadName] = portInfo + m[nadKey] = portInfo } else { - m = map[string]*lpInfo{nadName: portInfo} + m = map[string]*lpInfo{nadKey: portInfo} c.cache[podName] = m } return portInfo } -func (c *PortCache) remove(pod *corev1.Pod, nadName string) { +func (c *PortCache) remove(pod *corev1.Pod, nadKey string) { var logicalPort string podName := fmt.Sprintf("%s/%s", pod.Namespace, pod.Name) - if nadName == types.DefaultNetworkName { + if nadKey == types.DefaultNetworkName { logicalPort = util.GetLogicalPortName(pod.Namespace, pod.Name) } else { - logicalPort = util.GetUserDefinedNetworkLogicalPortName(pod.Namespace, pod.Name, nadName) + logicalPort = util.GetUserDefinedNetworkLogicalPortName(pod.Namespace, pod.Name, nadKey) } c.Lock() @@ -122,7 +123,7 @@ func (c *PortCache) remove(pod *corev1.Pod, nadName string) { klog.V(5).Infof("port-cache(%s): port not found in cache or already marked for removal", logicalPort) return } - info, ok := infoMap[nadName] + info, ok := infoMap[nadKey] if !ok || !info.expires.IsZero() { klog.V(5).Infof("port-cache(%s): port not found in cache or already marked for removal", logicalPort) return @@ -143,10 +144,10 @@ func (c *PortCache) remove(pod *corev1.Pod, nadName string) { // that was deleted and re-added before the timer expires. infoMap, ok := c.cache[podName] if ok { - if info, ok := infoMap[nadName]; ok && !info.expires.IsZero() { + if info, ok := infoMap[nadKey]; ok && !info.expires.IsZero() { if time.Now().After(info.expires) { klog.V(5).Infof("port-cache(%s): removing port", logicalPort) - delete(infoMap, nadName) + delete(infoMap, nadKey) if len(infoMap) == 0 { delete(c.cache, podName) } diff --git a/go-controller/pkg/ovn/routeimport/route_import.go b/go-controller/pkg/ovn/routeimport/route_import.go index 18c372c276..e99c948edd 100644 --- a/go-controller/pkg/ovn/routeimport/route_import.go +++ b/go-controller/pkg/ovn/routeimport/route_import.go @@ -343,11 +343,13 @@ func (c *controller) syncNetwork(network string) error { c.setTableForNetworkUnlocked(info.GetNetworkID(), table) c.Unlock() - // skip routes in the pod network - // TODO do not skip these routes in no overlay mode - ignoreSubnets := make([]*net.IPNet, len(info.Subnets())) - for i, subnet := range info.Subnets() { - ignoreSubnets[i] = subnet.CIDR + var ignoreSubnets []*net.IPNet + if info.Transport() != types.NetworkTransportNoOverlay { + // if the network is overlay mode, skip routes to the pod network + ignoreSubnets = make([]*net.IPNet, len(info.Subnets())) + for i, subnet := range info.Subnets() { + ignoreSubnets[i] = subnet.CIDR + } } expected, err := c.getBGPRoutes(table, ignoreSubnets) @@ -431,6 +433,7 @@ func (c *controller) getBGPRoutes(table int, ignoreSubnets []*net.IPNet) (sets.S routes := sets.New[route]() for _, nlroute := range nlroutes { if util.IsContainedInAnyCIDR(nlroute.Dst, ignoreSubnets...) { + c.log.V(5).Info("Ignore BGP route", "table", table, "route", stringer{nlroute}) continue } routes.Insert(routesFromNetlinkRoute(&nlroute)...) diff --git a/go-controller/pkg/ovn/routeimport/route_import_test.go b/go-controller/pkg/ovn/routeimport/route_import_test.go index cf71a392a8..b79f93101d 100644 --- a/go-controller/pkg/ovn/routeimport/route_import_test.go +++ b/go-controller/pkg/ovn/routeimport/route_import_test.go @@ -2,6 +2,7 @@ package routeimport import ( "errors" + "net" "sync" "testing" @@ -13,6 +14,7 @@ import ( "k8s.io/client-go/util/workqueue" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" controllerutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" ovntesting "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" @@ -26,16 +28,33 @@ import ( func Test_controller_syncNetwork(t *testing.T) { node := "testnode" + // Capture original global config values and restore after test + origClusterSubnets := config.Default.ClusterSubnets + t.Cleanup(func() { + config.Default.ClusterSubnets = origClusterSubnets + }) + defaultNetwork := &util.DefaultNetInfo{} defaultNetworkRouter := defaultNetwork.GetNetworkScopedGWRouterName(node) defaultNetworkRouterPort := types.GWRouterToExtSwitchPrefix + defaultNetworkRouter + config.Default.ClusterSubnets = []config.CIDRNetworkEntry{ + { + CIDR: &net.IPNet{ + IP: net.IPv4(10, 128, 0, 0), + Mask: net.CIDRMask(16, 32), + }, + HostSubnetLength: 24, + }, + } + udn := &multinetworkmocks.NetInfo{} udn.On("IsDefault").Return(false) udn.On("GetNetworkName").Return("udn") udn.On("GetNetworkID").Return(1) udn.On("Subnets").Return(nil) udn.On("GetNetworkScopedGWRouterName", node).Return("router") + udn.On("Transport").Return("") cudn := &multinetworkmocks.NetInfo{} cudn.On("IsDefault").Return(false) @@ -43,6 +62,7 @@ func Test_controller_syncNetwork(t *testing.T) { cudn.On("GetNetworkID").Return(2) cudn.On("Subnets").Return(nil) cudn.On("GetNetworkScopedGWRouterName", node).Return("router") + cudn.On("Transport").Return("") type fields struct { networkIDs map[int]string @@ -52,16 +72,17 @@ func Test_controller_syncNetwork(t *testing.T) { network string } tests := []struct { - name string - fields fields - args args - initial []libovsdb.TestData - expected []libovsdb.TestData - routes []netlink.Route - link netlink.Link - linkErr bool - routesErr bool - wantErr bool + name string + fields fields + args args + initial []libovsdb.TestData + expected []libovsdb.TestData + routes []netlink.Route + link netlink.Link + noOverlayEnabled bool + linkErr bool + routesErr bool + wantErr bool }{ { name: "ignored if network not known", @@ -168,11 +189,61 @@ func Test_controller_syncNetwork(t *testing.T) { &nbdb.LogicalRouterStaticRoute{UUID: "untouched-1", IPPrefix: "3.3.3.0/24", Nexthop: "3.3.3.2", ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, }, }, + { + name: "ignores host subnet routes as necessary in overlay mode", + args: args{"default"}, + fields: fields{ + networkIDs: map[int]string{0: "default"}, + networks: map[string]util.NetInfo{"default": defaultNetwork}, + }, + link: &netlink.Vrf{Table: unix.RT_TABLE_MAIN}, + initial: []libovsdb.TestData{ + &nbdb.LogicalRouter{Name: defaultNetwork.GetNetworkScopedGWRouterName(node), StaticRoutes: []string{"keep-1"}}, + &nbdb.LogicalRouterStaticRoute{UUID: "keep-1", IPPrefix: "1.1.1.0/24", Nexthop: "1.1.1.1", OutputPort: &defaultNetworkRouterPort, ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + }, + routes: []netlink.Route{ + {Dst: ovntesting.MustParseIPNet("1.1.1.0/24"), Gw: ovntesting.MustParseIP("1.1.1.1")}, + {Dst: ovntesting.MustParseIPNet("10.128.1.0/24"), Gw: ovntesting.MustParseIP("2.2.2.1")}, + }, + expected: []libovsdb.TestData{ + &nbdb.LogicalRouter{UUID: "router", Name: defaultNetwork.GetNetworkScopedGWRouterName(node), StaticRoutes: []string{"keep-1"}}, + &nbdb.LogicalRouterStaticRoute{UUID: "keep-1", IPPrefix: "1.1.1.0/24", Nexthop: "1.1.1.1", OutputPort: &defaultNetworkRouterPort, ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + }, + }, + { + name: "adds host subnet routes as necessary in no-overlay mode", + noOverlayEnabled: true, + args: args{"default"}, + fields: fields{ + networkIDs: map[int]string{0: "default"}, + networks: map[string]util.NetInfo{"default": defaultNetwork}, + }, + link: &netlink.Vrf{Table: unix.RT_TABLE_MAIN}, + initial: []libovsdb.TestData{ + &nbdb.LogicalRouter{Name: defaultNetwork.GetNetworkScopedGWRouterName(node), StaticRoutes: []string{"keep-1"}}, + &nbdb.LogicalRouterStaticRoute{UUID: "keep-1", IPPrefix: "1.1.1.0/24", Nexthop: "1.1.1.1", OutputPort: &defaultNetworkRouterPort, ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + }, + routes: []netlink.Route{ + {Dst: ovntesting.MustParseIPNet("1.1.1.0/24"), Gw: ovntesting.MustParseIP("1.1.1.1")}, + {Dst: ovntesting.MustParseIPNet("10.128.1.0/24"), Gw: ovntesting.MustParseIP("2.2.2.1")}, + }, + expected: []libovsdb.TestData{ + &nbdb.LogicalRouter{UUID: "router", Name: defaultNetwork.GetNetworkScopedGWRouterName(node), StaticRoutes: []string{"keep-1", "add-1"}}, + &nbdb.LogicalRouterStaticRoute{UUID: "keep-1", IPPrefix: "1.1.1.0/24", Nexthop: "1.1.1.1", OutputPort: &defaultNetworkRouterPort, ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + &nbdb.LogicalRouterStaticRoute{UUID: "add-1", IPPrefix: "10.128.1.0/24", Nexthop: "2.2.2.1", OutputPort: &defaultNetworkRouterPort, ExternalIDs: map[string]string{controllerExternalIDKey: controllerName}}, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := gomega.NewWithT(t) + // Capture and restore global config value for this subtest + origTransport := config.Default.Transport + t.Cleanup(func() { + config.Default.Transport = origTransport + }) + testError := errors.New("test forced error or incorrect test arguments") network := tt.fields.networks[tt.args.network] @@ -211,6 +282,10 @@ func Test_controller_syncNetwork(t *testing.T) { netlink: nlmock, } + if tt.noOverlayEnabled { + config.Default.Transport = types.NetworkTransportNoOverlay + } + err = c.syncNetwork(tt.args.network) if tt.wantErr { g.Expect(err).To(gomega.HaveOccurred()) diff --git a/go-controller/pkg/ovn/topology/topologyfactory.go b/go-controller/pkg/ovn/topology/topologyfactory.go index ead14e05b2..080b306b23 100644 --- a/go-controller/pkg/ovn/topology/topologyfactory.go +++ b/go-controller/pkg/ovn/topology/topologyfactory.go @@ -36,7 +36,7 @@ func (gtf *GatewayTopologyFactory) NewClusterRouterWithMulticastSupport( netInfo util.NetInfo, coopUUID string, ) (*nbdb.LogicalRouter, error) { - routerOptions := map[string]string{"mcast_relay": "true"} + routerOptions := map[string]string{"mcast_relay": "true", "always_learn_from_arp_request": "false"} return gtf.newClusterRouter(clusterRouterName, netInfo, coopUUID, routerOptions) } diff --git a/go-controller/pkg/ovn/topology/topologyfactory_test.go b/go-controller/pkg/ovn/topology/topologyfactory_test.go index 4d189e030a..dbed43ffb5 100644 --- a/go-controller/pkg/ovn/topology/topologyfactory_test.go +++ b/go-controller/pkg/ovn/topology/topologyfactory_test.go @@ -88,7 +88,7 @@ var _ = Describe("Topology factory", func() { ovntypes.TopologyExternalID: ovntypes.Layer3Topology, "k8s-cluster-router": "yes", } - expectedOptions := map[string]string{"mcast_relay": "true"} + expectedOptions := map[string]string{"mcast_relay": "true", "always_learn_from_arp_request": "false"} Expect(clusterRouter).To( WithTransform( removeUUID, diff --git a/go-controller/pkg/ovn/zone_interconnect/chassis_handler.go b/go-controller/pkg/ovn/zone_interconnect/chassis_handler.go index b838221892..13a370ef90 100644 --- a/go-controller/pkg/ovn/zone_interconnect/chassis_handler.go +++ b/go-controller/pkg/ovn/zone_interconnect/chassis_handler.go @@ -160,12 +160,18 @@ func (zch *ZoneChassisHandler) createOrUpdateNodeChassis(node *corev1.Node, isRe } chassis := sbdb.Chassis{ - Name: chassisID, - Hostname: node.Name, + Name: chassisID, OtherConfig: map[string]string{ "is-remote": strconv.FormatBool(isRemote), }, } + if isRemote { + // For debugging purposes we add KAPI node name as the chassis hostname. + // It is not used for anything other than a helpful hint for debugging. + // There is no need to set it for the local node, as ovn-controller will + // set it automatically from the OVS external_id:hostname field. + chassis.Hostname = node.Name + } return libovsdbops.CreateOrUpdateChassis(zch.sbClient, &chassis, encaps...) } diff --git a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go index 23a310c9ab..b269fd2052 100644 --- a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go +++ b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "strconv" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -173,12 +174,20 @@ func (zic *ZoneInterconnectHandler) createOrUpdateTransitSwitch(networkID int) e // ensureTransitSwitch sets up the global transit switch required for interoperability with other zones // Must wait for network id to be annotated to any node by cluster manager -func (zic *ZoneInterconnectHandler) ensureTransitSwitch(nodes []*corev1.Node) error { - if len(nodes) == 0 { // nothing to do - return nil - } +func (zic *ZoneInterconnectHandler) ensureTransitSwitch() error { start := time.Now() + // Get the transit switch. If its not present no cleanup to do + ts := &nbdb.LogicalSwitch{ + Name: zic.networkTransitSwitchName, + } + + _, err := libovsdbops.GetLogicalSwitch(zic.nbClient, ts) + if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { + return err + } + + // Create the transit switch if it doesn't exist if err := zic.createOrUpdateTransitSwitch(zic.GetNetworkID()); err != nil { return err } @@ -198,6 +207,10 @@ func (zic *ZoneInterconnectHandler) AddLocalZoneNode(node *corev1.Node) error { return fmt.Errorf("failed to get node id for node - %s", node.Name) } + if err := zic.ensureTransitSwitch(); err != nil { + return fmt.Errorf("ensuring transit switch for local zone node %s for the network %s failed : err - %w", node.Name, zic.GetNetworkName(), err) + } + if err := zic.createLocalZoneNodeResources(node, nodeID); err != nil { return fmt.Errorf("creating interconnect resources for local zone node %s for the network %s failed : err - %w", node.Name, zic.GetNetworkName(), err) } @@ -257,6 +270,10 @@ func (zic *ZoneInterconnectHandler) AddRemoteZoneNode(node *corev1.Node) error { } } + if err := zic.ensureTransitSwitch(); err != nil { + return fmt.Errorf("ensuring transit switch for remote zone node %s for the network %s failed : err - %w", node.Name, zic.GetNetworkName(), err) + } + klog.Infof("Creating interconnect resources for remote zone node %s for the network %s", node.Name, zic.GetNetworkName()) if err := zic.createRemoteZoneNodeResources(node, nodeID, nodeTransitSwitchPortIPs, nodeSubnets, nodeGRPIPs); err != nil { @@ -273,58 +290,94 @@ func (zic *ZoneInterconnectHandler) DeleteNode(node *corev1.Node) error { return zic.cleanupNode(node.Name) } -// SyncNodes ensures a transit switch exists and cleans up the interconnect -// resources present in the OVN Northbound db for the stale nodes -func (zic *ZoneInterconnectHandler) SyncNodes(objs []interface{}) error { +// CleanupStaleNodes cleans up the interconnect resources for stale nodes. +func (zic *ZoneInterconnectHandler) CleanupStaleNodes(objs []interface{}) error { + // Build set of current node names foundNodeNames := sets.New[string]() - foundNodes := make([]*corev1.Node, len(objs)) - for i, obj := range objs { + for _, obj := range objs { node, ok := obj.(*corev1.Node) if !ok { - return fmt.Errorf("spurious object in syncNodes: %v", obj) + return fmt.Errorf("spurious object in CleanupStaleNodes: %v", obj) } foundNodeNames.Insert(node.Name) - foundNodes[i] = node } + staleNodeNames := sets.New[string]() - // Get the transit switch. If its not present no cleanup to do + // Get the transit switch ts := &nbdb.LogicalSwitch{ Name: zic.networkTransitSwitchName, } - ts, err := libovsdbops.GetLogicalSwitch(zic.nbClient, ts) - if err != nil { - if errors.Is(err, libovsdbclient.ErrNotFound) { - // This can happen for the first time when interconnect is enabled. - // Let's ensure the transit switch exists - return zic.ensureTransitSwitch(foundNodes) - } + if err == nil { + // Transit switch exists - find stale nodes by checking transit switch ports + for _, p := range ts.Ports { + lp := &nbdb.LogicalSwitchPort{ + UUID: p, + } - return err - } + lp, err := libovsdbops.GetLogicalSwitchPort(zic.nbClient, lp) + if err != nil { + continue + } + + if lp.ExternalIDs == nil { + continue + } - staleNodeNames := []string{} - for _, p := range ts.Ports { - lp := &nbdb.LogicalSwitchPort{ - UUID: p, + lportNode := lp.ExternalIDs["node"] + if lportNode != "" && !foundNodeNames.Has(lportNode) { + staleNodeNames.Insert(lportNode) + } + } + } else if errors.Is(err, libovsdbclient.ErrNotFound) { + // Transit switch doesn't exist - discover nodes from cluster router resources + lr := &nbdb.LogicalRouter{Name: zic.networkClusterRouterName} + lr, err = libovsdbops.GetLogicalRouter(zic.nbClient, lr) + if err != nil { + if !errors.Is(err, libovsdbclient.ErrNotFound) { + return fmt.Errorf("failed to get cluster router: %w", err) + } + // Router doesn't exist, nothing to cleanup + return nil } - lp, err = libovsdbops.GetLogicalSwitchPort(zic.nbClient, lp) + // Discover remote zone nodes from static routes with ic-node external ID + p := func(route *nbdb.LogicalRouterStaticRoute) bool { + return route.ExternalIDs != nil && route.ExternalIDs["ic-node"] != "" + } + routes, err := libovsdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(zic.nbClient, lr, p) if err != nil { - continue + return fmt.Errorf("failed to get static routes for cluster router: %w", err) } - if lp.ExternalIDs == nil { - continue + for _, route := range routes { + nodeName := route.ExternalIDs["ic-node"] + if nodeName != "" && !foundNodeNames.Has(nodeName) { + staleNodeNames.Insert(nodeName) + } } - lportNode := lp.ExternalIDs["node"] - if !foundNodeNames.Has(lportNode) { - staleNodeNames = append(staleNodeNames, lportNode) + // Discover local zone nodes from router ports connecting to transit switch + routerPortPrefix := zic.GetNetworkScopedName(types.RouterToTransitSwitchPrefix) + for _, portUUID := range lr.Ports { + lrp, err := libovsdbops.GetLogicalRouterPort(zic.nbClient, &nbdb.LogicalRouterPort{UUID: portUUID}) + if err != nil { + continue + } + // Extract node name from port name (e.g., "rtots-node1" -> "node1") + if nodeName, found := strings.CutPrefix(lrp.Name, routerPortPrefix); found { + if nodeName != "" && !foundNodeNames.Has(nodeName) { + staleNodeNames.Insert(nodeName) + } + } } + } else { + // Unexpected error + return fmt.Errorf("unexpected error while getting transit switch: %w", err) } - for _, staleNodeName := range staleNodeNames { + // Cleanup stale interconnect resources + for _, staleNodeName := range staleNodeNames.UnsortedList() { if err := zic.cleanupNode(staleNodeName); err != nil { klog.Errorf("Failed to cleanup the interconnect resources from OVN Northbound db for the stale node %s: %v", staleNodeName, err) } @@ -333,10 +386,25 @@ func (zic *ZoneInterconnectHandler) SyncNodes(objs []interface{}) error { return nil } -// Cleanup deletes the transit switch for the network +// Cleanup deletes all interconnect resources for the network, including all node resources +// (ports, router ports, static routes) and the transit switch itself. This method is idempotent +// and safe to call multiple times. func (zic *ZoneInterconnectHandler) Cleanup() error { + klog.Infof("Cleaning up all interconnect resources for network %s", zic.GetNetworkName()) + + // First cleanup all node resources (ports, routes, etc.) + // Passing nil removes all nodes from the transit switch + if err := zic.CleanupStaleNodes(nil); err != nil { + return fmt.Errorf("failed to cleanup node resources: %w", err) + } + + // Then delete the transit switch klog.Infof("Deleting the transit switch %s for the network %s", zic.networkTransitSwitchName, zic.GetNetworkName()) - return libovsdbops.DeleteLogicalSwitch(zic.nbClient, zic.networkTransitSwitchName) + if err := libovsdbops.DeleteLogicalSwitch(zic.nbClient, zic.networkTransitSwitchName); err != nil && + !errors.Is(err, libovsdbclient.ErrNotFound) { + return fmt.Errorf("failed to delete transit switch: %w", err) + } + return nil } // AddTransitSwitchConfig is only used by the layer2 network controller @@ -384,7 +452,6 @@ func (zic *ZoneInterconnectHandler) addTransitSwitchConfig(sw *nbdb.LogicalSwitc } // createLocalZoneNodeResources creates the local zone node resources for interconnect -// - creates Transit switch if it doesn't yet exit // - creates a logical switch port of type "router" in the transit switch with the name as - .tstor- // Eg. if the node name is ovn-worker and the network is default, the name would be - tstor-ovn-worker // if the node name is ovn-worker and the network name is blue, the logical port name would be - blue.tstor-ovn-worker @@ -442,7 +509,6 @@ func (zic *ZoneInterconnectHandler) createLocalZoneNodeResources(node *corev1.No } // createRemoteZoneNodeResources creates the remote zone node resources -// - creates Transit switch if it doesn't yet exit // - creates a logical port of type "remote" in the transit switch with the name as - .tstor. // Eg. if the node name is ovn-worker and the network is default, the name would be - tstor.ovn-worker // if the node name is ovn-worker and the network name is blue, the logical port name would be - blue.tstor.ovn-worker @@ -460,9 +526,18 @@ func (zic *ZoneInterconnectHandler) createRemoteZoneNodeResources(node *corev1.N remotePortAddr = remotePortAddr + " " + tsNetwork } + chassisID, err := util.ParseNodeChassisIDAnnotation(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + // remote node may not have the annotation yet, suppress it + return types.NewSuppressedError(err) + } + return fmt.Errorf("failed to parse node chassis-id for node %s: %w", node.Name, err) + } + lspOptions := map[string]string{ libovsdbops.RequestedTnlKey: strconv.Itoa(nodeID), - libovsdbops.RequestedChassis: node.Name, + libovsdbops.RequestedChassis: chassisID, } // Store the node name in the external_ids column for book keeping externalIDs := map[string]string{ diff --git a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler_test.go b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler_test.go index e138037031..76f872e00b 100644 --- a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler_test.go +++ b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler_test.go @@ -35,8 +35,8 @@ const ( // ovnNodeZoneNameAnnotation is the node annotation name to store the node zone name. ovnNodeZoneNameAnnotation = "k8s.ovn.org/zone-name" - // ovnNodeChassisIDAnnotatin is the node annotation name to store the node chassis id. - ovnNodeChassisIDAnnotatin = "k8s.ovn.org/node-chassis-id" + // ovnNodeChassisIDAnnotation is the node annotation name to store the node chassis id. + ovnNodeChassisIDAnnotation = "k8s.ovn.org/node-chassis-id" // ovnNodeSubnetsAnnotation is the node annotation name to store the node subnets. ovnNodeSubnetsAnnotation = "k8s.ovn.org/node-subnets" @@ -298,7 +298,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node1", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", ovnNodeZoneNameAnnotation: "global", ovnNodeIDAnnotaton: "2", ovnNodeSubnetsAnnotation: "{\"default\":[\"10.244.2.0/24\"]}", @@ -315,7 +315,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac7", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac7", ovnNodeZoneNameAnnotation: "global", ovnNodeIDAnnotaton: "3", ovnNodeSubnetsAnnotation: "{\"default\":[\"10.244.3.0/24\"]}", @@ -332,7 +332,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac8", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac8", ovnNodeZoneNameAnnotation: "foo", ovnNodeIDAnnotaton: "4", ovnNodeSubnetsAnnotation: "{\"default\":[\"10.244.4.0/24\"]}", @@ -562,11 +562,11 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { err = checkInterconnectResources("global", types.DefaultNetworkName, libovsdbOvnNBClient, testNodesRouteInfo, &testNode1, &testNode2, &testNode3) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Call ICHandler SyncNodes function removing the testNode3 from the list of nodes + // Call ICHandler CleanupStaleNodes function removing the testNode3 from the list of nodes var kNodes []interface{} kNodes = append(kNodes, &testNode1) kNodes = append(kNodes, &testNode2) - err = zoneICHandler.SyncNodes(kNodes) + err = zoneICHandler.CleanupStaleNodes(kNodes) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = checkInterconnectResources("global", types.DefaultNetworkName, libovsdbOvnNBClient, testNodesRouteInfo, &testNode1, &testNode2) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -583,6 +583,239 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { }) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) + + ginkgo.It("CleanupStaleNodes with nil should cleanup all transit switch ports for no-overlay migration", func() { + app.Action = func(ctx *cli.Context) error { + dbSetup := libovsdbtest.TestSetup{ + NBData: initialNBDB, + SBData: initialSBDB, + } + + _, err := config.InitConfig(ctx, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + config.Kubernetes.HostNetworkNamespace = "" + + var libovsdbOvnNBClient, libovsdbOvnSBClient libovsdbclient.Client + libovsdbOvnNBClient, libovsdbOvnSBClient, libovsdbCleanup, err = libovsdbtest.NewNBSBTestHarness(dbSetup) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = createTransitSwitchPortBindings(libovsdbOvnSBClient, types.DefaultNetworkName, &testNode1, &testNode2, &testNode3) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + zoneICHandler := NewZoneInterconnectHandler(&util.DefaultNetInfo{}, libovsdbOvnNBClient, libovsdbOvnSBClient, nil) + gomega.Expect(zoneICHandler).NotTo(gomega.BeNil()) + + // Create transit switch and add nodes (simulating previous overlay configuration) + err = zoneICHandler.createOrUpdateTransitSwitch(0) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Set up nodes: testNode1 as local zone, testNode2 and testNode3 as remote zones + testNode2.Annotations[ovnNodeZoneNameAnnotation] = "remote-zone-1" + testNode3.Annotations[ovnNodeZoneNameAnnotation] = "remote-zone-2" + err = invokeICHandlerAddNodeFunction("global", zoneICHandler, &testNode1, &testNode2, &testNode3) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify transit switch exists with ports + ts, err := libovsdbops.GetLogicalSwitch(libovsdbOvnNBClient, &nbdb.LogicalSwitch{Name: types.TransitSwitch}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ts.Ports).NotTo(gomega.BeEmpty(), "Transit switch should have ports before cleanup") + + // Verify IC router ports exist (for local zone node) + clusterRouter, err := libovsdbops.GetLogicalRouter(libovsdbOvnNBClient, &nbdb.LogicalRouter{Name: types.OVNClusterRouter}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + icRouterPorts := 0 + for _, p := range clusterRouter.Ports { + lrp, err := libovsdbops.GetLogicalRouterPort(libovsdbOvnNBClient, &nbdb.LogicalRouterPort{UUID: p}) + if err != nil { + continue + } + if len(lrp.Name) >= len(types.RouterToTransitSwitchPrefix) && lrp.Name[:len(types.RouterToTransitSwitchPrefix)] == types.RouterToTransitSwitchPrefix { + icRouterPorts++ + } + } + gomega.Expect(icRouterPorts).To(gomega.Equal(1), "Should have router port for local zone node before cleanup") + + // Verify IC static routes exist (for remote zone nodes) + p := func(route *nbdb.LogicalRouterStaticRoute) bool { + return route.ExternalIDs != nil && route.ExternalIDs["ic-node"] != "" + } + routes, err := libovsdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(libovsdbOvnNBClient, clusterRouter, p) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(routes).NotTo(gomega.BeEmpty(), "Should have IC static routes for remote zone nodes before cleanup") + + // Call CleanupStaleNodes with nil to simulate no-overlay migration + // nil means "no current IC nodes", so all nodes become stale and should be cleaned up + err = zoneICHandler.CleanupStaleNodes(nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify all transit switch ports are cleaned up + ts, err = libovsdbops.GetLogicalSwitch(libovsdbOvnNBClient, &nbdb.LogicalSwitch{Name: types.TransitSwitch}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ts.Ports).To(gomega.BeEmpty(), "Transit switch ports should be cleaned up") + + // Verify all IC router ports are cleaned up (local zone node resources) + clusterRouter, err = libovsdbops.GetLogicalRouter(libovsdbOvnNBClient, &nbdb.LogicalRouter{Name: types.OVNClusterRouter}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + icRouterPorts = 0 + for _, p := range clusterRouter.Ports { + lrp, err := libovsdbops.GetLogicalRouterPort(libovsdbOvnNBClient, &nbdb.LogicalRouterPort{UUID: p}) + if err != nil { + continue + } + if len(lrp.Name) >= len(types.RouterToTransitSwitchPrefix) && lrp.Name[:len(types.RouterToTransitSwitchPrefix)] == types.RouterToTransitSwitchPrefix { + icRouterPorts++ + } + } + gomega.Expect(icRouterPorts).To(gomega.Equal(0), "All IC router ports should be cleaned up") + + // Verify all IC static routes are cleaned up (remote zone node resources) + routes, err = libovsdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(libovsdbOvnNBClient, clusterRouter, p) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(routes).To(gomega.BeEmpty(), "All IC static routes should be cleaned up") + + // Now call Cleanup to remove all interconnect resources (transit switch and any remaining nodes) + err = zoneICHandler.Cleanup() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify transit switch is deleted + _, err = libovsdbops.GetLogicalSwitch(libovsdbOvnNBClient, &nbdb.LogicalSwitch{Name: types.TransitSwitch}) + gomega.Expect(err).To(gomega.MatchError(libovsdbclient.ErrNotFound)) + + return nil + } + + err := app.Run([]string{ + app.Name, + "-cluster-subnets=" + clusterCIDR, + "-init-cluster-manager", + "-zone-join-switch-subnets=" + joinSubnetCIDR, + "-enable-interconnect", + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("CleanupStaleNodes with nil should cleanup orphaned IC resources when transit switch doesn't exist", func() { + app.Action = func(ctx *cli.Context) error { + dbSetup := libovsdbtest.TestSetup{ + NBData: initialNBDB, + SBData: initialSBDB, + } + + _, err := config.InitConfig(ctx, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + config.Kubernetes.HostNetworkNamespace = "" + + var libovsdbOvnNBClient, libovsdbOvnSBClient libovsdbclient.Client + libovsdbOvnNBClient, libovsdbOvnSBClient, libovsdbCleanup, err = libovsdbtest.NewNBSBTestHarness(dbSetup) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = createTransitSwitchPortBindings(libovsdbOvnSBClient, types.DefaultNetworkName, &testNode1, &testNode2, &testNode3) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + zoneICHandler := NewZoneInterconnectHandler(&util.DefaultNetInfo{}, libovsdbOvnNBClient, libovsdbOvnSBClient, nil) + gomega.Expect(zoneICHandler).NotTo(gomega.BeNil()) + + // Create transit switch and add nodes (simulating previous IC configuration) + err = zoneICHandler.createOrUpdateTransitSwitch(0) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Add testNode1 as local zone, testNode2 and testNode3 as remote zone + testNode2.Annotations[ovnNodeZoneNameAnnotation] = "remote" + testNode3.Annotations[ovnNodeZoneNameAnnotation] = "remote" + err = invokeICHandlerAddNodeFunction("global", zoneICHandler, &testNode1, &testNode2, &testNode3) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify IC resources exist + clusterRouter, err := libovsdbops.GetLogicalRouter(libovsdbOvnNBClient, &nbdb.LogicalRouter{Name: types.OVNClusterRouter}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Count IC router ports (for local zone nodes) + icRouterPorts := 0 + for _, p := range clusterRouter.Ports { + lrp, err := libovsdbops.GetLogicalRouterPort(libovsdbOvnNBClient, &nbdb.LogicalRouterPort{UUID: p}) + if err != nil { + continue + } + if len(lrp.Name) >= len(types.RouterToTransitSwitchPrefix) && lrp.Name[:len(types.RouterToTransitSwitchPrefix)] == types.RouterToTransitSwitchPrefix { + icRouterPorts++ + } + } + gomega.Expect(icRouterPorts).To(gomega.Equal(1), "Should have router port for local zone node (node1)") + + // Count IC static routes (for remote zone nodes) + p := func(route *nbdb.LogicalRouterStaticRoute) bool { + return route.ExternalIDs != nil && route.ExternalIDs["ic-node"] != "" + } + routes, err := libovsdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(libovsdbOvnNBClient, clusterRouter, p) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(routes).ToNot(gomega.BeEmpty(), "Should have IC static routes for remote zone nodes (node2, node3)") + + // Manually delete the transit switch to simulate the resource leak scenario + // This leaves orphaned router ports and static routes + err = libovsdbops.DeleteLogicalSwitch(libovsdbOvnNBClient, types.TransitSwitch) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify transit switch is gone + _, err = libovsdbops.GetLogicalSwitch(libovsdbOvnNBClient, &nbdb.LogicalSwitch{Name: types.TransitSwitch}) + gomega.Expect(err).To(gomega.MatchError(libovsdbclient.ErrNotFound)) + + // Verify orphaned resources still exist + clusterRouter, err = libovsdbops.GetLogicalRouter(libovsdbOvnNBClient, &nbdb.LogicalRouter{Name: types.OVNClusterRouter}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + icRouterPorts = 0 + for _, p := range clusterRouter.Ports { + lrp, err := libovsdbops.GetLogicalRouterPort(libovsdbOvnNBClient, &nbdb.LogicalRouterPort{UUID: p}) + if err != nil { + continue + } + if len(lrp.Name) >= len(types.RouterToTransitSwitchPrefix) && lrp.Name[:len(types.RouterToTransitSwitchPrefix)] == types.RouterToTransitSwitchPrefix { + icRouterPorts++ + } + } + gomega.Expect(icRouterPorts).To(gomega.Equal(1), "Router port should still exist before cleanup (the leak)") + + routes, err = libovsdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(libovsdbOvnNBClient, clusterRouter, p) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(routes).ToNot(gomega.BeEmpty(), "IC static routes should still exist before cleanup (the leak)") + + // Call CleanupStaleNodes with nil - should discover all nodes and clean them + err = zoneICHandler.CleanupStaleNodes(nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify all router ports are cleaned up + clusterRouter, err = libovsdbops.GetLogicalRouter(libovsdbOvnNBClient, &nbdb.LogicalRouter{Name: types.OVNClusterRouter}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + icRouterPorts = 0 + for _, p := range clusterRouter.Ports { + lrp, err := libovsdbops.GetLogicalRouterPort(libovsdbOvnNBClient, &nbdb.LogicalRouterPort{UUID: p}) + if err != nil { + continue + } + if len(lrp.Name) >= len(types.RouterToTransitSwitchPrefix) && lrp.Name[:len(types.RouterToTransitSwitchPrefix)] == types.RouterToTransitSwitchPrefix { + icRouterPorts++ + } + } + gomega.Expect(icRouterPorts).To(gomega.Equal(0), "All router ports should be cleaned up") + + // Verify all IC static routes are cleaned up + routes, err = libovsdbops.GetRouterLogicalRouterStaticRoutesWithPredicate(libovsdbOvnNBClient, clusterRouter, p) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(routes).To(gomega.BeEmpty(), "All IC static routes should be cleaned up") + + return nil + } + + err := app.Run([]string{ + app.Name, + "-cluster-subnets=" + clusterCIDR, + "-init-cluster-manager", + "-zone-join-switch-subnets=" + joinSubnetCIDR, + "-enable-interconnect", + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) }) ginkgo.Context("Secondary networks", func() { @@ -591,7 +824,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node1", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", ovnNodeZoneNameAnnotation: "global", ovnNodeIDAnnotaton: "2", ovnNodeSubnetsAnnotation: "{\"blue\":[\"10.244.2.0/24\"]}", @@ -608,7 +841,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac7", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac7", ovnNodeZoneNameAnnotation: "global", ovnNodeIDAnnotaton: "3", ovnNodeSubnetsAnnotation: "{\"blue\":[\"10.244.3.0/24\"]}", @@ -625,7 +858,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac8", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac8", ovnNodeZoneNameAnnotation: "foo", ovnNodeIDAnnotaton: "4", ovnNodeSubnetsAnnotation: "{\"blue\":[\"10.244.4.0/24\"]}", @@ -718,11 +951,11 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { err = checkInterconnectResources("global", "blue", libovsdbOvnNBClient, testNodesRouteInfo, &testNode1, &testNode2, &testNode3) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Call ICHandler SyncNodes function removing the testNode3 from the list of nodes + // Call ICHandler CleanupStaleNodes function removing the testNode3 from the list of nodes var kNodes []interface{} kNodes = append(kNodes, &testNode1) kNodes = append(kNodes, &testNode2) - err = zoneICHandler.SyncNodes(kNodes) + err = zoneICHandler.CleanupStaleNodes(kNodes) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = checkInterconnectResources("global", "blue", libovsdbOvnNBClient, testNodesRouteInfo, &testNode1, &testNode2) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -746,7 +979,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node1", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", ovnNodeZoneNameAnnotation: "global", ovnNodeIDAnnotaton: "2", ovnNodeSubnetsAnnotation: "{\"red\":[\"10.244.2.0/24\"], \"blue\":[\"11.244.2.0/24\"]}", @@ -763,7 +996,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac7", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac7", ovnNodeZoneNameAnnotation: "foo", ovnNodeIDAnnotaton: "3", ovnNodeSubnetsAnnotation: "{\"red\":[\"10.244.3.0/24\"], \"blue\":[\"11.244.3.0/24\"]}", @@ -780,7 +1013,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Annotations: map[string]string{ - ovnNodeChassisIDAnnotatin: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac8", + ovnNodeChassisIDAnnotation: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac8", ovnNodeZoneNameAnnotation: "foo", ovnNodeIDAnnotaton: "4", ovnNodeSubnetsAnnotation: "{\"red\":[\"10.244.4.0/24\"], \"blue\":[\"11.244.4.0/24\"]}", @@ -1004,6 +1237,11 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { // Set the node transit switch port ips testNode4.Annotations[ovnTransitSwitchPortAddrAnnotation] = "{\"ipv4\":\"100.88.0.5/16\"}" err = zoneICHandler.AddRemoteZoneNode(&testNode4) + gomega.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("k8s.ovn.org/node-chassis-id annotation not found for node node4"))) + + // Set chassis-id annotation + testNode4.Annotations[ovnNodeChassisIDAnnotation] = "c44f341d-2862-4fbe-8b93-10e98b0fa84f" + err = zoneICHandler.AddRemoteZoneNode(&testNode4) gomega.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("failed to create static route ops: unable to get logical router static routes with predicate on router ovn_cluster_router"))) // Create the cluster router diff --git a/go-controller/pkg/retry/obj_retry.go b/go-controller/pkg/retry/obj_retry.go index 970a3ae052..c9de84d8c2 100644 --- a/go-controller/pkg/retry/obj_retry.go +++ b/go-controller/pkg/retry/obj_retry.go @@ -41,6 +41,9 @@ type retryObjEntry struct { backoff time.Duration // number of times this object has been unsuccessfully added/updated/deleted failedAttempts uint8 + // infiniteRetry indicates whether this object should be retried indefinitely, regardless of the number of failed attempts + // Used for pods only right now + infiniteRetry bool } type EventHandler interface { @@ -146,6 +149,10 @@ func (r *RetryFramework) initRetryObjWithAddBackoff(obj interface{}, lockedKey s entry, _ := r.retryEntries.LoadOrStore(lockedKey, &retryObjEntry{backoff: backoff}) entry.timeStamp = time.Now() entry.newObj = obj + if _, isPod := obj.(*corev1.Pod); isPod { + // for pods we want to retry indefinitely + entry.infiniteRetry = true + } entry.failedAttempts = 0 entry.backoff = backoff return entry @@ -163,21 +170,33 @@ func (r *RetryFramework) initRetryObjWithUpdate(oldObj, newObj interface{}, lock // even if the object was loaded and changed before with the same lock, LoadOrStore will return reference to the same object entry.timeStamp = time.Now() entry.newObj = newObj + if _, isPod := newObj.(*corev1.Pod); isPod { + // for pods we want to retry indefinitely + entry.infiniteRetry = true + } entry.config = oldObj entry.failedAttempts = 0 return entry } -// InitRetryObjWithDelete creates a retry entry for an object that is being deleted, +// initRetryObjWithDelete creates a retry entry for an object that is being deleted, // so that, if it fails, the delete can be potentially retried later. // When applied to pods, we include the config object as well in case the namespace is removed // and the object is orphaned from the namespace. // The noRetryAdd boolean argument is to indicate whether to retry for addition -func (r *RetryFramework) InitRetryObjWithDelete(obj interface{}, lockedKey string, config interface{}, noRetryAdd bool) *retryObjEntry { +func (r *RetryFramework) initRetryObjWithDelete(obj interface{}, lockedKey string, config interface{}, noRetryAdd bool) *retryObjEntry { + return r.initRetryObjWithDeleteBackoff(obj, lockedKey, config, noRetryAdd, initialBackoff) +} + +func (r *RetryFramework) initRetryObjWithDeleteBackoff(obj interface{}, lockedKey string, config interface{}, noRetryAdd bool, backoff time.Duration) *retryObjEntry { // even if the object was loaded and changed before with the same lock, LoadOrStore will return reference to the same object entry, _ := r.retryEntries.LoadOrStore(lockedKey, &retryObjEntry{config: config, backoff: initialBackoff}) entry.timeStamp = time.Now() entry.oldObj = obj + if _, isPod := obj.(*corev1.Pod); isPod { + // for pods we want to retry indefinitely + entry.infiniteRetry = true + } if entry.config == nil { entry.config = config } @@ -186,9 +205,21 @@ func (r *RetryFramework) InitRetryObjWithDelete(obj interface{}, lockedKey strin // will not be retried for addition entry.newObj = nil } + entry.backoff = backoff return entry } +func (r *RetryFramework) AddRetryObjWithDeleteNoBackoff(obj interface{}) error { + key, err := GetResourceKey(obj) + if err != nil { + return fmt.Errorf("could not get the key of %s %v: %v", r.ResourceHandler.ObjType, obj, err) + } + r.DoWithLock(key, func(key string) { + r.initRetryObjWithDeleteBackoff(obj, key, nil, true, noBackoff) + }) + return nil +} + // AddRetryObjWithAddNoBackoff adds an object to be retried immediately for add. // It will lock the key, create or update retryObject, and unlock the key func (r *RetryFramework) AddRetryObjWithAddNoBackoff(obj interface{}) error { @@ -226,7 +257,10 @@ func (r *RetryFramework) removeDeleteFromRetryObj(entry *retryObjEntry) { // increaseFailedAttemptsCounter increases by one the counter of failed add/update/delete attempts // for the given key func (r *RetryFramework) increaseFailedAttemptsCounter(entry *retryObjEntry) { - entry.failedAttempts++ + // avoid overflowing the counter for infinite retries + if entry.failedAttempts < 255 { + entry.failedAttempts++ + } } // RequestRetryFramework allows a caller to immediately request to iterate through all objects that @@ -259,7 +293,7 @@ func (r *RetryFramework) resourceRetry(objKey string, now time.Time) { return } - if entry.failedAttempts >= MaxFailedAttempts { + if entry.failedAttempts >= MaxFailedAttempts && !entry.infiniteRetry { klog.Warningf("Dropping retry entry for %s %s: exceeded number of failed attempts", r.ResourceHandler.ObjType, objKey) r.DeleteRetryObj(key) @@ -324,8 +358,8 @@ func (r *RetryFramework) resourceRetry(objKey string, now time.Time) { klog.Errorf("%v retry: cannot update object that is not scheduled: %s", r.ResourceHandler.ObjType, objKey) } else if err := r.ResourceHandler.UpdateResource(entry.config, entry.newObj, true); err != nil { entry.timeStamp = time.Now() - entry.failedAttempts++ - if entry.failedAttempts >= MaxFailedAttempts { + r.increaseFailedAttemptsCounter(entry) + if entry.failedAttempts >= MaxFailedAttempts && !entry.infiniteRetry { klog.Errorf("Retry update failed final attempt for %s %s: error: %v", r.ResourceHandler.ObjType, objKey, err) } else { klog.Infof("%v retry update failed for %s, will try again later: %v", r.ResourceHandler.ObjType, objKey, err) @@ -345,8 +379,8 @@ func (r *RetryFramework) resourceRetry(objKey string, now time.Time) { klog.Errorf("%v retry: cannot delete object that was not scheduled %s", r.ResourceHandler.ObjType, objKey) } else if err := r.ResourceHandler.DeleteResource(entry.oldObj, entry.config); err != nil { entry.timeStamp = time.Now() - entry.failedAttempts++ - if entry.failedAttempts >= MaxFailedAttempts { + r.increaseFailedAttemptsCounter(entry) + if entry.failedAttempts >= MaxFailedAttempts && !entry.infiniteRetry { klog.Errorf("Retry delete failed final attempt for %s %s: error: %v", r.ResourceHandler.ObjType, objKey, err) } else { klog.Infof("Retry delete failed for %s %s, will try again later: %v", @@ -368,8 +402,8 @@ func (r *RetryFramework) resourceRetry(objKey string, now time.Time) { klog.Errorf("%v retry: cannot create object that is not scheduled %s", r.ResourceHandler.ObjType, objKey) } else if err := r.ResourceHandler.AddResource(entry.newObj, true); err != nil { entry.timeStamp = time.Now() - entry.failedAttempts++ - if entry.failedAttempts >= MaxFailedAttempts { + r.increaseFailedAttemptsCounter(entry) + if entry.failedAttempts >= MaxFailedAttempts && !entry.infiniteRetry { klog.Errorf("Retry add failed final attempt for %s %s: error: %v", r.ResourceHandler.ObjType, objKey, err) } else { klog.Infof("Retry add failed for %s %s, will try again later: %v", r.ResourceHandler.ObjType, objKey, err) @@ -464,7 +498,7 @@ func (r *RetryFramework) processObjectInTerminalState(obj interface{}, lockedKey klog.Infof("Detected object %s of type %s in terminal state (e.g. completed)"+ " during %s event: will remove it", lockedKey, r.ResourceHandler.ObjType, event) internalCacheEntry := r.ResourceHandler.GetInternalCacheEntry(obj) - retryEntry := r.InitRetryObjWithDelete(obj, lockedKey, internalCacheEntry, true) // set up the retry obj for deletion + retryEntry := r.initRetryObjWithDelete(obj, lockedKey, internalCacheEntry, true) // set up the retry obj for deletion if err := r.ResourceHandler.DeleteResource(obj, internalCacheEntry); err != nil { klog.Errorf("Failed to delete object %s of type %s in terminal state, during %s event: %v", lockedKey, r.ResourceHandler.ObjType, event, err) @@ -662,7 +696,7 @@ func (r *RetryFramework) WatchResourceFiltered(namespaceForFilteredHandler strin klog.Errorf("Failed to delete %s %s, during update: %v", r.ResourceHandler.ObjType, oldKey, err) r.ResourceHandler.RecordErrorEvent(old, "ErrorDeletingResource", err) - retryEntry := r.InitRetryObjWithDelete(old, key, nil, false) + retryEntry := r.initRetryObjWithDelete(old, key, nil, false) r.initRetryObjWithAdd(latest, key) r.increaseFailedAttemptsCounter(retryEntry) return @@ -741,9 +775,9 @@ func (r *RetryFramework) WatchResourceFiltered(namespaceForFilteredHandler strin } r.DoWithLock(key, func(key string) { internalCacheEntry := r.ResourceHandler.GetInternalCacheEntry(obj) - retryEntry := r.InitRetryObjWithDelete(obj, key, internalCacheEntry, false) // set up the retry obj for deletion + retryEntry := r.initRetryObjWithDelete(obj, key, internalCacheEntry, false) // set up the retry obj for deletion if err = r.ResourceHandler.DeleteResource(obj, internalCacheEntry); err != nil { - retryEntry.failedAttempts++ + r.increaseFailedAttemptsCounter(retryEntry) klog.Errorf("Failed to delete %s %s, error: %v", r.ResourceHandler.ObjType, key, err) return } diff --git a/go-controller/pkg/sbdb/datapath_binding.go b/go-controller/pkg/sbdb/datapath_binding.go index 295660e9c3..5907a5c239 100644 --- a/go-controller/pkg/sbdb/datapath_binding.go +++ b/go-controller/pkg/sbdb/datapath_binding.go @@ -7,12 +7,23 @@ import "github.com/ovn-kubernetes/libovsdb/model" const DatapathBindingTable = "Datapath_Binding" +type ( + DatapathBindingType = string +) + +var ( + DatapathBindingTypeLogicalSwitch DatapathBindingType = "logical-switch" + DatapathBindingTypeLogicalRouter DatapathBindingType = "logical-router" +) + // DatapathBinding defines an object in Datapath_Binding table type DatapathBinding struct { - UUID string `ovsdb:"_uuid"` - ExternalIDs map[string]string `ovsdb:"external_ids"` - LoadBalancers []string `ovsdb:"load_balancers"` - TunnelKey int `ovsdb:"tunnel_key"` + UUID string `ovsdb:"_uuid"` + ExternalIDs map[string]string `ovsdb:"external_ids"` + LoadBalancers []string `ovsdb:"load_balancers"` + NbUUID *string `ovsdb:"nb_uuid"` + TunnelKey int `ovsdb:"tunnel_key"` + Type *DatapathBindingType `ovsdb:"type"` } func (a *DatapathBinding) GetUUID() string { @@ -77,14 +88,60 @@ func equalDatapathBindingLoadBalancers(a, b []string) bool { return true } +func (a *DatapathBinding) GetNbUUID() *string { + return a.NbUUID +} + +func copyDatapathBindingNbUUID(a *string) *string { + if a == nil { + return nil + } + b := *a + return &b +} + +func equalDatapathBindingNbUUID(a, b *string) bool { + if (a == nil) != (b == nil) { + return false + } + if a == b { + return true + } + return *a == *b +} + func (a *DatapathBinding) GetTunnelKey() int { return a.TunnelKey } +func (a *DatapathBinding) GetType() *DatapathBindingType { + return a.Type +} + +func copyDatapathBindingType(a *DatapathBindingType) *DatapathBindingType { + if a == nil { + return nil + } + b := *a + return &b +} + +func equalDatapathBindingType(a, b *DatapathBindingType) bool { + if (a == nil) != (b == nil) { + return false + } + if a == b { + return true + } + return *a == *b +} + func (a *DatapathBinding) DeepCopyInto(b *DatapathBinding) { *b = *a b.ExternalIDs = copyDatapathBindingExternalIDs(a.ExternalIDs) b.LoadBalancers = copyDatapathBindingLoadBalancers(a.LoadBalancers) + b.NbUUID = copyDatapathBindingNbUUID(a.NbUUID) + b.Type = copyDatapathBindingType(a.Type) } func (a *DatapathBinding) DeepCopy() *DatapathBinding { @@ -106,7 +163,9 @@ func (a *DatapathBinding) Equals(b *DatapathBinding) bool { return a.UUID == b.UUID && equalDatapathBindingExternalIDs(a.ExternalIDs, b.ExternalIDs) && equalDatapathBindingLoadBalancers(a.LoadBalancers, b.LoadBalancers) && - a.TunnelKey == b.TunnelKey + equalDatapathBindingNbUUID(a.NbUUID, b.NbUUID) && + a.TunnelKey == b.TunnelKey && + equalDatapathBindingType(a.Type, b.Type) } func (a *DatapathBinding) EqualsModel(b model.Model) bool { diff --git a/go-controller/pkg/sbdb/encap.go b/go-controller/pkg/sbdb/encap.go index 4c524a52ca..9b1fe28afc 100644 --- a/go-controller/pkg/sbdb/encap.go +++ b/go-controller/pkg/sbdb/encap.go @@ -13,7 +13,6 @@ type ( var ( EncapTypeGeneve EncapType = "geneve" - EncapTypeSTT EncapType = "stt" EncapTypeVxlan EncapType = "vxlan" ) diff --git a/go-controller/pkg/sbdb/mirror.go b/go-controller/pkg/sbdb/mirror.go index b9139214ca..db0d65a011 100644 --- a/go-controller/pkg/sbdb/mirror.go +++ b/go-controller/pkg/sbdb/mirror.go @@ -19,6 +19,7 @@ var ( MirrorTypeGre MirrorType = "gre" MirrorTypeErspan MirrorType = "erspan" MirrorTypeLocal MirrorType = "local" + MirrorTypeLport MirrorType = "lport" ) // Mirror defines an object in Mirror table diff --git a/go-controller/pkg/sbdb/model.go b/go-controller/pkg/sbdb/model.go index 0d9fe177bf..9e728c8872 100644 --- a/go-controller/pkg/sbdb/model.go +++ b/go-controller/pkg/sbdb/model.go @@ -56,7 +56,7 @@ func FullDatabaseModel() (model.ClientDBModel, error) { var schema = `{ "name": "OVN_Southbound", - "version": "20.41.0", + "version": "21.5.0", "tables": { "ACL_ID": { "columns": { @@ -634,6 +634,15 @@ var schema = `{ "max": "unlimited" } }, + "nb_uuid": { + "type": { + "key": { + "type": "uuid" + }, + "min": 0, + "max": 1 + } + }, "tunnel_key": { "type": { "key": { @@ -642,6 +651,22 @@ var schema = `{ "maxInteger": 16777215 } } + }, + "type": { + "type": { + "key": { + "type": "string", + "enum": [ + "set", + [ + "logical-switch", + "logical-router" + ] + ] + }, + "min": 0, + "max": 1 + } } }, "indexes": [ @@ -730,7 +755,6 @@ var schema = `{ "set", [ "geneve", - "stt", "vxlan" ] ] @@ -1474,7 +1498,8 @@ var schema = `{ [ "gre", "erspan", - "local" + "local", + "lport" ] ] } @@ -1634,6 +1659,15 @@ var schema = `{ "max": "unlimited" } }, + "mirror_port": { + "type": { + "key": { + "type": "string" + }, + "min": 0, + "max": 1 + } + }, "mirror_rules": { "type": { "key": { @@ -1941,6 +1975,9 @@ var schema = `{ "max": "unlimited" } }, + "ic_learned": { + "type": "boolean" + }, "ip": { "type": "string" }, @@ -1984,6 +2021,9 @@ var schema = `{ "max": 1 } }, + "remote": { + "type": "boolean" + }, "src_ip": { "type": "string" }, diff --git a/go-controller/pkg/sbdb/port_binding.go b/go-controller/pkg/sbdb/port_binding.go index 48668023fc..18ba7f175c 100644 --- a/go-controller/pkg/sbdb/port_binding.go +++ b/go-controller/pkg/sbdb/port_binding.go @@ -20,6 +20,7 @@ type PortBinding struct { HaChassisGroup *string `ovsdb:"ha_chassis_group"` LogicalPort string `ovsdb:"logical_port"` MAC []string `ovsdb:"mac"` + MirrorPort *string `ovsdb:"mirror_port"` MirrorRules []string `ovsdb:"mirror_rules"` NatAddresses []string `ovsdb:"nat_addresses"` Options map[string]string `ovsdb:"options"` @@ -254,6 +255,28 @@ func equalPortBindingMAC(a, b []string) bool { return true } +func (a *PortBinding) GetMirrorPort() *string { + return a.MirrorPort +} + +func copyPortBindingMirrorPort(a *string) *string { + if a == nil { + return nil + } + b := *a + return &b +} + +func equalPortBindingMirrorPort(a, b *string) bool { + if (a == nil) != (b == nil) { + return false + } + if a == b { + return true + } + return *a == *b +} + func (a *PortBinding) GetMirrorRules() []string { return a.MirrorRules } @@ -524,6 +547,7 @@ func (a *PortBinding) DeepCopyInto(b *PortBinding) { b.GatewayChassis = copyPortBindingGatewayChassis(a.GatewayChassis) b.HaChassisGroup = copyPortBindingHaChassisGroup(a.HaChassisGroup) b.MAC = copyPortBindingMAC(a.MAC) + b.MirrorPort = copyPortBindingMirrorPort(a.MirrorPort) b.MirrorRules = copyPortBindingMirrorRules(a.MirrorRules) b.NatAddresses = copyPortBindingNatAddresses(a.NatAddresses) b.Options = copyPortBindingOptions(a.Options) @@ -563,6 +587,7 @@ func (a *PortBinding) Equals(b *PortBinding) bool { equalPortBindingHaChassisGroup(a.HaChassisGroup, b.HaChassisGroup) && a.LogicalPort == b.LogicalPort && equalPortBindingMAC(a.MAC, b.MAC) && + equalPortBindingMirrorPort(a.MirrorPort, b.MirrorPort) && equalPortBindingMirrorRules(a.MirrorRules, b.MirrorRules) && equalPortBindingNatAddresses(a.NatAddresses, b.NatAddresses) && equalPortBindingOptions(a.Options, b.Options) && diff --git a/go-controller/pkg/sbdb/service_monitor.go b/go-controller/pkg/sbdb/service_monitor.go index 189f09f659..1ee1af2d74 100644 --- a/go-controller/pkg/sbdb/service_monitor.go +++ b/go-controller/pkg/sbdb/service_monitor.go @@ -25,11 +25,13 @@ type ServiceMonitor struct { UUID string `ovsdb:"_uuid"` ChassisName string `ovsdb:"chassis_name"` ExternalIDs map[string]string `ovsdb:"external_ids"` + IcLearned bool `ovsdb:"ic_learned"` IP string `ovsdb:"ip"` LogicalPort string `ovsdb:"logical_port"` Options map[string]string `ovsdb:"options"` Port int `ovsdb:"port"` Protocol *ServiceMonitorProtocol `ovsdb:"protocol"` + Remote bool `ovsdb:"remote"` SrcIP string `ovsdb:"src_ip"` SrcMAC string `ovsdb:"src_mac"` Status *ServiceMonitorStatus `ovsdb:"status"` @@ -73,6 +75,10 @@ func equalServiceMonitorExternalIDs(a, b map[string]string) bool { return true } +func (a *ServiceMonitor) GetIcLearned() bool { + return a.IcLearned +} + func (a *ServiceMonitor) GetIP() string { return a.IP } @@ -137,6 +143,10 @@ func equalServiceMonitorProtocol(a, b *ServiceMonitorProtocol) bool { return *a == *b } +func (a *ServiceMonitor) GetRemote() bool { + return a.Remote +} + func (a *ServiceMonitor) GetSrcIP() string { return a.SrcIP } @@ -194,11 +204,13 @@ func (a *ServiceMonitor) Equals(b *ServiceMonitor) bool { return a.UUID == b.UUID && a.ChassisName == b.ChassisName && equalServiceMonitorExternalIDs(a.ExternalIDs, b.ExternalIDs) && + a.IcLearned == b.IcLearned && a.IP == b.IP && a.LogicalPort == b.LogicalPort && equalServiceMonitorOptions(a.Options, b.Options) && a.Port == b.Port && equalServiceMonitorProtocol(a.Protocol, b.Protocol) && + a.Remote == b.Remote && a.SrcIP == b.SrcIP && a.SrcMAC == b.SrcMAC && equalServiceMonitorStatus(a.Status, b.Status) diff --git a/go-controller/pkg/testing/libovsdb/matchers.go b/go-controller/pkg/testing/libovsdb/matchers.go index 1ff3977065..c1fd36a016 100644 --- a/go-controller/pkg/testing/libovsdb/matchers.go +++ b/go-controller/pkg/testing/libovsdb/matchers.go @@ -224,6 +224,26 @@ func HaveEmptyData() gomegatypes.GomegaMatcher { return gomega.WithTransform(transform, gomega.BeEmpty()) } +// HaveDataSubset asserts that all expected TestData objects exist in the actual database, +// but ignores any extra data. UUIDs are ignored when comparing. +func HaveDataSubset(expected ...TestData) gomegatypes.GomegaMatcher { + if len(expected) == 1 { + if e, ok := expected[0].([]TestData); ok { + expected = e + } + } + matchers := []*testDataMatcher{} + for _, e := range expected { + matchers = append(matchers, matchTestData(true, e)) + } + + transform := func(client libovsdbclient.Client) []TestData { + return getTestDataFromClientCache(client) + } + + return gomega.WithTransform(transform, gomega.ContainElements(matchers)) +} + func haveData(ignoreUUIDs, nameUUIDs bool, expected []TestData) gomegatypes.GomegaMatcher { if len(expected) == 1 { if e, ok := expected[0].([]TestData); ok { diff --git a/go-controller/pkg/testing/mocks/k8s.io/kubelet/pkg/apis/podresources/v1/PodResourcesListerClient.go b/go-controller/pkg/testing/mocks/k8s.io/kubelet/pkg/apis/podresources/v1/PodResourcesListerClient.go new file mode 100644 index 0000000000..4cd99e94a0 --- /dev/null +++ b/go-controller/pkg/testing/mocks/k8s.io/kubelet/pkg/apis/podresources/v1/PodResourcesListerClient.go @@ -0,0 +1,143 @@ +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package mocks + +import ( + context "context" + + grpc "google.golang.org/grpc" + + mock "github.com/stretchr/testify/mock" + + v1 "k8s.io/kubelet/pkg/apis/podresources/v1" +) + +// PodResourcesListerClient is an autogenerated mock type for the PodResourcesListerClient type +type PodResourcesListerClient struct { + mock.Mock +} + +// Get provides a mock function with given fields: ctx, in, opts +func (_m *PodResourcesListerClient) Get(ctx context.Context, in *v1.GetPodResourcesRequest, opts ...grpc.CallOption) (*v1.GetPodResourcesResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 *v1.GetPodResourcesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v1.GetPodResourcesRequest, ...grpc.CallOption) (*v1.GetPodResourcesResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *v1.GetPodResourcesRequest, ...grpc.CallOption) *v1.GetPodResourcesResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.GetPodResourcesResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *v1.GetPodResourcesRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAllocatableResources provides a mock function with given fields: ctx, in, opts +func (_m *PodResourcesListerClient) GetAllocatableResources(ctx context.Context, in *v1.AllocatableResourcesRequest, opts ...grpc.CallOption) (*v1.AllocatableResourcesResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for GetAllocatableResources") + } + + var r0 *v1.AllocatableResourcesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v1.AllocatableResourcesRequest, ...grpc.CallOption) (*v1.AllocatableResourcesResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *v1.AllocatableResourcesRequest, ...grpc.CallOption) *v1.AllocatableResourcesResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.AllocatableResourcesResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *v1.AllocatableResourcesRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// List provides a mock function with given fields: ctx, in, opts +func (_m *PodResourcesListerClient) List(ctx context.Context, in *v1.ListPodResourcesRequest, opts ...grpc.CallOption) (*v1.ListPodResourcesResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for List") + } + + var r0 *v1.ListPodResourcesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v1.ListPodResourcesRequest, ...grpc.CallOption) (*v1.ListPodResourcesResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *v1.ListPodResourcesRequest, ...grpc.CallOption) *v1.ListPodResourcesResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.ListPodResourcesResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *v1.ListPodResourcesRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewPodResourcesListerClient creates a new instance of PodResourcesListerClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewPodResourcesListerClient(t interface { + mock.TestingT + Cleanup(func()) +}) *PodResourcesListerClient { + mock := &PodResourcesListerClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index d3df30e380..fc979dec24 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -69,6 +69,12 @@ const ( TransitRouterToSwitchPrefix = "trtos-" SwitchToTransitRouterPrefix = "stotr-" + // Connect router prefix (for ClusterNetworkConnect feature) + ConnectRouterPrefix = "connect_router_" + // Connect router port prefixes (for ClusterNetworkConnect) + ConnectRouterToRouterPrefix = "crtor-" + RouterToConnectRouterPrefix = "rtocr-" + // DefaultACLTier Priorities // Default routed multicast allow acl rule priority @@ -116,6 +122,7 @@ const ( EgressSVCReroutePriority = 101 EgressIPReroutePriority = 100 EgressIPRerouteQoSRulePriority = 103 + NetworkConnectPolicyPriority = 9001 // priority of logical router policies on a nodes gateway router EgressIPSNATMarkPriority = 95 EgressLiveMigrationReroutePriority = 10 @@ -155,6 +162,9 @@ const ( PacketsPerSecond = "pktps" MeterAction = "drop" + // Default COPP object name + DefaultCOPPName = "ovnkube-default" + // OVN-K8S annotation & taint constants OvnK8sPrefix = "k8s.ovn.org" @@ -242,6 +252,11 @@ const ( NetworkRoleInfrastructure = "infrastructure-locked" NetworkRoleNone = "none" + // Network transport types - canonical format (lowercase) + NetworkTransportGeneve = "geneve" + NetworkTransportNoOverlay = "no-overlay" + NetworkTransportEVPN = "evpn" + // db index keys // PrimaryIDKey is used as a primary client index PrimaryIDKey = OvnK8sPrefix + "/id" diff --git a/go-controller/pkg/util/dns.go b/go-controller/pkg/util/dns.go index 9466ad16f5..86d8a9e054 100644 --- a/go-controller/pkg/util/dns.go +++ b/go-controller/pkg/util/dns.go @@ -16,8 +16,12 @@ import ( ) const ( - // defaultTTL is used if an invalid or zero TTL is provided. - defaultTTL = 30 * time.Minute + // defaultMinTTL is the minimum TTL value that will be used for a domain name if an invalid or zero TTL is found + defaultMinTTL = 5 * time.Second + // defaultMaxTTL is the maximum TTL value that will be used for a domain name if an invalid or zero TTL is found + defaultMaxTTL = 2 * time.Minute + // maxRetryBeforeBackoff is the maximum number of times to retry a DNS lookup before exponential backoff starts + maxRetryBeforeBackoff = 10 ) type dnsValue struct { @@ -27,6 +31,8 @@ type dnsValue struct { ttl time.Duration // Holds (last dns lookup time + ttl), tells when to refresh IPs next time nextQueryTime time.Time + // Number of times the DNS lookup has been retried before backoff starts + retryCount int } type DNS struct { @@ -105,11 +111,22 @@ func (d *DNS) updateOne(dns string) (bool, error) { return false, fmt.Errorf("DNS value not found in dnsMap for domain: %q", dns) } - ips, ttl, err := d.getIPsAndMinTTL(dns) - if err != nil { - res.nextQueryTime = time.Now().Add(defaultTTL) - d.dnsMap[dns] = res - return false, err + ips, ttl, retry, err := d.getIPsAndMinTTL(dns) + if retry { + // If the DNS lookup has been retried maxRetryCount times, use exponential backoff + // by doubling the previous TTL. The TTL is capped at defaultMaxTTL. + if res.retryCount >= maxRetryBeforeBackoff { + ttl = min(res.ttl*2, defaultMaxTTL) + } else { + // Increment the retry count + res.retryCount++ + } + // If no valid IPs were found, use the previous IPs as fallback. + if len(ips) == 0 { + ips = res.ips + } + } else { + res.retryCount = 0 } changed := false @@ -120,10 +137,10 @@ func (d *DNS) updateOne(dns string) (bool, error) { res.ttl = ttl res.nextQueryTime = time.Now().Add(res.ttl) d.dnsMap[dns] = res - return changed, nil + return changed, err } -func (d *DNS) getIPsAndMinTTL(domain string) ([]net.IP, time.Duration, error) { +func (d *DNS) getIPsAndMinTTL(domain string) ([]net.IP, time.Duration, bool, error) { ips := []net.IP{} ttlSet := false var ttlSeconds uint32 @@ -197,19 +214,27 @@ func (d *DNS) getIPsAndMinTTL(domain string) ([]net.IP, time.Duration, error) { } if !ttlSet || (len(ips) == 0) { - return nil, defaultTTL, fmt.Errorf("IPv4 or IPv6 addr not found for domain: %q, nameservers: %v", domain, d.nameservers) + return nil, defaultMinTTL, true, fmt.Errorf("IPv4 or IPv6 addr not found for domain: %q, nameservers: %v", domain, d.nameservers) } + ips = removeDuplicateIPs(ips) + ttl, err := time.ParseDuration(fmt.Sprintf("%ds", minTTL)) if err != nil { - utilruntime.HandleError(fmt.Errorf("invalid TTL value for domain: %q, err: %v, defaulting ttl=%s", domain, err, defaultTTL.String())) - ttl = defaultTTL + utilruntime.HandleError(fmt.Errorf("invalid TTL value for domain: %q, err: %v", domain, err)) + return ips, defaultMinTTL, true, nil } if ttl == 0 { - ttl = defaultTTL + // If the TTL is 0, return the default minimum TTL. The retry is set to false as this + // is not an error scenario. TTL being 0 is a valid scenario for some DNS servers + // and it means that the IP addresses should be refreshed everytime whenever the DNS + // name is being used. From the point of view of OVN-Kubernetes, the IP addresses are + // refreshed every defaultMinTTL. + klog.V(5).Infof("TTL value is 0 for domain: %q, defaulting ttl=%s", domain, defaultMinTTL.String()) + return ips, defaultMinTTL, false, nil } - return removeDuplicateIPs(ips), ttl, nil + return ips, ttl, false, nil } func (d *DNS) GetNextQueryTime() (time.Time, string, bool) { diff --git a/go-controller/pkg/util/dns_test.go b/go-controller/pkg/util/dns_test.go index a9d248042b..9f40c176ba 100644 --- a/go-controller/pkg/util/dns_test.go +++ b/go-controller/pkg/util/dns_test.go @@ -70,13 +70,16 @@ func TestGetIPsAndMinTTL(t *testing.T) { tests := []struct { desc string errExp bool + retry bool ipv4Mode bool ipv6Mode bool dnsOpsMockHelper []ovntest.TestifyMockHelper + expectedTTL time.Duration }{ { desc: "call to Exchange fails IPv4 only", errExp: true, + retry: true, ipv4Mode: true, ipv6Mode: false, dnsOpsMockHelper: []ovntest.TestifyMockHelper{ @@ -89,10 +92,12 @@ func TestGetIPsAndMinTTL(t *testing.T) { CallTimes: 1, }, }, + expectedTTL: defaultMinTTL, }, { desc: "Exchange returns correctly but Rcode != RcodeSuccess IPv4 only", errExp: true, + retry: true, ipv4Mode: true, ipv6Mode: false, dnsOpsMockHelper: []ovntest.TestifyMockHelper{ @@ -105,6 +110,46 @@ func TestGetIPsAndMinTTL(t *testing.T) { CallTimes: 1, }, }, + expectedTTL: defaultMinTTL, + }, + { + desc: "Exchange returns correctly but with TTL 0 IPv4 only", + errExp: false, + retry: false, + ipv4Mode: true, + ipv6Mode: false, + dnsOpsMockHelper: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "SetQuestion", OnCallMethodArgType: []string{"*dns.Msg", "string", "uint16"}, RetArgList: []interface{}{&dns.Msg{}}, CallTimes: 1}, + {OnCallMethodName: "Fqdn", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"www.test.com"}, CallTimes: 1}, + {OnCallMethodName: "Exchange", OnCallMethodArgType: []string{"*dns.Client", "*dns.Msg", "string"}, RetArgList: []interface{}{&dns.Msg{MsgHdr: dns.MsgHdr{Rcode: dns.RcodeSuccess}, Answer: []dns.RR{&dns.A{A: net.ParseIP("1.2.3.4")}}}, 0 * time.Second, nil}, CallTimes: 1}, + }, + expectedTTL: defaultMinTTL, + }, + { + desc: "Exchange returns correctly but no Answer IPv4 only", + errExp: true, + retry: true, + ipv4Mode: true, + ipv6Mode: false, + dnsOpsMockHelper: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "SetQuestion", OnCallMethodArgType: []string{"*dns.Msg", "string", "uint16"}, RetArgList: []interface{}{&dns.Msg{}}, CallTimes: 1}, + {OnCallMethodName: "Fqdn", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"www.test.com"}, CallTimes: 1}, + {OnCallMethodName: "Exchange", OnCallMethodArgType: []string{"*dns.Client", "*dns.Msg", "string"}, RetArgList: []interface{}{&dns.Msg{MsgHdr: dns.MsgHdr{Rcode: dns.RcodeSuccess}, Answer: []dns.RR{}}, 0 * time.Second, nil}, CallTimes: 1}, + }, + expectedTTL: defaultMinTTL, + }, + { + desc: "Exchange returns correctly but with non-zero TTL IPv4 only", + errExp: false, + retry: false, + ipv4Mode: true, + ipv6Mode: false, + dnsOpsMockHelper: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "SetQuestion", OnCallMethodArgType: []string{"*dns.Msg", "string", "uint16"}, RetArgList: []interface{}{&dns.Msg{}}, CallTimes: 1}, + {OnCallMethodName: "Fqdn", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"www.test.com"}, CallTimes: 1}, + {OnCallMethodName: "Exchange", OnCallMethodArgType: []string{"*dns.Client", "*dns.Msg", "string"}, RetArgList: []interface{}{&dns.Msg{MsgHdr: dns.MsgHdr{Rcode: dns.RcodeSuccess}, Answer: []dns.RR{&dns.A{Hdr: dns.RR_Header{Ttl: 100}, A: net.ParseIP("1.2.3.4")}}}, 0 * time.Second, nil}, CallTimes: 1}, + }, + expectedTTL: 100 * time.Second, }, } @@ -128,19 +173,22 @@ func TestGetIPsAndMinTTL(t *testing.T) { } config.IPv4Mode = tc.ipv4Mode config.IPv6Mode = tc.ipv6Mode - res, _, err := testDNS.getIPsAndMinTTL("www.test.com") - t.Log(res, err) + res, ttl, retry, err := testDNS.getIPsAndMinTTL("www.test.com") + t.Log(res, ttl, retry, err) if tc.errExp { require.Error(t, err) } else { require.NoError(t, err) } + assert.Equal(t, tc.retry, retry, "the exponentialBackoff variable should match the return from dns.getIPsAndMinTTL()") + assert.Equal(t, tc.expectedTTL, ttl, "the ttl variable should match the return from dns.getIPsAndMinTTL()") mockDNSOps.AssertExpectations(t) }) } } func TestUpdate(t *testing.T) { + config.IPv4Mode = true mockDNSOps := new(util_mocks.DNSOps) SetDNSLibOpsMockInst(mockDNSOps) @@ -252,6 +300,7 @@ func TestUpdate(t *testing.T) { } func TestAdd(t *testing.T) { + config.IPv4Mode = true dnsName := "www.testing.com" mockDNSOps := new(util_mocks.DNSOps) SetDNSLibOpsMockInst(mockDNSOps) @@ -319,3 +368,211 @@ func TestAdd(t *testing.T) { } } + +func TestIPsEqual(t *testing.T) { + tests := []struct { + desc string + oldips []net.IP + newips []net.IP + expEqual bool + }{ + { + desc: "oldips and newips are the same", + oldips: []net.IP{net.ParseIP("1.2.3.4")}, + newips: []net.IP{net.ParseIP("1.2.3.4")}, + expEqual: true, + }, + { + desc: "oldips and newips are different", + oldips: []net.IP{net.ParseIP("1.2.3.4")}, + newips: []net.IP{net.ParseIP("1.2.3.5")}, + expEqual: false, + }, + { + desc: "oldips and newips are different length", + oldips: []net.IP{net.ParseIP("1.2.3.4")}, + newips: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("1.2.3.5")}, + expEqual: false, + }, + { + desc: "oldips is nil and newips is not nil", + oldips: nil, + newips: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("1.2.3.5")}, + expEqual: false, + }, + { + desc: "oldips is empty and newips is not empty", + oldips: []net.IP{}, + newips: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("1.2.3.5")}, + expEqual: false, + }, + { + desc: "oldips is not nil and newips is nil", + oldips: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("1.2.3.5")}, + newips: nil, + expEqual: false, + }, + { + desc: "oldips is not empty and newips is empty", + oldips: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("1.2.3.5")}, + newips: []net.IP{}, + expEqual: false, + }, + { + desc: "oldips and newips are both nil", + oldips: nil, + newips: nil, + expEqual: true, + }, + { + desc: "oldips and newips are both empty", + oldips: []net.IP{}, + newips: []net.IP{}, + expEqual: true, + }, + { + desc: "oldips is nil and newips is empty", + oldips: nil, + newips: []net.IP{}, + expEqual: true, + }, + { + desc: "oldips is empty and newips is nil", + oldips: []net.IP{}, + newips: nil, + expEqual: true, + }, + } + for i, tc := range tests { + t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) { + res := ipsEqual(tc.oldips, tc.newips) + assert.Equal(t, tc.expEqual, res) + }) + } +} + +func TestUpdateOne(t *testing.T) { + config.IPv4Mode = true + dnsName := "www.testing.com" + newIP := net.ParseIP("1.2.3.4") + fqdnOpsMockHelper := ovntest.TestifyMockHelper{ + OnCallMethodName: "Fqdn", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{dnsName}, CallTimes: 1, + } + setQuestionOpsMockHelper := ovntest.TestifyMockHelper{ + OnCallMethodName: "SetQuestion", OnCallMethodArgType: []string{"*dns.Msg", "string", "uint16"}, RetArgList: []interface{}{&dns.Msg{}}, CallTimes: 1, + } + exchangeSuccessNoAnswerOpsMockHelper := ovntest.TestifyMockHelper{ + OnCallMethodName: "Exchange", OnCallMethodArgType: []string{"*dns.Client", "*dns.Msg", "string"}, RetArgList: []interface{}{&dns.Msg{MsgHdr: dns.MsgHdr{Rcode: dns.RcodeSuccess}, Answer: []dns.RR{}}, 0 * time.Second, nil}, CallTimes: 1, + } + exchangeSuccessZeroTTLOpsMockHelper := ovntest.TestifyMockHelper{ + OnCallMethodName: "Exchange", OnCallMethodArgType: []string{"*dns.Client", "*dns.Msg", "string"}, RetArgList: []interface{}{&dns.Msg{MsgHdr: dns.MsgHdr{Rcode: dns.RcodeSuccess}, Answer: []dns.RR{&dns.A{A: newIP}}}, 0 * time.Second, nil}, CallTimes: 1, + } + exchangeSuccessNonZeroTTLOpsMockHelper := ovntest.TestifyMockHelper{ + OnCallMethodName: "Exchange", OnCallMethodArgType: []string{"*dns.Client", "*dns.Msg", "string"}, RetArgList: []interface{}{&dns.Msg{MsgHdr: dns.MsgHdr{Rcode: dns.RcodeSuccess}, Answer: []dns.RR{&dns.A{Hdr: dns.RR_Header{Ttl: 100}, A: newIP}}}, 0 * time.Second, nil}, CallTimes: 1, + } + exchangeFailureOpsMockHelper := ovntest.TestifyMockHelper{ + OnCallMethodName: "Exchange", OnCallMethodArgType: []string{"*dns.Client", "*dns.Msg", "string"}, RetArgList: []interface{}{&dns.Msg{MsgHdr: dns.MsgHdr{Rcode: dns.RcodeServerFailure}}, 0 * time.Second, nil}, CallTimes: 1, + } + tests := []struct { + desc string + numCalls int + exchangeOpsMockHelper ovntest.TestifyMockHelper + expTTL time.Duration + }{ + { + desc: "when Exchange function returns with Rcode != RcodeSuccess, defaultMinTTL is used", + numCalls: 1, + exchangeOpsMockHelper: exchangeFailureOpsMockHelper, + expTTL: defaultMinTTL, + }, + { + desc: "when Exchange function returns successfully but without Answer, defaultMinTTL is used", + numCalls: 1, + exchangeOpsMockHelper: exchangeSuccessNoAnswerOpsMockHelper, + expTTL: defaultMinTTL, + }, + { + desc: "when TTL returned is 0 by Exchange function, defaultMinTTL is used", + numCalls: 1, + exchangeOpsMockHelper: exchangeSuccessZeroTTLOpsMockHelper, + expTTL: defaultMinTTL, + }, + { + desc: "when TTL returned is 0 by Exchange function 2 times, defaultMinTTL is used", + numCalls: 2, + exchangeOpsMockHelper: exchangeSuccessZeroTTLOpsMockHelper, + expTTL: defaultMinTTL, + }, + { + desc: "when TTL returned is 0 by Exchange function 11 times, defaultMinTTL is used", + numCalls: 11, + exchangeOpsMockHelper: exchangeSuccessZeroTTLOpsMockHelper, + expTTL: defaultMinTTL, + }, + { + desc: "when Exchange function returns with Rcode != RcodeSuccess twice, defaultMinTTL is used", + numCalls: 2, + exchangeOpsMockHelper: exchangeFailureOpsMockHelper, + expTTL: defaultMinTTL, + }, + { + desc: "when Exchange function returns with Rcode != RcodeSuccess 10 times, defaultMinTTL is used", + numCalls: 10, + exchangeOpsMockHelper: exchangeFailureOpsMockHelper, + expTTL: defaultMinTTL, + }, + { + desc: "when Exchange function returns with Rcode != RcodeSuccess 11 times, defaultMinTTL is doubled", + numCalls: 11, + exchangeOpsMockHelper: exchangeFailureOpsMockHelper, + expTTL: 2 * defaultMinTTL, + }, + { + desc: "when Exchange function returns with Rcode != RcodeSuccess 14 times, 16 (2^4) times defaultMinTTL is used", + numCalls: 14, + exchangeOpsMockHelper: exchangeFailureOpsMockHelper, + expTTL: 16 * defaultMinTTL, + }, + { + desc: "when Exchange function returns with Rcode != RcodeSuccess 15 times, defaultMaxTTL is used", + numCalls: 15, + exchangeOpsMockHelper: exchangeFailureOpsMockHelper, + expTTL: defaultMaxTTL, + }, + { + desc: "when TTL returned is non-zero by Exchange function, it is used", + numCalls: 1, + exchangeOpsMockHelper: exchangeSuccessNonZeroTTLOpsMockHelper, + expTTL: 100 * time.Second, + }, + } + for i, tc := range tests { + t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) { + mockDNSOps := new(util_mocks.DNSOps) + SetDNSLibOpsMockInst(mockDNSOps) + dnsOpsMockHelper := []ovntest.TestifyMockHelper{fqdnOpsMockHelper, setQuestionOpsMockHelper, tc.exchangeOpsMockHelper} + for index := 0; index < tc.numCalls; index++ { + for _, item := range dnsOpsMockHelper { + call := mockDNSOps.On(item.OnCallMethodName) + for _, arg := range item.OnCallMethodArgType { + call.Arguments = append(call.Arguments, mock.AnythingOfType(arg)) + } + for _, ret := range item.RetArgList { + call.ReturnArguments = append(call.ReturnArguments, ret) + } + call.Once() + } + } + dns := DNS{ + dnsMap: make(map[string]dnsValue), + nameservers: []string{"1.1.1.1"}, + } + dns.dnsMap[dnsName] = dnsValue{} + for i := 0; i < tc.numCalls; i++ { + _, _ = dns.updateOne(dnsName) + } + assert.Equal(t, tc.expTTL, dns.dnsMap[dnsName].ttl) + mockDNSOps.AssertExpectations(t) + }) + } +} diff --git a/go-controller/pkg/util/dpu_annotations.go b/go-controller/pkg/util/dpu_annotations.go index 1f91772c2d..076f0a5d2a 100644 --- a/go-controller/pkg/util/dpu_annotations.go +++ b/go-controller/pkg/util/dpu_annotations.go @@ -85,7 +85,7 @@ func UnmarshalPodDPUConnDetailsAllNetworks(annotations map[string]string) (map[s // MarshalPodDPUConnDetails adds the pod's connection details of the specified NAD to the corresponding pod annotation; // if dcd is nil, delete the pod's connection details of the specified NAD -func MarshalPodDPUConnDetails(annotations map[string]string, dcd *DPUConnectionDetails, nadName string) (map[string]string, error) { +func MarshalPodDPUConnDetails(annotations map[string]string, dcd *DPUConnectionDetails, nadKey string) (map[string]string, error) { if annotations == nil { annotations = make(map[string]string) } @@ -93,19 +93,19 @@ func MarshalPodDPUConnDetails(annotations map[string]string, dcd *DPUConnectionD if err != nil { return nil, err } - dc, ok := podDcds[nadName] + dc, ok := podDcds[nadKey] if dcd != nil { if ok && dc == *dcd { return nil, newAnnotationAlreadySetError("OVN pod %s annotation for NAD %s already exists in %v", - DPUConnectionDetailsAnnot, nadName, annotations) + DPUConnectionDetailsAnnot, nadKey, annotations) } - podDcds[nadName] = *dcd + podDcds[nadKey] = *dcd } else { if !ok { return nil, newAnnotationAlreadySetError("OVN pod %s annotation for NAD %s already removed", - DPUConnectionDetailsAnnot, nadName) + DPUConnectionDetailsAnnot, nadKey) } - delete(podDcds, nadName) + delete(podDcds, nadKey) } bytes, err := json.Marshal(podDcds) @@ -117,7 +117,7 @@ func MarshalPodDPUConnDetails(annotations map[string]string, dcd *DPUConnectionD } // UnmarshalPodDPUConnDetails returns dpu connection details for the specified NAD -func UnmarshalPodDPUConnDetails(annotations map[string]string, nadName string) (*DPUConnectionDetails, error) { +func UnmarshalPodDPUConnDetails(annotations map[string]string, nadKey string) (*DPUConnectionDetails, error) { ovnAnnotation, ok := annotations[DPUConnectionDetailsAnnot] if !ok { return nil, newAnnotationNotSetError("could not find OVN pod %s annotation in %v", @@ -129,10 +129,10 @@ func UnmarshalPodDPUConnDetails(annotations map[string]string, nadName string) ( return nil, err } - dcd, ok := podDcds[nadName] + dcd, ok := podDcds[nadKey] if !ok { - return nil, newAnnotationNotSetError("no OVN %s annotation for network %s: %q", - DPUConnectionDetailsAnnot, nadName, ovnAnnotation) + return nil, newAnnotationNotSetError("no OVN %s annotation for NAD %s: %q", + DPUConnectionDetailsAnnot, nadKey, ovnAnnotation) } return &dcd, nil } @@ -158,7 +158,7 @@ func UnmarshalPodDPUConnStatusAllNetworks(annotations map[string]string) (map[st // MarshalPodDPUConnStatus adds the pod's connection status of the specified NAD to the corresponding pod annotation. // if scs is nil, delete the pod's connection status of the specified NAD -func MarshalPodDPUConnStatus(annotations map[string]string, scs *DPUConnectionStatus, nadName string) (map[string]string, error) { +func MarshalPodDPUConnStatus(annotations map[string]string, scs *DPUConnectionStatus, nadKey string) (map[string]string, error) { if annotations == nil { annotations = make(map[string]string) } @@ -166,19 +166,19 @@ func MarshalPodDPUConnStatus(annotations map[string]string, scs *DPUConnectionSt if err != nil { return nil, err } - sc, ok := podScss[nadName] + sc, ok := podScss[nadKey] if scs != nil { if ok && sc == *scs { - return nil, newAnnotationAlreadySetError("OVN pod %s annotation for NAD %s already exists in %v", - DPUConnectionStatusAnnot, nadName, annotations) + return nil, newAnnotationAlreadySetError("OVN pod %s annotation for NAD key %s already exists in %v", + DPUConnectionStatusAnnot, nadKey, annotations) } - podScss[nadName] = *scs + podScss[nadKey] = *scs } else { if !ok { - return nil, newAnnotationAlreadySetError("OVN pod %s annotation for NAD %s already removed", - DPUConnectionStatusAnnot, nadName) + return nil, newAnnotationAlreadySetError("OVN pod %s annotation for NAD key %s already removed", + DPUConnectionStatusAnnot, nadKey) } - delete(podScss, nadName) + delete(podScss, nadKey) } bytes, err := json.Marshal(podScss) if err != nil { @@ -189,7 +189,7 @@ func MarshalPodDPUConnStatus(annotations map[string]string, scs *DPUConnectionSt } // UnmarshalPodDPUConnStatus returns DPU connection status for the specified NAD -func UnmarshalPodDPUConnStatus(annotations map[string]string, nadName string) (*DPUConnectionStatus, error) { +func UnmarshalPodDPUConnStatus(annotations map[string]string, nadKey string) (*DPUConnectionStatus, error) { ovnAnnotation, ok := annotations[DPUConnectionStatusAnnot] if !ok { return nil, newAnnotationNotSetError("could not find OVN pod annotation in %v", annotations) @@ -199,20 +199,20 @@ func UnmarshalPodDPUConnStatus(annotations map[string]string, nadName string) (* if err != nil { return nil, err } - scs, ok := podScss[nadName] + scs, ok := podScss[nadKey] if !ok { - return nil, newAnnotationNotSetError("no OVN %s annotation for network %s: %q", - DPUConnectionStatusAnnot, nadName, ovnAnnotation) + return nil, newAnnotationNotSetError("no OVN %s annotation for NAD %s: %q", + DPUConnectionStatusAnnot, nadKey, ovnAnnotation) } return &scs, nil } // UpdatePodDPUConnStatusWithRetry updates the DPU connection status annotation // on the pod retrying on conflict -func UpdatePodDPUConnStatusWithRetry(podLister listers.PodLister, kube kube.Interface, pod *corev1.Pod, dpuConnStatus *DPUConnectionStatus, nadName string) error { +func UpdatePodDPUConnStatusWithRetry(podLister listers.PodLister, kube kube.Interface, pod *corev1.Pod, dpuConnStatus *DPUConnectionStatus, nadKey string) error { updatePodAnnotationNoRollback := func(pod *corev1.Pod) (*corev1.Pod, func(), error) { var err error - pod.Annotations, err = MarshalPodDPUConnStatus(pod.Annotations, dpuConnStatus, nadName) + pod.Annotations, err = MarshalPodDPUConnStatus(pod.Annotations, dpuConnStatus, nadKey) if err != nil { return nil, nil, err } @@ -229,10 +229,10 @@ func UpdatePodDPUConnStatusWithRetry(podLister listers.PodLister, kube kube.Inte // UpdatePodDPUConnDetailsWithRetry updates the DPU connection details // annotation on the pod retrying on conflict -func UpdatePodDPUConnDetailsWithRetry(podLister listers.PodLister, kube kube.Interface, pod *corev1.Pod, dpuConnDetails *DPUConnectionDetails, nadName string) error { +func UpdatePodDPUConnDetailsWithRetry(podLister listers.PodLister, kube kube.Interface, pod *corev1.Pod, dpuConnDetails *DPUConnectionDetails, nadKey string) error { updatePodAnnotationNoRollback := func(pod *corev1.Pod) (*corev1.Pod, func(), error) { var err error - pod.Annotations, err = MarshalPodDPUConnDetails(pod.Annotations, dpuConnDetails, nadName) + pod.Annotations, err = MarshalPodDPUConnDetails(pod.Annotations, dpuConnDetails, nadKey) if err != nil { return nil, nil, err } diff --git a/go-controller/pkg/util/external_gw_conntrack.go b/go-controller/pkg/util/external_gw_conntrack.go index 8f1a53b61d..b34a5b182e 100644 --- a/go-controller/pkg/util/external_gw_conntrack.go +++ b/go-controller/pkg/util/external_gw_conntrack.go @@ -311,7 +311,7 @@ func SyncConntrackForExternalGateways(gwIPsToKeep sets.Set[string], isPodInLocal } } - podIPs, err := GetPodIPsOfNetwork(pod, &DefaultNetInfo{}) + podIPs, err := GetPodIPsOfNetwork(pod, &DefaultNetInfo{}, nil) if err != nil && !errors.Is(err, ErrNoPodIPFound) { errs = append(errs, fmt.Errorf("unable to fetch IP for pod %s/%s: %v", pod.Namespace, pod.Name, err)) } diff --git a/go-controller/pkg/util/fake_client.go b/go-controller/pkg/util/fake_client.go index 0ca981e849..e78010d572 100644 --- a/go-controller/pkg/util/fake_client.go +++ b/go-controller/pkg/util/fake_client.go @@ -39,6 +39,8 @@ import ( routeadvertisementsfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/clientset/versioned/fake" udnv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1" udnfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/fake" + vtepv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" + vtepfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned/fake" ) func GetOVNClientset(objects ...runtime.Object) *OVNClientset { @@ -58,6 +60,7 @@ func GetOVNClientset(objects ...runtime.Object) *OVNClientset { raObjects := []runtime.Object{} frrObjects := []runtime.Object{} networkConnectObjects := []runtime.Object{} + vtepObjects := []runtime.Object{} for _, object := range objects { switch object.(type) { case *egressip.EgressIP: @@ -90,6 +93,8 @@ func GetOVNClientset(objects ...runtime.Object) *OVNClientset { networkQoSObjects = append(networkQoSObjects, object) case *networkconnect.ClusterNetworkConnect: networkConnectObjects = append(networkConnectObjects, object) + case *vtepv1.VTEP: + vtepObjects = append(vtepObjects, object) default: v1Objects = append(v1Objects, object) } @@ -119,6 +124,7 @@ func GetOVNClientset(objects ...runtime.Object) *OVNClientset { FRRClient: frrfake.NewSimpleClientset(frrObjects...), NetworkQoSClient: networkqosfake.NewSimpleClientset(networkQoSObjects...), NetworkConnectClient: networkconnectfake.NewSimpleClientset(networkConnectObjects...), + VTEPClient: vtepfake.NewSimpleClientset(vtepObjects...), } } diff --git a/go-controller/pkg/util/kube.go b/go-controller/pkg/util/kube.go index 9c0c54e550..b5e314d315 100644 --- a/go-controller/pkg/util/kube.go +++ b/go-controller/pkg/util/kube.go @@ -54,6 +54,7 @@ import ( networkqosclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1/apis/clientset/versioned" routeadvertisementsclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/clientset/versioned" userdefinednetworkclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned" + vtepclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/clientset/versioned" ) // OVNClientset is a wrapper around all clientsets used by OVN-Kubernetes @@ -75,6 +76,7 @@ type OVNClientset struct { RouteAdvertisementsClient routeadvertisementsclientset.Interface FRRClient frrclientset.Interface NetworkQoSClient networkqosclientset.Interface + VTEPClient vtepclientset.Interface } // OVNMasterClientset @@ -95,6 +97,7 @@ type OVNMasterClientset struct { RouteAdvertisementsClient routeadvertisementsclientset.Interface FRRClient frrclientset.Interface NetworkQoSClient networkqosclientset.Interface + VTEPClient vtepclientset.Interface } // OVNKubeControllerClientset @@ -113,6 +116,7 @@ type OVNKubeControllerClientset struct { UserDefinedNetworkClient userdefinednetworkclientset.Interface RouteAdvertisementsClient routeadvertisementsclientset.Interface NetworkQoSClient networkqosclientset.Interface + NetworkConnectClient networkconnectclientset.Interface } type OVNNodeClientset struct { @@ -142,6 +146,7 @@ type OVNClusterManagerClientset struct { RouteAdvertisementsClient routeadvertisementsclientset.Interface FRRClient frrclientset.Interface NetworkQoSClient networkqosclientset.Interface + VTEPClient vtepclientset.Interface } const ( @@ -172,6 +177,7 @@ func (cs *OVNClientset) GetMasterClientset() *OVNMasterClientset { RouteAdvertisementsClient: cs.RouteAdvertisementsClient, FRRClient: cs.FRRClient, NetworkQoSClient: cs.NetworkQoSClient, + VTEPClient: cs.VTEPClient, } } @@ -210,6 +216,7 @@ func (cs *OVNClientset) GetOVNKubeControllerClientset() *OVNKubeControllerClient UserDefinedNetworkClient: cs.UserDefinedNetworkClient, RouteAdvertisementsClient: cs.RouteAdvertisementsClient, NetworkQoSClient: cs.NetworkQoSClient, + NetworkConnectClient: cs.NetworkConnectClient, } } @@ -231,6 +238,7 @@ func (cs *OVNClientset) GetClusterManagerClientset() *OVNClusterManagerClientset RouteAdvertisementsClient: cs.RouteAdvertisementsClient, FRRClient: cs.FRRClient, NetworkQoSClient: cs.NetworkQoSClient, + VTEPClient: cs.VTEPClient, } } @@ -545,6 +553,11 @@ func NewOVNClientset(conf *config.KubernetesConfig) (*OVNClientset, error) { return nil, err } + vtepClientset, err := vtepclientset.NewForConfig(kconfig) + if err != nil { + return nil, err + } + return &OVNClientset{ KubeClient: kclientset, ANPClient: anpClientset, @@ -563,6 +576,7 @@ func NewOVNClientset(conf *config.KubernetesConfig) (*OVNClientset, error) { RouteAdvertisementsClient: routeAdvertisementsClientset, FRRClient: frrClientset, NetworkQoSClient: networkqosClientset, + VTEPClient: vtepClientset, }, nil } diff --git a/go-controller/pkg/util/mocks/NetLinkOps.go b/go-controller/pkg/util/mocks/NetLinkOps.go index 75819232a4..4aaa9a7112 100644 --- a/go-controller/pkg/util/mocks/NetLinkOps.go +++ b/go-controller/pkg/util/mocks/NetLinkOps.go @@ -704,6 +704,24 @@ func (_m *NetLinkOps) RuleListFiltered(family int, filter *netlink.Rule, filterM return r0, r1 } +// RuleAdd provides a mock function with given fields: rule +func (_m *NetLinkOps) RuleAdd(rule *netlink.Rule) error { + ret := _m.Called(rule) + + if len(ret) == 0 { + panic("no return value specified for RuleAdd") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*netlink.Rule) error); ok { + r0 = rf(rule) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // NewNetLinkOps creates a new instance of NetLinkOps. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewNetLinkOps(t interface { diff --git a/go-controller/pkg/util/mocks/multinetwork/NetInfo.go b/go-controller/pkg/util/mocks/multinetwork/NetInfo.go index 50edd3ef0a..edaef77470 100644 --- a/go-controller/pkg/util/mocks/multinetwork/NetInfo.go +++ b/go-controller/pkg/util/mocks/multinetwork/NetInfo.go @@ -34,25 +34,127 @@ func (_m *NetInfo) AllowsPersistentIPs() bool { return r0 } -// EqualNADs provides a mock function with given fields: nads -func (_m *NetInfo) EqualNADs(nads ...string) bool { - _va := make([]interface{}, len(nads)) - for _i := range nads { - _va[_i] = nads[_i] +// EVPNIPVRFRouteTarget provides a mock function with no fields +func (_m *NetInfo) EVPNIPVRFRouteTarget() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for EVPNIPVRFRouteTarget") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) } - var _ca []interface{} - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) + + return r0 +} + +// EVPNIPVRFVID provides a mock function with no fields +func (_m *NetInfo) EVPNIPVRFVID() int { + ret := _m.Called() if len(ret) == 0 { - panic("no return value specified for EqualNADs") + panic("no return value specified for EVPNIPVRFVID") } - var r0 bool - if rf, ok := ret.Get(0).(func(...string) bool); ok { - r0 = rf(nads...) + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() } else { - r0 = ret.Get(0).(bool) + r0 = ret.Get(0).(int) + } + + return r0 +} + +// EVPNIPVRFVNI provides a mock function with no fields +func (_m *NetInfo) EVPNIPVRFVNI() int32 { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for EVPNIPVRFVNI") + } + + var r0 int32 + if rf, ok := ret.Get(0).(func() int32); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int32) + } + + return r0 +} + +// EVPNMACVRFRouteTarget provides a mock function with no fields +func (_m *NetInfo) EVPNMACVRFRouteTarget() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for EVPNMACVRFRouteTarget") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// EVPNMACVRFVID provides a mock function with no fields +func (_m *NetInfo) EVPNMACVRFVID() int { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for EVPNMACVRFVID") + } + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// EVPNMACVRFVNI provides a mock function with no fields +func (_m *NetInfo) EVPNMACVRFVNI() int32 { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for EVPNMACVRFVNI") + } + + var r0 int32 + if rf, ok := ret.Get(0).(func() int32); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int32) + } + + return r0 +} + +// EVPNVTEPName provides a mock function with no fields +func (_m *NetInfo) EVPNVTEPName() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for EVPNVTEPName") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) } return r0 @@ -158,26 +260,6 @@ func (_m *NetInfo) GetNADNamespaces() []string { return r0 } -// GetNADs provides a mock function with no fields -func (_m *NetInfo) GetNADs() []string { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for GetNADs") - } - - var r0 []string - if rf, ok := ret.Get(0).(func() []string); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - return r0 -} - // GetNetInfo provides a mock function with no fields func (_m *NetInfo) GetNetInfo() util.NetInfo { ret := _m.Called() @@ -414,6 +496,24 @@ func (_m *NetInfo) GetNetworkScopedPatchPortName(bridgeID string, nodeName strin return r0 } +// GetNetworkScopedRouterToSwitchPortName provides a mock function with given fields: nodeName +func (_m *NetInfo) GetNetworkScopedRouterToSwitchPortName(nodeName string) string { + ret := _m.Called(nodeName) + + if len(ret) == 0 { + panic("no return value specified for GetNetworkScopedRouterToSwitchPortName") + } + + var r0 string + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(nodeName) + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + // GetNetworkScopedSwitchName provides a mock function with given fields: nodeName func (_m *NetInfo) GetNetworkScopedSwitchName(nodeName string) string { ret := _m.Called(nodeName) @@ -532,24 +632,6 @@ func (_m *NetInfo) GetTunnelKeys() []int { return r0 } -// HasNAD provides a mock function with given fields: nadName -func (_m *NetInfo) HasNAD(nadName string) bool { - ret := _m.Called(nadName) - - if len(ret) == 0 { - panic("no return value specified for HasNAD") - } - - var r0 bool - if rf, ok := ret.Get(0).(func(string) bool); ok { - r0 = rf(nadName) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - // IPMode provides a mock function with no fields func (_m *NetInfo) IPMode() (bool, bool) { ret := _m.Called() @@ -844,6 +926,24 @@ func (_m *NetInfo) TransitSubnets() []*net.IPNet { return r0 } +// Transport provides a mock function with no fields +func (_m *NetInfo) Transport() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Transport") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + // Vlan provides a mock function with no fields func (_m *NetInfo) Vlan() uint { ret := _m.Called() diff --git a/go-controller/pkg/util/multi_network.go b/go-controller/pkg/util/multi_network.go index 10bde09120..878f8666cd 100644 --- a/go-controller/pkg/util/multi_network.go +++ b/go-controller/pkg/util/multi_network.go @@ -18,8 +18,8 @@ import ( "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" - k8sapitypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" knet "k8s.io/utils/net" @@ -56,13 +56,19 @@ type NetInfo interface { Vlan() uint AllowsPersistentIPs() bool PhysicalNetworkName() string + Transport() string + EVPNVTEPName() string + EVPNMACVRFVNI() int32 + EVPNMACVRFRouteTarget() string + EVPNMACVRFVID() int + EVPNIPVRFVNI() int32 + EVPNIPVRFRouteTarget() string + EVPNIPVRFVID() int GetNodeGatewayIP(hostSubnet *net.IPNet) *net.IPNet GetNodeManagementIP(hostSubnet *net.IPNet) *net.IPNet // dynamic information, can change over time - GetNADs() []string - EqualNADs(nads ...string) bool - HasNAD(nadName string) bool + // GetPodNetworkAdvertisedVRFs returns the target VRFs where the pod network // is advertised per node, through a map of node names to slice of VRFs. GetPodNetworkAdvertisedVRFs() map[string][]string @@ -93,6 +99,7 @@ type NetInfo interface { GetNetworkScopedExtPortName(bridgeID, nodeName string) string GetNetworkScopedLoadBalancerName(lbName string) string GetNetworkScopedLoadBalancerGroupName(lbGroupName string) string + GetNetworkScopedRouterToSwitchPortName(nodeName string) string // GetNetInfo is an identity method used to get the specific NetInfo // implementation @@ -395,32 +402,6 @@ func (nInfo *mutableNetInfo) GetEgressIPAdvertisedNodes() []string { return maps.Keys(nInfo.eipAdvertisements) } -// GetNADs returns all the NADs associated with this network -func (nInfo *mutableNetInfo) GetNADs() []string { - nInfo.RLock() - defer nInfo.RUnlock() - return nInfo.getNads().UnsortedList() -} - -// EqualNADs checks if the NADs associated with nInfo are the same as the ones -// passed in the nads slice. -func (nInfo *mutableNetInfo) EqualNADs(nads ...string) bool { - nInfo.RLock() - defer nInfo.RUnlock() - if nInfo.getNads().Len() != len(nads) { - return false - } - return nInfo.getNads().HasAll(nads...) -} - -// HasNAD returns true if the given NAD exists, used -// to check if the network needs to be plumbed over -func (nInfo *mutableNetInfo) HasNAD(nadName string) bool { - nInfo.RLock() - defer nInfo.RUnlock() - return nInfo.getNads().Has(nadName) -} - // SetNADs replaces the NADs associated with the network func (nInfo *mutableNetInfo) SetNADs(nadNames ...string) { nInfo.Lock() @@ -480,6 +461,8 @@ func (nInfo *mutableNetInfo) getNamespaces() sets.Set[string] { } func (nInfo *mutableNetInfo) GetNADNamespaces() []string { + nInfo.RLock() + defer nInfo.RUnlock() return nInfo.getNamespaces().UnsortedList() } @@ -570,6 +553,10 @@ func (nInfo *DefaultNetInfo) GetNetworkScopedLoadBalancerGroupName(lbGroupName s return nInfo.GetNetworkScopedName(lbGroupName) } +func (nInfo *DefaultNetInfo) GetNetworkScopedRouterToSwitchPortName(nodeName string) string { + return types.RouterToSwitchPrefix + nInfo.GetNetworkScopedSwitchName(nodeName) +} + func (nInfo *DefaultNetInfo) canReconcile(netInfo NetInfo) bool { _, ok := netInfo.(*DefaultNetInfo) return ok @@ -673,6 +660,46 @@ func (nInfo *DefaultNetInfo) PhysicalNetworkName() string { return "" } +// Transport returns the transport protocol for east-west traffic +func (nInfo *DefaultNetInfo) Transport() string { + return config.Default.Transport +} + +// EVPNVTEPName returns empty as EVPN is not supported on the default network +func (nInfo *DefaultNetInfo) EVPNVTEPName() string { + return "" +} + +// EVPNMACVRFVNI returns 0 as EVPN is not supported on the default network +func (nInfo *DefaultNetInfo) EVPNMACVRFVNI() int32 { + return 0 +} + +// EVPNMACVRFRouteTarget returns empty as EVPN is not supported on the default network +func (nInfo *DefaultNetInfo) EVPNMACVRFRouteTarget() string { + return "" +} + +// EVPNIPVRFVNI returns 0 as EVPN is not supported on the default network +func (nInfo *DefaultNetInfo) EVPNIPVRFVNI() int32 { + return 0 +} + +// EVPNIPVRFRouteTarget returns empty as EVPN is not supported on the default network +func (nInfo *DefaultNetInfo) EVPNIPVRFRouteTarget() string { + return "" +} + +// EVPNMACVRFVID returns 0 as EVPN is not supported on the default network +func (nInfo *DefaultNetInfo) EVPNMACVRFVID() int { + return 0 +} + +// EVPNIPVRFVID returns 0 as EVPN is not supported on the default network +func (nInfo *DefaultNetInfo) EVPNIPVRFVID() int { + return 0 +} + func (nInfo *DefaultNetInfo) GetNodeGatewayIP(hostSubnet *net.IPNet) *net.IPNet { return GetNodeGatewayIfAddr(hostSubnet) } @@ -705,6 +732,9 @@ type userDefinedNetInfo struct { physicalNetworkName string defaultGatewayIPs []net.IP managementIPs []net.IP + + transport string + evpn *ovncnitypes.EVPNConfig } func (nInfo *userDefinedNetInfo) GetNetInfo() NetInfo { @@ -793,6 +823,18 @@ func (nInfo *userDefinedNetInfo) GetNetworkScopedLoadBalancerGroupName(lbGroupNa return nInfo.GetNetworkScopedName(lbGroupName) } +// GetNetworkScopedRouterToSwitchPortName returns the port name from router to switch. +// For Layer2 topology, this is the transit router to switch port (trtos-). +// For Layer3 topology, this is the router to switch port (rtos-). +// Not Applicable for Localnet topology. +func (nInfo *userDefinedNetInfo) GetNetworkScopedRouterToSwitchPortName(nodeName string) string { + switchName := nInfo.GetNetworkScopedSwitchName(nodeName) + if nInfo.TopologyType() == types.Layer2Topology { + return types.TransitRouterToSwitchPrefix + switchName + } + return types.RouterToSwitchPrefix + switchName +} + // getPrefix returns if the logical entities prefix for this network func (nInfo *userDefinedNetInfo) getPrefix() string { return GetUserDefinedNetworkPrefix(nInfo.netName) @@ -823,6 +865,70 @@ func (nInfo *userDefinedNetInfo) PhysicalNetworkName() string { return nInfo.physicalNetworkName } +// Transport returns the transport protocol for east-west traffic +func (nInfo *userDefinedNetInfo) Transport() string { + if nInfo.transport == "" { + return types.NetworkTransportGeneve + } + return nInfo.transport +} + +// EVPNVTEPName returns the name of the VTEP CR for EVPN +func (nInfo *userDefinedNetInfo) EVPNVTEPName() string { + if nInfo.evpn == nil { + return "" + } + return nInfo.evpn.VTEP +} + +// EVPNMACVRFVNI returns the MAC-VRF VNI for EVPN +func (nInfo *userDefinedNetInfo) EVPNMACVRFVNI() int32 { + if nInfo.evpn == nil || nInfo.evpn.MACVRF == nil { + return 0 + } + return nInfo.evpn.MACVRF.VNI +} + +// EVPNMACVRFRouteTarget returns the MAC-VRF route target for EVPN +func (nInfo *userDefinedNetInfo) EVPNMACVRFRouteTarget() string { + if nInfo.evpn == nil || nInfo.evpn.MACVRF == nil { + return "" + } + return nInfo.evpn.MACVRF.RouteTarget +} + +// EVPNIPVRFVNI returns the IP-VRF VNI for EVPN +func (nInfo *userDefinedNetInfo) EVPNIPVRFVNI() int32 { + if nInfo.evpn == nil || nInfo.evpn.IPVRF == nil { + return 0 + } + return nInfo.evpn.IPVRF.VNI +} + +// EVPNIPVRFRouteTarget returns the IP-VRF route target for EVPN +func (nInfo *userDefinedNetInfo) EVPNIPVRFRouteTarget() string { + if nInfo.evpn == nil || nInfo.evpn.IPVRF == nil { + return "" + } + return nInfo.evpn.IPVRF.RouteTarget +} + +// EVPNMACVRFVID returns the MAC-VRF VID for EVPN +func (nInfo *userDefinedNetInfo) EVPNMACVRFVID() int { + if nInfo.evpn == nil || nInfo.evpn.MACVRF == nil { + return 0 + } + return nInfo.evpn.MACVRF.VID +} + +// EVPNIPVRFVID returns the IP-VRF VID for EVPN +func (nInfo *userDefinedNetInfo) EVPNIPVRFVID() int { + if nInfo.evpn == nil || nInfo.evpn.IPVRF == nil { + return 0 + } + return nInfo.evpn.IPVRF.VID +} + func (nInfo *userDefinedNetInfo) GetNodeGatewayIP(hostSubnet *net.IPNet) *net.IPNet { if IsPreconfiguredUDNAddressesEnabled() && nInfo.TopologyType() == types.Layer2Topology && nInfo.IsPrimaryNetwork() { isIPV6 := knet.IsIPv6CIDR(hostSubnet) @@ -936,6 +1042,24 @@ func (nInfo *userDefinedNetInfo) canReconcile(other NetInfo) bool { if nInfo.physicalNetworkName != other.PhysicalNetworkName() { return false } + if nInfo.Transport() != other.Transport() { + return false + } + if nInfo.EVPNVTEPName() != other.EVPNVTEPName() { + return false + } + if nInfo.EVPNMACVRFVNI() != other.EVPNMACVRFVNI() { + return false + } + if nInfo.EVPNMACVRFRouteTarget() != other.EVPNMACVRFRouteTarget() { + return false + } + if nInfo.EVPNIPVRFVNI() != other.EVPNIPVRFVNI() { + return false + } + if nInfo.EVPNIPVRFRouteTarget() != other.EVPNIPVRFRouteTarget() { + return false + } lessCIDRNetworkEntry := func(a, b config.CIDRNetworkEntry) bool { return a.String() < b.String() } if !cmp.Equal(nInfo.subnets, other.Subnets(), cmpopts.SortSlices(lessCIDRNetworkEntry)) { @@ -978,6 +1102,8 @@ func (nInfo *userDefinedNetInfo) copy() *userDefinedNetInfo { physicalNetworkName: nInfo.physicalNetworkName, defaultGatewayIPs: nInfo.defaultGatewayIPs, managementIPs: nInfo.managementIPs, + transport: nInfo.transport, + evpn: nInfo.evpn, } // copy mutables c.mutableNetInfo.copyFrom(&nInfo.mutableNetInfo) @@ -1001,6 +1127,8 @@ func newLayer3NetConfInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) subnets: subnets, joinSubnets: joinSubnets, mtu: netconf.MTU, + transport: netconf.Transport, + evpn: netconf.EVPN, mutableNetInfo: mutableNetInfo{ id: types.InvalidID, nads: sets.Set[string]{}, @@ -1076,6 +1204,8 @@ func newLayer2NetConfInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) allowPersistentIPs: netconf.AllowPersistentIPs, defaultGatewayIPs: defaultGatewayIPs, managementIPs: managementIPs, + transport: netconf.Transport, + evpn: netconf.EVPN, mutableNetInfo: mutableNetInfo{ id: types.InvalidID, nads: sets.Set[string]{}, @@ -1233,6 +1363,35 @@ func GetNADName(namespace, name string) string { return fmt.Sprintf("%s/%s", namespace, name) } +// GetIndexedNADKey returns key of NetAttachDefInfo.NetAttachDefs map for multiple identical NADs, also used as Pod annotation key +// the resulted nadKey example is like "ns/nad", "ns/nad/1", "ns/nad/2" etc... +func GetIndexedNADKey(nadName string, n int) string { + if n == 0 { + return nadName + } + return fmt.Sprintf("%s/%d", nadName, n) +} + +// GetNadFromIndexedNADKey returns NAD name part from a nadName key (with or without index) +func GetNadFromIndexedNADKey(nadKey string) (string, int, error) { + parts := strings.Split(nadKey, "/") + + if len(parts) == 3 { + // Attempt to parse the integer part + num, err := strconv.Atoi(parts[2]) + if err != nil { + return "", 0, fmt.Errorf("malformed index for NAD key %s: %v", nadKey, err) + } + return fmt.Sprintf("%s/%s", parts[0], parts[1]), num, nil + } + + if len(parts) == 2 { + return fmt.Sprintf("%s/%s", parts[0], parts[1]), 0, nil + } + + return "", 0, fmt.Errorf("malformed NAD key %s, expect in the form of namespace/name{/index}", nadKey) +} + // GetUserDefinedNetworkPrefix gets the string used as prefix of the logical entities // of the User Defined Network of the given network name, in the form of _. // @@ -1377,6 +1536,18 @@ func ValidateNetConf(nadName string, netconf *ovncnitypes.NetConf) error { return fmt.Errorf("error parsing Network Attachment Definition %s: %w", nadName, ErrorUnsupportedIPAMKey) } + // Validate transport if specified + if netconf.Transport != "" && + netconf.Transport != types.NetworkTransportGeneve && + netconf.Transport != types.NetworkTransportNoOverlay && + netconf.Transport != types.NetworkTransportEVPN { + return fmt.Errorf("invalid transport %q: must be one of %q", netconf.Transport, []string{ + types.NetworkTransportGeneve, + types.NetworkTransportNoOverlay, + types.NetworkTransportEVPN, + }) + } + if netconf.JoinSubnet != "" && netconf.Topology == types.LocalnetTopology { return fmt.Errorf("localnet topology does not allow specifying join-subnet as services are not supported") } @@ -1475,17 +1646,26 @@ func SubnetOverlapCheck(netconf *ovncnitypes.NetConf) (*net.IPNet, *net.IPNet, e return nil, nil, nil } -// GetPodNADToNetworkMapping sees if the given pod needs to plumb over this given network specified by netconf, +// getPodNADToNetworkMapping sees if the given pod needs to plumb over this given network specified by netconf, // and return the matching NetworkSelectionElement if any exists. // // Return value: // -// bool: if this Pod is on this Network; true or false -// map[string]*nettypes.NetworkSelectionElement: all NetworkSelectionElement that pod is requested -// for the specified network, key is NADName. Note multiple NADs of the same network are allowed -// on one pod, as long as they are of different NADName. -// error: error in case of failure -func GetPodNADToNetworkMapping(pod *corev1.Pod, nInfo NetInfo) (bool, map[string]*nettypes.NetworkSelectionElement, error) { +// bool: if this Pod is on this Network; true or false +// map[string]*nettypes.NetworkSelectionElement: all NetworkSelectionElement that the pod requests +// for the specified network, keyed by NAD key. NAD keys are of the form "namespace/name" +// for the first attachment of a NAD, and "namespace/name/" (idx start from 1) for +// additional attachments of the same NAD on the same pod. Note multiple NADs of the same +// network are allowed on one pod. They can be of different NAD Name or the same NAD Name. +// error: error in case of failure +// +// getNetworkNameForNADKey may be nil only for default-network lookups. +// For UDN lookups, it must be provided to validate NAD keys. +func getPodNADToNetworkMapping( + pod *corev1.Pod, + nInfo NetInfo, + getNetworkNameForNADKey func(nadKey string) string, +) (bool, map[string]*nettypes.NetworkSelectionElement, error) { if pod.Spec.HostNetwork { return false, nil, nil } @@ -1504,6 +1684,40 @@ func GetPodNADToNetworkMapping(pod *corev1.Pod, nInfo NetInfo) (bool, map[string return true, networkSelections, nil } + if getNetworkNameForNADKey == nil { + return false, nil, fmt.Errorf("UDN mapping requires a network resolver") + } + + return getPodNADToNetworkMappingWithPredicate(pod, nInfo, func(nadName string) bool { + networkName := getNetworkNameForNADKey(nadName) + return networkName != "" && networkName == nInfo.GetNetworkName() + }) +} + +// GetUDNPodNADToNetworkMapping returns pod network selections for the specified UDN, +// validating NAD keys via the provided resolver. +func GetUDNPodNADToNetworkMapping( + pod *corev1.Pod, + nInfo NetInfo, + getNetworkNameForNADKey func(nadKey string) string, +) (bool, map[string]*nettypes.NetworkSelectionElement, error) { + if getNetworkNameForNADKey == nil { + return false, nil, fmt.Errorf("UDN mapping requires a network resolver") + } + return getPodNADToNetworkMapping(pod, nInfo, getNetworkNameForNADKey) +} + +// GetDefaultPodNADToNetworkMapping returns default-network selections for a pod. +// This should only be used by the default network controller. +func GetDefaultPodNADToNetworkMapping(pod *corev1.Pod) (bool, map[string]*nettypes.NetworkSelectionElement, error) { + return getPodNADToNetworkMapping(pod, &DefaultNetInfo{}, nil) +} + +func getPodNADToNetworkMappingWithPredicate( + pod *corev1.Pod, + nInfo NetInfo, + nadMatches func(nadKey string) bool, +) (bool, map[string]*nettypes.NetworkSelectionElement, error) { // For non-default network controller, try to see if its name exists in the Pod's k8s.v1.cni.cncf.io/networks, if no, // return false; allNetworks, err := GetK8sPodAllNetworkSelections(pod) @@ -1511,18 +1725,31 @@ func GetPodNADToNetworkMapping(pod *corev1.Pod, nInfo NetInfo) (bool, map[string return false, nil, err } + networkSelections := map[string]*nettypes.NetworkSelectionElement{} + // Get map of per-NAD NetworkSelectionElement, if there are multiple NetworkSelectionElements of the same NAD, + // calculate numbers of network elements of that same NAD. + nNADs := map[string]int{} for _, network := range allNetworks { - nadName := GetNADName(network.Namespace, network.Name) - if nInfo.HasNAD(nadName) { - if nInfo.IsPrimaryNetwork() { - return false, nil, fmt.Errorf("unexpected primary network %q specified with a NetworkSelectionElement %+v", nInfo.GetNetworkName(), network) - } - if _, ok := networkSelections[nadName]; ok { - return false, nil, fmt.Errorf("unexpected error: more than one of the same NAD %s specified for pod %s", - nadName, podDesc) - } - networkSelections[nadName] = network + nadNamespace := network.Namespace + if nadNamespace == "" { + nadNamespace = pod.Namespace + } + nadName := GetNADName(nadNamespace, network.Name) + if !nadMatches(nadName) { + continue + } + if nInfo.IsPrimaryNetwork() { + return false, nil, fmt.Errorf("unexpected primary network %q specified with a NetworkSelectionElement %+v", nInfo.GetNetworkName(), network) } + + // for multiple NetworkSelectionElements of the same NAD, set its nadName to indexed nadName + cnt := nNADs[nadName] + if cnt > 0 && nInfo.TopologyType() == types.LocalnetTopology { + return false, nil, fmt.Errorf("pod %s/%s cannot have same networkSelectionElement %s of type %s multiple times", + pod.Namespace, pod.Name, nadName, types.LocalnetTopology) + } + nNADs[nadName] = cnt + 1 + networkSelections[GetIndexedNADKey(nadName, cnt)] = network } if len(networkSelections) == 0 { @@ -1546,11 +1773,18 @@ func overrideActiveNSEWithDefaultNSE(defaultNSE, activeNSE *nettypes.NetworkSele return nil } -// GetPodNADToNetworkMappingWithActiveNetwork will call `GetPodNADToNetworkMapping` passing "nInfo" which correspond -// to the NetInfo representing the NAD, the resulting NetworkSelectingElements will be decorated with the ones -// from found active network -func GetPodNADToNetworkMappingWithActiveNetwork(pod *corev1.Pod, nInfo NetInfo, activeNetwork NetInfo) (bool, map[string]*nettypes.NetworkSelectionElement, error) { - on, networkSelections, err := GetPodNADToNetworkMapping(pod, nInfo) +// GetPodNADToNetworkMappingWithActiveNetwork resolves the pod's NAD attachments using nInfo (the NAD's NetInfo). +// If activeNetwork is provided and matches nInfo's network, it adds the namespace's active primary NAD selection +// (and any requested default-network IP/MAC details) to the returned mapping. +func GetPodNADToNetworkMappingWithActiveNetwork( + pod *corev1.Pod, + nInfo NetInfo, + activeNetwork NetInfo, + getNetworkNameForNADKey func(nadKey string) string, + getPrimaryNADForNamespace func(namespace string) (string, error), +) (bool, map[string]*nettypes.NetworkSelectionElement, error) { + // scan network selection elements using the resolver to validate attachments + on, networkSelections, err := getPodNADToNetworkMapping(pod, nInfo, getNetworkNameForNADKey) if err != nil { return false, nil, err } @@ -1565,15 +1799,24 @@ func GetPodNADToNetworkMappingWithActiveNetwork(pod *corev1.Pod, nInfo NetInfo, return on, networkSelections, nil } - // Add the active network to the NSE map if it is configured - activeNetworkNADs := activeNetwork.GetNADs() - if len(activeNetworkNADs) < 1 { - return false, nil, fmt.Errorf("missing NADs at active network %q for namespace %q", activeNetwork.GetNetworkName(), pod.Namespace) + if getPrimaryNADForNamespace == nil { + return false, nil, fmt.Errorf("missing primary NAD resolver for network %q", nInfo.GetNetworkName()) + } + + primaryNADKey, err := getPrimaryNADForNamespace(pod.Namespace) + if err != nil { + return false, nil, fmt.Errorf("failed to get primary NAD for namespace %q: %w", pod.Namespace, err) + } + if primaryNADKey == types.DefaultNetworkName { + return false, nil, fmt.Errorf("no primary NAD found for namespace %q", pod.Namespace) + } + if networkName := getNetworkNameForNADKey(primaryNADKey); networkName == "" || networkName != nInfo.GetNetworkName() { + return false, nil, fmt.Errorf("primary NAD %q does not match network %q for namespace %q", primaryNADKey, nInfo.GetNetworkName(), pod.Namespace) } - activeNADKey := getNADWithNamespace(activeNetworkNADs, pod.Namespace) - if activeNADKey == nil { - return false, nil, fmt.Errorf("no active NAD found for namespace %q", pod.Namespace) + nadNamespace, nadName, err := cache.SplitMetaNamespaceKey(primaryNADKey) + if err != nil { + return false, nil, fmt.Errorf("failed to split NAD key %q: %w", primaryNADKey, err) } if len(networkSelections) == 0 { @@ -1581,8 +1824,8 @@ func GetPodNADToNetworkMappingWithActiveNetwork(pod *corev1.Pod, nInfo NetInfo, } activeNSE := &nettypes.NetworkSelectionElement{ - Namespace: activeNADKey.Namespace, - Name: activeNADKey.Name, + Namespace: nadNamespace, + Name: nadName, } isPersistentIPsPrimaryNetwork := nInfo.IsPrimaryNetwork() && AllowsPersistentIPs(nInfo) @@ -1620,26 +1863,10 @@ func GetPodNADToNetworkMappingWithActiveNetwork(pod *corev1.Pod, nInfo NetInfo, } } - networkSelections[activeNADKey.String()] = activeNSE + networkSelections[primaryNADKey] = activeNSE return true, networkSelections, nil } -// getNADWithNamespace returns the first occurrence of NAD key with the given namespace name. -func getNADWithNamespace(nads []string, targetNamespace string) *k8sapitypes.NamespacedName { - for _, nad := range nads { - nsName := strings.Split(nad, "/") - if len(nsName) != 2 { - continue - } - ns, name := nsName[0], nsName[1] - if ns != targetNamespace { - continue - } - return &k8sapitypes.NamespacedName{Namespace: ns, Name: name} - } - return nil -} - func IsMultiNetworkPoliciesSupportEnabled() bool { return config.OVNKubernetesFeature.EnableMultiNetwork && config.OVNKubernetesFeature.EnableMultiNetworkPolicy } @@ -1658,6 +1885,10 @@ func IsRouteAdvertisementsEnabled() bool { return config.OVNKubernetesFeature.EnableMultiNetwork && config.OVNKubernetesFeature.EnableRouteAdvertisements } +func IsEVPNEnabled() bool { + return IsRouteAdvertisementsEnabled() && config.OVNKubernetesFeature.EnableEVPN +} + // IsPreconfiguredUDNAddressesEnabled indicates if user defined IPs / MAC // addresses can be set in primary UDNs func IsPreconfiguredUDNAddressesEnabled() bool { @@ -1771,50 +2002,67 @@ func CanServeNamespace(network NetInfo, namespace string) bool { // is otherwise locked for all intents and purposes. // // (4) "none" if the pod has no networks on this controller -func GetNetworkRole(controllerNetInfo NetInfo, getActiveNetworkForNamespace func(namespace string) (NetInfo, error), pod *corev1.Pod) (string, error) { +func GetNetworkRole( + controllerNetInfo NetInfo, + getPrimaryNADForNamespace func(namespace string) (string, error), + getNetworkNameForNADKey func(nadKey string) string, + pod *corev1.Pod, +) (string, error) { + if getNetworkNameForNADKey == nil { + return "", fmt.Errorf("getNetworkNameForNADKey is required") + } // no network segmentation enabled, and is default controller, must be default network if !IsNetworkSegmentationSupportEnabled() && controllerNetInfo.IsDefault() { return types.NetworkRolePrimary, nil } - var activeNetwork NetInfo var err error // controller is serving primary network or is default, we need to get the active network if controllerNetInfo.IsPrimaryNetwork() || controllerNetInfo.IsDefault() { - activeNetwork, err = getActiveNetworkForNamespace(pod.Namespace) + // check if primary NAD exists + primaryNAD, err := getPrimaryNADForNamespace(pod.Namespace) if err != nil { return "", err } - - // if active network for pod matches controller network, then primary interface is handled by this controller - if activeNetwork.GetNetworkName() == controllerNetInfo.GetNetworkName() { - return types.NetworkRolePrimary, nil - } - - // otherwise, if this is the default controller, and the pod active network does not match the default network - // we know the role for this default controller is infra locked if controllerNetInfo.IsDefault() { + if primaryNAD == types.DefaultNetworkName { + return types.NetworkRolePrimary, nil + } return types.NetworkRoleInfrastructure, nil } - // this is a primary network controller, and it does not match the pod's active network - // the controller must not be serving this pod + if networkName := getNetworkNameForNADKey(primaryNAD); networkName != "" { + if networkName == controllerNetInfo.GetNetworkName() { + return types.NetworkRolePrimary, nil + } + return types.NetworkRoleNone, nil + } + + // this is a primary network controller, and it does not have the pod's primary NAD return types.NetworkRoleNone, nil } // at this point the controller must be a secondary network - on, _, err := GetPodNADToNetworkMapping(pod, controllerNetInfo.GetNetInfo()) + allNetworks, err := GetK8sPodAllNetworkSelections(pod) if err != nil { return "", fmt.Errorf("failed to get pod network mapping: %w", err) } - - if !on { - return types.NetworkRoleNone, nil + for _, network := range allNetworks { + nadNamespace := network.Namespace + if nadNamespace == "" { + nadNamespace = pod.Namespace + } + nadKey := GetNADName(nadNamespace, network.Name) + networkName := getNetworkNameForNADKey(nadKey) + if networkName == "" { + continue + } + if networkName == controllerNetInfo.GetNetworkName() { + return types.NetworkRoleSecondary, nil + } } - - // must be secondary role - return types.NetworkRoleSecondary, nil + return types.NetworkRoleNone, nil } // (C)UDN network name generation functions must ensure the absence of name conflicts between all (C)UDNs. diff --git a/go-controller/pkg/util/multi_network_test.go b/go-controller/pkg/util/multi_network_test.go index e0335e0f5e..2861650ba8 100644 --- a/go-controller/pkg/util/multi_network_test.go +++ b/go-controller/pkg/util/multi_network_test.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" @@ -922,6 +923,7 @@ func TestGetPodNADToNetworkMapping(t *testing.T) { inputPodAnnotations map[string]string expectedError error expectedIsAttachmentRequested bool + expectedNetworkMapping map[string]struct{} } tests := []testConfig{ @@ -935,6 +937,7 @@ func TestGetPodNADToNetworkMapping(t *testing.T) { NADName: GetNADName(namespaceName, attachmentName), }, expectedIsAttachmentRequested: false, + expectedNetworkMapping: map[string]struct{}{}, }, { desc: "Looking for a network present in the pod's attachment requests", @@ -947,9 +950,10 @@ func TestGetPodNADToNetworkMapping(t *testing.T) { nadv1.NetworkAttachmentAnnot: GetNADName(namespaceName, attachmentName), }, expectedIsAttachmentRequested: true, + expectedNetworkMapping: map[string]struct{}{GetNADName(namespaceName, attachmentName): {}}, }, { - desc: "Multiple attachments to the same network in the same pod are not supported", + desc: "Multiple attachments to the same network in the same pod", inputNamespace: namespaceName, inputNetConf: &ovncnitypes.NetConf{ NetConf: cnitypes.NetConf{Name: networkName}, @@ -959,7 +963,11 @@ func TestGetPodNADToNetworkMapping(t *testing.T) { inputPodAnnotations: map[string]string{ nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%[1]s,%[1]s", GetNADName(namespaceName, attachmentName)), }, - expectedError: fmt.Errorf("unexpected error: more than one of the same NAD ns1/attachment1 specified for pod ns1/test-pod"), + expectedIsAttachmentRequested: true, + expectedNetworkMapping: map[string]struct{}{ + GetNADName(namespaceName, attachmentName): {}, + GetIndexedNADKey(GetNADName(namespaceName, attachmentName), 1): {}, + }, }, { desc: "Attaching to a secondary network to a user defined primary network is not supported", @@ -973,7 +981,8 @@ func TestGetPodNADToNetworkMapping(t *testing.T) { inputPodAnnotations: map[string]string{ nadv1.NetworkAttachmentAnnot: GetNADName(namespaceName, attachmentName), }, - expectedError: fmt.Errorf("unexpected primary network \"l3-network\" specified with a NetworkSelectionElement &{Name:attachment1 Namespace:ns1 IPRequest:[] MacRequest: InfinibandGUIDRequest: InterfaceRequest: PortMappingsRequest:[] BandwidthRequest: CNIArgs: GatewayRequest:[] IPAMClaimReference:}"), + expectedError: fmt.Errorf("unexpected primary network \"l3-network\" specified with a NetworkSelectionElement &{Name:attachment1 Namespace:ns1 IPRequest:[] MacRequest: InfinibandGUIDRequest: InterfaceRequest: PortMappingsRequest:[] BandwidthRequest: CNIArgs: GatewayRequest:[] IPAMClaimReference:}"), + expectedNetworkMapping: map[string]struct{}{}, }, } @@ -996,12 +1005,33 @@ func TestGetPodNADToNetworkMapping(t *testing.T) { }, } - isAttachmentRequested, _, err := GetPodNADToNetworkMapping(pod, netInfo) + var resolver func(string) string + if netInfo.IsUserDefinedNetwork() { + expectedNADKey := test.inputNetConf.NADName + if expectedNADKey == "" { + t.Fatalf("missing NAD name for user-defined network %q", netInfo.GetNetworkName()) + } + resolver = func(nadKey string) string { + if nadKey == expectedNADKey { + return netInfo.GetNetworkName() + } + return "" + } + } - if err != nil { + isAttachmentRequested, networkMap, err := getPodNADToNetworkMapping(pod, netInfo, resolver) + if test.expectedError != nil { + g.Expect(err).To(gomega.HaveOccurred()) g.Expect(err).To(gomega.MatchError(test.expectedError)) + } else { + g.Expect(err).NotTo(gomega.HaveOccurred()) + } + actualNetworkMapping := map[string]struct{}{} + for nadName := range networkMap { + actualNetworkMapping[nadName] = struct{}{} } g.Expect(isAttachmentRequested).To(gomega.Equal(test.expectedIsAttachmentRequested)) + g.Expect(actualNetworkMapping).To(gomega.Equal(test.expectedNetworkMapping)) }) } } @@ -1344,7 +1374,7 @@ func TestGetPodNADToNetworkMappingWithActiveNetwork(t *testing.T) { { desc: "should fail when no nad of the active network found on the pod namespace", inputNamespace: "non-existent-ns", - expectedError: fmt.Errorf(`no active NAD found for namespace "non-existent-ns"`), + expectedError: fmt.Errorf(`failed to get primary NAD for namespace "non-existent-ns": no active NAD found for namespace "non-existent-ns"`), inputNetConf: &ovncnitypes.NetConf{ NetConf: cnitypes.NetConf{Name: networkName}, NADName: GetNADName(namespaceName, attachmentName), @@ -1488,15 +1518,55 @@ func TestGetPodNADToNetworkMappingWithActiveNetwork(t *testing.T) { pod.Namespace = test.inputNamespace } + expectedNADKey := test.inputNetConf.NADName + if expectedNADKey == "" { + t.Fatalf("missing NAD name for user-defined network %q", netInfo.GetNetworkName()) + } + nadNetworkNames := map[string]string{ + expectedNADKey: netInfo.GetNetworkName(), + } + primaryNADByNamespace := map[string]string{} + if primaryUDNNetInfo != nil { + primaryNADKeys := make([]string, 0, 1+len(test.injectPrimaryUDNNADs)) + if test.inputPrimaryUDNConfig != nil && test.inputPrimaryUDNConfig.NADName != "" { + primaryNADKeys = append(primaryNADKeys, test.inputPrimaryUDNConfig.NADName) + } + primaryNADKeys = append(primaryNADKeys, test.injectPrimaryUDNNADs...) + for _, nadKey := range primaryNADKeys { + nadNamespace, _, err := cache.SplitMetaNamespaceKey(nadKey) + if err != nil { + t.Fatalf("failed to split NAD key %q: %v", nadKey, err) + } + primaryNADByNamespace[nadNamespace] = nadKey + nadNetworkNames[nadKey] = primaryUDNNetInfo.GetNetworkName() + } + } + resolver := func(nadKey string) string { + if networkName, ok := nadNetworkNames[nadKey]; ok { + return networkName + } + return "" + } + isAttachmentRequested, networkSelectionElements, err := GetPodNADToNetworkMappingWithActiveNetwork( pod, netInfo, primaryUDNNetInfo, + resolver, + func(namespace string) (string, error) { + if primaryUDNNetInfo == nil { + return ovntypes.DefaultNetworkName, nil + } + if nadKey, ok := primaryNADByNamespace[namespace]; ok { + return nadKey, nil + } + return "", fmt.Errorf("no active NAD found for namespace %q", namespace) + }, ) if test.expectedError != nil { g.Expect(err).To(gomega.HaveOccurred(), "unexpected success operation, epecting error") - g.Expect(err).To(gomega.MatchError(test.expectedError)) + g.Expect(err.Error()).To(gomega.Equal(test.expectedError.Error())) } else { g.Expect(err).ToNot(gomega.HaveOccurred()) g.Expect(isAttachmentRequested).To(gomega.Equal(test.expectedIsAttachmentRequested)) @@ -1759,6 +1829,13 @@ func TestAreNetworksCompatible(t *testing.T) { expectedResult: false, expectationDescription: "we should reconcile on physical network name updates", }, + { + desc: "empty transport and geneve config should be compatible", + aNetwork: &userDefinedNetInfo{transport: ""}, + anotherNetwork: &userDefinedNetInfo{transport: "geneve"}, + expectedResult: true, + expectationDescription: "networks with no EVPN config should be compatible", + }, } for _, test := range tests { @@ -1927,6 +2004,277 @@ func TestGetNodeManagementIP(t *testing.T) { } } +func TestEVPNConfig(t *testing.T) { + type testConfig struct { + desc string + inputNetConf *ovncnitypes.NetConf + expectedTransport string + expectedVTEPName string + expectedMACVRFVNI int32 + expectedMACVRFRouteTarget string + expectedMACVRFVID int + expectedIPVRFVNI int32 + expectedIPVRFRouteTarget string + expectedIPVRFVID int + } + + tests := []testConfig{ + { + desc: "default network has no EVPN config", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: ovntypes.DefaultNetworkName}, + Topology: ovntypes.Layer3Topology, + }, + expectedTransport: "geneve", + expectedVTEPName: "", + expectedMACVRFVNI: 0, + expectedMACVRFRouteTarget: "", + expectedIPVRFVNI: 0, + expectedIPVRFRouteTarget: "", + }, + { + desc: "layer3 network without EVPN config", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "l3-network"}, + Topology: ovntypes.Layer3Topology, + }, + expectedTransport: "geneve", + expectedVTEPName: "", + expectedMACVRFVNI: 0, + expectedMACVRFRouteTarget: "", + expectedIPVRFVNI: 0, + expectedIPVRFRouteTarget: "", + }, + { + desc: "layer3 network with EVPN transport and IP-VRF only", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "evpn-l3-network"}, + Topology: ovntypes.Layer3Topology, + Transport: "evpn", + EVPN: &ovncnitypes.EVPNConfig{ + VTEP: "my-vtep", + IPVRF: &ovncnitypes.VRFConfig{ + VNI: 2000, + RouteTarget: "65000:2000", + }, + }, + }, + expectedTransport: "evpn", + expectedVTEPName: "my-vtep", + expectedMACVRFVNI: 0, + expectedMACVRFRouteTarget: "", + expectedIPVRFVNI: 2000, + expectedIPVRFRouteTarget: "65000:2000", + }, + { + desc: "layer2 network with EVPN transport and MAC-VRF only", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "evpn-l2-network"}, + Topology: ovntypes.Layer2Topology, + Transport: "evpn", + EVPN: &ovncnitypes.EVPNConfig{ + VTEP: "my-vtep", + MACVRF: &ovncnitypes.VRFConfig{ + VNI: 100, + RouteTarget: "65000:100", + }, + }, + }, + expectedTransport: "evpn", + expectedVTEPName: "my-vtep", + expectedMACVRFVNI: 100, + expectedMACVRFRouteTarget: "65000:100", + expectedIPVRFVNI: 0, + expectedIPVRFRouteTarget: "", + }, + { + desc: "layer2 network with EVPN transport and both MAC-VRF and IP-VRF (symmetric IRB)", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "evpn-l2-symmetric"}, + Topology: ovntypes.Layer2Topology, + Transport: "evpn", + EVPN: &ovncnitypes.EVPNConfig{ + VTEP: "symmetric-vtep", + MACVRF: &ovncnitypes.VRFConfig{ + VNI: 100, + RouteTarget: "65000:100", + }, + IPVRF: &ovncnitypes.VRFConfig{ + VNI: 1000, + RouteTarget: "65000:1000", + }, + }, + }, + expectedTransport: "evpn", + expectedVTEPName: "symmetric-vtep", + expectedMACVRFVNI: 100, + expectedMACVRFRouteTarget: "65000:100", + expectedIPVRFVNI: 1000, + expectedIPVRFRouteTarget: "65000:1000", + }, + { + desc: "layer2 network with EVPN transport including VIDs (allocated by controller)", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "evpn-with-vids"}, + Topology: ovntypes.Layer2Topology, + Transport: "evpn", + EVPN: &ovncnitypes.EVPNConfig{ + VTEP: "vid-vtep", + MACVRF: &ovncnitypes.VRFConfig{ + VNI: 100, + RouteTarget: "65000:100", + VID: 12, + }, + IPVRF: &ovncnitypes.VRFConfig{ + VNI: 1000, + RouteTarget: "65000:1000", + VID: 13, + }, + }, + }, + expectedTransport: "evpn", + expectedVTEPName: "vid-vtep", + expectedMACVRFVNI: 100, + expectedMACVRFRouteTarget: "65000:100", + expectedMACVRFVID: 12, + expectedIPVRFVNI: 1000, + expectedIPVRFRouteTarget: "65000:1000", + expectedIPVRFVID: 13, + }, + { + desc: "EVPN config with VNI only (no route target)", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "evpn-minimal"}, + Topology: ovntypes.Layer2Topology, + Transport: "evpn", + EVPN: &ovncnitypes.EVPNConfig{ + VTEP: "minimal-vtep", + MACVRF: &ovncnitypes.VRFConfig{ + VNI: 500, + }, + }, + }, + expectedTransport: "evpn", + expectedVTEPName: "minimal-vtep", + expectedMACVRFVNI: 500, + expectedMACVRFRouteTarget: "", + expectedIPVRFVNI: 0, + expectedIPVRFRouteTarget: "", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + g := gomega.NewWithT(t) + netInfo, err := NewNetInfo(test.inputNetConf) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + g.Expect(netInfo.Transport()).To(gomega.Equal(test.expectedTransport), "Transport mismatch") + g.Expect(netInfo.EVPNVTEPName()).To(gomega.Equal(test.expectedVTEPName), "VTEP name mismatch") + g.Expect(netInfo.EVPNMACVRFVNI()).To(gomega.Equal(test.expectedMACVRFVNI), "MAC-VRF VNI mismatch") + g.Expect(netInfo.EVPNMACVRFRouteTarget()).To(gomega.Equal(test.expectedMACVRFRouteTarget), "MAC-VRF RouteTarget mismatch") + g.Expect(netInfo.EVPNMACVRFVID()).To(gomega.Equal(test.expectedMACVRFVID), "MAC-VRF VID mismatch") + g.Expect(netInfo.EVPNIPVRFVNI()).To(gomega.Equal(test.expectedIPVRFVNI), "IP-VRF VNI mismatch") + g.Expect(netInfo.EVPNIPVRFRouteTarget()).To(gomega.Equal(test.expectedIPVRFRouteTarget), "IP-VRF RouteTarget mismatch") + g.Expect(netInfo.EVPNIPVRFVID()).To(gomega.Equal(test.expectedIPVRFVID), "IP-VRF VID mismatch") + }) + } +} + +func TestEVPNNetworkCompatibility(t *testing.T) { + tests := []struct { + desc string + aNetwork NetInfo + anotherNetwork NetInfo + expectedResult bool + expectationDescription string + }{ + { + desc: "same EVPN config should be compatible", + aNetwork: &userDefinedNetInfo{transport: "evpn", evpn: &ovncnitypes.EVPNConfig{VTEP: "vtep1"}}, + anotherNetwork: &userDefinedNetInfo{transport: "evpn", evpn: &ovncnitypes.EVPNConfig{VTEP: "vtep1"}}, + expectedResult: true, + expectationDescription: "networks with same EVPN config should be compatible", + }, + { + desc: "different transport should not be compatible", + aNetwork: &userDefinedNetInfo{transport: "evpn"}, + anotherNetwork: &userDefinedNetInfo{transport: "no-overlay"}, + expectedResult: false, + expectationDescription: "networks with different transport should not be compatible", + }, + { + desc: "different VTEP name should not be compatible", + aNetwork: &userDefinedNetInfo{transport: "evpn", evpn: &ovncnitypes.EVPNConfig{VTEP: "vtep1"}}, + anotherNetwork: &userDefinedNetInfo{transport: "evpn", evpn: &ovncnitypes.EVPNConfig{VTEP: "vtep2"}}, + expectedResult: false, + expectationDescription: "networks with different VTEP name should not be compatible", + }, + { + desc: "different MAC-VRF VNI should not be compatible", + aNetwork: &userDefinedNetInfo{ + transport: "evpn", + evpn: &ovncnitypes.EVPNConfig{ + VTEP: "vtep1", + MACVRF: &ovncnitypes.VRFConfig{VNI: 100}, + }, + }, + anotherNetwork: &userDefinedNetInfo{ + transport: "evpn", + evpn: &ovncnitypes.EVPNConfig{ + VTEP: "vtep1", + MACVRF: &ovncnitypes.VRFConfig{VNI: 200}, + }, + }, + expectedResult: false, + expectationDescription: "networks with different MAC-VRF VNI should not be compatible", + }, + { + desc: "different IP-VRF route target should not be compatible", + aNetwork: &userDefinedNetInfo{ + transport: "evpn", + evpn: &ovncnitypes.EVPNConfig{ + VTEP: "vtep1", + IPVRF: &ovncnitypes.VRFConfig{VNI: 1000, RouteTarget: "65000:1000"}, + }, + }, + anotherNetwork: &userDefinedNetInfo{ + transport: "evpn", + evpn: &ovncnitypes.EVPNConfig{ + VTEP: "vtep1", + IPVRF: &ovncnitypes.VRFConfig{VNI: 1000, RouteTarget: "65001:1000"}, + }, + }, + expectedResult: false, + expectationDescription: "networks with different IP-VRF route target should not be compatible", + }, + { + desc: "both nil EVPN config should be compatible", + aNetwork: &userDefinedNetInfo{transport: "geneve"}, + anotherNetwork: &userDefinedNetInfo{transport: "geneve"}, + expectedResult: true, + expectationDescription: "networks with no EVPN config should be compatible", + }, + { + desc: "one nil EVPN config should not be compatible", + aNetwork: &userDefinedNetInfo{transport: "evpn", evpn: &ovncnitypes.EVPNConfig{VTEP: "vtep1"}}, + anotherNetwork: &userDefinedNetInfo{transport: "evpn"}, + expectedResult: false, + expectationDescription: "network with EVPN config vs without should not be compatible", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + g := gomega.NewWithT(t) + g.Expect(AreNetworksCompatible(test.aNetwork, test.anotherNetwork)).To( + gomega.Equal(test.expectedResult), + test.expectationDescription, + ) + }) + } +} + func TestGetNodeGatewayIP(t *testing.T) { testCases := []struct { name string diff --git a/go-controller/pkg/util/ndp/na.go b/go-controller/pkg/util/ndp/na.go index b83310c27c..01164bc610 100644 --- a/go-controller/pkg/util/ndp/na.go +++ b/go-controller/pkg/util/ndp/na.go @@ -4,8 +4,10 @@ import ( "fmt" "net" "net/netip" + "strconv" "github.com/mdlayher/ndp" + "golang.org/x/net/icmp" "golang.org/x/net/ipv6" "k8s.io/klog/v2" @@ -73,12 +75,15 @@ func SendUnsolicitedNeighborAdvertisement(interfaceName string, na NeighborAdver return fmt.Errorf("failed to convert IP %s to netip.Addr", targetIP.String()) } - // Use unspecified address to handle cases where the advertised IP is not assigned to the interface. - c, _, err := ndp.Listen(iface, ndp.Unspecified) + // Use icmp.ListenPacket instead of ndp.Listen because ndp.Listen uses the interface name + // for the IPv6 zone, which Go's net package caches. If the interface is recreated with the + // same name but a different index, the cached zone becomes stale. Using the index directly + // avoids this issue. Unspecified address handles cases where the IP isn't assigned to the interface. + ic, err := icmp.ListenPacket("ip6:ipv6-icmp", netip.IPv6Unspecified().WithZone(strconv.Itoa(iface.Index)).String()) if err != nil { return fmt.Errorf("failed to create NDP connection on %s: %w", interfaceName, err) } - defer c.Close() + defer ic.Close() // Unsolicited neighbor advertisement from a host, should override any existing cache entries una := &ndp.NeighborAdvertisement{ @@ -93,9 +98,17 @@ func SendUnsolicitedNeighborAdvertisement(interfaceName string, na NeighborAdver }, }, } + rawUNA, err := ndp.MarshalMessage(una) + if err != nil { + return fmt.Errorf("failed to marshal UNA message: %w", err) + } // rfc4861 - hop Limit 255 for unsolicited neighbor advertisements as per RFC, send to all-nodes multicast address - if err := c.WriteTo(una, &ipv6.ControlMessage{HopLimit: ndp.HopLimit}, netip.IPv6LinkLocalAllNodes()); err != nil { + _, err = ic.IPv6PacketConn().WriteTo(rawUNA, &ipv6.ControlMessage{HopLimit: ndp.HopLimit}, &net.IPAddr{ + IP: netip.IPv6LinkLocalAllNodes().AsSlice(), + Zone: strconv.Itoa(iface.Index), + }) + if err != nil { return fmt.Errorf("failed to send an unsolicited neighbor advertisement for IP %s over interface %s: %w", targetIP.String(), interfaceName, err) } diff --git a/go-controller/pkg/util/net_linux.go b/go-controller/pkg/util/net_linux.go index f35f3abfef..63625e9460 100644 --- a/go-controller/pkg/util/net_linux.go +++ b/go-controller/pkg/util/net_linux.go @@ -50,6 +50,7 @@ type NetLinkOps interface { RouteReplace(route *netlink.Route) error RouteListFiltered(family int, filter *netlink.Route, filterMask uint64) ([]netlink.Route, error) RuleListFiltered(family int, filter *netlink.Rule, filterMask uint64) ([]netlink.Rule, error) + RuleAdd(rule *netlink.Rule) error NeighAdd(neigh *netlink.Neigh) error NeighDel(neigh *netlink.Neigh) error NeighList(linkIndex, family int) ([]netlink.Neigh, error) @@ -179,6 +180,10 @@ func (defaultNetLinkOps) RuleListFiltered(family int, filter *netlink.Rule, filt return netlink.RuleListFiltered(family, filter, filterMask) } +func (defaultNetLinkOps) RuleAdd(rule *netlink.Rule) error { + return netlink.RuleAdd(rule) +} + func (defaultNetLinkOps) NeighAdd(neigh *netlink.Neigh) error { return netlink.NeighAdd(neigh) } diff --git a/go-controller/pkg/util/net_unit_test.go b/go-controller/pkg/util/net_unit_test.go index a4a62747b0..d4b94938ab 100644 --- a/go-controller/pkg/util/net_unit_test.go +++ b/go-controller/pkg/util/net_unit_test.go @@ -72,7 +72,7 @@ func TestGetOVSPortMACAddress(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} diff --git a/go-controller/pkg/util/network_connect_annotation.go b/go-controller/pkg/util/network_connect.go similarity index 78% rename from go-controller/pkg/util/network_connect_annotation.go rename to go-controller/pkg/util/network_connect.go index 01f97667ca..d7aefbb8c1 100644 --- a/go-controller/pkg/util/network_connect_annotation.go +++ b/go-controller/pkg/util/network_connect.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "strconv" + "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -13,6 +14,7 @@ import ( networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" networkconnectapply "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1/apis/applyconfiguration/clusternetworkconnect/v1" networkconnectclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1/apis/clientset/versioned" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" ) const ( @@ -22,6 +24,26 @@ const ( networkConnectRouterTunnelKeyFieldManager = "ovn-kubernetes-network-connect-controller-tunnel-key-annotation" ) +// ComputeNetworkOwner returns a unique owner key for a network based on its topology type and ID. +// This is used for tracking network ownership in external IDs and annotations. +func ComputeNetworkOwner(networkType string, networkID int) string { + return fmt.Sprintf("%s_%d", networkType, networkID) +} + +// ParseNetworkOwner parses an owner key like "layer3_1" into topology type and network ID. +func ParseNetworkOwner(owner string) (topologyType string, networkID int, err error) { + if strings.HasPrefix(owner, ovntypes.Layer3Topology+"_") { + topologyType = ovntypes.Layer3Topology + _, err = fmt.Sscanf(owner, ovntypes.Layer3Topology+"_%d", &networkID) + } else if strings.HasPrefix(owner, ovntypes.Layer2Topology+"_") { + topologyType = ovntypes.Layer2Topology + _, err = fmt.Sscanf(owner, ovntypes.Layer2Topology+"_%d", &networkID) + } else { + err = fmt.Errorf("unknown owner format: %s", owner) + } + return +} + type NetworkConnectSubnetAnnotation struct { IPv4 string `json:"ipv4,omitempty"` IPv6 string `json:"ipv6,omitempty"` @@ -108,6 +130,16 @@ func ParseNetworkConnectSubnetAnnotation(cnc *networkconnectv1.ClusterNetworkCon return result, nil } +func NetworkConnectSubnetAnnotationChanged(oldObj, newObj *networkconnectv1.ClusterNetworkConnect) bool { + if oldObj == nil && newObj == nil { + return false + } + if oldObj == nil || newObj == nil { + return true + } + return oldObj.Annotations[ovnNetworkConnectSubnetAnnotation] != newObj.Annotations[ovnNetworkConnectSubnetAnnotation] +} + // UpdateNetworkConnectRouterTunnelKeyAnnotation updates the router tunnel key annotation for the given CNC and given tunnel ID. // It uses the Apply method to patch the annotation and has its own manager field to avoid conflicts with other annotation patches // like the subnet annotation patch above. @@ -147,3 +179,13 @@ func ParseNetworkConnectTunnelKeyAnnotation(cnc *networkconnectv1.ClusterNetwork return tunnelID, nil } + +func NetworkConnectTunnelKeyAnnotationsChanged(oldObj, newObj *networkconnectv1.ClusterNetworkConnect) bool { + if oldObj == nil && newObj == nil { + return false + } + if oldObj == nil || newObj == nil { + return true + } + return oldObj.Annotations[OvnConnectRouterTunnelKeyAnnotation] != newObj.Annotations[OvnConnectRouterTunnelKeyAnnotation] +} diff --git a/go-controller/pkg/util/network_connect_annotation_unit_test.go b/go-controller/pkg/util/network_connect_unit_test.go similarity index 85% rename from go-controller/pkg/util/network_connect_annotation_unit_test.go rename to go-controller/pkg/util/network_connect_unit_test.go index 3faa3e1d4a..897a0648a3 100644 --- a/go-controller/pkg/util/network_connect_annotation_unit_test.go +++ b/go-controller/pkg/util/network_connect_unit_test.go @@ -14,6 +14,7 @@ import ( networkconnectv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1" networkconnectfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/clusternetworkconnect/v1/apis/clientset/versioned/fake" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" ) func TestUpdateNetworkConnectSubnetAnnotation(t *testing.T) { @@ -318,3 +319,70 @@ func TestUpdateNetworkConnectSubnetAnnotation_PreservesExistingAnnotations(t *te // Verify the new subnet annotation was added assert.Contains(t, updatedCNC.Annotations, ovnNetworkConnectSubnetAnnotation) } + +func TestParseNetworkOwner(t *testing.T) { + tests := []struct { + name string + owner string + expectedTopology string + expectedID int + expectError bool + }{ + { + name: "layer3 topology with ID 1", + owner: "layer3_1", + expectedTopology: ovntypes.Layer3Topology, + expectedID: 1, + expectError: false, + }, + { + name: "layer3 topology with ID 100", + owner: "layer3_100", + expectedTopology: ovntypes.Layer3Topology, + expectedID: 100, + expectError: false, + }, + { + name: "layer2 topology with ID 1", + owner: "layer2_1", + expectedTopology: ovntypes.Layer2Topology, + expectedID: 1, + expectError: false, + }, + { + name: "layer2 topology with ID 50", + owner: "layer2_50", + expectedTopology: ovntypes.Layer2Topology, + expectedID: 50, + expectError: false, + }, + { + name: "unknown topology type", + owner: "unknown_1", + expectError: true, + }, + { + name: "invalid format - no underscore", + owner: "layer31", + expectError: true, + }, + { + name: "invalid format - empty", + owner: "", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + topology, networkID, err := ParseNetworkOwner(tt.owner) + if tt.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedTopology, topology) + assert.Equal(t, tt.expectedID, networkID) + } + }) + } +} diff --git a/go-controller/pkg/util/nicstobridge_test.go b/go-controller/pkg/util/nicstobridge_test.go index a259073f09..01ba0e2120 100644 --- a/go-controller/pkg/util/nicstobridge_test.go +++ b/go-controller/pkg/util/nicstobridge_test.go @@ -20,7 +20,7 @@ func TestGetNicName(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -408,7 +408,7 @@ func TestNicToBridge(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} @@ -577,7 +577,7 @@ func TestBridgeToNic(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} diff --git a/go-controller/pkg/util/ovs.go b/go-controller/pkg/util/ovs.go index 8e2c3b04e0..b32f73999b 100644 --- a/go-controller/pkg/util/ovs.go +++ b/go-controller/pkg/util/ovs.go @@ -174,7 +174,7 @@ func (runsvc *defaultExecRunner) RunCmd(cmd kexec.Cmd, cmdPath string, envVars [ return stdout, stderr, err } -var runCmdExecRunner ExecRunner = &defaultExecRunner{} +var RunCmdExecRunner ExecRunner = &defaultExecRunner{} // SetExec validates executable paths and saves the given exec interface // to be used for running various OVS and OVN utilites @@ -301,17 +301,17 @@ func ResetRunner() { var runCounter uint64 func runCmd(cmd kexec.Cmd, cmdPath string, args ...string) (*bytes.Buffer, *bytes.Buffer, error) { - return runCmdExecRunner.RunCmd(cmd, cmdPath, []string{}, args...) + return RunCmdExecRunner.RunCmd(cmd, cmdPath, []string{}, args...) } func run(cmdPath string, args ...string) (*bytes.Buffer, *bytes.Buffer, error) { cmd := runner.exec.Command(cmdPath, args...) - return runCmdExecRunner.RunCmd(cmd, cmdPath, []string{}, args...) + return RunCmdExecRunner.RunCmd(cmd, cmdPath, []string{}, args...) } func runWithEnvVars(cmdPath string, envVars []string, args ...string) (*bytes.Buffer, *bytes.Buffer, error) { cmd := runner.exec.Command(cmdPath, args...) - return runCmdExecRunner.RunCmd(cmd, cmdPath, envVars, args...) + return RunCmdExecRunner.RunCmd(cmd, cmdPath, envVars, args...) } // RunOVSOfctl runs a command via ovs-ofctl. @@ -911,10 +911,10 @@ func GetOVSPortPodInfo(hostIfName string) (bool, string, string, error) { return false, "", "", nil } sandbox := GetExternalIDValByKey(stdout, "sandbox") - nadName := GetExternalIDValByKey(stdout, types.NADExternalID) + nadkey := GetExternalIDValByKey(stdout, types.NADExternalID) // if network_name does not exists, it is default network - if nadName == "" { - nadName = types.DefaultNetworkName + if nadkey == "" { + nadkey = types.DefaultNetworkName } - return true, sandbox, nadName, nil + return true, sandbox, nadkey, nil } diff --git a/go-controller/pkg/util/ovs_unit_test.go b/go-controller/pkg/util/ovs_unit_test.go index 96ab123f84..2b8e633949 100644 --- a/go-controller/pkg/util/ovs_unit_test.go +++ b/go-controller/pkg/util/ovs_unit_test.go @@ -99,7 +99,9 @@ func TestRunOVNretry(t *testing.T) { // Variables below are defined in ovs.go ovnCmdRetryCount = 0 ovnCmdRetryInterval = 1 * time.Millisecond - runCmdExecRunner = mockExecRunner + // below is defined in ovs.go + RunCmdExecRunner = mockExecRunner + // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} // Used for "test path when PID changes" test case @@ -308,7 +310,7 @@ func TestRunOVNNorthAppCtl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} // note runner.ovndir is defined in ovs.go file and so is ovnRunDir var with an initial value @@ -390,7 +392,7 @@ func TestRunOVNControllerAppCtl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} // note runner.ovndir is defined in ovs.go file and so is ovnRunDir var with an initial value @@ -472,7 +474,7 @@ func TestRunOvsVswitchdAppCtl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} @@ -549,7 +551,7 @@ func TestDefaultExecRunner_RunCmd(t *testing.T) { mockCmd := new(mock_k8s_io_utils_exec.Cmd) // tests in other files in the package would set runCmdExecRunner to mocks.ExecRunner, // for this test we want to ensure the non-mock instance is used - runCmdExecRunner = &defaultExecRunner{} + RunCmdExecRunner = &defaultExecRunner{} tests := []struct { desc string @@ -582,7 +584,7 @@ func TestDefaultExecRunner_RunCmd(t *testing.T) { if tc.cmd != nil { ovntest.ProcessMockFnList(&tc.cmd.(*mock_k8s_io_utils_exec.Cmd).Mock, tc.onRetArgsCmdList) } - _, _, e := runCmdExecRunner.RunCmd(tc.cmd, tc.cmdPath, tc.envVars, tc.cmdArg) + _, _, e := RunCmdExecRunner.RunCmd(tc.cmd, tc.cmdPath, tc.envVars, tc.cmdArg) assert.Equal(t, tc.expectedErr, e) mockCmd.AssertExpectations(t) @@ -902,7 +904,7 @@ func TestRunOVSOfctl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -964,7 +966,7 @@ OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0 mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1039,7 +1041,7 @@ func TestRunOVSVsctl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1080,7 +1082,7 @@ func TestRunOVSAppctlWithTimeout(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1124,7 +1126,7 @@ func TestRunOVNAppctlWithTimeout(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1168,7 +1170,7 @@ func TestRunOVNNbctlWithTimeout(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1214,7 +1216,7 @@ func TestRunOVNNbctl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1257,7 +1259,7 @@ func TestRunOVNSbctlWithTimeout(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1303,7 +1305,7 @@ func TestRunOVNSbctl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1346,7 +1348,7 @@ func TestRunOVSDBClient(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1389,7 +1391,7 @@ func TestRunOVSDBTool(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go. - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1432,7 +1434,7 @@ func TestRunOVSDBClientOVNNB(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1475,7 +1477,7 @@ func TestRunOVNNBAppCtl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1518,7 +1520,7 @@ func TestRunOVNSBAppCtl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1561,7 +1563,7 @@ func TestRunIP(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1596,7 +1598,7 @@ func TestRunSysctl(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1631,7 +1633,7 @@ func TestAddOFFlowWithSpecificAction(t *testing.T) { mockCmd := new(mock_k8s_io_utils_exec.Cmd) mockExecRunner := new(mocks.ExecRunner) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1678,7 +1680,7 @@ func TestReplaceOFFlows(t *testing.T) { mockCmd := new(mock_k8s_io_utils_exec.Cmd) mockExecRunner := new(mocks.ExecRunner) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1725,7 +1727,7 @@ func TestGetOVNDBServerInfo(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} tests := []struct { @@ -1780,7 +1782,7 @@ func TestDetectSCTPSupport(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} diff --git a/go-controller/pkg/util/pod_annotation.go b/go-controller/pkg/util/pod_annotation.go index 614bcf473c..022ca66cc3 100644 --- a/go-controller/pkg/util/pod_annotation.go +++ b/go-controller/pkg/util/pod_annotation.go @@ -12,7 +12,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" listers "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" utilnet "k8s.io/utils/net" "sigs.k8s.io/yaml" @@ -145,7 +144,7 @@ type OpenPort struct { } // MarshalPodAnnotation adds the pod's network details of the specified network to the corresponding pod annotation. -func MarshalPodAnnotation(annotations map[string]string, podInfo *PodAnnotation, nadName string) (map[string]string, error) { +func MarshalPodAnnotation(annotations map[string]string, podInfo *PodAnnotation, nadKey string) (map[string]string, error) { if annotations == nil { annotations = make(map[string]string) } @@ -171,7 +170,7 @@ func MarshalPodAnnotation(annotations map[string]string, podInfo *PodAnnotation, pa.IPs = append(pa.IPs, ip.String()) } - existingPa, ok := podNetworks[nadName] + existingPa, ok := podNetworks[nadKey] if ok { if len(pa.IPs) != len(existingPa.IPs) { return nil, ErrOverridePodIPs @@ -205,7 +204,7 @@ func MarshalPodAnnotation(annotations map[string]string, podInfo *PodAnnotation, pa.GatewayIPv6LLA = podInfo.GatewayIPv6LLA.String() } - podNetworks[nadName] = pa + podNetworks[nadKey] = pa bytes, err := json.Marshal(podNetworks) if err != nil { return nil, fmt.Errorf("failed marshaling podNetworks map %v", podNetworks) @@ -215,7 +214,7 @@ func MarshalPodAnnotation(annotations map[string]string, podInfo *PodAnnotation, } // UnmarshalPodAnnotation returns the Pod's network info of the given network from pod.Annotations -func UnmarshalPodAnnotation(annotations map[string]string, nadName string) (*PodAnnotation, error) { +func UnmarshalPodAnnotation(annotations map[string]string, nadKey string) (*PodAnnotation, error) { var err error ovnAnnotation, ok := annotations[OvnPodAnnotationName] if !ok { @@ -227,10 +226,10 @@ func UnmarshalPodAnnotation(annotations map[string]string, nadName string) (*Pod return nil, err } - tempA, ok := podNetworks[nadName] + tempA, ok := podNetworks[nadKey] if !ok { - return nil, newAnnotationNotSetError("no ovn pod annotation for NAD %s: %q", - nadName, ovnAnnotation) + return nil, newAnnotationNotSetError("no ovn pod annotation for NAD key %s: %q", + nadKey, ovnAnnotation) } a := &tempA @@ -318,10 +317,10 @@ func UnmarshalPodAnnotationAllNetworks(annotations map[string]string) (map[strin return podNetworks, nil } -// GetPodCIDRsWithFullMask returns the pod's IP addresses in a CIDR with FullMask format -// Internally it calls GetPodIPsOfNetwork -func GetPodCIDRsWithFullMask(pod *corev1.Pod, nInfo NetInfo) ([]*net.IPNet, error) { - podIPs, err := GetPodIPsOfNetwork(pod, nInfo) +// GetPodCIDRsWithFullMask returns the pod's IP addresses in a CIDR with FullMask format. +// Internally it calls GetPodIPsOfNetwork. +func GetPodCIDRsWithFullMask(pod *corev1.Pod, nInfo NetInfo, getNetworkNameForNADKey func(nadKey string) string) ([]*net.IPNet, error) { + podIPs, err := GetPodIPsOfNetwork(pod, nInfo, getNetworkNameForNADKey) if err != nil { return nil, err } @@ -339,17 +338,21 @@ func GetPodCIDRsWithFullMask(pod *corev1.Pod, nInfo NetInfo) ([]*net.IPNet, erro // GetPodIPsOfNetwork returns the pod's IP addresses, first from the OVN annotation // and then falling back to the Pod Status IPs. This function is intended to // also return IPs for HostNetwork and other non-OVN-IPAM-ed pods. -func GetPodIPsOfNetwork(pod *corev1.Pod, nInfo NetInfo) ([]net.IP, error) { +// getNetworkNameForNADKey is required for user defined networks. +func GetPodIPsOfNetwork(pod *corev1.Pod, nInfo NetInfo, getNetworkNameForNADKey func(nadKey string) string) ([]net.IP, error) { if nInfo.IsUserDefinedNetwork() { - return SecondaryNetworkPodIPs(pod, nInfo) + if getNetworkNameForNADKey == nil { + return nil, fmt.Errorf("missing NAD resolver for network %q", nInfo.GetNetworkName()) + } + return SecondaryNetworkPodIPs(pod, nInfo, getNetworkNameForNADKey) } return DefaultNetworkPodIPs(pod) } // GetPodCIDRsWithFullMaskOfNetwork returns the pod's IP addresses in a CIDR with FullMask format // from a pod network annotation 'k8s.ovn.org/pod-networks' using key nadName. -func GetPodCIDRsWithFullMaskOfNetwork(pod *corev1.Pod, nadName string) []*net.IPNet { - ips := getAnnotatedPodIPs(pod, nadName) +func GetPodCIDRsWithFullMaskOfNetwork(pod *corev1.Pod, nadKey string) []*net.IPNet { + ips := getAnnotatedPodIPs(pod, nadKey) ipNets := make([]*net.IPNet, 0, len(ips)) for _, ip := range ips { ipNet := net.IPNet{ @@ -392,57 +395,62 @@ func DefaultNetworkPodIPs(pod *corev1.Pod) ([]net.IP, error) { return []net.IP{ip}, nil } -func SecondaryNetworkPodIPs(pod *corev1.Pod, networkInfo NetInfo) ([]net.IP, error) { +func SecondaryNetworkPodIPs(pod *corev1.Pod, networkInfo NetInfo, getNetworkNameForNADKey func(nadKey string) string) ([]net.IP, error) { ips := []net.IP{} - podNadNames, err := PodNadNames(pod, networkInfo) + if getNetworkNameForNADKey == nil { + return nil, fmt.Errorf("missing NAD resolver for network %q", networkInfo.GetNetworkName()) + } + podNADKeys, err := PodNADKeys(pod, networkInfo, getNetworkNameForNADKey) if err != nil { return nil, err } - for _, nadName := range podNadNames { - ips = append(ips, getAnnotatedPodIPs(pod, nadName)...) + for _, nadKey := range podNADKeys { + ips = append(ips, getAnnotatedPodIPs(pod, nadKey)...) } return ips, nil } -// PodNadNames returns pod's NAD names associated with given network specified by netconf. -// If netinfo belongs to user defined primary network, then retrieve NAD names from -// netinfo.GetNADs() which is serving pod's namespace. -// For all other cases, retrieve NAD names for the pod based on NetworkSelectionElement. -func PodNadNames(pod *corev1.Pod, netinfo NetInfo) ([]string, error) { +// PodNADKeys returns pod's NAD keys associated with given network specified by netconf. +// For primary UDNs, retrieve NAD names for the pod based on its OVN annotations. +// For secondary UDNs, retrieve NAD names for the pod based on NetworkSelectionElement. +func PodNADKeys(pod *corev1.Pod, netinfo NetInfo, getNetworkNameForNADKey func(nadKey string) string) ([]string, error) { + if netinfo.IsUserDefinedNetwork() && getNetworkNameForNADKey == nil { + return nil, fmt.Errorf("missing NAD resolver for network %q", netinfo.GetNetworkName()) + } + if netinfo.IsPrimaryNetwork() { - return GetPrimaryNetworkNADNamesForNamespaceFromNetInfo(pod.Namespace, netinfo) + podNetworks, err := UnmarshalPodAnnotationAllNetworks(pod.Annotations) + if err != nil { + return nil, err + } + nadKeys := make([]string, 0, len(podNetworks)) + for nadKey := range podNetworks { + networkName := getNetworkNameForNADKey(nadKey) + if networkName == "" || networkName != netinfo.GetNetworkName() { + continue + } + nadKeys = append(nadKeys, nadKey) + } + return nadKeys, nil } - on, networkMap, err := GetPodNADToNetworkMapping(pod, netinfo) + + on, networkMap, err := getPodNADToNetworkMapping(pod, netinfo, getNetworkNameForNADKey) // skip pods that are not on this network if err != nil { return nil, err } else if !on { return []string{}, nil } - nadNames := make([]string, 0, len(networkMap)) - for nadName := range networkMap { - nadNames = append(nadNames, nadName) - } - return nadNames, nil -} - -func GetPrimaryNetworkNADNamesForNamespaceFromNetInfo(namespace string, netinfo NetInfo) ([]string, error) { - for _, nadName := range netinfo.GetNADs() { - ns, _, err := cache.SplitMetaNamespaceKey(nadName) - if err != nil { - return nil, fmt.Errorf("error parsing nad name %s from network %s: %v", nadName, netinfo.GetNetworkName(), err) - } - if ns != namespace { - continue - } - return []string{nadName}, nil + nadKeys := make([]string, 0, len(networkMap)) + for nadKey := range networkMap { + nadKeys = append(nadKeys, nadKey) } - return []string{}, nil + return nadKeys, nil } -func getAnnotatedPodIPs(pod *corev1.Pod, nadName string) []net.IP { +func getAnnotatedPodIPs(pod *corev1.Pod, nadKey string) []net.IP { var ips []net.IP - annotation, _ := UnmarshalPodAnnotation(pod.Annotations, nadName) + annotation, _ := UnmarshalPodAnnotation(pod.Annotations, nadKey) if annotation != nil { // Use the OVN annotation if valid for _, ip := range annotation.IPs { @@ -490,10 +498,10 @@ func GetK8sPodAllNetworkSelections(pod *corev1.Pod) ([]*nadapi.NetworkSelectionE // UpdatePodAnnotationWithRetry updates the pod annotation on the pod retrying // on conflict -func UpdatePodAnnotationWithRetry(podLister listers.PodLister, kube kube.Interface, pod *corev1.Pod, podAnnotation *PodAnnotation, nadName string) error { +func UpdatePodAnnotationWithRetry(podLister listers.PodLister, kube kube.Interface, pod *corev1.Pod, podAnnotation *PodAnnotation, nadKey string) error { updatePodAnnotationNoRollback := func(pod *corev1.Pod) (*corev1.Pod, func(), error) { var err error - pod.Annotations, err = MarshalPodAnnotation(pod.Annotations, podAnnotation, nadName) + pod.Annotations, err = MarshalPodAnnotation(pod.Annotations, podAnnotation, nadKey) if err != nil { return nil, nil, err } diff --git a/go-controller/pkg/util/pod_annotation_unit_test.go b/go-controller/pkg/util/pod_annotation_unit_test.go index 7a07629bc8..f02a4ae3c4 100644 --- a/go-controller/pkg/util/pod_annotation_unit_test.go +++ b/go-controller/pkg/util/pod_annotation_unit_test.go @@ -369,7 +369,17 @@ func TestGetPodIPsOfNetwork(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) { - res1, e := GetPodIPsOfNetwork(tc.inpPod, tc.networkInfo) + var resolver func(nadKey string) string + if tc.networkInfo.IsUserDefinedNetwork() { + expectedNADKey := GetNADName(namespace, secondaryNetworkName) + resolver = func(nadKey string) string { + if nadKey == expectedNADKey { + return tc.networkInfo.GetNetworkName() + } + return "" + } + } + res1, e := GetPodIPsOfNetwork(tc.inpPod, tc.networkInfo, resolver) t.Log(res1, e) if tc.errAssert { require.Error(t, e) @@ -383,7 +393,7 @@ func TestGetPodIPsOfNetwork(t *testing.T) { assert.Equal(t, tc.outExp, res1) } if len(tc.outExp) > 0 { - res2, e := GetPodCIDRsWithFullMask(tc.inpPod, tc.networkInfo) + res2, e := GetPodCIDRsWithFullMask(tc.inpPod, tc.networkInfo, resolver) t.Log(res2, e) if tc.errAssert { assert.Error(t, e) diff --git a/go-controller/pkg/util/util_unit_test.go b/go-controller/pkg/util/util_unit_test.go index 222e4f40c2..2e2d622023 100644 --- a/go-controller/pkg/util/util_unit_test.go +++ b/go-controller/pkg/util/util_unit_test.go @@ -59,7 +59,7 @@ func TestGetNodeChassisID(t *testing.T) { mockExecRunner := new(mocks.ExecRunner) mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below is defined in ovs.go - runCmdExecRunner = mockExecRunner + RunCmdExecRunner = mockExecRunner // note runner is defined in ovs.go file runner = &execHelper{exec: mockKexecIface} diff --git a/go-controller/vendor/github.com/go-openapi/jsonpointer/.golangci.yml b/go-controller/vendor/github.com/go-openapi/jsonpointer/.golangci.yml index 22f8d21cca..d2fafb8a2b 100644 --- a/go-controller/vendor/github.com/go-openapi/jsonpointer/.golangci.yml +++ b/go-controller/vendor/github.com/go-openapi/jsonpointer/.golangci.yml @@ -1,12 +1,6 @@ linters-settings: - govet: - check-shadowing: true - golint: - min-confidence: 0 gocyclo: min-complexity: 45 - maligned: - suggest-new: true dupl: threshold: 200 goconst: @@ -16,7 +10,7 @@ linters-settings: linters: enable-all: true disable: - - maligned + - recvcheck - unparam - lll - gochecknoinits @@ -29,9 +23,6 @@ linters: - wrapcheck - testpackage - nlreturn - - gomnd - - exhaustivestruct - - goerr113 - errorlint - nestif - godot @@ -39,7 +30,6 @@ linters: - paralleltest - tparallel - thelper - - ifshort - exhaustruct - varnamelen - gci @@ -52,10 +42,15 @@ linters: - forcetypeassert - cyclop # deprecated linters - - deadcode - - interfacer - - scopelint - - varcheck - - structcheck - - golint - - nosnakecase + #- deadcode + #- interfacer + #- scopelint + #- varcheck + #- structcheck + #- golint + #- nosnakecase + #- maligned + #- goerr113 + #- ifshort + #- gomnd + #- exhaustivestruct diff --git a/go-controller/vendor/github.com/go-openapi/jsonpointer/errors.go b/go-controller/vendor/github.com/go-openapi/jsonpointer/errors.go new file mode 100644 index 0000000000..b84343d9d7 --- /dev/null +++ b/go-controller/vendor/github.com/go-openapi/jsonpointer/errors.go @@ -0,0 +1,18 @@ +package jsonpointer + +type pointerError string + +func (e pointerError) Error() string { + return string(e) +} + +const ( + // ErrPointer is an error raised by the jsonpointer package + ErrPointer pointerError = "JSON pointer error" + + // ErrInvalidStart states that a JSON pointer must start with a separator ("/") + ErrInvalidStart pointerError = `JSON pointer must be empty or start with a "` + pointerSeparator + + // ErrUnsupportedValueType indicates that a value of the wrong type is being set + ErrUnsupportedValueType pointerError = "only structs, pointers, maps and slices are supported for setting values" +) diff --git a/go-controller/vendor/github.com/go-openapi/jsonpointer/pointer.go b/go-controller/vendor/github.com/go-openapi/jsonpointer/pointer.go index d970c7cf44..a08cd68ac0 100644 --- a/go-controller/vendor/github.com/go-openapi/jsonpointer/pointer.go +++ b/go-controller/vendor/github.com/go-openapi/jsonpointer/pointer.go @@ -39,9 +39,6 @@ import ( const ( emptyPointer = `` pointerSeparator = `/` - - invalidStart = `JSON pointer must be empty or start with a "` + pointerSeparator - notFound = `Can't find the pointer in the document` ) var jsonPointableType = reflect.TypeOf(new(JSONPointable)).Elem() @@ -80,7 +77,7 @@ func (p *Pointer) parse(jsonPointerString string) error { if jsonPointerString != emptyPointer { if !strings.HasPrefix(jsonPointerString, pointerSeparator) { - err = errors.New(invalidStart) + err = errors.Join(ErrInvalidStart, ErrPointer) } else { referenceTokens := strings.Split(jsonPointerString, pointerSeparator) p.referenceTokens = append(p.referenceTokens, referenceTokens[1:]...) @@ -128,7 +125,7 @@ func getSingleImpl(node any, decodedToken string, nameProvider *swag.NameProvide rValue := reflect.Indirect(reflect.ValueOf(node)) kind := rValue.Kind() if isNil(node) { - return nil, kind, fmt.Errorf("nil value has not field %q", decodedToken) + return nil, kind, fmt.Errorf("nil value has no field %q: %w", decodedToken, ErrPointer) } switch typed := node.(type) { @@ -146,7 +143,7 @@ func getSingleImpl(node any, decodedToken string, nameProvider *swag.NameProvide case reflect.Struct: nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken) if !ok { - return nil, kind, fmt.Errorf("object has no field %q", decodedToken) + return nil, kind, fmt.Errorf("object has no field %q: %w", decodedToken, ErrPointer) } fld := rValue.FieldByName(nm) return fld.Interface(), kind, nil @@ -158,7 +155,7 @@ func getSingleImpl(node any, decodedToken string, nameProvider *swag.NameProvide if mv.IsValid() { return mv.Interface(), kind, nil } - return nil, kind, fmt.Errorf("object has no key %q", decodedToken) + return nil, kind, fmt.Errorf("object has no key %q: %w", decodedToken, ErrPointer) case reflect.Slice: tokenIndex, err := strconv.Atoi(decodedToken) @@ -167,14 +164,14 @@ func getSingleImpl(node any, decodedToken string, nameProvider *swag.NameProvide } sLength := rValue.Len() if tokenIndex < 0 || tokenIndex >= sLength { - return nil, kind, fmt.Errorf("index out of bounds array[0,%d] index '%d'", sLength-1, tokenIndex) + return nil, kind, fmt.Errorf("index out of bounds array[0,%d] index '%d': %w", sLength-1, tokenIndex, ErrPointer) } elem := rValue.Index(tokenIndex) return elem.Interface(), kind, nil default: - return nil, kind, fmt.Errorf("invalid token reference %q", decodedToken) + return nil, kind, fmt.Errorf("invalid token reference %q: %w", decodedToken, ErrPointer) } } @@ -194,7 +191,7 @@ func setSingleImpl(node, data any, decodedToken string, nameProvider *swag.NameP case reflect.Struct: nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken) if !ok { - return fmt.Errorf("object has no field %q", decodedToken) + return fmt.Errorf("object has no field %q: %w", decodedToken, ErrPointer) } fld := rValue.FieldByName(nm) if fld.IsValid() { @@ -214,18 +211,18 @@ func setSingleImpl(node, data any, decodedToken string, nameProvider *swag.NameP } sLength := rValue.Len() if tokenIndex < 0 || tokenIndex >= sLength { - return fmt.Errorf("index out of bounds array[0,%d] index '%d'", sLength, tokenIndex) + return fmt.Errorf("index out of bounds array[0,%d] index '%d': %w", sLength, tokenIndex, ErrPointer) } elem := rValue.Index(tokenIndex) if !elem.CanSet() { - return fmt.Errorf("can't set slice index %s to %v", decodedToken, data) + return fmt.Errorf("can't set slice index %s to %v: %w", decodedToken, data, ErrPointer) } elem.Set(reflect.ValueOf(data)) return nil default: - return fmt.Errorf("invalid token reference %q", decodedToken) + return fmt.Errorf("invalid token reference %q: %w", decodedToken, ErrPointer) } } @@ -244,7 +241,6 @@ func (p *Pointer) get(node any, nameProvider *swag.NameProvider) (any, reflect.K } for _, token := range p.referenceTokens { - decodedToken := Unescape(token) r, knd, err := getSingleImpl(node, decodedToken, nameProvider) @@ -264,7 +260,10 @@ func (p *Pointer) set(node, data any, nameProvider *swag.NameProvider) error { knd := reflect.ValueOf(node).Kind() if knd != reflect.Ptr && knd != reflect.Struct && knd != reflect.Map && knd != reflect.Slice && knd != reflect.Array { - return errors.New("only structs, pointers, maps and slices are supported for setting values") + return errors.Join( + ErrUnsupportedValueType, + ErrPointer, + ) } if nameProvider == nil { @@ -307,7 +306,7 @@ func (p *Pointer) set(node, data any, nameProvider *swag.NameProvider) error { case reflect.Struct: nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken) if !ok { - return fmt.Errorf("object has no field %q", decodedToken) + return fmt.Errorf("object has no field %q: %w", decodedToken, ErrPointer) } fld := rValue.FieldByName(nm) if fld.CanAddr() && fld.Kind() != reflect.Interface && fld.Kind() != reflect.Map && fld.Kind() != reflect.Slice && fld.Kind() != reflect.Ptr { @@ -321,7 +320,7 @@ func (p *Pointer) set(node, data any, nameProvider *swag.NameProvider) error { mv := rValue.MapIndex(kv) if !mv.IsValid() { - return fmt.Errorf("object has no key %q", decodedToken) + return fmt.Errorf("object has no key %q: %w", decodedToken, ErrPointer) } if mv.CanAddr() && mv.Kind() != reflect.Interface && mv.Kind() != reflect.Map && mv.Kind() != reflect.Slice && mv.Kind() != reflect.Ptr { node = mv.Addr().Interface() @@ -336,7 +335,7 @@ func (p *Pointer) set(node, data any, nameProvider *swag.NameProvider) error { } sLength := rValue.Len() if tokenIndex < 0 || tokenIndex >= sLength { - return fmt.Errorf("index out of bounds array[0,%d] index '%d'", sLength, tokenIndex) + return fmt.Errorf("index out of bounds array[0,%d] index '%d': %w", sLength, tokenIndex, ErrPointer) } elem := rValue.Index(tokenIndex) @@ -347,7 +346,7 @@ func (p *Pointer) set(node, data any, nameProvider *swag.NameProvider) error { node = elem.Interface() default: - return fmt.Errorf("invalid token reference %q", decodedToken) + return fmt.Errorf("invalid token reference %q: %w", decodedToken, ErrPointer) } } @@ -404,10 +403,10 @@ func (p *Pointer) Offset(document string) (int64, error) { return 0, err } default: - return 0, fmt.Errorf("invalid token %#v", tk) + return 0, fmt.Errorf("invalid token %#v: %w", tk, ErrPointer) } default: - return 0, fmt.Errorf("invalid token %#v", tk) + return 0, fmt.Errorf("invalid token %#v: %w", tk, ErrPointer) } } return offset, nil @@ -437,16 +436,16 @@ func offsetSingleObject(dec *json.Decoder, decodedToken string) (int64, error) { return offset, nil } default: - return 0, fmt.Errorf("invalid token %#v", tk) + return 0, fmt.Errorf("invalid token %#v: %w", tk, ErrPointer) } } - return 0, fmt.Errorf("token reference %q not found", decodedToken) + return 0, fmt.Errorf("token reference %q not found: %w", decodedToken, ErrPointer) } func offsetSingleArray(dec *json.Decoder, decodedToken string) (int64, error) { idx, err := strconv.Atoi(decodedToken) if err != nil { - return 0, fmt.Errorf("token reference %q is not a number: %v", decodedToken, err) + return 0, fmt.Errorf("token reference %q is not a number: %v: %w", decodedToken, err, ErrPointer) } var i int for i = 0; i < idx && dec.More(); i++ { @@ -470,7 +469,7 @@ func offsetSingleArray(dec *json.Decoder, decodedToken string) (int64, error) { } if !dec.More() { - return 0, fmt.Errorf("token reference %q not found", decodedToken) + return 0, fmt.Errorf("token reference %q not found: %w", decodedToken, ErrPointer) } return dec.InputOffset(), nil } diff --git a/go-controller/vendor/github.com/go-openapi/swag/.golangci.yml b/go-controller/vendor/github.com/go-openapi/swag/.golangci.yml index 80e2be0042..d2fafb8a2b 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/.golangci.yml +++ b/go-controller/vendor/github.com/go-openapi/swag/.golangci.yml @@ -1,22 +1,17 @@ linters-settings: - govet: - check-shadowing: true - golint: - min-confidence: 0 gocyclo: min-complexity: 45 - maligned: - suggest-new: true dupl: threshold: 200 goconst: - min-len: 3 + min-len: 2 min-occurrences: 3 linters: enable-all: true disable: - - maligned + - recvcheck + - unparam - lll - gochecknoinits - gochecknoglobals @@ -28,9 +23,6 @@ linters: - wrapcheck - testpackage - nlreturn - - gomnd - - exhaustivestruct - - goerr113 - errorlint - nestif - godot @@ -38,7 +30,6 @@ linters: - paralleltest - tparallel - thelper - - ifshort - exhaustruct - varnamelen - gci @@ -51,10 +42,15 @@ linters: - forcetypeassert - cyclop # deprecated linters - - deadcode - - interfacer - - scopelint - - varcheck - - structcheck - - golint - - nosnakecase + #- deadcode + #- interfacer + #- scopelint + #- varcheck + #- structcheck + #- golint + #- nosnakecase + #- maligned + #- goerr113 + #- ifshort + #- gomnd + #- exhaustivestruct diff --git a/go-controller/vendor/github.com/go-openapi/swag/errors.go b/go-controller/vendor/github.com/go-openapi/swag/errors.go new file mode 100644 index 0000000000..6c67fbf92e --- /dev/null +++ b/go-controller/vendor/github.com/go-openapi/swag/errors.go @@ -0,0 +1,15 @@ +package swag + +type swagError string + +const ( + // ErrYAML is an error raised by YAML utilities + ErrYAML swagError = "yaml error" + + // ErrLoader is an error raised by the file loader utility + ErrLoader swagError = "loader error" +) + +func (e swagError) Error() string { + return string(e) +} diff --git a/go-controller/vendor/github.com/go-openapi/swag/json.go b/go-controller/vendor/github.com/go-openapi/swag/json.go index 7e9902ca31..c7caa9908f 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/json.go +++ b/go-controller/vendor/github.com/go-openapi/swag/json.go @@ -126,7 +126,8 @@ func ConcatJSON(blobs ...[]byte) []byte { continue // don't know how to concatenate non container objects } - if len(b) < 3 { // yep empty but also the last one, so closing this thing + const minLengthIfNotEmpty = 3 + if len(b) < minLengthIfNotEmpty { // yep empty but also the last one, so closing this thing if i == last && a > 0 { if err := buf.WriteByte(closing); err != nil { log.Println(err) diff --git a/go-controller/vendor/github.com/go-openapi/swag/loading.go b/go-controller/vendor/github.com/go-openapi/swag/loading.go index 783442fddf..658a24b789 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/loading.go +++ b/go-controller/vendor/github.com/go-openapi/swag/loading.go @@ -168,7 +168,7 @@ func loadHTTPBytes(timeout time.Duration) func(path string) ([]byte, error) { } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("could not access document at %q [%s] ", path, resp.Status) + return nil, fmt.Errorf("could not access document at %q [%s]: %w", path, resp.Status, ErrLoader) } return io.ReadAll(resp.Body) diff --git a/go-controller/vendor/github.com/go-openapi/swag/yaml.go b/go-controller/vendor/github.com/go-openapi/swag/yaml.go index f59e025932..575346539a 100644 --- a/go-controller/vendor/github.com/go-openapi/swag/yaml.go +++ b/go-controller/vendor/github.com/go-openapi/swag/yaml.go @@ -16,7 +16,6 @@ package swag import ( "encoding/json" - "errors" "fmt" "path/filepath" "reflect" @@ -51,7 +50,7 @@ func BytesToYAMLDoc(data []byte) (interface{}, error) { return nil, err } if document.Kind != yaml.DocumentNode || len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { - return nil, errors.New("only YAML documents that are objects are supported") + return nil, fmt.Errorf("only YAML documents that are objects are supported: %w", ErrYAML) } return &document, nil } @@ -69,31 +68,32 @@ func yamlNode(root *yaml.Node) (interface{}, error) { case yaml.AliasNode: return yamlNode(root.Alias) default: - return nil, fmt.Errorf("unsupported YAML node type: %v", root.Kind) + return nil, fmt.Errorf("unsupported YAML node type: %v: %w", root.Kind, ErrYAML) } } func yamlDocument(node *yaml.Node) (interface{}, error) { if len(node.Content) != 1 { - return nil, fmt.Errorf("unexpected YAML Document node content length: %d", len(node.Content)) + return nil, fmt.Errorf("unexpected YAML Document node content length: %d: %w", len(node.Content), ErrYAML) } return yamlNode(node.Content[0]) } func yamlMapping(node *yaml.Node) (interface{}, error) { - m := make(JSONMapSlice, len(node.Content)/2) + const sensibleAllocDivider = 2 + m := make(JSONMapSlice, len(node.Content)/sensibleAllocDivider) var j int for i := 0; i < len(node.Content); i += 2 { var nmi JSONMapItem k, err := yamlStringScalarC(node.Content[i]) if err != nil { - return nil, fmt.Errorf("unable to decode YAML map key: %w", err) + return nil, fmt.Errorf("unable to decode YAML map key: %w: %w", err, ErrYAML) } nmi.Key = k v, err := yamlNode(node.Content[i+1]) if err != nil { - return nil, fmt.Errorf("unable to process YAML map value for key %q: %w", k, err) + return nil, fmt.Errorf("unable to process YAML map value for key %q: %w: %w", k, err, ErrYAML) } nmi.Value = v m[j] = nmi @@ -109,7 +109,7 @@ func yamlSequence(node *yaml.Node) (interface{}, error) { v, err := yamlNode(node.Content[i]) if err != nil { - return nil, fmt.Errorf("unable to decode YAML sequence value: %w", err) + return nil, fmt.Errorf("unable to decode YAML sequence value: %w: %w", err, ErrYAML) } s = append(s, v) } @@ -132,19 +132,19 @@ func yamlScalar(node *yaml.Node) (interface{}, error) { case yamlBoolScalar: b, err := strconv.ParseBool(node.Value) if err != nil { - return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting bool content: %w", node.Value, err) + return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting bool content: %w: %w", node.Value, err, ErrYAML) } return b, nil case yamlIntScalar: i, err := strconv.ParseInt(node.Value, 10, 64) if err != nil { - return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting integer content: %w", node.Value, err) + return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting integer content: %w: %w", node.Value, err, ErrYAML) } return i, nil case yamlFloatScalar: f, err := strconv.ParseFloat(node.Value, 64) if err != nil { - return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting float content: %w", node.Value, err) + return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting float content: %w: %w", node.Value, err, ErrYAML) } return f, nil case yamlTimestamp: @@ -152,19 +152,19 @@ func yamlScalar(node *yaml.Node) (interface{}, error) { case yamlNull: return nil, nil //nolint:nilnil default: - return nil, fmt.Errorf("YAML tag %q is not supported", node.LongTag()) + return nil, fmt.Errorf("YAML tag %q is not supported: %w", node.LongTag(), ErrYAML) } } func yamlStringScalarC(node *yaml.Node) (string, error) { if node.Kind != yaml.ScalarNode { - return "", fmt.Errorf("expecting a string scalar but got %q", node.Kind) + return "", fmt.Errorf("expecting a string scalar but got %q: %w", node.Kind, ErrYAML) } switch node.LongTag() { case yamlStringScalar, yamlIntScalar, yamlFloatScalar: return node.Value, nil default: - return "", fmt.Errorf("YAML tag %q is not supported as map key", node.LongTag()) + return "", fmt.Errorf("YAML tag %q is not supported as map key: %w", node.LongTag(), ErrYAML) } } @@ -349,7 +349,7 @@ func json2yaml(item interface{}) (*yaml.Node, error) { Value: strconv.FormatBool(val), }, nil default: - return nil, fmt.Errorf("unhandled type: %T", val) + return nil, fmt.Errorf("unhandled type: %T: %w", val, ErrYAML) } } @@ -416,7 +416,7 @@ func transformData(input interface{}) (out interface{}, err error) { case int64: return strconv.FormatInt(k, 10), nil default: - return "", fmt.Errorf("unexpected map key type, got: %T", k) + return "", fmt.Errorf("unexpected map key type, got: %T: %w", k, ErrYAML) } } diff --git a/go-controller/vendor/github.com/inconshreveable/mousetrap/LICENSE b/go-controller/vendor/github.com/inconshreveable/mousetrap/LICENSE new file mode 100644 index 0000000000..5f920e9732 --- /dev/null +++ b/go-controller/vendor/github.com/inconshreveable/mousetrap/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Alan Shreve (@inconshreveable) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/go-controller/vendor/github.com/inconshreveable/mousetrap/README.md b/go-controller/vendor/github.com/inconshreveable/mousetrap/README.md new file mode 100644 index 0000000000..7a950d1774 --- /dev/null +++ b/go-controller/vendor/github.com/inconshreveable/mousetrap/README.md @@ -0,0 +1,23 @@ +# mousetrap + +mousetrap is a tiny library that answers a single question. + +On a Windows machine, was the process invoked by someone double clicking on +the executable file while browsing in explorer? + +### Motivation + +Windows developers unfamiliar with command line tools will often "double-click" +the executable for a tool. Because most CLI tools print the help and then exit +when invoked without arguments, this is often very frustrating for those users. + +mousetrap provides a way to detect these invocations so that you can provide +more helpful behavior and instructions on how to run the CLI tool. To see what +this looks like, both from an organizational and a technical perspective, see +https://inconshreveable.com/09-09-2014/sweat-the-small-stuff/ + +### The interface + +The library exposes a single interface: + + func StartedByExplorer() (bool) diff --git a/go-controller/vendor/github.com/inconshreveable/mousetrap/trap_others.go b/go-controller/vendor/github.com/inconshreveable/mousetrap/trap_others.go new file mode 100644 index 0000000000..06a91f0868 --- /dev/null +++ b/go-controller/vendor/github.com/inconshreveable/mousetrap/trap_others.go @@ -0,0 +1,16 @@ +//go:build !windows +// +build !windows + +package mousetrap + +// StartedByExplorer returns true if the program was invoked by the user +// double-clicking on the executable from explorer.exe +// +// It is conservative and returns false if any of the internal calls fail. +// It does not guarantee that the program was run from a terminal. It only can tell you +// whether it was launched from explorer.exe +// +// On non-Windows platforms, it always returns false. +func StartedByExplorer() bool { + return false +} diff --git a/go-controller/vendor/github.com/inconshreveable/mousetrap/trap_windows.go b/go-controller/vendor/github.com/inconshreveable/mousetrap/trap_windows.go new file mode 100644 index 0000000000..0c56880216 --- /dev/null +++ b/go-controller/vendor/github.com/inconshreveable/mousetrap/trap_windows.go @@ -0,0 +1,42 @@ +package mousetrap + +import ( + "syscall" + "unsafe" +) + +func getProcessEntry(pid int) (*syscall.ProcessEntry32, error) { + snapshot, err := syscall.CreateToolhelp32Snapshot(syscall.TH32CS_SNAPPROCESS, 0) + if err != nil { + return nil, err + } + defer syscall.CloseHandle(snapshot) + var procEntry syscall.ProcessEntry32 + procEntry.Size = uint32(unsafe.Sizeof(procEntry)) + if err = syscall.Process32First(snapshot, &procEntry); err != nil { + return nil, err + } + for { + if procEntry.ProcessID == uint32(pid) { + return &procEntry, nil + } + err = syscall.Process32Next(snapshot, &procEntry) + if err != nil { + return nil, err + } + } +} + +// StartedByExplorer returns true if the program was invoked by the user double-clicking +// on the executable from explorer.exe +// +// It is conservative and returns false if any of the internal calls fail. +// It does not guarantee that the program was run from a terminal. It only can tell you +// whether it was launched from explorer.exe +func StartedByExplorer() bool { + pe, err := getProcessEntry(syscall.Getppid()) + if err != nil { + return false + } + return "explorer.exe" == syscall.UTF16ToString(pe.ExeFile[:]) +} diff --git a/go-controller/vendor/github.com/mailru/easyjson/jlexer/bytestostr.go b/go-controller/vendor/github.com/mailru/easyjson/jlexer/bytestostr.go index ff7b27c5b2..e68108f868 100644 --- a/go-controller/vendor/github.com/mailru/easyjson/jlexer/bytestostr.go +++ b/go-controller/vendor/github.com/mailru/easyjson/jlexer/bytestostr.go @@ -8,7 +8,6 @@ package jlexer import ( - "reflect" "unsafe" ) @@ -18,7 +17,5 @@ import ( // chunk may be either blocked from being freed by GC because of a single string or the buffer.Data // may be garbage-collected even when the string exists. func bytesToStr(data []byte) string { - h := (*reflect.SliceHeader)(unsafe.Pointer(&data)) - shdr := reflect.StringHeader{Data: h.Data, Len: h.Len} - return *(*string)(unsafe.Pointer(&shdr)) + return *(*string)(unsafe.Pointer(&data)) } diff --git a/go-controller/vendor/github.com/mailru/easyjson/jlexer/lexer.go b/go-controller/vendor/github.com/mailru/easyjson/jlexer/lexer.go index b5f5e26132..a27705b12b 100644 --- a/go-controller/vendor/github.com/mailru/easyjson/jlexer/lexer.go +++ b/go-controller/vendor/github.com/mailru/easyjson/jlexer/lexer.go @@ -19,21 +19,21 @@ import ( "github.com/josharian/intern" ) -// tokenKind determines type of a token. -type tokenKind byte +// TokenKind determines type of a token. +type TokenKind byte const ( - tokenUndef tokenKind = iota // No token. - tokenDelim // Delimiter: one of '{', '}', '[' or ']'. - tokenString // A string literal, e.g. "abc\u1234" - tokenNumber // Number literal, e.g. 1.5e5 - tokenBool // Boolean literal: true or false. - tokenNull // null keyword. + TokenUndef TokenKind = iota // No token. + TokenDelim // Delimiter: one of '{', '}', '[' or ']'. + TokenString // A string literal, e.g. "abc\u1234" + TokenNumber // Number literal, e.g. 1.5e5 + TokenBool // Boolean literal: true or false. + TokenNull // null keyword. ) // token describes a single token: type, position in the input and value. type token struct { - kind tokenKind // Type of a token. + kind TokenKind // Type of a token. boolValue bool // Value if a boolean literal token. byteValueCloned bool // true if byteValue was allocated and does not refer to original json body @@ -47,7 +47,7 @@ type Lexer struct { start int // Start of the current token. pos int // Current unscanned position in the input stream. - token token // Last scanned token, if token.kind != tokenUndef. + token token // Last scanned token, if token.kind != TokenUndef. firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. @@ -59,7 +59,7 @@ type Lexer struct { // FetchToken scans the input for the next token. func (r *Lexer) FetchToken() { - r.token.kind = tokenUndef + r.token.kind = TokenUndef r.start = r.pos // Check if r.Data has r.pos element @@ -90,7 +90,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } - r.token.kind = tokenString + r.token.kind = TokenString r.fetchString() return @@ -99,7 +99,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } r.firstElement = true - r.token.kind = tokenDelim + r.token.kind = TokenDelim r.token.delimValue = r.Data[r.pos] r.pos++ return @@ -109,7 +109,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } r.wantSep = 0 - r.token.kind = tokenDelim + r.token.kind = TokenDelim r.token.delimValue = r.Data[r.pos] r.pos++ return @@ -118,7 +118,7 @@ func (r *Lexer) FetchToken() { if r.wantSep != 0 { r.errSyntax() } - r.token.kind = tokenNumber + r.token.kind = TokenNumber r.fetchNumber() return @@ -127,7 +127,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } - r.token.kind = tokenNull + r.token.kind = TokenNull r.fetchNull() return @@ -136,7 +136,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } - r.token.kind = tokenBool + r.token.kind = TokenBool r.token.boolValue = true r.fetchTrue() return @@ -146,7 +146,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } - r.token.kind = tokenBool + r.token.kind = TokenBool r.token.boolValue = false r.fetchFalse() return @@ -391,7 +391,7 @@ func (r *Lexer) fetchString() { // scanToken scans the next token if no token is currently available in the lexer. func (r *Lexer) scanToken() { - if r.token.kind != tokenUndef || r.fatalError != nil { + if r.token.kind != TokenUndef || r.fatalError != nil { return } @@ -400,7 +400,7 @@ func (r *Lexer) scanToken() { // consume resets the current token to allow scanning the next one. func (r *Lexer) consume() { - r.token.kind = tokenUndef + r.token.kind = TokenUndef r.token.byteValueCloned = false r.token.delimValue = 0 } @@ -443,10 +443,10 @@ func (r *Lexer) errInvalidToken(expected string) { switch expected { case "[": r.token.delimValue = ']' - r.token.kind = tokenDelim + r.token.kind = TokenDelim case "{": r.token.delimValue = '}' - r.token.kind = tokenDelim + r.token.kind = TokenDelim } r.addNonfatalError(&LexerError{ Reason: fmt.Sprintf("expected %s", expected), @@ -475,7 +475,7 @@ func (r *Lexer) GetPos() int { // Delim consumes a token and verifies that it is the given delimiter. func (r *Lexer) Delim(c byte) { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } @@ -489,7 +489,7 @@ func (r *Lexer) Delim(c byte) { // IsDelim returns true if there was no scanning error and next token is the given delimiter. func (r *Lexer) IsDelim(c byte) bool { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } return !r.Ok() || r.token.delimValue == c @@ -497,10 +497,10 @@ func (r *Lexer) IsDelim(c byte) bool { // Null verifies that the next token is null and consumes it. func (r *Lexer) Null() { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenNull { + if !r.Ok() || r.token.kind != TokenNull { r.errInvalidToken("null") } r.consume() @@ -508,15 +508,15 @@ func (r *Lexer) Null() { // IsNull returns true if the next token is a null keyword. func (r *Lexer) IsNull() bool { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - return r.Ok() && r.token.kind == tokenNull + return r.Ok() && r.token.kind == TokenNull } // Skip skips a single token. func (r *Lexer) Skip() { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } r.consume() @@ -621,10 +621,10 @@ func (r *Lexer) Consumed() { } func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenString { + if !r.Ok() || r.token.kind != TokenString { r.errInvalidToken("string") return "", nil } @@ -664,10 +664,10 @@ func (r *Lexer) UnsafeFieldName(skipUnescape bool) string { // String reads a string literal. func (r *Lexer) String() string { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenString { + if !r.Ok() || r.token.kind != TokenString { r.errInvalidToken("string") return "" } @@ -687,10 +687,10 @@ func (r *Lexer) String() string { // StringIntern reads a string literal, and performs string interning on it. func (r *Lexer) StringIntern() string { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenString { + if !r.Ok() || r.token.kind != TokenString { r.errInvalidToken("string") return "" } @@ -705,10 +705,10 @@ func (r *Lexer) StringIntern() string { // Bytes reads a string literal and base64 decodes it into a byte slice. func (r *Lexer) Bytes() []byte { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenString { + if !r.Ok() || r.token.kind != TokenString { r.errInvalidToken("string") return nil } @@ -731,10 +731,10 @@ func (r *Lexer) Bytes() []byte { // Bool reads a true or false boolean keyword. func (r *Lexer) Bool() bool { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenBool { + if !r.Ok() || r.token.kind != TokenBool { r.errInvalidToken("bool") return false } @@ -744,10 +744,10 @@ func (r *Lexer) Bool() bool { } func (r *Lexer) number() string { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenNumber { + if !r.Ok() || r.token.kind != TokenNumber { r.errInvalidToken("number") return "" } @@ -1151,7 +1151,7 @@ func (r *Lexer) GetNonFatalErrors() []*LexerError { // JsonNumber fetches and json.Number from 'encoding/json' package. // Both int, float or string, contains them are valid values func (r *Lexer) JsonNumber() json.Number { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } if !r.Ok() { @@ -1160,11 +1160,11 @@ func (r *Lexer) JsonNumber() json.Number { } switch r.token.kind { - case tokenString: + case TokenString: return json.Number(r.String()) - case tokenNumber: + case TokenNumber: return json.Number(r.Raw()) - case tokenNull: + case TokenNull: r.Null() return json.Number("") default: @@ -1175,7 +1175,7 @@ func (r *Lexer) JsonNumber() json.Number { // Interface fetches an interface{} analogous to the 'encoding/json' package. func (r *Lexer) Interface() interface{} { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } @@ -1183,13 +1183,13 @@ func (r *Lexer) Interface() interface{} { return nil } switch r.token.kind { - case tokenString: + case TokenString: return r.String() - case tokenNumber: + case TokenNumber: return r.Float64() - case tokenBool: + case TokenBool: return r.Bool() - case tokenNull: + case TokenNull: r.Null() return nil } @@ -1242,3 +1242,16 @@ func (r *Lexer) WantColon() { r.wantSep = ':' r.firstElement = false } + +// CurrentToken returns current token kind if there were no errors and TokenUndef otherwise +func (r *Lexer) CurrentToken() TokenKind { + if r.token.kind == TokenUndef && r.Ok() { + r.FetchToken() + } + + if !r.Ok() { + return TokenUndef + } + + return r.token.kind +} diff --git a/go-controller/vendor/github.com/mailru/easyjson/jwriter/writer.go b/go-controller/vendor/github.com/mailru/easyjson/jwriter/writer.go index 2c5b20105b..34b0ade468 100644 --- a/go-controller/vendor/github.com/mailru/easyjson/jwriter/writer.go +++ b/go-controller/vendor/github.com/mailru/easyjson/jwriter/writer.go @@ -67,6 +67,18 @@ func (w *Writer) RawString(s string) { w.Buffer.AppendString(s) } +// RawBytesString appends string from bytes to the buffer. +func (w *Writer) RawBytesString(data []byte, err error) { + switch { + case w.Error != nil: + return + case err != nil: + w.Error = err + default: + w.String(string(data)) + } +} + // Raw appends raw binary data to the buffer or sets the error if it is given. Useful for // calling with results of MarshalJSON-like functions. func (w *Writer) Raw(data []byte, err error) { diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/bgp_session_state_types.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/bgp_session_state_types.go new file mode 100644 index 0000000000..fca6383340 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/bgp_session_state_types.go @@ -0,0 +1,64 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// BGPSessionStateSpec defines the desired state of BGPSessionState. +type BGPSessionStateSpec struct { +} + +// BGPSessionStateStatus defines the observed state of BGPSessionState. +type BGPSessionStateStatus struct { + BGPStatus string `json:"bgpStatus,omitempty"` + BFDStatus string `json:"bfdStatus,omitempty"` + Node string `json:"node,omitempty"` + Peer string `json:"peer,omitempty"` + VRF string `json:"vrf,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// BGPSessionState exposes the status of a BGP Session from the FRR instance running on the node. +// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=`.status.node` +// +kubebuilder:printcolumn:name="Peer",type=string,JSONPath=`.status.peer` +// +kubebuilder:printcolumn:name="VRF",type=string,JSONPath=`.status.vrf` +// +kubebuilder:printcolumn:name="BGP",type=string,JSONPath=`.status.bgpStatus` +// +kubebuilder:printcolumn:name="BFD",type=string,JSONPath=`.status.bfdStatus` +type BGPSessionState struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BGPSessionStateSpec `json:"spec,omitempty"` + Status BGPSessionStateStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// BGPSessionStateList contains a list of BGPSessionState. +type BGPSessionStateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BGPSessionState `json:"items"` +} + +func init() { + SchemeBuilder.Register(&BGPSessionState{}, &BGPSessionStateList{}) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go index 421f38a1e9..ec725e3444 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go @@ -117,7 +117,18 @@ type Neighbor struct { SourceAddress string `json:"sourceaddress,omitempty"` // Address is the IP address to establish the session with. - Address string `json:"address"` + // +optional + Address string `json:"address,omitempty"` + + // Interface is the node interface over which the unnumbered BGP peering will + // be established. No API validation takes place as that string value + // represents an interface name on the host and if user provides an invalid + // value, only the actual BGP session will not be established. + // Address and Interface are mutually exclusive and one of them must be specified. + // Note: when enabling unnumbered, the neighbor will be enabled for both + // IPv4 and IPv6 address families. + // +optional + Interface string `json:"interface,omitempty"` // Port is the port to dial when establishing the session. // Defaults to 179. @@ -181,9 +192,16 @@ type Neighbor struct { ToReceive Receive `json:"toReceive,omitempty"` // To set if we want to disable MP BGP that will separate IPv4 and IPv6 route exchanges into distinct BGP sessions. + // Deprecated: DisableMP is deprecated in favor of dualStackAddressFamily. // +optional // +kubebuilder:default:=false DisableMP bool `json:"disableMP,omitempty"` + + // To set if we want to enable the neighbor not only for the ipfamily related to its session, + // but also the other one. This allows to advertise/receive IPv4 prefixes over IPv6 sessions and vice versa. + // +optional + // +kubebuilder:default:=false + DualStackAddressFamily bool `json:"dualStackAddressFamily,omitempty"` } // Advertise represents a list of prefixes to advertise to the given neighbor. diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go index 982781e7a5..4670081537 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go @@ -160,6 +160,95 @@ func (in *BGPConfig) DeepCopy() *BGPConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BGPSessionState) DeepCopyInto(out *BGPSessionState) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BGPSessionState. +func (in *BGPSessionState) DeepCopy() *BGPSessionState { + if in == nil { + return nil + } + out := new(BGPSessionState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BGPSessionState) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BGPSessionStateList) DeepCopyInto(out *BGPSessionStateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BGPSessionState, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BGPSessionStateList. +func (in *BGPSessionStateList) DeepCopy() *BGPSessionStateList { + if in == nil { + return nil + } + out := new(BGPSessionStateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BGPSessionStateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BGPSessionStateSpec) DeepCopyInto(out *BGPSessionStateSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BGPSessionStateSpec. +func (in *BGPSessionStateSpec) DeepCopy() *BGPSessionStateSpec { + if in == nil { + return nil + } + out := new(BGPSessionStateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BGPSessionStateStatus) DeepCopyInto(out *BGPSessionStateStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BGPSessionStateStatus. +func (in *BGPSessionStateStatus) DeepCopy() *BGPSessionStateStatus { + if in == nil { + return nil + } + out := new(BGPSessionStateStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CommunityPrefixes) DeepCopyInto(out *CommunityPrefixes) { *out = *in diff --git a/go-controller/vendor/github.com/prometheus/common/model/alert.go b/go-controller/vendor/github.com/prometheus/common/model/alert.go index bd3a39e3e1..460f554f29 100644 --- a/go-controller/vendor/github.com/prometheus/common/model/alert.go +++ b/go-controller/vendor/github.com/prometheus/common/model/alert.go @@ -65,7 +65,7 @@ func (a *Alert) Resolved() bool { return a.ResolvedAt(time.Now()) } -// ResolvedAt returns true off the activity interval ended before +// ResolvedAt returns true iff the activity interval ended before // the given timestamp. func (a *Alert) ResolvedAt(ts time.Time) bool { if a.EndsAt.IsZero() { diff --git a/go-controller/vendor/github.com/prometheus/common/model/labels.go b/go-controller/vendor/github.com/prometheus/common/model/labels.go index 73b7aa3e60..f4a387605f 100644 --- a/go-controller/vendor/github.com/prometheus/common/model/labels.go +++ b/go-controller/vendor/github.com/prometheus/common/model/labels.go @@ -22,7 +22,7 @@ import ( ) const ( - // AlertNameLabel is the name of the label containing the an alert's name. + // AlertNameLabel is the name of the label containing the alert's name. AlertNameLabel = "alertname" // ExportedLabelPrefix is the prefix to prepend to the label names present in diff --git a/go-controller/vendor/github.com/prometheus/common/model/metric.go b/go-controller/vendor/github.com/prometheus/common/model/metric.go index 5766107cf9..a6b01755bd 100644 --- a/go-controller/vendor/github.com/prometheus/common/model/metric.go +++ b/go-controller/vendor/github.com/prometheus/common/model/metric.go @@ -27,13 +27,25 @@ import ( ) var ( - // NameValidationScheme determines the method of name validation to be used by - // all calls to IsValidMetricName() and LabelName IsValid(). Setting UTF-8 - // mode in isolation from other components that don't support UTF-8 may result - // in bugs or other undefined behavior. This value can be set to - // LegacyValidation during startup if a binary is not UTF-8-aware binaries. To - // avoid need for locking, this value should be set once, ideally in an - // init(), before multiple goroutines are started. + // NameValidationScheme determines the global default method of the name + // validation to be used by all calls to IsValidMetricName() and LabelName + // IsValid(). + // + // Deprecated: This variable should not be used and might be removed in the + // far future. If you wish to stick to the legacy name validation use + // `IsValidLegacyMetricName()` and `LabelName.IsValidLegacy()` methods + // instead. This variable is here as an escape hatch for emergency cases, + // given the recent change from `LegacyValidation` to `UTF8Validation`, e.g., + // to delay UTF-8 migrations in time or aid in debugging unforeseen results of + // the change. In such a case, a temporary assignment to `LegacyValidation` + // value in the `init()` function in your main.go or so, could be considered. + // + // Historically we opted for a global variable for feature gating different + // validation schemes in operations that were not otherwise easily adjustable + // (e.g. Labels yaml unmarshaling). That could have been a mistake, a separate + // Labels structure or package might have been a better choice. Given the + // change was made and many upgraded the common already, we live this as-is + // with this warning and learning for the future. NameValidationScheme = UTF8Validation // NameEscapingScheme defines the default way that names will be escaped when @@ -50,7 +62,7 @@ var ( type ValidationScheme int const ( - // LegacyValidation is a setting that requirets that metric and label names + // LegacyValidation is a setting that requires that all metric and label names // conform to the original Prometheus character requirements described by // MetricNameRE and LabelNameRE. LegacyValidation ValidationScheme = iota diff --git a/go-controller/vendor/github.com/prometheus/procfs/.golangci.yml b/go-controller/vendor/github.com/prometheus/procfs/.golangci.yml index 126df9e67a..3c3bf910fd 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/.golangci.yml +++ b/go-controller/vendor/github.com/prometheus/procfs/.golangci.yml @@ -1,22 +1,45 @@ ---- +version: "2" linters: enable: - - errcheck - - godot - - gosimple - - govet - - ineffassign - - misspell - - revive - - staticcheck - - testifylint - - unused - -linter-settings: - godot: - capital: true - exclude: - # Ignore "See: URL" - - 'See:' - misspell: - locale: US + - forbidigo + - godot + - misspell + - revive + - testifylint + settings: + forbidigo: + forbid: + - pattern: ^fmt\.Print.*$ + msg: Do not commit print statements. + godot: + exclude: + # Ignore "See: URL". + - 'See:' + capital: true + misspell: + locale: US + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/prometheus/procfs + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/go-controller/vendor/github.com/prometheus/procfs/Makefile.common b/go-controller/vendor/github.com/prometheus/procfs/Makefile.common index 1617292350..0ed55c2ba2 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/Makefile.common +++ b/go-controller/vendor/github.com/prometheus/procfs/Makefile.common @@ -33,7 +33,7 @@ GOHOSTOS ?= $(shell $(GO) env GOHOSTOS) GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH) GO_VERSION ?= $(shell $(GO) version) -GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) +GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION))Error Parsing File PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.') PROMU := $(FIRST_GOPATH)/bin/promu @@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.59.0 +GOLANGCI_LINT_VERSION ?= v2.0.2 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) @@ -275,3 +275,9 @@ $(1)_precheck: exit 1; \ fi endef + +govulncheck: install-govulncheck + govulncheck ./... + +install-govulncheck: + command -v govulncheck > /dev/null || go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/go-controller/vendor/github.com/prometheus/procfs/README.md b/go-controller/vendor/github.com/prometheus/procfs/README.md index 1224816c2a..0718239cf1 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/README.md +++ b/go-controller/vendor/github.com/prometheus/procfs/README.md @@ -47,15 +47,15 @@ However, most of the API includes unit tests which can be run with `make test`. The procfs library includes a set of test fixtures which include many example files from the `/proc` and `/sys` filesystems. These fixtures are included as a [ttar](https://github.com/ideaship/ttar) file which is extracted automatically during testing. To add/update the test fixtures, first -ensure the `fixtures` directory is up to date by removing the existing directory and then -extracting the ttar file using `make fixtures/.unpacked` or just `make test`. +ensure the `testdata/fixtures` directory is up to date by removing the existing directory and then +extracting the ttar file using `make testdata/fixtures/.unpacked` or just `make test`. ```bash rm -rf testdata/fixtures make test ``` -Next, make the required changes to the extracted files in the `fixtures` directory. When +Next, make the required changes to the extracted files in the `testdata/fixtures` directory. When the changes are complete, run `make update_fixtures` to create a new `fixtures.ttar` file based on the updated `fixtures` directory. And finally, verify the changes using `git diff testdata/fixtures.ttar`. diff --git a/go-controller/vendor/github.com/prometheus/procfs/arp.go b/go-controller/vendor/github.com/prometheus/procfs/arp.go index cdcc8a7ccc..2e53344151 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/arp.go +++ b/go-controller/vendor/github.com/prometheus/procfs/arp.go @@ -23,9 +23,9 @@ import ( // Learned from include/uapi/linux/if_arp.h. const ( - // completed entry (ha valid). + // Completed entry (ha valid). ATFComplete = 0x02 - // permanent entry. + // Permanent entry. ATFPermanent = 0x04 // Publish entry. ATFPublish = 0x08 diff --git a/go-controller/vendor/github.com/prometheus/procfs/fs.go b/go-controller/vendor/github.com/prometheus/procfs/fs.go index 4980c875bf..9bdaccc7c8 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/fs.go +++ b/go-controller/vendor/github.com/prometheus/procfs/fs.go @@ -24,8 +24,14 @@ type FS struct { isReal bool } -// DefaultMountPoint is the common mount point of the proc filesystem. -const DefaultMountPoint = fs.DefaultProcMountPoint +const ( + // DefaultMountPoint is the common mount point of the proc filesystem. + DefaultMountPoint = fs.DefaultProcMountPoint + + // SectorSize represents the size of a sector in bytes. + // It is specific to Linux block I/O operations. + SectorSize = 512 +) // NewDefaultFS returns a new proc FS mounted under the default proc mountPoint. // It will error if the mount point directory can't be read or is a file. diff --git a/go-controller/vendor/github.com/prometheus/procfs/fs_statfs_notype.go b/go-controller/vendor/github.com/prometheus/procfs/fs_statfs_notype.go index 134767d69a..1b5bdbdf84 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/fs_statfs_notype.go +++ b/go-controller/vendor/github.com/prometheus/procfs/fs_statfs_notype.go @@ -17,7 +17,7 @@ package procfs // isRealProc returns true on architectures that don't have a Type argument -// in their Statfs_t struct -func isRealProc(mountPoint string) (bool, error) { +// in their Statfs_t struct. +func isRealProc(_ string) (bool, error) { return true, nil } diff --git a/go-controller/vendor/github.com/prometheus/procfs/fscache.go b/go-controller/vendor/github.com/prometheus/procfs/fscache.go index cf2e3eaa03..7db8633077 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/fscache.go +++ b/go-controller/vendor/github.com/prometheus/procfs/fscache.go @@ -162,7 +162,7 @@ type Fscacheinfo struct { ReleaseRequestsAgainstPagesStoredByTimeLockGranted uint64 // Number of release reqs ignored due to in-progress store ReleaseRequestsIgnoredDueToInProgressStore uint64 - // Number of page stores cancelled due to release req + // Number of page stores canceled due to release req PageStoresCancelledByReleaseRequests uint64 VmscanWaiting uint64 // Number of times async ops added to pending queues @@ -171,11 +171,11 @@ type Fscacheinfo struct { OpsRunning uint64 // Number of times async ops queued for processing OpsEnqueued uint64 - // Number of async ops cancelled + // Number of async ops canceled OpsCancelled uint64 // Number of async ops rejected due to object lookup/create failure OpsRejected uint64 - // Number of async ops initialised + // Number of async ops initialized OpsInitialised uint64 // Number of async ops queued for deferred release OpsDeferred uint64 diff --git a/go-controller/vendor/github.com/prometheus/procfs/internal/fs/fs.go b/go-controller/vendor/github.com/prometheus/procfs/internal/fs/fs.go index 3c18c7610e..3a43e83915 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/internal/fs/fs.go +++ b/go-controller/vendor/github.com/prometheus/procfs/internal/fs/fs.go @@ -28,6 +28,9 @@ const ( // DefaultConfigfsMountPoint is the common mount point of the configfs. DefaultConfigfsMountPoint = "/sys/kernel/config" + + // DefaultSelinuxMountPoint is the common mount point of the selinuxfs. + DefaultSelinuxMountPoint = "/sys/fs/selinux" ) // FS represents a pseudo-filesystem, normally /proc or /sys, which provides an diff --git a/go-controller/vendor/github.com/prometheus/procfs/internal/util/parse.go b/go-controller/vendor/github.com/prometheus/procfs/internal/util/parse.go index 14272dc788..5a7d2df06a 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/internal/util/parse.go +++ b/go-controller/vendor/github.com/prometheus/procfs/internal/util/parse.go @@ -14,6 +14,7 @@ package util import ( + "errors" "os" "strconv" "strings" @@ -110,3 +111,16 @@ func ParseBool(b string) *bool { } return &truth } + +// ReadHexFromFile reads a file and attempts to parse a uint64 from a hexadecimal format 0xXX. +func ReadHexFromFile(path string) (uint64, error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, err + } + hexString := strings.TrimSpace(string(data)) + if !strings.HasPrefix(hexString, "0x") { + return 0, errors.New("invalid format: hex string does not start with '0x'") + } + return strconv.ParseUint(hexString[2:], 16, 64) +} diff --git a/go-controller/vendor/github.com/prometheus/procfs/internal/util/sysreadfile.go b/go-controller/vendor/github.com/prometheus/procfs/internal/util/sysreadfile.go index 1ab875ceec..d5404a6d72 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/internal/util/sysreadfile.go +++ b/go-controller/vendor/github.com/prometheus/procfs/internal/util/sysreadfile.go @@ -20,6 +20,8 @@ package util import ( "bytes" "os" + "strconv" + "strings" "syscall" ) @@ -48,3 +50,21 @@ func SysReadFile(file string) (string, error) { return string(bytes.TrimSpace(b[:n])), nil } + +// SysReadUintFromFile reads a file using SysReadFile and attempts to parse a uint64 from it. +func SysReadUintFromFile(path string) (uint64, error) { + data, err := SysReadFile(path) + if err != nil { + return 0, err + } + return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) +} + +// SysReadIntFromFile reads a file using SysReadFile and attempts to parse a int64 from it. +func SysReadIntFromFile(path string) (int64, error) { + data, err := SysReadFile(path) + if err != nil { + return 0, err + } + return strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) +} diff --git a/go-controller/vendor/github.com/prometheus/procfs/mountstats.go b/go-controller/vendor/github.com/prometheus/procfs/mountstats.go index 75a3b6c810..50caa73274 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/mountstats.go +++ b/go-controller/vendor/github.com/prometheus/procfs/mountstats.go @@ -45,11 +45,11 @@ const ( fieldTransport11TCPLen = 13 fieldTransport11UDPLen = 10 - // kernel version >= 4.14 MaxLen + // Kernel version >= 4.14 MaxLen // See: https://elixir.bootlin.com/linux/v6.4.8/source/net/sunrpc/xprtrdma/xprt_rdma.h#L393 fieldTransport11RDMAMaxLen = 28 - // kernel version <= 4.2 MinLen + // Kernel version <= 4.2 MinLen // See: https://elixir.bootlin.com/linux/v4.2.8/source/net/sunrpc/xprtrdma/xprt_rdma.h#L331 fieldTransport11RDMAMinLen = 20 ) @@ -601,11 +601,12 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats switch statVersion { case statVersion10: var expectedLength int - if protocol == "tcp" { + switch protocol { + case "tcp": expectedLength = fieldTransport10TCPLen - } else if protocol == "udp" { + case "udp": expectedLength = fieldTransport10UDPLen - } else { + default: return nil, fmt.Errorf("%w: Invalid NFS protocol \"%s\" in stats 1.0 statement: %v", ErrFileParse, protocol, ss) } if len(ss) != expectedLength { @@ -613,13 +614,14 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats } case statVersion11: var expectedLength int - if protocol == "tcp" { + switch protocol { + case "tcp": expectedLength = fieldTransport11TCPLen - } else if protocol == "udp" { + case "udp": expectedLength = fieldTransport11UDPLen - } else if protocol == "rdma" { + case "rdma": expectedLength = fieldTransport11RDMAMinLen - } else { + default: return nil, fmt.Errorf("%w: invalid NFS protocol \"%s\" in stats 1.1 statement: %v", ErrFileParse, protocol, ss) } if (len(ss) != expectedLength && (protocol == "tcp" || protocol == "udp")) || @@ -655,11 +657,12 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats // For the udp RPC transport there is no connection count, connect idle time, // or idle time (fields #3, #4, and #5); all other fields are the same. So // we set them to 0 here. - if protocol == "udp" { + switch protocol { + case "udp": ns = append(ns[:2], append(make([]uint64, 3), ns[2:]...)...) - } else if protocol == "tcp" { + case "tcp": ns = append(ns[:fieldTransport11TCPLen], make([]uint64, fieldTransport11RDMAMaxLen-fieldTransport11TCPLen+3)...) - } else if protocol == "rdma" { + case "rdma": ns = append(ns[:fieldTransport10TCPLen], append(make([]uint64, 3), ns[fieldTransport10TCPLen:]...)...) } diff --git a/go-controller/vendor/github.com/prometheus/procfs/net_dev_snmp6.go b/go-controller/vendor/github.com/prometheus/procfs/net_dev_snmp6.go new file mode 100644 index 0000000000..f50b38e352 --- /dev/null +++ b/go-controller/vendor/github.com/prometheus/procfs/net_dev_snmp6.go @@ -0,0 +1,96 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "bufio" + "errors" + "io" + "os" + "strconv" + "strings" +) + +// NetDevSNMP6 is parsed from files in /proc/net/dev_snmp6/ or /proc//net/dev_snmp6/. +// The outer map's keys are interface names and the inner map's keys are stat names. +// +// If you'd like a total across all interfaces, please use the Snmp6() method of the Proc type. +type NetDevSNMP6 map[string]map[string]uint64 + +// Returns kernel/system statistics read from interface files within the /proc/net/dev_snmp6/ +// directory. +func (fs FS) NetDevSNMP6() (NetDevSNMP6, error) { + return newNetDevSNMP6(fs.proc.Path("net/dev_snmp6")) +} + +// Returns kernel/system statistics read from interface files within the /proc//net/dev_snmp6/ +// directory. +func (p Proc) NetDevSNMP6() (NetDevSNMP6, error) { + return newNetDevSNMP6(p.path("net/dev_snmp6")) +} + +// newNetDevSNMP6 creates a new NetDevSNMP6 from the contents of the given directory. +func newNetDevSNMP6(dir string) (NetDevSNMP6, error) { + netDevSNMP6 := make(NetDevSNMP6) + + // The net/dev_snmp6 folders contain one file per interface + ifaceFiles, err := os.ReadDir(dir) + if err != nil { + // On systems with IPv6 disabled, this directory won't exist. + // Do nothing. + if errors.Is(err, os.ErrNotExist) { + return netDevSNMP6, err + } + return netDevSNMP6, err + } + + for _, iFaceFile := range ifaceFiles { + f, err := os.Open(dir + "/" + iFaceFile.Name()) + if err != nil { + return netDevSNMP6, err + } + defer f.Close() + + netDevSNMP6[iFaceFile.Name()], err = parseNetDevSNMP6Stats(f) + if err != nil { + return netDevSNMP6, err + } + } + + return netDevSNMP6, nil +} + +func parseNetDevSNMP6Stats(r io.Reader) (map[string]uint64, error) { + m := make(map[string]uint64) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + stat := strings.Fields(scanner.Text()) + if len(stat) < 2 { + continue + } + key, val := stat[0], stat[1] + + // Expect stat name to contain "6" or be "ifIndex" + if strings.Contains(key, "6") || key == "ifIndex" { + v, err := strconv.ParseUint(val, 10, 64) + if err != nil { + return m, err + } + + m[key] = v + } + } + return m, scanner.Err() +} diff --git a/go-controller/vendor/github.com/prometheus/procfs/net_ip_socket.go b/go-controller/vendor/github.com/prometheus/procfs/net_ip_socket.go index b70f1fc7a4..19e3378f72 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/net_ip_socket.go +++ b/go-controller/vendor/github.com/prometheus/procfs/net_ip_socket.go @@ -25,7 +25,7 @@ import ( ) const ( - // readLimit is used by io.LimitReader while reading the content of the + // Maximum size limit used by io.LimitReader while reading the content of the // /proc/net/udp{,6} files. The number of lines inside such a file is dynamic // as each line represents a single used socket. // In theory, the number of available sockets is 65535 (2^16 - 1) per IP. @@ -50,12 +50,12 @@ type ( // UsedSockets shows the total number of parsed lines representing the // number of used sockets. UsedSockets uint64 - // Drops shows the total number of dropped packets of all UPD sockets. + // Drops shows the total number of dropped packets of all UDP sockets. Drops *uint64 } - // netIPSocketLine represents the fields parsed from a single line - // in /proc/net/{t,u}dp{,6}. Fields which are not used by IPSocket are skipped. + // A single line parser for fields from /proc/net/{t,u}dp{,6}. + // Fields which are not used by IPSocket are skipped. // Drops is non-nil for udp{,6}, but nil for tcp{,6}. // For the proc file format details, see https://linux.die.net/man/5/proc. netIPSocketLine struct { diff --git a/go-controller/vendor/github.com/prometheus/procfs/net_protocols.go b/go-controller/vendor/github.com/prometheus/procfs/net_protocols.go index b6c77b709f..8d4b1ac05b 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/net_protocols.go +++ b/go-controller/vendor/github.com/prometheus/procfs/net_protocols.go @@ -115,22 +115,24 @@ func (ps NetProtocolStats) parseLine(rawLine string) (*NetProtocolStatLine, erro if err != nil { return nil, err } - if fields[4] == enabled { + switch fields[4] { + case enabled: line.Pressure = 1 - } else if fields[4] == disabled { + case disabled: line.Pressure = 0 - } else { + default: line.Pressure = -1 } line.MaxHeader, err = strconv.ParseUint(fields[5], 10, 64) if err != nil { return nil, err } - if fields[6] == enabled { + switch fields[6] { + case enabled: line.Slab = true - } else if fields[6] == disabled { + case disabled: line.Slab = false - } else { + default: return nil, fmt.Errorf("%w: capability for protocol: %s", ErrFileParse, line.Name) } line.ModuleName = fields[7] @@ -168,11 +170,12 @@ func (pc *NetProtocolCapabilities) parseCapabilities(capabilities []string) erro } for i := 0; i < len(capabilities); i++ { - if capabilities[i] == "y" { + switch capabilities[i] { + case "y": *capabilityFields[i] = true - } else if capabilities[i] == "n" { + case "n": *capabilityFields[i] = false - } else { + default: return fmt.Errorf("%w: capability block for protocol: position %d", ErrFileParse, i) } } diff --git a/go-controller/vendor/github.com/prometheus/procfs/net_tcp.go b/go-controller/vendor/github.com/prometheus/procfs/net_tcp.go index 5277629557..0396d72015 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/net_tcp.go +++ b/go-controller/vendor/github.com/prometheus/procfs/net_tcp.go @@ -25,24 +25,28 @@ type ( // NetTCP returns the IPv4 kernel/networking statistics for TCP datagrams // read from /proc/net/tcp. +// Deprecated: Use github.com/mdlayher/netlink#Conn (with syscall.AF_INET) instead. func (fs FS) NetTCP() (NetTCP, error) { return newNetTCP(fs.proc.Path("net/tcp")) } // NetTCP6 returns the IPv6 kernel/networking statistics for TCP datagrams // read from /proc/net/tcp6. +// Deprecated: Use github.com/mdlayher/netlink#Conn (with syscall.AF_INET6) instead. func (fs FS) NetTCP6() (NetTCP, error) { return newNetTCP(fs.proc.Path("net/tcp6")) } // NetTCPSummary returns already computed statistics like the total queue lengths // for TCP datagrams read from /proc/net/tcp. +// Deprecated: Use github.com/mdlayher/netlink#Conn (with syscall.AF_INET) instead. func (fs FS) NetTCPSummary() (*NetTCPSummary, error) { return newNetTCPSummary(fs.proc.Path("net/tcp")) } // NetTCP6Summary returns already computed statistics like the total queue lengths // for TCP datagrams read from /proc/net/tcp6. +// Deprecated: Use github.com/mdlayher/netlink#Conn (with syscall.AF_INET6) instead. func (fs FS) NetTCP6Summary() (*NetTCPSummary, error) { return newNetTCPSummary(fs.proc.Path("net/tcp6")) } diff --git a/go-controller/vendor/github.com/prometheus/procfs/net_unix.go b/go-controller/vendor/github.com/prometheus/procfs/net_unix.go index d868cebdaa..d7e0cacb4c 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/net_unix.go +++ b/go-controller/vendor/github.com/prometheus/procfs/net_unix.go @@ -121,12 +121,12 @@ func parseNetUNIX(r io.Reader) (*NetUNIX, error) { return &nu, nil } -func (u *NetUNIX) parseLine(line string, hasInode bool, min int) (*NetUNIXLine, error) { +func (u *NetUNIX) parseLine(line string, hasInode bool, minFields int) (*NetUNIXLine, error) { fields := strings.Fields(line) l := len(fields) - if l < min { - return nil, fmt.Errorf("%w: expected at least %d fields but got %d", ErrFileParse, min, l) + if l < minFields { + return nil, fmt.Errorf("%w: expected at least %d fields but got %d", ErrFileParse, minFields, l) } // Field offsets are as follows: @@ -172,7 +172,7 @@ func (u *NetUNIX) parseLine(line string, hasInode bool, min int) (*NetUNIXLine, } // Path field is optional. - if l > min { + if l > minFields { // Path occurs at either index 6 or 7 depending on whether inode is // already present. pathIdx := 7 diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc.go b/go-controller/vendor/github.com/prometheus/procfs/proc.go index 142796368f..368187fa88 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc.go @@ -37,9 +37,9 @@ type Proc struct { type Procs []Proc var ( - ErrFileParse = errors.New("Error Parsing File") - ErrFileRead = errors.New("Error Reading File") - ErrMountPoint = errors.New("Error Accessing Mount point") + ErrFileParse = errors.New("error parsing file") + ErrFileRead = errors.New("error reading file") + ErrMountPoint = errors.New("error accessing mount point") ) func (p Procs) Len() int { return len(p) } @@ -79,7 +79,7 @@ func (fs FS) Self() (Proc, error) { if err != nil { return Proc{}, err } - pid, err := strconv.Atoi(strings.Replace(p, string(fs.proc), "", -1)) + pid, err := strconv.Atoi(strings.ReplaceAll(p, string(fs.proc), "")) if err != nil { return Proc{}, err } diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc_cgroup.go b/go-controller/vendor/github.com/prometheus/procfs/proc_cgroup.go index daeed7f571..4a64347c03 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc_cgroup.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc_cgroup.go @@ -24,7 +24,7 @@ import ( ) // Cgroup models one line from /proc/[pid]/cgroup. Each Cgroup struct describes the placement of a PID inside a -// specific control hierarchy. The kernel has two cgroup APIs, v1 and v2. v1 has one hierarchy per available resource +// specific control hierarchy. The kernel has two cgroup APIs, v1 and v2. The v1 has one hierarchy per available resource // controller, while v2 has one unified hierarchy shared by all controllers. Regardless of v1 or v2, all hierarchies // contain all running processes, so the question answerable with a Cgroup struct is 'where is this process in // this hierarchy' (where==what path on the specific cgroupfs). By prefixing this path with the mount point of diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc_io.go b/go-controller/vendor/github.com/prometheus/procfs/proc_io.go index 776f349717..d15b66ddb6 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc_io.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc_io.go @@ -50,7 +50,7 @@ func (p Proc) IO() (ProcIO, error) { ioFormat := "rchar: %d\nwchar: %d\nsyscr: %d\nsyscw: %d\n" + "read_bytes: %d\nwrite_bytes: %d\n" + - "cancelled_write_bytes: %d\n" + "cancelled_write_bytes: %d\n" //nolint:misspell _, err = fmt.Sscanf(string(data), ioFormat, &pio.RChar, &pio.WChar, &pio.SyscR, &pio.SyscW, &pio.ReadBytes, &pio.WriteBytes, &pio.CancelledWriteBytes) diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc_netstat.go b/go-controller/vendor/github.com/prometheus/procfs/proc_netstat.go index 8e3ff4d794..4248c1716e 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc_netstat.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc_netstat.go @@ -209,232 +209,232 @@ func parseProcNetstat(r io.Reader, fileName string) (ProcNetstat, error) { case "TcpExt": switch key { case "SyncookiesSent": - procNetstat.TcpExt.SyncookiesSent = &value + procNetstat.SyncookiesSent = &value case "SyncookiesRecv": - procNetstat.TcpExt.SyncookiesRecv = &value + procNetstat.SyncookiesRecv = &value case "SyncookiesFailed": - procNetstat.TcpExt.SyncookiesFailed = &value + procNetstat.SyncookiesFailed = &value case "EmbryonicRsts": - procNetstat.TcpExt.EmbryonicRsts = &value + procNetstat.EmbryonicRsts = &value case "PruneCalled": - procNetstat.TcpExt.PruneCalled = &value + procNetstat.PruneCalled = &value case "RcvPruned": - procNetstat.TcpExt.RcvPruned = &value + procNetstat.RcvPruned = &value case "OfoPruned": - procNetstat.TcpExt.OfoPruned = &value + procNetstat.OfoPruned = &value case "OutOfWindowIcmps": - procNetstat.TcpExt.OutOfWindowIcmps = &value + procNetstat.OutOfWindowIcmps = &value case "LockDroppedIcmps": - procNetstat.TcpExt.LockDroppedIcmps = &value + procNetstat.LockDroppedIcmps = &value case "ArpFilter": - procNetstat.TcpExt.ArpFilter = &value + procNetstat.ArpFilter = &value case "TW": - procNetstat.TcpExt.TW = &value + procNetstat.TW = &value case "TWRecycled": - procNetstat.TcpExt.TWRecycled = &value + procNetstat.TWRecycled = &value case "TWKilled": - procNetstat.TcpExt.TWKilled = &value + procNetstat.TWKilled = &value case "PAWSActive": - procNetstat.TcpExt.PAWSActive = &value + procNetstat.PAWSActive = &value case "PAWSEstab": - procNetstat.TcpExt.PAWSEstab = &value + procNetstat.PAWSEstab = &value case "DelayedACKs": - procNetstat.TcpExt.DelayedACKs = &value + procNetstat.DelayedACKs = &value case "DelayedACKLocked": - procNetstat.TcpExt.DelayedACKLocked = &value + procNetstat.DelayedACKLocked = &value case "DelayedACKLost": - procNetstat.TcpExt.DelayedACKLost = &value + procNetstat.DelayedACKLost = &value case "ListenOverflows": - procNetstat.TcpExt.ListenOverflows = &value + procNetstat.ListenOverflows = &value case "ListenDrops": - procNetstat.TcpExt.ListenDrops = &value + procNetstat.ListenDrops = &value case "TCPHPHits": - procNetstat.TcpExt.TCPHPHits = &value + procNetstat.TCPHPHits = &value case "TCPPureAcks": - procNetstat.TcpExt.TCPPureAcks = &value + procNetstat.TCPPureAcks = &value case "TCPHPAcks": - procNetstat.TcpExt.TCPHPAcks = &value + procNetstat.TCPHPAcks = &value case "TCPRenoRecovery": - procNetstat.TcpExt.TCPRenoRecovery = &value + procNetstat.TCPRenoRecovery = &value case "TCPSackRecovery": - procNetstat.TcpExt.TCPSackRecovery = &value + procNetstat.TCPSackRecovery = &value case "TCPSACKReneging": - procNetstat.TcpExt.TCPSACKReneging = &value + procNetstat.TCPSACKReneging = &value case "TCPSACKReorder": - procNetstat.TcpExt.TCPSACKReorder = &value + procNetstat.TCPSACKReorder = &value case "TCPRenoReorder": - procNetstat.TcpExt.TCPRenoReorder = &value + procNetstat.TCPRenoReorder = &value case "TCPTSReorder": - procNetstat.TcpExt.TCPTSReorder = &value + procNetstat.TCPTSReorder = &value case "TCPFullUndo": - procNetstat.TcpExt.TCPFullUndo = &value + procNetstat.TCPFullUndo = &value case "TCPPartialUndo": - procNetstat.TcpExt.TCPPartialUndo = &value + procNetstat.TCPPartialUndo = &value case "TCPDSACKUndo": - procNetstat.TcpExt.TCPDSACKUndo = &value + procNetstat.TCPDSACKUndo = &value case "TCPLossUndo": - procNetstat.TcpExt.TCPLossUndo = &value + procNetstat.TCPLossUndo = &value case "TCPLostRetransmit": - procNetstat.TcpExt.TCPLostRetransmit = &value + procNetstat.TCPLostRetransmit = &value case "TCPRenoFailures": - procNetstat.TcpExt.TCPRenoFailures = &value + procNetstat.TCPRenoFailures = &value case "TCPSackFailures": - procNetstat.TcpExt.TCPSackFailures = &value + procNetstat.TCPSackFailures = &value case "TCPLossFailures": - procNetstat.TcpExt.TCPLossFailures = &value + procNetstat.TCPLossFailures = &value case "TCPFastRetrans": - procNetstat.TcpExt.TCPFastRetrans = &value + procNetstat.TCPFastRetrans = &value case "TCPSlowStartRetrans": - procNetstat.TcpExt.TCPSlowStartRetrans = &value + procNetstat.TCPSlowStartRetrans = &value case "TCPTimeouts": - procNetstat.TcpExt.TCPTimeouts = &value + procNetstat.TCPTimeouts = &value case "TCPLossProbes": - procNetstat.TcpExt.TCPLossProbes = &value + procNetstat.TCPLossProbes = &value case "TCPLossProbeRecovery": - procNetstat.TcpExt.TCPLossProbeRecovery = &value + procNetstat.TCPLossProbeRecovery = &value case "TCPRenoRecoveryFail": - procNetstat.TcpExt.TCPRenoRecoveryFail = &value + procNetstat.TCPRenoRecoveryFail = &value case "TCPSackRecoveryFail": - procNetstat.TcpExt.TCPSackRecoveryFail = &value + procNetstat.TCPSackRecoveryFail = &value case "TCPRcvCollapsed": - procNetstat.TcpExt.TCPRcvCollapsed = &value + procNetstat.TCPRcvCollapsed = &value case "TCPDSACKOldSent": - procNetstat.TcpExt.TCPDSACKOldSent = &value + procNetstat.TCPDSACKOldSent = &value case "TCPDSACKOfoSent": - procNetstat.TcpExt.TCPDSACKOfoSent = &value + procNetstat.TCPDSACKOfoSent = &value case "TCPDSACKRecv": - procNetstat.TcpExt.TCPDSACKRecv = &value + procNetstat.TCPDSACKRecv = &value case "TCPDSACKOfoRecv": - procNetstat.TcpExt.TCPDSACKOfoRecv = &value + procNetstat.TCPDSACKOfoRecv = &value case "TCPAbortOnData": - procNetstat.TcpExt.TCPAbortOnData = &value + procNetstat.TCPAbortOnData = &value case "TCPAbortOnClose": - procNetstat.TcpExt.TCPAbortOnClose = &value + procNetstat.TCPAbortOnClose = &value case "TCPDeferAcceptDrop": - procNetstat.TcpExt.TCPDeferAcceptDrop = &value + procNetstat.TCPDeferAcceptDrop = &value case "IPReversePathFilter": - procNetstat.TcpExt.IPReversePathFilter = &value + procNetstat.IPReversePathFilter = &value case "TCPTimeWaitOverflow": - procNetstat.TcpExt.TCPTimeWaitOverflow = &value + procNetstat.TCPTimeWaitOverflow = &value case "TCPReqQFullDoCookies": - procNetstat.TcpExt.TCPReqQFullDoCookies = &value + procNetstat.TCPReqQFullDoCookies = &value case "TCPReqQFullDrop": - procNetstat.TcpExt.TCPReqQFullDrop = &value + procNetstat.TCPReqQFullDrop = &value case "TCPRetransFail": - procNetstat.TcpExt.TCPRetransFail = &value + procNetstat.TCPRetransFail = &value case "TCPRcvCoalesce": - procNetstat.TcpExt.TCPRcvCoalesce = &value + procNetstat.TCPRcvCoalesce = &value case "TCPRcvQDrop": - procNetstat.TcpExt.TCPRcvQDrop = &value + procNetstat.TCPRcvQDrop = &value case "TCPOFOQueue": - procNetstat.TcpExt.TCPOFOQueue = &value + procNetstat.TCPOFOQueue = &value case "TCPOFODrop": - procNetstat.TcpExt.TCPOFODrop = &value + procNetstat.TCPOFODrop = &value case "TCPOFOMerge": - procNetstat.TcpExt.TCPOFOMerge = &value + procNetstat.TCPOFOMerge = &value case "TCPChallengeACK": - procNetstat.TcpExt.TCPChallengeACK = &value + procNetstat.TCPChallengeACK = &value case "TCPSYNChallenge": - procNetstat.TcpExt.TCPSYNChallenge = &value + procNetstat.TCPSYNChallenge = &value case "TCPFastOpenActive": - procNetstat.TcpExt.TCPFastOpenActive = &value + procNetstat.TCPFastOpenActive = &value case "TCPFastOpenActiveFail": - procNetstat.TcpExt.TCPFastOpenActiveFail = &value + procNetstat.TCPFastOpenActiveFail = &value case "TCPFastOpenPassive": - procNetstat.TcpExt.TCPFastOpenPassive = &value + procNetstat.TCPFastOpenPassive = &value case "TCPFastOpenPassiveFail": - procNetstat.TcpExt.TCPFastOpenPassiveFail = &value + procNetstat.TCPFastOpenPassiveFail = &value case "TCPFastOpenListenOverflow": - procNetstat.TcpExt.TCPFastOpenListenOverflow = &value + procNetstat.TCPFastOpenListenOverflow = &value case "TCPFastOpenCookieReqd": - procNetstat.TcpExt.TCPFastOpenCookieReqd = &value + procNetstat.TCPFastOpenCookieReqd = &value case "TCPFastOpenBlackhole": - procNetstat.TcpExt.TCPFastOpenBlackhole = &value + procNetstat.TCPFastOpenBlackhole = &value case "TCPSpuriousRtxHostQueues": - procNetstat.TcpExt.TCPSpuriousRtxHostQueues = &value + procNetstat.TCPSpuriousRtxHostQueues = &value case "BusyPollRxPackets": - procNetstat.TcpExt.BusyPollRxPackets = &value + procNetstat.BusyPollRxPackets = &value case "TCPAutoCorking": - procNetstat.TcpExt.TCPAutoCorking = &value + procNetstat.TCPAutoCorking = &value case "TCPFromZeroWindowAdv": - procNetstat.TcpExt.TCPFromZeroWindowAdv = &value + procNetstat.TCPFromZeroWindowAdv = &value case "TCPToZeroWindowAdv": - procNetstat.TcpExt.TCPToZeroWindowAdv = &value + procNetstat.TCPToZeroWindowAdv = &value case "TCPWantZeroWindowAdv": - procNetstat.TcpExt.TCPWantZeroWindowAdv = &value + procNetstat.TCPWantZeroWindowAdv = &value case "TCPSynRetrans": - procNetstat.TcpExt.TCPSynRetrans = &value + procNetstat.TCPSynRetrans = &value case "TCPOrigDataSent": - procNetstat.TcpExt.TCPOrigDataSent = &value + procNetstat.TCPOrigDataSent = &value case "TCPHystartTrainDetect": - procNetstat.TcpExt.TCPHystartTrainDetect = &value + procNetstat.TCPHystartTrainDetect = &value case "TCPHystartTrainCwnd": - procNetstat.TcpExt.TCPHystartTrainCwnd = &value + procNetstat.TCPHystartTrainCwnd = &value case "TCPHystartDelayDetect": - procNetstat.TcpExt.TCPHystartDelayDetect = &value + procNetstat.TCPHystartDelayDetect = &value case "TCPHystartDelayCwnd": - procNetstat.TcpExt.TCPHystartDelayCwnd = &value + procNetstat.TCPHystartDelayCwnd = &value case "TCPACKSkippedSynRecv": - procNetstat.TcpExt.TCPACKSkippedSynRecv = &value + procNetstat.TCPACKSkippedSynRecv = &value case "TCPACKSkippedPAWS": - procNetstat.TcpExt.TCPACKSkippedPAWS = &value + procNetstat.TCPACKSkippedPAWS = &value case "TCPACKSkippedSeq": - procNetstat.TcpExt.TCPACKSkippedSeq = &value + procNetstat.TCPACKSkippedSeq = &value case "TCPACKSkippedFinWait2": - procNetstat.TcpExt.TCPACKSkippedFinWait2 = &value + procNetstat.TCPACKSkippedFinWait2 = &value case "TCPACKSkippedTimeWait": - procNetstat.TcpExt.TCPACKSkippedTimeWait = &value + procNetstat.TCPACKSkippedTimeWait = &value case "TCPACKSkippedChallenge": - procNetstat.TcpExt.TCPACKSkippedChallenge = &value + procNetstat.TCPACKSkippedChallenge = &value case "TCPWinProbe": - procNetstat.TcpExt.TCPWinProbe = &value + procNetstat.TCPWinProbe = &value case "TCPKeepAlive": - procNetstat.TcpExt.TCPKeepAlive = &value + procNetstat.TCPKeepAlive = &value case "TCPMTUPFail": - procNetstat.TcpExt.TCPMTUPFail = &value + procNetstat.TCPMTUPFail = &value case "TCPMTUPSuccess": - procNetstat.TcpExt.TCPMTUPSuccess = &value + procNetstat.TCPMTUPSuccess = &value case "TCPWqueueTooBig": - procNetstat.TcpExt.TCPWqueueTooBig = &value + procNetstat.TCPWqueueTooBig = &value } case "IpExt": switch key { case "InNoRoutes": - procNetstat.IpExt.InNoRoutes = &value + procNetstat.InNoRoutes = &value case "InTruncatedPkts": - procNetstat.IpExt.InTruncatedPkts = &value + procNetstat.InTruncatedPkts = &value case "InMcastPkts": - procNetstat.IpExt.InMcastPkts = &value + procNetstat.InMcastPkts = &value case "OutMcastPkts": - procNetstat.IpExt.OutMcastPkts = &value + procNetstat.OutMcastPkts = &value case "InBcastPkts": - procNetstat.IpExt.InBcastPkts = &value + procNetstat.InBcastPkts = &value case "OutBcastPkts": - procNetstat.IpExt.OutBcastPkts = &value + procNetstat.OutBcastPkts = &value case "InOctets": - procNetstat.IpExt.InOctets = &value + procNetstat.InOctets = &value case "OutOctets": - procNetstat.IpExt.OutOctets = &value + procNetstat.OutOctets = &value case "InMcastOctets": - procNetstat.IpExt.InMcastOctets = &value + procNetstat.InMcastOctets = &value case "OutMcastOctets": - procNetstat.IpExt.OutMcastOctets = &value + procNetstat.OutMcastOctets = &value case "InBcastOctets": - procNetstat.IpExt.InBcastOctets = &value + procNetstat.InBcastOctets = &value case "OutBcastOctets": - procNetstat.IpExt.OutBcastOctets = &value + procNetstat.OutBcastOctets = &value case "InCsumErrors": - procNetstat.IpExt.InCsumErrors = &value + procNetstat.InCsumErrors = &value case "InNoECTPkts": - procNetstat.IpExt.InNoECTPkts = &value + procNetstat.InNoECTPkts = &value case "InECT1Pkts": - procNetstat.IpExt.InECT1Pkts = &value + procNetstat.InECT1Pkts = &value case "InECT0Pkts": - procNetstat.IpExt.InECT0Pkts = &value + procNetstat.InECT0Pkts = &value case "InCEPkts": - procNetstat.IpExt.InCEPkts = &value + procNetstat.InCEPkts = &value case "ReasmOverlaps": - procNetstat.IpExt.ReasmOverlaps = &value + procNetstat.ReasmOverlaps = &value } } } diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc_smaps.go b/go-controller/vendor/github.com/prometheus/procfs/proc_smaps.go index 09060e8208..9a297afcf8 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc_smaps.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc_smaps.go @@ -19,7 +19,6 @@ package procfs import ( "bufio" "errors" - "fmt" "os" "regexp" "strconv" @@ -29,7 +28,7 @@ import ( ) var ( - // match the header line before each mapped zone in `/proc/pid/smaps`. + // Match the header line before each mapped zone in `/proc/pid/smaps`. procSMapsHeaderLine = regexp.MustCompile(`^[a-f0-9].*$`) ) @@ -117,7 +116,6 @@ func (p Proc) procSMapsRollupManual() (ProcSMapsRollup, error) { func (s *ProcSMapsRollup) parseLine(line string) error { kv := strings.SplitN(line, ":", 2) if len(kv) != 2 { - fmt.Println(line) return errors.New("invalid net/dev line, missing colon") } diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc_snmp.go b/go-controller/vendor/github.com/prometheus/procfs/proc_snmp.go index b9d2cf642a..4bdc90b07e 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc_snmp.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc_snmp.go @@ -173,138 +173,138 @@ func parseSnmp(r io.Reader, fileName string) (ProcSnmp, error) { case "Ip": switch key { case "Forwarding": - procSnmp.Ip.Forwarding = &value + procSnmp.Forwarding = &value case "DefaultTTL": - procSnmp.Ip.DefaultTTL = &value + procSnmp.DefaultTTL = &value case "InReceives": - procSnmp.Ip.InReceives = &value + procSnmp.InReceives = &value case "InHdrErrors": - procSnmp.Ip.InHdrErrors = &value + procSnmp.InHdrErrors = &value case "InAddrErrors": - procSnmp.Ip.InAddrErrors = &value + procSnmp.InAddrErrors = &value case "ForwDatagrams": - procSnmp.Ip.ForwDatagrams = &value + procSnmp.ForwDatagrams = &value case "InUnknownProtos": - procSnmp.Ip.InUnknownProtos = &value + procSnmp.InUnknownProtos = &value case "InDiscards": - procSnmp.Ip.InDiscards = &value + procSnmp.InDiscards = &value case "InDelivers": - procSnmp.Ip.InDelivers = &value + procSnmp.InDelivers = &value case "OutRequests": - procSnmp.Ip.OutRequests = &value + procSnmp.OutRequests = &value case "OutDiscards": - procSnmp.Ip.OutDiscards = &value + procSnmp.OutDiscards = &value case "OutNoRoutes": - procSnmp.Ip.OutNoRoutes = &value + procSnmp.OutNoRoutes = &value case "ReasmTimeout": - procSnmp.Ip.ReasmTimeout = &value + procSnmp.ReasmTimeout = &value case "ReasmReqds": - procSnmp.Ip.ReasmReqds = &value + procSnmp.ReasmReqds = &value case "ReasmOKs": - procSnmp.Ip.ReasmOKs = &value + procSnmp.ReasmOKs = &value case "ReasmFails": - procSnmp.Ip.ReasmFails = &value + procSnmp.ReasmFails = &value case "FragOKs": - procSnmp.Ip.FragOKs = &value + procSnmp.FragOKs = &value case "FragFails": - procSnmp.Ip.FragFails = &value + procSnmp.FragFails = &value case "FragCreates": - procSnmp.Ip.FragCreates = &value + procSnmp.FragCreates = &value } case "Icmp": switch key { case "InMsgs": - procSnmp.Icmp.InMsgs = &value + procSnmp.InMsgs = &value case "InErrors": procSnmp.Icmp.InErrors = &value case "InCsumErrors": procSnmp.Icmp.InCsumErrors = &value case "InDestUnreachs": - procSnmp.Icmp.InDestUnreachs = &value + procSnmp.InDestUnreachs = &value case "InTimeExcds": - procSnmp.Icmp.InTimeExcds = &value + procSnmp.InTimeExcds = &value case "InParmProbs": - procSnmp.Icmp.InParmProbs = &value + procSnmp.InParmProbs = &value case "InSrcQuenchs": - procSnmp.Icmp.InSrcQuenchs = &value + procSnmp.InSrcQuenchs = &value case "InRedirects": - procSnmp.Icmp.InRedirects = &value + procSnmp.InRedirects = &value case "InEchos": - procSnmp.Icmp.InEchos = &value + procSnmp.InEchos = &value case "InEchoReps": - procSnmp.Icmp.InEchoReps = &value + procSnmp.InEchoReps = &value case "InTimestamps": - procSnmp.Icmp.InTimestamps = &value + procSnmp.InTimestamps = &value case "InTimestampReps": - procSnmp.Icmp.InTimestampReps = &value + procSnmp.InTimestampReps = &value case "InAddrMasks": - procSnmp.Icmp.InAddrMasks = &value + procSnmp.InAddrMasks = &value case "InAddrMaskReps": - procSnmp.Icmp.InAddrMaskReps = &value + procSnmp.InAddrMaskReps = &value case "OutMsgs": - procSnmp.Icmp.OutMsgs = &value + procSnmp.OutMsgs = &value case "OutErrors": - procSnmp.Icmp.OutErrors = &value + procSnmp.OutErrors = &value case "OutDestUnreachs": - procSnmp.Icmp.OutDestUnreachs = &value + procSnmp.OutDestUnreachs = &value case "OutTimeExcds": - procSnmp.Icmp.OutTimeExcds = &value + procSnmp.OutTimeExcds = &value case "OutParmProbs": - procSnmp.Icmp.OutParmProbs = &value + procSnmp.OutParmProbs = &value case "OutSrcQuenchs": - procSnmp.Icmp.OutSrcQuenchs = &value + procSnmp.OutSrcQuenchs = &value case "OutRedirects": - procSnmp.Icmp.OutRedirects = &value + procSnmp.OutRedirects = &value case "OutEchos": - procSnmp.Icmp.OutEchos = &value + procSnmp.OutEchos = &value case "OutEchoReps": - procSnmp.Icmp.OutEchoReps = &value + procSnmp.OutEchoReps = &value case "OutTimestamps": - procSnmp.Icmp.OutTimestamps = &value + procSnmp.OutTimestamps = &value case "OutTimestampReps": - procSnmp.Icmp.OutTimestampReps = &value + procSnmp.OutTimestampReps = &value case "OutAddrMasks": - procSnmp.Icmp.OutAddrMasks = &value + procSnmp.OutAddrMasks = &value case "OutAddrMaskReps": - procSnmp.Icmp.OutAddrMaskReps = &value + procSnmp.OutAddrMaskReps = &value } case "IcmpMsg": switch key { case "InType3": - procSnmp.IcmpMsg.InType3 = &value + procSnmp.InType3 = &value case "OutType3": - procSnmp.IcmpMsg.OutType3 = &value + procSnmp.OutType3 = &value } case "Tcp": switch key { case "RtoAlgorithm": - procSnmp.Tcp.RtoAlgorithm = &value + procSnmp.RtoAlgorithm = &value case "RtoMin": - procSnmp.Tcp.RtoMin = &value + procSnmp.RtoMin = &value case "RtoMax": - procSnmp.Tcp.RtoMax = &value + procSnmp.RtoMax = &value case "MaxConn": - procSnmp.Tcp.MaxConn = &value + procSnmp.MaxConn = &value case "ActiveOpens": - procSnmp.Tcp.ActiveOpens = &value + procSnmp.ActiveOpens = &value case "PassiveOpens": - procSnmp.Tcp.PassiveOpens = &value + procSnmp.PassiveOpens = &value case "AttemptFails": - procSnmp.Tcp.AttemptFails = &value + procSnmp.AttemptFails = &value case "EstabResets": - procSnmp.Tcp.EstabResets = &value + procSnmp.EstabResets = &value case "CurrEstab": - procSnmp.Tcp.CurrEstab = &value + procSnmp.CurrEstab = &value case "InSegs": - procSnmp.Tcp.InSegs = &value + procSnmp.InSegs = &value case "OutSegs": - procSnmp.Tcp.OutSegs = &value + procSnmp.OutSegs = &value case "RetransSegs": - procSnmp.Tcp.RetransSegs = &value + procSnmp.RetransSegs = &value case "InErrs": - procSnmp.Tcp.InErrs = &value + procSnmp.InErrs = &value case "OutRsts": - procSnmp.Tcp.OutRsts = &value + procSnmp.OutRsts = &value case "InCsumErrors": procSnmp.Tcp.InCsumErrors = &value } diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc_snmp6.go b/go-controller/vendor/github.com/prometheus/procfs/proc_snmp6.go index 3059cc6a13..fb7fd3995b 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc_snmp6.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc_snmp6.go @@ -182,161 +182,161 @@ func parseSNMP6Stats(r io.Reader) (ProcSnmp6, error) { case "Ip6": switch key { case "InReceives": - procSnmp6.Ip6.InReceives = &value + procSnmp6.InReceives = &value case "InHdrErrors": - procSnmp6.Ip6.InHdrErrors = &value + procSnmp6.InHdrErrors = &value case "InTooBigErrors": - procSnmp6.Ip6.InTooBigErrors = &value + procSnmp6.InTooBigErrors = &value case "InNoRoutes": - procSnmp6.Ip6.InNoRoutes = &value + procSnmp6.InNoRoutes = &value case "InAddrErrors": - procSnmp6.Ip6.InAddrErrors = &value + procSnmp6.InAddrErrors = &value case "InUnknownProtos": - procSnmp6.Ip6.InUnknownProtos = &value + procSnmp6.InUnknownProtos = &value case "InTruncatedPkts": - procSnmp6.Ip6.InTruncatedPkts = &value + procSnmp6.InTruncatedPkts = &value case "InDiscards": - procSnmp6.Ip6.InDiscards = &value + procSnmp6.InDiscards = &value case "InDelivers": - procSnmp6.Ip6.InDelivers = &value + procSnmp6.InDelivers = &value case "OutForwDatagrams": - procSnmp6.Ip6.OutForwDatagrams = &value + procSnmp6.OutForwDatagrams = &value case "OutRequests": - procSnmp6.Ip6.OutRequests = &value + procSnmp6.OutRequests = &value case "OutDiscards": - procSnmp6.Ip6.OutDiscards = &value + procSnmp6.OutDiscards = &value case "OutNoRoutes": - procSnmp6.Ip6.OutNoRoutes = &value + procSnmp6.OutNoRoutes = &value case "ReasmTimeout": - procSnmp6.Ip6.ReasmTimeout = &value + procSnmp6.ReasmTimeout = &value case "ReasmReqds": - procSnmp6.Ip6.ReasmReqds = &value + procSnmp6.ReasmReqds = &value case "ReasmOKs": - procSnmp6.Ip6.ReasmOKs = &value + procSnmp6.ReasmOKs = &value case "ReasmFails": - procSnmp6.Ip6.ReasmFails = &value + procSnmp6.ReasmFails = &value case "FragOKs": - procSnmp6.Ip6.FragOKs = &value + procSnmp6.FragOKs = &value case "FragFails": - procSnmp6.Ip6.FragFails = &value + procSnmp6.FragFails = &value case "FragCreates": - procSnmp6.Ip6.FragCreates = &value + procSnmp6.FragCreates = &value case "InMcastPkts": - procSnmp6.Ip6.InMcastPkts = &value + procSnmp6.InMcastPkts = &value case "OutMcastPkts": - procSnmp6.Ip6.OutMcastPkts = &value + procSnmp6.OutMcastPkts = &value case "InOctets": - procSnmp6.Ip6.InOctets = &value + procSnmp6.InOctets = &value case "OutOctets": - procSnmp6.Ip6.OutOctets = &value + procSnmp6.OutOctets = &value case "InMcastOctets": - procSnmp6.Ip6.InMcastOctets = &value + procSnmp6.InMcastOctets = &value case "OutMcastOctets": - procSnmp6.Ip6.OutMcastOctets = &value + procSnmp6.OutMcastOctets = &value case "InBcastOctets": - procSnmp6.Ip6.InBcastOctets = &value + procSnmp6.InBcastOctets = &value case "OutBcastOctets": - procSnmp6.Ip6.OutBcastOctets = &value + procSnmp6.OutBcastOctets = &value case "InNoECTPkts": - procSnmp6.Ip6.InNoECTPkts = &value + procSnmp6.InNoECTPkts = &value case "InECT1Pkts": - procSnmp6.Ip6.InECT1Pkts = &value + procSnmp6.InECT1Pkts = &value case "InECT0Pkts": - procSnmp6.Ip6.InECT0Pkts = &value + procSnmp6.InECT0Pkts = &value case "InCEPkts": - procSnmp6.Ip6.InCEPkts = &value + procSnmp6.InCEPkts = &value } case "Icmp6": switch key { case "InMsgs": - procSnmp6.Icmp6.InMsgs = &value + procSnmp6.InMsgs = &value case "InErrors": procSnmp6.Icmp6.InErrors = &value case "OutMsgs": - procSnmp6.Icmp6.OutMsgs = &value + procSnmp6.OutMsgs = &value case "OutErrors": - procSnmp6.Icmp6.OutErrors = &value + procSnmp6.OutErrors = &value case "InCsumErrors": procSnmp6.Icmp6.InCsumErrors = &value case "InDestUnreachs": - procSnmp6.Icmp6.InDestUnreachs = &value + procSnmp6.InDestUnreachs = &value case "InPktTooBigs": - procSnmp6.Icmp6.InPktTooBigs = &value + procSnmp6.InPktTooBigs = &value case "InTimeExcds": - procSnmp6.Icmp6.InTimeExcds = &value + procSnmp6.InTimeExcds = &value case "InParmProblems": - procSnmp6.Icmp6.InParmProblems = &value + procSnmp6.InParmProblems = &value case "InEchos": - procSnmp6.Icmp6.InEchos = &value + procSnmp6.InEchos = &value case "InEchoReplies": - procSnmp6.Icmp6.InEchoReplies = &value + procSnmp6.InEchoReplies = &value case "InGroupMembQueries": - procSnmp6.Icmp6.InGroupMembQueries = &value + procSnmp6.InGroupMembQueries = &value case "InGroupMembResponses": - procSnmp6.Icmp6.InGroupMembResponses = &value + procSnmp6.InGroupMembResponses = &value case "InGroupMembReductions": - procSnmp6.Icmp6.InGroupMembReductions = &value + procSnmp6.InGroupMembReductions = &value case "InRouterSolicits": - procSnmp6.Icmp6.InRouterSolicits = &value + procSnmp6.InRouterSolicits = &value case "InRouterAdvertisements": - procSnmp6.Icmp6.InRouterAdvertisements = &value + procSnmp6.InRouterAdvertisements = &value case "InNeighborSolicits": - procSnmp6.Icmp6.InNeighborSolicits = &value + procSnmp6.InNeighborSolicits = &value case "InNeighborAdvertisements": - procSnmp6.Icmp6.InNeighborAdvertisements = &value + procSnmp6.InNeighborAdvertisements = &value case "InRedirects": - procSnmp6.Icmp6.InRedirects = &value + procSnmp6.InRedirects = &value case "InMLDv2Reports": - procSnmp6.Icmp6.InMLDv2Reports = &value + procSnmp6.InMLDv2Reports = &value case "OutDestUnreachs": - procSnmp6.Icmp6.OutDestUnreachs = &value + procSnmp6.OutDestUnreachs = &value case "OutPktTooBigs": - procSnmp6.Icmp6.OutPktTooBigs = &value + procSnmp6.OutPktTooBigs = &value case "OutTimeExcds": - procSnmp6.Icmp6.OutTimeExcds = &value + procSnmp6.OutTimeExcds = &value case "OutParmProblems": - procSnmp6.Icmp6.OutParmProblems = &value + procSnmp6.OutParmProblems = &value case "OutEchos": - procSnmp6.Icmp6.OutEchos = &value + procSnmp6.OutEchos = &value case "OutEchoReplies": - procSnmp6.Icmp6.OutEchoReplies = &value + procSnmp6.OutEchoReplies = &value case "OutGroupMembQueries": - procSnmp6.Icmp6.OutGroupMembQueries = &value + procSnmp6.OutGroupMembQueries = &value case "OutGroupMembResponses": - procSnmp6.Icmp6.OutGroupMembResponses = &value + procSnmp6.OutGroupMembResponses = &value case "OutGroupMembReductions": - procSnmp6.Icmp6.OutGroupMembReductions = &value + procSnmp6.OutGroupMembReductions = &value case "OutRouterSolicits": - procSnmp6.Icmp6.OutRouterSolicits = &value + procSnmp6.OutRouterSolicits = &value case "OutRouterAdvertisements": - procSnmp6.Icmp6.OutRouterAdvertisements = &value + procSnmp6.OutRouterAdvertisements = &value case "OutNeighborSolicits": - procSnmp6.Icmp6.OutNeighborSolicits = &value + procSnmp6.OutNeighborSolicits = &value case "OutNeighborAdvertisements": - procSnmp6.Icmp6.OutNeighborAdvertisements = &value + procSnmp6.OutNeighborAdvertisements = &value case "OutRedirects": - procSnmp6.Icmp6.OutRedirects = &value + procSnmp6.OutRedirects = &value case "OutMLDv2Reports": - procSnmp6.Icmp6.OutMLDv2Reports = &value + procSnmp6.OutMLDv2Reports = &value case "InType1": - procSnmp6.Icmp6.InType1 = &value + procSnmp6.InType1 = &value case "InType134": - procSnmp6.Icmp6.InType134 = &value + procSnmp6.InType134 = &value case "InType135": - procSnmp6.Icmp6.InType135 = &value + procSnmp6.InType135 = &value case "InType136": - procSnmp6.Icmp6.InType136 = &value + procSnmp6.InType136 = &value case "InType143": - procSnmp6.Icmp6.InType143 = &value + procSnmp6.InType143 = &value case "OutType133": - procSnmp6.Icmp6.OutType133 = &value + procSnmp6.OutType133 = &value case "OutType135": - procSnmp6.Icmp6.OutType135 = &value + procSnmp6.OutType135 = &value case "OutType136": - procSnmp6.Icmp6.OutType136 = &value + procSnmp6.OutType136 = &value case "OutType143": - procSnmp6.Icmp6.OutType143 = &value + procSnmp6.OutType143 = &value } case "Udp6": switch key { @@ -355,7 +355,7 @@ func parseSNMP6Stats(r io.Reader) (ProcSnmp6, error) { case "InCsumErrors": procSnmp6.Udp6.InCsumErrors = &value case "IgnoredMulti": - procSnmp6.Udp6.IgnoredMulti = &value + procSnmp6.IgnoredMulti = &value } case "UdpLite6": switch key { diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc_status.go b/go-controller/vendor/github.com/prometheus/procfs/proc_status.go index a055197c63..dd8aa56885 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc_status.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc_status.go @@ -146,7 +146,11 @@ func (s *ProcStatus) fillStatus(k string, vString string, vUint uint64, vUintByt } } case "NSpid": - s.NSpids = calcNSPidsList(vString) + nspids, err := calcNSPidsList(vString) + if err != nil { + return err + } + s.NSpids = nspids case "VmPeak": s.VmPeak = vUintBytes case "VmSize": @@ -222,17 +226,17 @@ func calcCpusAllowedList(cpuString string) []uint64 { return g } -func calcNSPidsList(nspidsString string) []uint64 { - s := strings.Split(nspidsString, " ") +func calcNSPidsList(nspidsString string) ([]uint64, error) { + s := strings.Split(nspidsString, "\t") var nspids []uint64 for _, nspid := range s { - nspid, _ := strconv.ParseUint(nspid, 10, 64) - if nspid == 0 { - continue + nspid, err := strconv.ParseUint(nspid, 10, 64) + if err != nil { + return nil, err } nspids = append(nspids, nspid) } - return nspids + return nspids, nil } diff --git a/go-controller/vendor/github.com/prometheus/procfs/proc_sys.go b/go-controller/vendor/github.com/prometheus/procfs/proc_sys.go index 5eefbe2ef8..3810d1ac99 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/proc_sys.go +++ b/go-controller/vendor/github.com/prometheus/procfs/proc_sys.go @@ -21,7 +21,7 @@ import ( ) func sysctlToPath(sysctl string) string { - return strings.Replace(sysctl, ".", "/", -1) + return strings.ReplaceAll(sysctl, ".", "/") } func (fs FS) SysctlStrings(sysctl string) ([]string, error) { diff --git a/go-controller/vendor/github.com/prometheus/procfs/softirqs.go b/go-controller/vendor/github.com/prometheus/procfs/softirqs.go index 28708e0745..403e6ae708 100644 --- a/go-controller/vendor/github.com/prometheus/procfs/softirqs.go +++ b/go-controller/vendor/github.com/prometheus/procfs/softirqs.go @@ -68,8 +68,8 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { if len(parts) < 2 { continue } - switch { - case parts[0] == "HI:": + switch parts[0] { + case "HI:": perCPU := parts[1:] softirqs.Hi = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -77,7 +77,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (HI%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "TIMER:": + case "TIMER:": perCPU := parts[1:] softirqs.Timer = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -85,7 +85,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (TIMER%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "NET_TX:": + case "NET_TX:": perCPU := parts[1:] softirqs.NetTx = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -93,7 +93,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (NET_TX%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "NET_RX:": + case "NET_RX:": perCPU := parts[1:] softirqs.NetRx = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -101,7 +101,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (NET_RX%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "BLOCK:": + case "BLOCK:": perCPU := parts[1:] softirqs.Block = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -109,7 +109,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (BLOCK%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "IRQ_POLL:": + case "IRQ_POLL:": perCPU := parts[1:] softirqs.IRQPoll = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -117,7 +117,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (IRQ_POLL%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "TASKLET:": + case "TASKLET:": perCPU := parts[1:] softirqs.Tasklet = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -125,7 +125,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (TASKLET%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "SCHED:": + case "SCHED:": perCPU := parts[1:] softirqs.Sched = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -133,7 +133,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (SCHED%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "HRTIMER:": + case "HRTIMER:": perCPU := parts[1:] softirqs.HRTimer = make([]uint64, len(perCPU)) for i, count := range perCPU { @@ -141,7 +141,7 @@ func parseSoftirqs(r io.Reader) (Softirqs, error) { return Softirqs{}, fmt.Errorf("%w: couldn't parse %q (HRTIMER%d): %w", ErrFileParse, count, i, err) } } - case parts[0] == "RCU:": + case "RCU:": perCPU := parts[1:] softirqs.RCU = make([]uint64, len(perCPU)) for i, count := range perCPU { diff --git a/go-controller/vendor/github.com/spf13/cobra/.gitignore b/go-controller/vendor/github.com/spf13/cobra/.gitignore new file mode 100644 index 0000000000..c7b459e4dd --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/.gitignore @@ -0,0 +1,39 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +# Vim files https://github.com/github/gitignore/blob/master/Global/Vim.gitignore +# swap +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +# session +Session.vim +# temporary +.netrwhist +*~ +# auto-generated tag files +tags + +*.exe +cobra.test +bin + +.idea/ +*.iml diff --git a/go-controller/vendor/github.com/spf13/cobra/.golangci.yml b/go-controller/vendor/github.com/spf13/cobra/.golangci.yml new file mode 100644 index 0000000000..2c8f4808c1 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/.golangci.yml @@ -0,0 +1,57 @@ +# Copyright 2013-2023 The Cobra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +run: + deadline: 5m + +linters: + disable-all: true + enable: + #- bodyclose + # - deadcode ! deprecated since v1.49.0; replaced by 'unused' + #- depguard + #- dogsled + #- dupl + - errcheck + #- exhaustive + #- funlen + #- gochecknoinits + - goconst + - gocritic + #- gocyclo + - gofmt + - goimports + #- gomnd + #- goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + #- lll + - misspell + #- nakedret + #- noctx + - nolintlint + #- rowserrcheck + #- scopelint + - staticcheck + #- structcheck ! deprecated since v1.49.0; replaced by 'unused' + - stylecheck + #- typecheck + - unconvert + #- unparam + - unused + # - varcheck ! deprecated since v1.49.0; replaced by 'unused' + #- whitespace + fast: false diff --git a/go-controller/vendor/github.com/spf13/cobra/.mailmap b/go-controller/vendor/github.com/spf13/cobra/.mailmap new file mode 100644 index 0000000000..94ec53068a --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/.mailmap @@ -0,0 +1,3 @@ +Steve Francia +Bjørn Erik Pedersen +Fabiano Franz diff --git a/go-controller/vendor/github.com/spf13/cobra/CONDUCT.md b/go-controller/vendor/github.com/spf13/cobra/CONDUCT.md new file mode 100644 index 0000000000..9d16f88fd1 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/CONDUCT.md @@ -0,0 +1,37 @@ +## Cobra User Contract + +### Versioning +Cobra will follow a steady release cadence. Non breaking changes will be released as minor versions quarterly. Patch bug releases are at the discretion of the maintainers. Users can expect security patch fixes to be released within relatively short order of a CVE becoming known. For more information on security patch fixes see the CVE section below. Releases will follow [Semantic Versioning](https://semver.org/). Users tracking the Master branch should expect unpredictable breaking changes as the project continues to move forward. For stability, it is highly recommended to use a release. + +### Backward Compatibility +We will maintain two major releases in a moving window. The N-1 release will only receive bug fixes and security updates and will be dropped once N+1 is released. + +### Deprecation +Deprecation of Go versions or dependent packages will only occur in major releases. To reduce the change of this taking users by surprise, any large deprecation will be preceded by an announcement in the [#cobra slack channel](https://gophers.slack.com/archives/CD3LP1199) and an Issue on Github. + +### CVE +Maintainers will make every effort to release security patches in the case of a medium to high severity CVE directly impacting the library. The speed in which these patches reach a release is up to the discretion of the maintainers. A low severity CVE may be a lower priority than a high severity one. + +### Communication +Cobra maintainers will use GitHub issues and the [#cobra slack channel](https://gophers.slack.com/archives/CD3LP1199) as the primary means of communication with the community. This is to foster open communication with all users and contributors. + +### Breaking Changes +Breaking changes are generally allowed in the master branch, as this is the branch used to develop the next release of Cobra. + +There may be times, however, when master is closed for breaking changes. This is likely to happen as we near the release of a new version. + +Breaking changes are not allowed in release branches, as these represent minor versions that have already been released. These version have consumers who expect the APIs, behaviors, etc, to remain stable during the lifetime of the patch stream for the minor release. + +Examples of breaking changes include: +- Removing or renaming exported constant, variable, type, or function. +- Updating the version of critical libraries such as `spf13/pflag`, `spf13/viper` etc... + - Some version updates may be acceptable for picking up bug fixes, but maintainers must exercise caution when reviewing. + +There may, at times, need to be exceptions where breaking changes are allowed in release branches. These are at the discretion of the project's maintainers, and must be carefully considered before merging. + +### CI Testing +Maintainers will ensure the Cobra test suite utilizes the current supported versions of Golang. + +### Disclaimer +Changes to this document and the contents therein are at the discretion of the maintainers. +None of the contents of this document are legally binding in any way to the maintainers or the users. diff --git a/go-controller/vendor/github.com/spf13/cobra/CONTRIBUTING.md b/go-controller/vendor/github.com/spf13/cobra/CONTRIBUTING.md new file mode 100644 index 0000000000..6f356e6a82 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/CONTRIBUTING.md @@ -0,0 +1,50 @@ +# Contributing to Cobra + +Thank you so much for contributing to Cobra. We appreciate your time and help. +Here are some guidelines to help you get started. + +## Code of Conduct + +Be kind and respectful to the members of the community. Take time to educate +others who are seeking help. Harassment of any kind will not be tolerated. + +## Questions + +If you have questions regarding Cobra, feel free to ask it in the community +[#cobra Slack channel][cobra-slack] + +## Filing a bug or feature + +1. Before filing an issue, please check the existing issues to see if a + similar one was already opened. If there is one already opened, feel free + to comment on it. +1. If you believe you've found a bug, please provide detailed steps of + reproduction, the version of Cobra and anything else you believe will be + useful to help troubleshoot it (e.g. OS environment, environment variables, + etc...). Also state the current behavior vs. the expected behavior. +1. If you'd like to see a feature or an enhancement please open an issue with + a clear title and description of what the feature is and why it would be + beneficial to the project and its users. + +## Submitting changes + +1. CLA: Upon submitting a Pull Request (PR), contributors will be prompted to + sign a CLA. Please sign the CLA :slightly_smiling_face: +1. Tests: If you are submitting code, please ensure you have adequate tests + for the feature. Tests can be run via `go test ./...` or `make test`. +1. Since this is golang project, ensure the new code is properly formatted to + ensure code consistency. Run `make all`. + +### Quick steps to contribute + +1. Fork the project. +1. Download your fork to your PC (`git clone https://github.com/your_username/cobra && cd cobra`) +1. Create your feature branch (`git checkout -b my-new-feature`) +1. Make changes and run tests (`make test`) +1. Add them to staging (`git add .`) +1. Commit your changes (`git commit -m 'Add some feature'`) +1. Push to the branch (`git push origin my-new-feature`) +1. Create new pull request + + +[cobra-slack]: https://gophers.slack.com/archives/CD3LP1199 diff --git a/go-controller/vendor/github.com/spf13/cobra/LICENSE.txt b/go-controller/vendor/github.com/spf13/cobra/LICENSE.txt new file mode 100644 index 0000000000..298f0e2665 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/LICENSE.txt @@ -0,0 +1,174 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/go-controller/vendor/github.com/spf13/cobra/MAINTAINERS b/go-controller/vendor/github.com/spf13/cobra/MAINTAINERS new file mode 100644 index 0000000000..4c5ac3dd99 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/MAINTAINERS @@ -0,0 +1,13 @@ +maintainers: +- spf13 +- johnSchnake +- jpmcb +- marckhouzam +inactive: +- anthonyfok +- bep +- bogem +- broady +- eparis +- jharshman +- wfernandes diff --git a/go-controller/vendor/github.com/spf13/cobra/Makefile b/go-controller/vendor/github.com/spf13/cobra/Makefile new file mode 100644 index 0000000000..0da8d7aa08 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/Makefile @@ -0,0 +1,35 @@ +BIN="./bin" +SRC=$(shell find . -name "*.go") + +ifeq (, $(shell which golangci-lint)) +$(warning "could not find golangci-lint in $(PATH), run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh") +endif + +.PHONY: fmt lint test install_deps clean + +default: all + +all: fmt test + +fmt: + $(info ******************** checking formatting ********************) + @test -z $(shell gofmt -l $(SRC)) || (gofmt -d $(SRC); exit 1) + +lint: + $(info ******************** running lint tools ********************) + golangci-lint run -v + +test: install_deps + $(info ******************** running tests ********************) + go test -v ./... + +richtest: install_deps + $(info ******************** running tests with kyoh86/richgo ********************) + richgo test -v ./... + +install_deps: + $(info ******************** downloading dependencies ********************) + go get -v ./... + +clean: + rm -rf $(BIN) diff --git a/go-controller/vendor/github.com/spf13/cobra/README.md b/go-controller/vendor/github.com/spf13/cobra/README.md new file mode 100644 index 0000000000..71757151c3 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/README.md @@ -0,0 +1,113 @@ + +![cobra logo](https://github.com/user-attachments/assets/cbc3adf8-0dff-46e9-a88d-5e2d971c169e) + +Cobra is a library for creating powerful modern CLI applications. + +Cobra is used in many Go projects such as [Kubernetes](https://kubernetes.io/), +[Hugo](https://gohugo.io), and [GitHub CLI](https://github.com/cli/cli) to +name a few. [This list](site/content/projects_using_cobra.md) contains a more extensive list of projects using Cobra. + +[![](https://img.shields.io/github/actions/workflow/status/spf13/cobra/test.yml?branch=main&longCache=true&label=Test&logo=github%20actions&logoColor=fff)](https://github.com/spf13/cobra/actions?query=workflow%3ATest) +[![Go Reference](https://pkg.go.dev/badge/github.com/spf13/cobra.svg)](https://pkg.go.dev/github.com/spf13/cobra) +[![Go Report Card](https://goreportcard.com/badge/github.com/spf13/cobra)](https://goreportcard.com/report/github.com/spf13/cobra) +[![Slack](https://img.shields.io/badge/Slack-cobra-brightgreen)](https://gophers.slack.com/archives/CD3LP1199) + +# Overview + +Cobra is a library providing a simple interface to create powerful modern CLI +interfaces similar to git & go tools. + +Cobra provides: +* Easy subcommand-based CLIs: `app server`, `app fetch`, etc. +* Fully POSIX-compliant flags (including short & long versions) +* Nested subcommands +* Global, local and cascading flags +* Intelligent suggestions (`app srver`... did you mean `app server`?) +* Automatic help generation for commands and flags +* Grouping help for subcommands +* Automatic help flag recognition of `-h`, `--help`, etc. +* Automatically generated shell autocomplete for your application (bash, zsh, fish, powershell) +* Automatically generated man pages for your application +* Command aliases so you can change things without breaking them +* The flexibility to define your own help, usage, etc. +* Optional seamless integration with [viper](https://github.com/spf13/viper) for 12-factor apps + +# Concepts + +Cobra is built on a structure of commands, arguments & flags. + +**Commands** represent actions, **Args** are things and **Flags** are modifiers for those actions. + +The best applications read like sentences when used, and as a result, users +intuitively know how to interact with them. + +The pattern to follow is +`APPNAME VERB NOUN --ADJECTIVE` + or +`APPNAME COMMAND ARG --FLAG`. + +A few good real world examples may better illustrate this point. + +In the following example, 'server' is a command, and 'port' is a flag: + + hugo server --port=1313 + +In this command we are telling Git to clone the url bare. + + git clone URL --bare + +## Commands + +Command is the central point of the application. Each interaction that +the application supports will be contained in a Command. A command can +have children commands and optionally run an action. + +In the example above, 'server' is the command. + +[More about cobra.Command](https://pkg.go.dev/github.com/spf13/cobra#Command) + +## Flags + +A flag is a way to modify the behavior of a command. Cobra supports +fully POSIX-compliant flags as well as the Go [flag package](https://golang.org/pkg/flag/). +A Cobra command can define flags that persist through to children commands +and flags that are only available to that command. + +In the example above, 'port' is the flag. + +Flag functionality is provided by the [pflag +library](https://github.com/spf13/pflag), a fork of the flag standard library +which maintains the same interface while adding POSIX compliance. + +# Installing +Using Cobra is easy. First, use `go get` to install the latest version +of the library. + +``` +go get -u github.com/spf13/cobra@latest +``` + +Next, include Cobra in your application: + +```go +import "github.com/spf13/cobra" +``` + +# Usage +`cobra-cli` is a command line program to generate cobra applications and command files. +It will bootstrap your application scaffolding to rapidly +develop a Cobra-based application. It is the easiest way to incorporate Cobra into your application. + +It can be installed by running: + +``` +go install github.com/spf13/cobra-cli@latest +``` + +For complete details on using the Cobra-CLI generator, please read [The Cobra Generator README](https://github.com/spf13/cobra-cli/blob/main/README.md) + +For complete details on using the Cobra library, please read [The Cobra User Guide](site/content/user_guide.md). + +# License + +Cobra is released under the Apache 2.0 license. See [LICENSE.txt](LICENSE.txt) diff --git a/go-controller/vendor/github.com/spf13/cobra/active_help.go b/go-controller/vendor/github.com/spf13/cobra/active_help.go new file mode 100644 index 0000000000..b3e2dadfed --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/active_help.go @@ -0,0 +1,60 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "fmt" + "os" +) + +const ( + activeHelpMarker = "_activeHelp_ " + // The below values should not be changed: programs will be using them explicitly + // in their user documentation, and users will be using them explicitly. + activeHelpEnvVarSuffix = "ACTIVE_HELP" + activeHelpGlobalEnvVar = configEnvVarGlobalPrefix + "_" + activeHelpEnvVarSuffix + activeHelpGlobalDisable = "0" +) + +// AppendActiveHelp adds the specified string to the specified array to be used as ActiveHelp. +// Such strings will be processed by the completion script and will be shown as ActiveHelp +// to the user. +// The array parameter should be the array that will contain the completions. +// This function can be called multiple times before and/or after completions are added to +// the array. Each time this function is called with the same array, the new +// ActiveHelp line will be shown below the previous ones when completion is triggered. +func AppendActiveHelp(compArray []Completion, activeHelpStr string) []Completion { + return append(compArray, fmt.Sprintf("%s%s", activeHelpMarker, activeHelpStr)) +} + +// GetActiveHelpConfig returns the value of the ActiveHelp environment variable +// _ACTIVE_HELP where is the name of the root command in upper +// case, with all non-ASCII-alphanumeric characters replaced by `_`. +// It will always return "0" if the global environment variable COBRA_ACTIVE_HELP +// is set to "0". +func GetActiveHelpConfig(cmd *Command) string { + activeHelpCfg := os.Getenv(activeHelpGlobalEnvVar) + if activeHelpCfg != activeHelpGlobalDisable { + activeHelpCfg = os.Getenv(activeHelpEnvVar(cmd.Root().Name())) + } + return activeHelpCfg +} + +// activeHelpEnvVar returns the name of the program-specific ActiveHelp environment +// variable. It has the format _ACTIVE_HELP where is the name of the +// root command in upper case, with all non-ASCII-alphanumeric characters replaced by `_`. +func activeHelpEnvVar(name string) string { + return configEnvVar(name, activeHelpEnvVarSuffix) +} diff --git a/go-controller/vendor/github.com/spf13/cobra/args.go b/go-controller/vendor/github.com/spf13/cobra/args.go new file mode 100644 index 0000000000..ed1e70ceaa --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/args.go @@ -0,0 +1,131 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "fmt" + "strings" +) + +type PositionalArgs func(cmd *Command, args []string) error + +// legacyArgs validation has the following behaviour: +// - root commands with no subcommands can take arbitrary arguments +// - root commands with subcommands will do subcommand validity checking +// - subcommands will always accept arbitrary arguments +func legacyArgs(cmd *Command, args []string) error { + // no subcommand, always take args + if !cmd.HasSubCommands() { + return nil + } + + // root command with subcommands, do subcommand checking. + if !cmd.HasParent() && len(args) > 0 { + return fmt.Errorf("unknown command %q for %q%s", args[0], cmd.CommandPath(), cmd.findSuggestions(args[0])) + } + return nil +} + +// NoArgs returns an error if any args are included. +func NoArgs(cmd *Command, args []string) error { + if len(args) > 0 { + return fmt.Errorf("unknown command %q for %q", args[0], cmd.CommandPath()) + } + return nil +} + +// OnlyValidArgs returns an error if there are any positional args that are not in +// the `ValidArgs` field of `Command` +func OnlyValidArgs(cmd *Command, args []string) error { + if len(cmd.ValidArgs) > 0 { + // Remove any description that may be included in ValidArgs. + // A description is following a tab character. + validArgs := make([]string, 0, len(cmd.ValidArgs)) + for _, v := range cmd.ValidArgs { + validArgs = append(validArgs, strings.SplitN(v, "\t", 2)[0]) + } + for _, v := range args { + if !stringInSlice(v, validArgs) { + return fmt.Errorf("invalid argument %q for %q%s", v, cmd.CommandPath(), cmd.findSuggestions(args[0])) + } + } + } + return nil +} + +// ArbitraryArgs never returns an error. +func ArbitraryArgs(cmd *Command, args []string) error { + return nil +} + +// MinimumNArgs returns an error if there is not at least N args. +func MinimumNArgs(n int) PositionalArgs { + return func(cmd *Command, args []string) error { + if len(args) < n { + return fmt.Errorf("requires at least %d arg(s), only received %d", n, len(args)) + } + return nil + } +} + +// MaximumNArgs returns an error if there are more than N args. +func MaximumNArgs(n int) PositionalArgs { + return func(cmd *Command, args []string) error { + if len(args) > n { + return fmt.Errorf("accepts at most %d arg(s), received %d", n, len(args)) + } + return nil + } +} + +// ExactArgs returns an error if there are not exactly n args. +func ExactArgs(n int) PositionalArgs { + return func(cmd *Command, args []string) error { + if len(args) != n { + return fmt.Errorf("accepts %d arg(s), received %d", n, len(args)) + } + return nil + } +} + +// RangeArgs returns an error if the number of args is not within the expected range. +func RangeArgs(min int, max int) PositionalArgs { + return func(cmd *Command, args []string) error { + if len(args) < min || len(args) > max { + return fmt.Errorf("accepts between %d and %d arg(s), received %d", min, max, len(args)) + } + return nil + } +} + +// MatchAll allows combining several PositionalArgs to work in concert. +func MatchAll(pargs ...PositionalArgs) PositionalArgs { + return func(cmd *Command, args []string) error { + for _, parg := range pargs { + if err := parg(cmd, args); err != nil { + return err + } + } + return nil + } +} + +// ExactValidArgs returns an error if there are not exactly N positional args OR +// there are any positional args that are not in the `ValidArgs` field of `Command` +// +// Deprecated: use MatchAll(ExactArgs(n), OnlyValidArgs) instead +func ExactValidArgs(n int) PositionalArgs { + return MatchAll(ExactArgs(n), OnlyValidArgs) +} diff --git a/go-controller/vendor/github.com/spf13/cobra/bash_completions.go b/go-controller/vendor/github.com/spf13/cobra/bash_completions.go new file mode 100644 index 0000000000..f4d198cbcb --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/bash_completions.go @@ -0,0 +1,709 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "bytes" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/spf13/pflag" +) + +// Annotations for Bash completion. +const ( + BashCompFilenameExt = "cobra_annotation_bash_completion_filename_extensions" + BashCompCustom = "cobra_annotation_bash_completion_custom" + BashCompOneRequiredFlag = "cobra_annotation_bash_completion_one_required_flag" + BashCompSubdirsInDir = "cobra_annotation_bash_completion_subdirs_in_dir" +) + +func writePreamble(buf io.StringWriter, name string) { + WriteStringAndCheck(buf, fmt.Sprintf("# bash completion for %-36s -*- shell-script -*-\n", name)) + WriteStringAndCheck(buf, fmt.Sprintf(` +__%[1]s_debug() +{ + if [[ -n ${BASH_COMP_DEBUG_FILE:-} ]]; then + echo "$*" >> "${BASH_COMP_DEBUG_FILE}" + fi +} + +# Homebrew on Macs have version 1.3 of bash-completion which doesn't include +# _init_completion. This is a very minimal version of that function. +__%[1]s_init_completion() +{ + COMPREPLY=() + _get_comp_words_by_ref "$@" cur prev words cword +} + +__%[1]s_index_of_word() +{ + local w word=$1 + shift + index=0 + for w in "$@"; do + [[ $w = "$word" ]] && return + index=$((index+1)) + done + index=-1 +} + +__%[1]s_contains_word() +{ + local w word=$1; shift + for w in "$@"; do + [[ $w = "$word" ]] && return + done + return 1 +} + +__%[1]s_handle_go_custom_completion() +{ + __%[1]s_debug "${FUNCNAME[0]}: cur is ${cur}, words[*] is ${words[*]}, #words[@] is ${#words[@]}" + + local shellCompDirectiveError=%[3]d + local shellCompDirectiveNoSpace=%[4]d + local shellCompDirectiveNoFileComp=%[5]d + local shellCompDirectiveFilterFileExt=%[6]d + local shellCompDirectiveFilterDirs=%[7]d + + local out requestComp lastParam lastChar comp directive args + + # Prepare the command to request completions for the program. + # Calling ${words[0]} instead of directly %[1]s allows handling aliases + args=("${words[@]:1}") + # Disable ActiveHelp which is not supported for bash completion v1 + requestComp="%[8]s=0 ${words[0]} %[2]s ${args[*]}" + + lastParam=${words[$((${#words[@]}-1))]} + lastChar=${lastParam:$((${#lastParam}-1)):1} + __%[1]s_debug "${FUNCNAME[0]}: lastParam ${lastParam}, lastChar ${lastChar}" + + if [ -z "${cur}" ] && [ "${lastChar}" != "=" ]; then + # If the last parameter is complete (there is a space following it) + # We add an extra empty parameter so we can indicate this to the go method. + __%[1]s_debug "${FUNCNAME[0]}: Adding extra empty parameter" + requestComp="${requestComp} \"\"" + fi + + __%[1]s_debug "${FUNCNAME[0]}: calling ${requestComp}" + # Use eval to handle any environment variables and such + out=$(eval "${requestComp}" 2>/dev/null) + + # Extract the directive integer at the very end of the output following a colon (:) + directive=${out##*:} + # Remove the directive + out=${out%%:*} + if [ "${directive}" = "${out}" ]; then + # There is not directive specified + directive=0 + fi + __%[1]s_debug "${FUNCNAME[0]}: the completion directive is: ${directive}" + __%[1]s_debug "${FUNCNAME[0]}: the completions are: ${out}" + + if [ $((directive & shellCompDirectiveError)) -ne 0 ]; then + # Error code. No completion. + __%[1]s_debug "${FUNCNAME[0]}: received error from custom completion go code" + return + else + if [ $((directive & shellCompDirectiveNoSpace)) -ne 0 ]; then + if [[ $(type -t compopt) = "builtin" ]]; then + __%[1]s_debug "${FUNCNAME[0]}: activating no space" + compopt -o nospace + fi + fi + if [ $((directive & shellCompDirectiveNoFileComp)) -ne 0 ]; then + if [[ $(type -t compopt) = "builtin" ]]; then + __%[1]s_debug "${FUNCNAME[0]}: activating no file completion" + compopt +o default + fi + fi + fi + + if [ $((directive & shellCompDirectiveFilterFileExt)) -ne 0 ]; then + # File extension filtering + local fullFilter filter filteringCmd + # Do not use quotes around the $out variable or else newline + # characters will be kept. + for filter in ${out}; do + fullFilter+="$filter|" + done + + filteringCmd="_filedir $fullFilter" + __%[1]s_debug "File filtering command: $filteringCmd" + $filteringCmd + elif [ $((directive & shellCompDirectiveFilterDirs)) -ne 0 ]; then + # File completion for directories only + local subdir + # Use printf to strip any trailing newline + subdir=$(printf "%%s" "${out}") + if [ -n "$subdir" ]; then + __%[1]s_debug "Listing directories in $subdir" + __%[1]s_handle_subdirs_in_dir_flag "$subdir" + else + __%[1]s_debug "Listing directories in ." + _filedir -d + fi + else + while IFS='' read -r comp; do + COMPREPLY+=("$comp") + done < <(compgen -W "${out}" -- "$cur") + fi +} + +__%[1]s_handle_reply() +{ + __%[1]s_debug "${FUNCNAME[0]}" + local comp + case $cur in + -*) + if [[ $(type -t compopt) = "builtin" ]]; then + compopt -o nospace + fi + local allflags + if [ ${#must_have_one_flag[@]} -ne 0 ]; then + allflags=("${must_have_one_flag[@]}") + else + allflags=("${flags[*]} ${two_word_flags[*]}") + fi + while IFS='' read -r comp; do + COMPREPLY+=("$comp") + done < <(compgen -W "${allflags[*]}" -- "$cur") + if [[ $(type -t compopt) = "builtin" ]]; then + [[ "${COMPREPLY[0]}" == *= ]] || compopt +o nospace + fi + + # complete after --flag=abc + if [[ $cur == *=* ]]; then + if [[ $(type -t compopt) = "builtin" ]]; then + compopt +o nospace + fi + + local index flag + flag="${cur%%=*}" + __%[1]s_index_of_word "${flag}" "${flags_with_completion[@]}" + COMPREPLY=() + if [[ ${index} -ge 0 ]]; then + PREFIX="" + cur="${cur#*=}" + ${flags_completion[${index}]} + if [ -n "${ZSH_VERSION:-}" ]; then + # zsh completion needs --flag= prefix + eval "COMPREPLY=( \"\${COMPREPLY[@]/#/${flag}=}\" )" + fi + fi + fi + + if [[ -z "${flag_parsing_disabled}" ]]; then + # If flag parsing is enabled, we have completed the flags and can return. + # If flag parsing is disabled, we may not know all (or any) of the flags, so we fallthrough + # to possibly call handle_go_custom_completion. + return 0; + fi + ;; + esac + + # check if we are handling a flag with special work handling + local index + __%[1]s_index_of_word "${prev}" "${flags_with_completion[@]}" + if [[ ${index} -ge 0 ]]; then + ${flags_completion[${index}]} + return + fi + + # we are parsing a flag and don't have a special handler, no completion + if [[ ${cur} != "${words[cword]}" ]]; then + return + fi + + local completions + completions=("${commands[@]}") + if [[ ${#must_have_one_noun[@]} -ne 0 ]]; then + completions+=("${must_have_one_noun[@]}") + elif [[ -n "${has_completion_function}" ]]; then + # if a go completion function is provided, defer to that function + __%[1]s_handle_go_custom_completion + fi + if [[ ${#must_have_one_flag[@]} -ne 0 ]]; then + completions+=("${must_have_one_flag[@]}") + fi + while IFS='' read -r comp; do + COMPREPLY+=("$comp") + done < <(compgen -W "${completions[*]}" -- "$cur") + + if [[ ${#COMPREPLY[@]} -eq 0 && ${#noun_aliases[@]} -gt 0 && ${#must_have_one_noun[@]} -ne 0 ]]; then + while IFS='' read -r comp; do + COMPREPLY+=("$comp") + done < <(compgen -W "${noun_aliases[*]}" -- "$cur") + fi + + if [[ ${#COMPREPLY[@]} -eq 0 ]]; then + if declare -F __%[1]s_custom_func >/dev/null; then + # try command name qualified custom func + __%[1]s_custom_func + else + # otherwise fall back to unqualified for compatibility + declare -F __custom_func >/dev/null && __custom_func + fi + fi + + # available in bash-completion >= 2, not always present on macOS + if declare -F __ltrim_colon_completions >/dev/null; then + __ltrim_colon_completions "$cur" + fi + + # If there is only 1 completion and it is a flag with an = it will be completed + # but we don't want a space after the = + if [[ "${#COMPREPLY[@]}" -eq "1" ]] && [[ $(type -t compopt) = "builtin" ]] && [[ "${COMPREPLY[0]}" == --*= ]]; then + compopt -o nospace + fi +} + +# The arguments should be in the form "ext1|ext2|extn" +__%[1]s_handle_filename_extension_flag() +{ + local ext="$1" + _filedir "@(${ext})" +} + +__%[1]s_handle_subdirs_in_dir_flag() +{ + local dir="$1" + pushd "${dir}" >/dev/null 2>&1 && _filedir -d && popd >/dev/null 2>&1 || return +} + +__%[1]s_handle_flag() +{ + __%[1]s_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" + + # if a command required a flag, and we found it, unset must_have_one_flag() + local flagname=${words[c]} + local flagvalue="" + # if the word contained an = + if [[ ${words[c]} == *"="* ]]; then + flagvalue=${flagname#*=} # take in as flagvalue after the = + flagname=${flagname%%=*} # strip everything after the = + flagname="${flagname}=" # but put the = back + fi + __%[1]s_debug "${FUNCNAME[0]}: looking for ${flagname}" + if __%[1]s_contains_word "${flagname}" "${must_have_one_flag[@]}"; then + must_have_one_flag=() + fi + + # if you set a flag which only applies to this command, don't show subcommands + if __%[1]s_contains_word "${flagname}" "${local_nonpersistent_flags[@]}"; then + commands=() + fi + + # keep flag value with flagname as flaghash + # flaghash variable is an associative array which is only supported in bash > 3. + if [[ -z "${BASH_VERSION:-}" || "${BASH_VERSINFO[0]:-}" -gt 3 ]]; then + if [ -n "${flagvalue}" ] ; then + flaghash[${flagname}]=${flagvalue} + elif [ -n "${words[ $((c+1)) ]}" ] ; then + flaghash[${flagname}]=${words[ $((c+1)) ]} + else + flaghash[${flagname}]="true" # pad "true" for bool flag + fi + fi + + # skip the argument to a two word flag + if [[ ${words[c]} != *"="* ]] && __%[1]s_contains_word "${words[c]}" "${two_word_flags[@]}"; then + __%[1]s_debug "${FUNCNAME[0]}: found a flag ${words[c]}, skip the next argument" + c=$((c+1)) + # if we are looking for a flags value, don't show commands + if [[ $c -eq $cword ]]; then + commands=() + fi + fi + + c=$((c+1)) + +} + +__%[1]s_handle_noun() +{ + __%[1]s_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" + + if __%[1]s_contains_word "${words[c]}" "${must_have_one_noun[@]}"; then + must_have_one_noun=() + elif __%[1]s_contains_word "${words[c]}" "${noun_aliases[@]}"; then + must_have_one_noun=() + fi + + nouns+=("${words[c]}") + c=$((c+1)) +} + +__%[1]s_handle_command() +{ + __%[1]s_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" + + local next_command + if [[ -n ${last_command} ]]; then + next_command="_${last_command}_${words[c]//:/__}" + else + if [[ $c -eq 0 ]]; then + next_command="_%[1]s_root_command" + else + next_command="_${words[c]//:/__}" + fi + fi + c=$((c+1)) + __%[1]s_debug "${FUNCNAME[0]}: looking for ${next_command}" + declare -F "$next_command" >/dev/null && $next_command +} + +__%[1]s_handle_word() +{ + if [[ $c -ge $cword ]]; then + __%[1]s_handle_reply + return + fi + __%[1]s_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" + if [[ "${words[c]}" == -* ]]; then + __%[1]s_handle_flag + elif __%[1]s_contains_word "${words[c]}" "${commands[@]}"; then + __%[1]s_handle_command + elif [[ $c -eq 0 ]]; then + __%[1]s_handle_command + elif __%[1]s_contains_word "${words[c]}" "${command_aliases[@]}"; then + # aliashash variable is an associative array which is only supported in bash > 3. + if [[ -z "${BASH_VERSION:-}" || "${BASH_VERSINFO[0]:-}" -gt 3 ]]; then + words[c]=${aliashash[${words[c]}]} + __%[1]s_handle_command + else + __%[1]s_handle_noun + fi + else + __%[1]s_handle_noun + fi + __%[1]s_handle_word +} + +`, name, ShellCompNoDescRequestCmd, + ShellCompDirectiveError, ShellCompDirectiveNoSpace, ShellCompDirectiveNoFileComp, + ShellCompDirectiveFilterFileExt, ShellCompDirectiveFilterDirs, activeHelpEnvVar(name))) +} + +func writePostscript(buf io.StringWriter, name string) { + name = strings.ReplaceAll(name, ":", "__") + WriteStringAndCheck(buf, fmt.Sprintf("__start_%s()\n", name)) + WriteStringAndCheck(buf, fmt.Sprintf(`{ + local cur prev words cword split + declare -A flaghash 2>/dev/null || : + declare -A aliashash 2>/dev/null || : + if declare -F _init_completion >/dev/null 2>&1; then + _init_completion -s || return + else + __%[1]s_init_completion -n "=" || return + fi + + local c=0 + local flag_parsing_disabled= + local flags=() + local two_word_flags=() + local local_nonpersistent_flags=() + local flags_with_completion=() + local flags_completion=() + local commands=("%[1]s") + local command_aliases=() + local must_have_one_flag=() + local must_have_one_noun=() + local has_completion_function="" + local last_command="" + local nouns=() + local noun_aliases=() + + __%[1]s_handle_word +} + +`, name)) + WriteStringAndCheck(buf, fmt.Sprintf(`if [[ $(type -t compopt) = "builtin" ]]; then + complete -o default -F __start_%s %s +else + complete -o default -o nospace -F __start_%s %s +fi + +`, name, name, name, name)) + WriteStringAndCheck(buf, "# ex: ts=4 sw=4 et filetype=sh\n") +} + +func writeCommands(buf io.StringWriter, cmd *Command) { + WriteStringAndCheck(buf, " commands=()\n") + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() && c != cmd.helpCommand { + continue + } + WriteStringAndCheck(buf, fmt.Sprintf(" commands+=(%q)\n", c.Name())) + writeCmdAliases(buf, c) + } + WriteStringAndCheck(buf, "\n") +} + +func writeFlagHandler(buf io.StringWriter, name string, annotations map[string][]string, cmd *Command) { + for key, value := range annotations { + switch key { + case BashCompFilenameExt: + WriteStringAndCheck(buf, fmt.Sprintf(" flags_with_completion+=(%q)\n", name)) + + var ext string + if len(value) > 0 { + ext = fmt.Sprintf("__%s_handle_filename_extension_flag ", cmd.Root().Name()) + strings.Join(value, "|") + } else { + ext = "_filedir" + } + WriteStringAndCheck(buf, fmt.Sprintf(" flags_completion+=(%q)\n", ext)) + case BashCompCustom: + WriteStringAndCheck(buf, fmt.Sprintf(" flags_with_completion+=(%q)\n", name)) + + if len(value) > 0 { + handlers := strings.Join(value, "; ") + WriteStringAndCheck(buf, fmt.Sprintf(" flags_completion+=(%q)\n", handlers)) + } else { + WriteStringAndCheck(buf, " flags_completion+=(:)\n") + } + case BashCompSubdirsInDir: + WriteStringAndCheck(buf, fmt.Sprintf(" flags_with_completion+=(%q)\n", name)) + + var ext string + if len(value) == 1 { + ext = fmt.Sprintf("__%s_handle_subdirs_in_dir_flag ", cmd.Root().Name()) + value[0] + } else { + ext = "_filedir -d" + } + WriteStringAndCheck(buf, fmt.Sprintf(" flags_completion+=(%q)\n", ext)) + } + } +} + +const cbn = "\")\n" + +func writeShortFlag(buf io.StringWriter, flag *pflag.Flag, cmd *Command) { + name := flag.Shorthand + format := " " + if len(flag.NoOptDefVal) == 0 { + format += "two_word_" + } + format += "flags+=(\"-%s" + cbn + WriteStringAndCheck(buf, fmt.Sprintf(format, name)) + writeFlagHandler(buf, "-"+name, flag.Annotations, cmd) +} + +func writeFlag(buf io.StringWriter, flag *pflag.Flag, cmd *Command) { + name := flag.Name + format := " flags+=(\"--%s" + if len(flag.NoOptDefVal) == 0 { + format += "=" + } + format += cbn + WriteStringAndCheck(buf, fmt.Sprintf(format, name)) + if len(flag.NoOptDefVal) == 0 { + format = " two_word_flags+=(\"--%s" + cbn + WriteStringAndCheck(buf, fmt.Sprintf(format, name)) + } + writeFlagHandler(buf, "--"+name, flag.Annotations, cmd) +} + +func writeLocalNonPersistentFlag(buf io.StringWriter, flag *pflag.Flag) { + name := flag.Name + format := " local_nonpersistent_flags+=(\"--%[1]s" + cbn + if len(flag.NoOptDefVal) == 0 { + format += " local_nonpersistent_flags+=(\"--%[1]s=" + cbn + } + WriteStringAndCheck(buf, fmt.Sprintf(format, name)) + if len(flag.Shorthand) > 0 { + WriteStringAndCheck(buf, fmt.Sprintf(" local_nonpersistent_flags+=(\"-%s\")\n", flag.Shorthand)) + } +} + +// prepareCustomAnnotationsForFlags setup annotations for go completions for registered flags +func prepareCustomAnnotationsForFlags(cmd *Command) { + flagCompletionMutex.RLock() + defer flagCompletionMutex.RUnlock() + for flag := range flagCompletionFunctions { + // Make sure the completion script calls the __*_go_custom_completion function for + // every registered flag. We need to do this here (and not when the flag was registered + // for completion) so that we can know the root command name for the prefix + // of ___go_custom_completion + if flag.Annotations == nil { + flag.Annotations = map[string][]string{} + } + flag.Annotations[BashCompCustom] = []string{fmt.Sprintf("__%[1]s_handle_go_custom_completion", cmd.Root().Name())} + } +} + +func writeFlags(buf io.StringWriter, cmd *Command) { + prepareCustomAnnotationsForFlags(cmd) + WriteStringAndCheck(buf, ` flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + +`) + + if cmd.DisableFlagParsing { + WriteStringAndCheck(buf, " flag_parsing_disabled=1\n") + } + + localNonPersistentFlags := cmd.LocalNonPersistentFlags() + cmd.NonInheritedFlags().VisitAll(func(flag *pflag.Flag) { + if nonCompletableFlag(flag) { + return + } + writeFlag(buf, flag, cmd) + if len(flag.Shorthand) > 0 { + writeShortFlag(buf, flag, cmd) + } + // localNonPersistentFlags are used to stop the completion of subcommands when one is set + // if TraverseChildren is true we should allow to complete subcommands + if localNonPersistentFlags.Lookup(flag.Name) != nil && !cmd.Root().TraverseChildren { + writeLocalNonPersistentFlag(buf, flag) + } + }) + cmd.InheritedFlags().VisitAll(func(flag *pflag.Flag) { + if nonCompletableFlag(flag) { + return + } + writeFlag(buf, flag, cmd) + if len(flag.Shorthand) > 0 { + writeShortFlag(buf, flag, cmd) + } + }) + + WriteStringAndCheck(buf, "\n") +} + +func writeRequiredFlag(buf io.StringWriter, cmd *Command) { + WriteStringAndCheck(buf, " must_have_one_flag=()\n") + flags := cmd.NonInheritedFlags() + flags.VisitAll(func(flag *pflag.Flag) { + if nonCompletableFlag(flag) { + return + } + if _, ok := flag.Annotations[BashCompOneRequiredFlag]; ok { + format := " must_have_one_flag+=(\"--%s" + if flag.Value.Type() != "bool" { + format += "=" + } + format += cbn + WriteStringAndCheck(buf, fmt.Sprintf(format, flag.Name)) + + if len(flag.Shorthand) > 0 { + WriteStringAndCheck(buf, fmt.Sprintf(" must_have_one_flag+=(\"-%s"+cbn, flag.Shorthand)) + } + } + }) +} + +func writeRequiredNouns(buf io.StringWriter, cmd *Command) { + WriteStringAndCheck(buf, " must_have_one_noun=()\n") + sort.Strings(cmd.ValidArgs) + for _, value := range cmd.ValidArgs { + // Remove any description that may be included following a tab character. + // Descriptions are not supported by bash completion. + value = strings.SplitN(value, "\t", 2)[0] + WriteStringAndCheck(buf, fmt.Sprintf(" must_have_one_noun+=(%q)\n", value)) + } + if cmd.ValidArgsFunction != nil { + WriteStringAndCheck(buf, " has_completion_function=1\n") + } +} + +func writeCmdAliases(buf io.StringWriter, cmd *Command) { + if len(cmd.Aliases) == 0 { + return + } + + sort.Strings(cmd.Aliases) + + WriteStringAndCheck(buf, fmt.Sprint(` if [[ -z "${BASH_VERSION:-}" || "${BASH_VERSINFO[0]:-}" -gt 3 ]]; then`, "\n")) + for _, value := range cmd.Aliases { + WriteStringAndCheck(buf, fmt.Sprintf(" command_aliases+=(%q)\n", value)) + WriteStringAndCheck(buf, fmt.Sprintf(" aliashash[%q]=%q\n", value, cmd.Name())) + } + WriteStringAndCheck(buf, ` fi`) + WriteStringAndCheck(buf, "\n") +} +func writeArgAliases(buf io.StringWriter, cmd *Command) { + WriteStringAndCheck(buf, " noun_aliases=()\n") + sort.Strings(cmd.ArgAliases) + for _, value := range cmd.ArgAliases { + WriteStringAndCheck(buf, fmt.Sprintf(" noun_aliases+=(%q)\n", value)) + } +} + +func gen(buf io.StringWriter, cmd *Command) { + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() && c != cmd.helpCommand { + continue + } + gen(buf, c) + } + commandName := cmd.CommandPath() + commandName = strings.ReplaceAll(commandName, " ", "_") + commandName = strings.ReplaceAll(commandName, ":", "__") + + if cmd.Root() == cmd { + WriteStringAndCheck(buf, fmt.Sprintf("_%s_root_command()\n{\n", commandName)) + } else { + WriteStringAndCheck(buf, fmt.Sprintf("_%s()\n{\n", commandName)) + } + + WriteStringAndCheck(buf, fmt.Sprintf(" last_command=%q\n", commandName)) + WriteStringAndCheck(buf, "\n") + WriteStringAndCheck(buf, " command_aliases=()\n") + WriteStringAndCheck(buf, "\n") + + writeCommands(buf, cmd) + writeFlags(buf, cmd) + writeRequiredFlag(buf, cmd) + writeRequiredNouns(buf, cmd) + writeArgAliases(buf, cmd) + WriteStringAndCheck(buf, "}\n\n") +} + +// GenBashCompletion generates bash completion file and writes to the passed writer. +func (c *Command) GenBashCompletion(w io.Writer) error { + buf := new(bytes.Buffer) + writePreamble(buf, c.Name()) + if len(c.BashCompletionFunction) > 0 { + buf.WriteString(c.BashCompletionFunction + "\n") + } + gen(buf, c) + writePostscript(buf, c.Name()) + + _, err := buf.WriteTo(w) + return err +} + +func nonCompletableFlag(flag *pflag.Flag) bool { + return flag.Hidden || len(flag.Deprecated) > 0 +} + +// GenBashCompletionFile generates bash completion file. +func (c *Command) GenBashCompletionFile(filename string) error { + outFile, err := os.Create(filename) + if err != nil { + return err + } + defer outFile.Close() + + return c.GenBashCompletion(outFile) +} diff --git a/go-controller/vendor/github.com/spf13/cobra/bash_completionsV2.go b/go-controller/vendor/github.com/spf13/cobra/bash_completionsV2.go new file mode 100644 index 0000000000..d2397aa366 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/bash_completionsV2.go @@ -0,0 +1,484 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "bytes" + "fmt" + "io" + "os" +) + +func (c *Command) genBashCompletion(w io.Writer, includeDesc bool) error { + buf := new(bytes.Buffer) + genBashComp(buf, c.Name(), includeDesc) + _, err := buf.WriteTo(w) + return err +} + +func genBashComp(buf io.StringWriter, name string, includeDesc bool) { + compCmd := ShellCompRequestCmd + if !includeDesc { + compCmd = ShellCompNoDescRequestCmd + } + + WriteStringAndCheck(buf, fmt.Sprintf(`# bash completion V2 for %-36[1]s -*- shell-script -*- + +__%[1]s_debug() +{ + if [[ -n ${BASH_COMP_DEBUG_FILE-} ]]; then + echo "$*" >> "${BASH_COMP_DEBUG_FILE}" + fi +} + +# Macs have bash3 for which the bash-completion package doesn't include +# _init_completion. This is a minimal version of that function. +__%[1]s_init_completion() +{ + COMPREPLY=() + _get_comp_words_by_ref "$@" cur prev words cword +} + +# This function calls the %[1]s program to obtain the completion +# results and the directive. It fills the 'out' and 'directive' vars. +__%[1]s_get_completion_results() { + local requestComp lastParam lastChar args + + # Prepare the command to request completions for the program. + # Calling ${words[0]} instead of directly %[1]s allows handling aliases + args=("${words[@]:1}") + requestComp="${words[0]} %[2]s ${args[*]}" + + lastParam=${words[$((${#words[@]}-1))]} + lastChar=${lastParam:$((${#lastParam}-1)):1} + __%[1]s_debug "lastParam ${lastParam}, lastChar ${lastChar}" + + if [[ -z ${cur} && ${lastChar} != = ]]; then + # If the last parameter is complete (there is a space following it) + # We add an extra empty parameter so we can indicate this to the go method. + __%[1]s_debug "Adding extra empty parameter" + requestComp="${requestComp} ''" + fi + + # When completing a flag with an = (e.g., %[1]s -n=) + # bash focuses on the part after the =, so we need to remove + # the flag part from $cur + if [[ ${cur} == -*=* ]]; then + cur="${cur#*=}" + fi + + __%[1]s_debug "Calling ${requestComp}" + # Use eval to handle any environment variables and such + out=$(eval "${requestComp}" 2>/dev/null) + + # Extract the directive integer at the very end of the output following a colon (:) + directive=${out##*:} + # Remove the directive + out=${out%%:*} + if [[ ${directive} == "${out}" ]]; then + # There is not directive specified + directive=0 + fi + __%[1]s_debug "The completion directive is: ${directive}" + __%[1]s_debug "The completions are: ${out}" +} + +__%[1]s_process_completion_results() { + local shellCompDirectiveError=%[3]d + local shellCompDirectiveNoSpace=%[4]d + local shellCompDirectiveNoFileComp=%[5]d + local shellCompDirectiveFilterFileExt=%[6]d + local shellCompDirectiveFilterDirs=%[7]d + local shellCompDirectiveKeepOrder=%[8]d + + if (((directive & shellCompDirectiveError) != 0)); then + # Error code. No completion. + __%[1]s_debug "Received error from custom completion go code" + return + else + if (((directive & shellCompDirectiveNoSpace) != 0)); then + if [[ $(type -t compopt) == builtin ]]; then + __%[1]s_debug "Activating no space" + compopt -o nospace + else + __%[1]s_debug "No space directive not supported in this version of bash" + fi + fi + if (((directive & shellCompDirectiveKeepOrder) != 0)); then + if [[ $(type -t compopt) == builtin ]]; then + # no sort isn't supported for bash less than < 4.4 + if [[ ${BASH_VERSINFO[0]} -lt 4 || ( ${BASH_VERSINFO[0]} -eq 4 && ${BASH_VERSINFO[1]} -lt 4 ) ]]; then + __%[1]s_debug "No sort directive not supported in this version of bash" + else + __%[1]s_debug "Activating keep order" + compopt -o nosort + fi + else + __%[1]s_debug "No sort directive not supported in this version of bash" + fi + fi + if (((directive & shellCompDirectiveNoFileComp) != 0)); then + if [[ $(type -t compopt) == builtin ]]; then + __%[1]s_debug "Activating no file completion" + compopt +o default + else + __%[1]s_debug "No file completion directive not supported in this version of bash" + fi + fi + fi + + # Separate activeHelp from normal completions + local completions=() + local activeHelp=() + __%[1]s_extract_activeHelp + + if (((directive & shellCompDirectiveFilterFileExt) != 0)); then + # File extension filtering + local fullFilter="" filter filteringCmd + + # Do not use quotes around the $completions variable or else newline + # characters will be kept. + for filter in ${completions[*]}; do + fullFilter+="$filter|" + done + + filteringCmd="_filedir $fullFilter" + __%[1]s_debug "File filtering command: $filteringCmd" + $filteringCmd + elif (((directive & shellCompDirectiveFilterDirs) != 0)); then + # File completion for directories only + + local subdir + subdir=${completions[0]} + if [[ -n $subdir ]]; then + __%[1]s_debug "Listing directories in $subdir" + pushd "$subdir" >/dev/null 2>&1 && _filedir -d && popd >/dev/null 2>&1 || return + else + __%[1]s_debug "Listing directories in ." + _filedir -d + fi + else + __%[1]s_handle_completion_types + fi + + __%[1]s_handle_special_char "$cur" : + __%[1]s_handle_special_char "$cur" = + + # Print the activeHelp statements before we finish + __%[1]s_handle_activeHelp +} + +__%[1]s_handle_activeHelp() { + # Print the activeHelp statements + if ((${#activeHelp[*]} != 0)); then + if [ -z $COMP_TYPE ]; then + # Bash v3 does not set the COMP_TYPE variable. + printf "\n"; + printf "%%s\n" "${activeHelp[@]}" + printf "\n" + __%[1]s_reprint_commandLine + return + fi + + # Only print ActiveHelp on the second TAB press + if [ $COMP_TYPE -eq 63 ]; then + printf "\n" + printf "%%s\n" "${activeHelp[@]}" + + if ((${#COMPREPLY[*]} == 0)); then + # When there are no completion choices from the program, file completion + # may kick in if the program has not disabled it; in such a case, we want + # to know if any files will match what the user typed, so that we know if + # there will be completions presented, so that we know how to handle ActiveHelp. + # To find out, we actually trigger the file completion ourselves; + # the call to _filedir will fill COMPREPLY if files match. + if (((directive & shellCompDirectiveNoFileComp) == 0)); then + __%[1]s_debug "Listing files" + _filedir + fi + fi + + if ((${#COMPREPLY[*]} != 0)); then + # If there are completion choices to be shown, print a delimiter. + # Re-printing the command-line will automatically be done + # by the shell when it prints the completion choices. + printf -- "--" + else + # When there are no completion choices at all, we need + # to re-print the command-line since the shell will + # not be doing it itself. + __%[1]s_reprint_commandLine + fi + elif [ $COMP_TYPE -eq 37 ] || [ $COMP_TYPE -eq 42 ]; then + # For completion type: menu-complete/menu-complete-backward and insert-completions + # the completions are immediately inserted into the command-line, so we first + # print the activeHelp message and reprint the command-line since the shell won't. + printf "\n" + printf "%%s\n" "${activeHelp[@]}" + + __%[1]s_reprint_commandLine + fi + fi +} + +__%[1]s_reprint_commandLine() { + # The prompt format is only available from bash 4.4. + # We test if it is available before using it. + if (x=${PS1@P}) 2> /dev/null; then + printf "%%s" "${PS1@P}${COMP_LINE[@]}" + else + # Can't print the prompt. Just print the + # text the user had typed, it is workable enough. + printf "%%s" "${COMP_LINE[@]}" + fi +} + +# Separate activeHelp lines from real completions. +# Fills the $activeHelp and $completions arrays. +__%[1]s_extract_activeHelp() { + local activeHelpMarker="%[9]s" + local endIndex=${#activeHelpMarker} + + while IFS='' read -r comp; do + [[ -z $comp ]] && continue + + if [[ ${comp:0:endIndex} == $activeHelpMarker ]]; then + comp=${comp:endIndex} + __%[1]s_debug "ActiveHelp found: $comp" + if [[ -n $comp ]]; then + activeHelp+=("$comp") + fi + else + # Not an activeHelp line but a normal completion + completions+=("$comp") + fi + done <<<"${out}" +} + +__%[1]s_handle_completion_types() { + __%[1]s_debug "__%[1]s_handle_completion_types: COMP_TYPE is $COMP_TYPE" + + case $COMP_TYPE in + 37|42) + # Type: menu-complete/menu-complete-backward and insert-completions + # If the user requested inserting one completion at a time, or all + # completions at once on the command-line we must remove the descriptions. + # https://github.com/spf13/cobra/issues/1508 + + # If there are no completions, we don't need to do anything + (( ${#completions[@]} == 0 )) && return 0 + + local tab=$'\t' + + # Strip any description and escape the completion to handled special characters + IFS=$'\n' read -ra completions -d '' < <(printf "%%q\n" "${completions[@]%%%%$tab*}") + + # Only consider the completions that match + IFS=$'\n' read -ra COMPREPLY -d '' < <(IFS=$'\n'; compgen -W "${completions[*]}" -- "${cur}") + + # compgen looses the escaping so we need to escape all completions again since they will + # all be inserted on the command-line. + IFS=$'\n' read -ra COMPREPLY -d '' < <(printf "%%q\n" "${COMPREPLY[@]}") + ;; + + *) + # Type: complete (normal completion) + __%[1]s_handle_standard_completion_case + ;; + esac +} + +__%[1]s_handle_standard_completion_case() { + local tab=$'\t' + + # If there are no completions, we don't need to do anything + (( ${#completions[@]} == 0 )) && return 0 + + # Short circuit to optimize if we don't have descriptions + if [[ "${completions[*]}" != *$tab* ]]; then + # First, escape the completions to handle special characters + IFS=$'\n' read -ra completions -d '' < <(printf "%%q\n" "${completions[@]}") + # Only consider the completions that match what the user typed + IFS=$'\n' read -ra COMPREPLY -d '' < <(IFS=$'\n'; compgen -W "${completions[*]}" -- "${cur}") + + # compgen looses the escaping so, if there is only a single completion, we need to + # escape it again because it will be inserted on the command-line. If there are multiple + # completions, we don't want to escape them because they will be printed in a list + # and we don't want to show escape characters in that list. + if (( ${#COMPREPLY[@]} == 1 )); then + COMPREPLY[0]=$(printf "%%q" "${COMPREPLY[0]}") + fi + return 0 + fi + + local longest=0 + local compline + # Look for the longest completion so that we can format things nicely + while IFS='' read -r compline; do + [[ -z $compline ]] && continue + + # Before checking if the completion matches what the user typed, + # we need to strip any description and escape the completion to handle special + # characters because those escape characters are part of what the user typed. + # Don't call "printf" in a sub-shell because it will be much slower + # since we are in a loop. + printf -v comp "%%q" "${compline%%%%$tab*}" &>/dev/null || comp=$(printf "%%q" "${compline%%%%$tab*}") + + # Only consider the completions that match + [[ $comp == "$cur"* ]] || continue + + # The completions matches. Add it to the list of full completions including + # its description. We don't escape the completion because it may get printed + # in a list if there are more than one and we don't want show escape characters + # in that list. + COMPREPLY+=("$compline") + + # Strip any description before checking the length, and again, don't escape + # the completion because this length is only used when printing the completions + # in a list and we don't want show escape characters in that list. + comp=${compline%%%%$tab*} + if ((${#comp}>longest)); then + longest=${#comp} + fi + done < <(printf "%%s\n" "${completions[@]}") + + # If there is a single completion left, remove the description text and escape any special characters + if ((${#COMPREPLY[*]} == 1)); then + __%[1]s_debug "COMPREPLY[0]: ${COMPREPLY[0]}" + COMPREPLY[0]=$(printf "%%q" "${COMPREPLY[0]%%%%$tab*}") + __%[1]s_debug "Removed description from single completion, which is now: ${COMPREPLY[0]}" + else + # Format the descriptions + __%[1]s_format_comp_descriptions $longest + fi +} + +__%[1]s_handle_special_char() +{ + local comp="$1" + local char=$2 + if [[ "$comp" == *${char}* && "$COMP_WORDBREAKS" == *${char}* ]]; then + local word=${comp%%"${comp##*${char}}"} + local idx=${#COMPREPLY[*]} + while ((--idx >= 0)); do + COMPREPLY[idx]=${COMPREPLY[idx]#"$word"} + done + fi +} + +__%[1]s_format_comp_descriptions() +{ + local tab=$'\t' + local comp desc maxdesclength + local longest=$1 + + local i ci + for ci in ${!COMPREPLY[*]}; do + comp=${COMPREPLY[ci]} + # Properly format the description string which follows a tab character if there is one + if [[ "$comp" == *$tab* ]]; then + __%[1]s_debug "Original comp: $comp" + desc=${comp#*$tab} + comp=${comp%%%%$tab*} + + # $COLUMNS stores the current shell width. + # Remove an extra 4 because we add 2 spaces and 2 parentheses. + maxdesclength=$(( COLUMNS - longest - 4 )) + + # Make sure we can fit a description of at least 8 characters + # if we are to align the descriptions. + if ((maxdesclength > 8)); then + # Add the proper number of spaces to align the descriptions + for ((i = ${#comp} ; i < longest ; i++)); do + comp+=" " + done + else + # Don't pad the descriptions so we can fit more text after the completion + maxdesclength=$(( COLUMNS - ${#comp} - 4 )) + fi + + # If there is enough space for any description text, + # truncate the descriptions that are too long for the shell width + if ((maxdesclength > 0)); then + if ((${#desc} > maxdesclength)); then + desc=${desc:0:$(( maxdesclength - 1 ))} + desc+="…" + fi + comp+=" ($desc)" + fi + COMPREPLY[ci]=$comp + __%[1]s_debug "Final comp: $comp" + fi + done +} + +__start_%[1]s() +{ + local cur prev words cword split + + COMPREPLY=() + + # Call _init_completion from the bash-completion package + # to prepare the arguments properly + if declare -F _init_completion >/dev/null 2>&1; then + _init_completion -n =: || return + else + __%[1]s_init_completion -n =: || return + fi + + __%[1]s_debug + __%[1]s_debug "========= starting completion logic ==========" + __%[1]s_debug "cur is ${cur}, words[*] is ${words[*]}, #words[@] is ${#words[@]}, cword is $cword" + + # The user could have moved the cursor backwards on the command-line. + # We need to trigger completion from the $cword location, so we need + # to truncate the command-line ($words) up to the $cword location. + words=("${words[@]:0:$cword+1}") + __%[1]s_debug "Truncated words[*]: ${words[*]}," + + local out directive + __%[1]s_get_completion_results + __%[1]s_process_completion_results +} + +if [[ $(type -t compopt) = "builtin" ]]; then + complete -o default -F __start_%[1]s %[1]s +else + complete -o default -o nospace -F __start_%[1]s %[1]s +fi + +# ex: ts=4 sw=4 et filetype=sh +`, name, compCmd, + ShellCompDirectiveError, ShellCompDirectiveNoSpace, ShellCompDirectiveNoFileComp, + ShellCompDirectiveFilterFileExt, ShellCompDirectiveFilterDirs, ShellCompDirectiveKeepOrder, + activeHelpMarker)) +} + +// GenBashCompletionFileV2 generates Bash completion version 2. +func (c *Command) GenBashCompletionFileV2(filename string, includeDesc bool) error { + outFile, err := os.Create(filename) + if err != nil { + return err + } + defer outFile.Close() + + return c.GenBashCompletionV2(outFile, includeDesc) +} + +// GenBashCompletionV2 generates Bash completion file version 2 +// and writes it to the passed writer. +func (c *Command) GenBashCompletionV2(w io.Writer, includeDesc bool) error { + return c.genBashCompletion(w, includeDesc) +} diff --git a/go-controller/vendor/github.com/spf13/cobra/cobra.go b/go-controller/vendor/github.com/spf13/cobra/cobra.go new file mode 100644 index 0000000000..d9cd2414e2 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/cobra.go @@ -0,0 +1,246 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Commands similar to git, go tools and other modern CLI tools +// inspired by go, go-Commander, gh and subcommand + +package cobra + +import ( + "fmt" + "io" + "os" + "reflect" + "strconv" + "strings" + "text/template" + "time" + "unicode" +) + +var templateFuncs = template.FuncMap{ + "trim": strings.TrimSpace, + "trimRightSpace": trimRightSpace, + "trimTrailingWhitespaces": trimRightSpace, + "appendIfNotPresent": appendIfNotPresent, + "rpad": rpad, + "gt": Gt, + "eq": Eq, +} + +var initializers []func() +var finalizers []func() + +const ( + defaultPrefixMatching = false + defaultCommandSorting = true + defaultCaseInsensitive = false + defaultTraverseRunHooks = false +) + +// EnablePrefixMatching allows setting automatic prefix matching. Automatic prefix matching can be a dangerous thing +// to automatically enable in CLI tools. +// Set this to true to enable it. +var EnablePrefixMatching = defaultPrefixMatching + +// EnableCommandSorting controls sorting of the slice of commands, which is turned on by default. +// To disable sorting, set it to false. +var EnableCommandSorting = defaultCommandSorting + +// EnableCaseInsensitive allows case-insensitive commands names. (case sensitive by default) +var EnableCaseInsensitive = defaultCaseInsensitive + +// EnableTraverseRunHooks executes persistent pre-run and post-run hooks from all parents. +// By default this is disabled, which means only the first run hook to be found is executed. +var EnableTraverseRunHooks = defaultTraverseRunHooks + +// MousetrapHelpText enables an information splash screen on Windows +// if the CLI is started from explorer.exe. +// To disable the mousetrap, just set this variable to blank string (""). +// Works only on Microsoft Windows. +var MousetrapHelpText = `This is a command line tool. + +You need to open cmd.exe and run it from there. +` + +// MousetrapDisplayDuration controls how long the MousetrapHelpText message is displayed on Windows +// if the CLI is started from explorer.exe. Set to 0 to wait for the return key to be pressed. +// To disable the mousetrap, just set MousetrapHelpText to blank string (""). +// Works only on Microsoft Windows. +var MousetrapDisplayDuration = 5 * time.Second + +// AddTemplateFunc adds a template function that's available to Usage and Help +// template generation. +func AddTemplateFunc(name string, tmplFunc interface{}) { + templateFuncs[name] = tmplFunc +} + +// AddTemplateFuncs adds multiple template functions that are available to Usage and +// Help template generation. +func AddTemplateFuncs(tmplFuncs template.FuncMap) { + for k, v := range tmplFuncs { + templateFuncs[k] = v + } +} + +// OnInitialize sets the passed functions to be run when each command's +// Execute method is called. +func OnInitialize(y ...func()) { + initializers = append(initializers, y...) +} + +// OnFinalize sets the passed functions to be run when each command's +// Execute method is terminated. +func OnFinalize(y ...func()) { + finalizers = append(finalizers, y...) +} + +// FIXME Gt is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra. + +// Gt takes two types and checks whether the first type is greater than the second. In case of types Arrays, Chans, +// Maps and Slices, Gt will compare their lengths. Ints are compared directly while strings are first parsed as +// ints and then compared. +func Gt(a interface{}, b interface{}) bool { + var left, right int64 + av := reflect.ValueOf(a) + + switch av.Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice: + left = int64(av.Len()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + left = av.Int() + case reflect.String: + left, _ = strconv.ParseInt(av.String(), 10, 64) + } + + bv := reflect.ValueOf(b) + + switch bv.Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice: + right = int64(bv.Len()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + right = bv.Int() + case reflect.String: + right, _ = strconv.ParseInt(bv.String(), 10, 64) + } + + return left > right +} + +// FIXME Eq is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra. + +// Eq takes two types and checks whether they are equal. Supported types are int and string. Unsupported types will panic. +func Eq(a interface{}, b interface{}) bool { + av := reflect.ValueOf(a) + bv := reflect.ValueOf(b) + + switch av.Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice: + panic("Eq called on unsupported type") + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return av.Int() == bv.Int() + case reflect.String: + return av.String() == bv.String() + } + return false +} + +func trimRightSpace(s string) string { + return strings.TrimRightFunc(s, unicode.IsSpace) +} + +// FIXME appendIfNotPresent is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra. + +// appendIfNotPresent will append stringToAppend to the end of s, but only if it's not yet present in s. +func appendIfNotPresent(s, stringToAppend string) string { + if strings.Contains(s, stringToAppend) { + return s + } + return s + " " + stringToAppend +} + +// rpad adds padding to the right of a string. +func rpad(s string, padding int) string { + formattedString := fmt.Sprintf("%%-%ds", padding) + return fmt.Sprintf(formattedString, s) +} + +func tmpl(text string) *tmplFunc { + return &tmplFunc{ + tmpl: text, + fn: func(w io.Writer, data interface{}) error { + t := template.New("top") + t.Funcs(templateFuncs) + template.Must(t.Parse(text)) + return t.Execute(w, data) + }, + } +} + +// ld compares two strings and returns the levenshtein distance between them. +func ld(s, t string, ignoreCase bool) int { + if ignoreCase { + s = strings.ToLower(s) + t = strings.ToLower(t) + } + d := make([][]int, len(s)+1) + for i := range d { + d[i] = make([]int, len(t)+1) + d[i][0] = i + } + for j := range d[0] { + d[0][j] = j + } + for j := 1; j <= len(t); j++ { + for i := 1; i <= len(s); i++ { + if s[i-1] == t[j-1] { + d[i][j] = d[i-1][j-1] + } else { + min := d[i-1][j] + if d[i][j-1] < min { + min = d[i][j-1] + } + if d[i-1][j-1] < min { + min = d[i-1][j-1] + } + d[i][j] = min + 1 + } + } + + } + return d[len(s)][len(t)] +} + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +// CheckErr prints the msg with the prefix 'Error:' and exits with error code 1. If the msg is nil, it does nothing. +func CheckErr(msg interface{}) { + if msg != nil { + fmt.Fprintln(os.Stderr, "Error:", msg) + os.Exit(1) + } +} + +// WriteStringAndCheck writes a string into a buffer, and checks if the error is not nil. +func WriteStringAndCheck(b io.StringWriter, s string) { + _, err := b.WriteString(s) + CheckErr(err) +} diff --git a/go-controller/vendor/github.com/spf13/cobra/command.go b/go-controller/vendor/github.com/spf13/cobra/command.go new file mode 100644 index 0000000000..dbb2c298ba --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/command.go @@ -0,0 +1,2067 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cobra is a commander providing a simple interface to create powerful modern CLI interfaces. +// In addition to providing an interface, Cobra simultaneously provides a controller to organize your application code. +package cobra + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + flag "github.com/spf13/pflag" +) + +const ( + FlagSetByCobraAnnotation = "cobra_annotation_flag_set_by_cobra" + CommandDisplayNameAnnotation = "cobra_annotation_command_display_name" + + helpFlagName = "help" + helpCommandName = "help" +) + +// FParseErrWhitelist configures Flag parse errors to be ignored +type FParseErrWhitelist flag.ParseErrorsWhitelist + +// Group Structure to manage groups for commands +type Group struct { + ID string + Title string +} + +// Command is just that, a command for your application. +// E.g. 'go run ...' - 'run' is the command. Cobra requires +// you to define the usage and description as part of your command +// definition to ensure usability. +type Command struct { + // Use is the one-line usage message. + // Recommended syntax is as follows: + // [ ] identifies an optional argument. Arguments that are not enclosed in brackets are required. + // ... indicates that you can specify multiple values for the previous argument. + // | indicates mutually exclusive information. You can use the argument to the left of the separator or the + // argument to the right of the separator. You cannot use both arguments in a single use of the command. + // { } delimits a set of mutually exclusive arguments when one of the arguments is required. If the arguments are + // optional, they are enclosed in brackets ([ ]). + // Example: add [-F file | -D dir]... [-f format] profile + Use string + + // Aliases is an array of aliases that can be used instead of the first word in Use. + Aliases []string + + // SuggestFor is an array of command names for which this command will be suggested - + // similar to aliases but only suggests. + SuggestFor []string + + // Short is the short description shown in the 'help' output. + Short string + + // The group id under which this subcommand is grouped in the 'help' output of its parent. + GroupID string + + // Long is the long message shown in the 'help ' output. + Long string + + // Example is examples of how to use the command. + Example string + + // ValidArgs is list of all valid non-flag arguments that are accepted in shell completions + ValidArgs []Completion + // ValidArgsFunction is an optional function that provides valid non-flag arguments for shell completion. + // It is a dynamic version of using ValidArgs. + // Only one of ValidArgs and ValidArgsFunction can be used for a command. + ValidArgsFunction CompletionFunc + + // Expected arguments + Args PositionalArgs + + // ArgAliases is List of aliases for ValidArgs. + // These are not suggested to the user in the shell completion, + // but accepted if entered manually. + ArgAliases []string + + // BashCompletionFunction is custom bash functions used by the legacy bash autocompletion generator. + // For portability with other shells, it is recommended to instead use ValidArgsFunction + BashCompletionFunction string + + // Deprecated defines, if this command is deprecated and should print this string when used. + Deprecated string + + // Annotations are key/value pairs that can be used by applications to identify or + // group commands or set special options. + Annotations map[string]string + + // Version defines the version for this command. If this value is non-empty and the command does not + // define a "version" flag, a "version" boolean flag will be added to the command and, if specified, + // will print content of the "Version" variable. A shorthand "v" flag will also be added if the + // command does not define one. + Version string + + // The *Run functions are executed in the following order: + // * PersistentPreRun() + // * PreRun() + // * Run() + // * PostRun() + // * PersistentPostRun() + // All functions get the same args, the arguments after the command name. + // The *PreRun and *PostRun functions will only be executed if the Run function of the current + // command has been declared. + // + // PersistentPreRun: children of this command will inherit and execute. + PersistentPreRun func(cmd *Command, args []string) + // PersistentPreRunE: PersistentPreRun but returns an error. + PersistentPreRunE func(cmd *Command, args []string) error + // PreRun: children of this command will not inherit. + PreRun func(cmd *Command, args []string) + // PreRunE: PreRun but returns an error. + PreRunE func(cmd *Command, args []string) error + // Run: Typically the actual work function. Most commands will only implement this. + Run func(cmd *Command, args []string) + // RunE: Run but returns an error. + RunE func(cmd *Command, args []string) error + // PostRun: run after the Run command. + PostRun func(cmd *Command, args []string) + // PostRunE: PostRun but returns an error. + PostRunE func(cmd *Command, args []string) error + // PersistentPostRun: children of this command will inherit and execute after PostRun. + PersistentPostRun func(cmd *Command, args []string) + // PersistentPostRunE: PersistentPostRun but returns an error. + PersistentPostRunE func(cmd *Command, args []string) error + + // groups for subcommands + commandgroups []*Group + + // args is actual args parsed from flags. + args []string + // flagErrorBuf contains all error messages from pflag. + flagErrorBuf *bytes.Buffer + // flags is full set of flags. + flags *flag.FlagSet + // pflags contains persistent flags. + pflags *flag.FlagSet + // lflags contains local flags. + // This field does not represent internal state, it's used as a cache to optimise LocalFlags function call + lflags *flag.FlagSet + // iflags contains inherited flags. + // This field does not represent internal state, it's used as a cache to optimise InheritedFlags function call + iflags *flag.FlagSet + // parentsPflags is all persistent flags of cmd's parents. + parentsPflags *flag.FlagSet + // globNormFunc is the global normalization function + // that we can use on every pflag set and children commands + globNormFunc func(f *flag.FlagSet, name string) flag.NormalizedName + + // usageFunc is usage func defined by user. + usageFunc func(*Command) error + // usageTemplate is usage template defined by user. + usageTemplate *tmplFunc + // flagErrorFunc is func defined by user and it's called when the parsing of + // flags returns an error. + flagErrorFunc func(*Command, error) error + // helpTemplate is help template defined by user. + helpTemplate *tmplFunc + // helpFunc is help func defined by user. + helpFunc func(*Command, []string) + // helpCommand is command with usage 'help'. If it's not defined by user, + // cobra uses default help command. + helpCommand *Command + // helpCommandGroupID is the group id for the helpCommand + helpCommandGroupID string + + // completionCommandGroupID is the group id for the completion command + completionCommandGroupID string + + // versionTemplate is the version template defined by user. + versionTemplate *tmplFunc + + // errPrefix is the error message prefix defined by user. + errPrefix string + + // inReader is a reader defined by the user that replaces stdin + inReader io.Reader + // outWriter is a writer defined by the user that replaces stdout + outWriter io.Writer + // errWriter is a writer defined by the user that replaces stderr + errWriter io.Writer + + // FParseErrWhitelist flag parse errors to be ignored + FParseErrWhitelist FParseErrWhitelist + + // CompletionOptions is a set of options to control the handling of shell completion + CompletionOptions CompletionOptions + + // commandsAreSorted defines, if command slice are sorted or not. + commandsAreSorted bool + // commandCalledAs is the name or alias value used to call this command. + commandCalledAs struct { + name string + called bool + } + + ctx context.Context + + // commands is the list of commands supported by this program. + commands []*Command + // parent is a parent command for this command. + parent *Command + // Max lengths of commands' string lengths for use in padding. + commandsMaxUseLen int + commandsMaxCommandPathLen int + commandsMaxNameLen int + + // TraverseChildren parses flags on all parents before executing child command. + TraverseChildren bool + + // Hidden defines, if this command is hidden and should NOT show up in the list of available commands. + Hidden bool + + // SilenceErrors is an option to quiet errors down stream. + SilenceErrors bool + + // SilenceUsage is an option to silence usage when an error occurs. + SilenceUsage bool + + // DisableFlagParsing disables the flag parsing. + // If this is true all flags will be passed to the command as arguments. + DisableFlagParsing bool + + // DisableAutoGenTag defines, if gen tag ("Auto generated by spf13/cobra...") + // will be printed by generating docs for this command. + DisableAutoGenTag bool + + // DisableFlagsInUseLine will disable the addition of [flags] to the usage + // line of a command when printing help or generating docs + DisableFlagsInUseLine bool + + // DisableSuggestions disables the suggestions based on Levenshtein distance + // that go along with 'unknown command' messages. + DisableSuggestions bool + + // SuggestionsMinimumDistance defines minimum levenshtein distance to display suggestions. + // Must be > 0. + SuggestionsMinimumDistance int +} + +// Context returns underlying command context. If command was executed +// with ExecuteContext or the context was set with SetContext, the +// previously set context will be returned. Otherwise, nil is returned. +// +// Notice that a call to Execute and ExecuteC will replace a nil context of +// a command with a context.Background, so a background context will be +// returned by Context after one of these functions has been called. +func (c *Command) Context() context.Context { + return c.ctx +} + +// SetContext sets context for the command. This context will be overwritten by +// Command.ExecuteContext or Command.ExecuteContextC. +func (c *Command) SetContext(ctx context.Context) { + c.ctx = ctx +} + +// SetArgs sets arguments for the command. It is set to os.Args[1:] by default, if desired, can be overridden +// particularly useful when testing. +func (c *Command) SetArgs(a []string) { + c.args = a +} + +// SetOutput sets the destination for usage and error messages. +// If output is nil, os.Stderr is used. +// +// Deprecated: Use SetOut and/or SetErr instead +func (c *Command) SetOutput(output io.Writer) { + c.outWriter = output + c.errWriter = output +} + +// SetOut sets the destination for usage messages. +// If newOut is nil, os.Stdout is used. +func (c *Command) SetOut(newOut io.Writer) { + c.outWriter = newOut +} + +// SetErr sets the destination for error messages. +// If newErr is nil, os.Stderr is used. +func (c *Command) SetErr(newErr io.Writer) { + c.errWriter = newErr +} + +// SetIn sets the source for input data +// If newIn is nil, os.Stdin is used. +func (c *Command) SetIn(newIn io.Reader) { + c.inReader = newIn +} + +// SetUsageFunc sets usage function. Usage can be defined by application. +func (c *Command) SetUsageFunc(f func(*Command) error) { + c.usageFunc = f +} + +// SetUsageTemplate sets usage template. Can be defined by Application. +func (c *Command) SetUsageTemplate(s string) { + if s == "" { + c.usageTemplate = nil + return + } + c.usageTemplate = tmpl(s) +} + +// SetFlagErrorFunc sets a function to generate an error when flag parsing +// fails. +func (c *Command) SetFlagErrorFunc(f func(*Command, error) error) { + c.flagErrorFunc = f +} + +// SetHelpFunc sets help function. Can be defined by Application. +func (c *Command) SetHelpFunc(f func(*Command, []string)) { + c.helpFunc = f +} + +// SetHelpCommand sets help command. +func (c *Command) SetHelpCommand(cmd *Command) { + c.helpCommand = cmd +} + +// SetHelpCommandGroupID sets the group id of the help command. +func (c *Command) SetHelpCommandGroupID(groupID string) { + if c.helpCommand != nil { + c.helpCommand.GroupID = groupID + } + // helpCommandGroupID is used if no helpCommand is defined by the user + c.helpCommandGroupID = groupID +} + +// SetCompletionCommandGroupID sets the group id of the completion command. +func (c *Command) SetCompletionCommandGroupID(groupID string) { + // completionCommandGroupID is used if no completion command is defined by the user + c.Root().completionCommandGroupID = groupID +} + +// SetHelpTemplate sets help template to be used. Application can use it to set custom template. +func (c *Command) SetHelpTemplate(s string) { + if s == "" { + c.helpTemplate = nil + return + } + c.helpTemplate = tmpl(s) +} + +// SetVersionTemplate sets version template to be used. Application can use it to set custom template. +func (c *Command) SetVersionTemplate(s string) { + if s == "" { + c.versionTemplate = nil + return + } + c.versionTemplate = tmpl(s) +} + +// SetErrPrefix sets error message prefix to be used. Application can use it to set custom prefix. +func (c *Command) SetErrPrefix(s string) { + c.errPrefix = s +} + +// SetGlobalNormalizationFunc sets a normalization function to all flag sets and also to child commands. +// The user should not have a cyclic dependency on commands. +func (c *Command) SetGlobalNormalizationFunc(n func(f *flag.FlagSet, name string) flag.NormalizedName) { + c.Flags().SetNormalizeFunc(n) + c.PersistentFlags().SetNormalizeFunc(n) + c.globNormFunc = n + + for _, command := range c.commands { + command.SetGlobalNormalizationFunc(n) + } +} + +// OutOrStdout returns output to stdout. +func (c *Command) OutOrStdout() io.Writer { + return c.getOut(os.Stdout) +} + +// OutOrStderr returns output to stderr +func (c *Command) OutOrStderr() io.Writer { + return c.getOut(os.Stderr) +} + +// ErrOrStderr returns output to stderr +func (c *Command) ErrOrStderr() io.Writer { + return c.getErr(os.Stderr) +} + +// InOrStdin returns input to stdin +func (c *Command) InOrStdin() io.Reader { + return c.getIn(os.Stdin) +} + +func (c *Command) getOut(def io.Writer) io.Writer { + if c.outWriter != nil { + return c.outWriter + } + if c.HasParent() { + return c.parent.getOut(def) + } + return def +} + +func (c *Command) getErr(def io.Writer) io.Writer { + if c.errWriter != nil { + return c.errWriter + } + if c.HasParent() { + return c.parent.getErr(def) + } + return def +} + +func (c *Command) getIn(def io.Reader) io.Reader { + if c.inReader != nil { + return c.inReader + } + if c.HasParent() { + return c.parent.getIn(def) + } + return def +} + +// UsageFunc returns either the function set by SetUsageFunc for this command +// or a parent, or it returns a default usage function. +func (c *Command) UsageFunc() (f func(*Command) error) { + if c.usageFunc != nil { + return c.usageFunc + } + if c.HasParent() { + return c.Parent().UsageFunc() + } + return func(c *Command) error { + c.mergePersistentFlags() + fn := c.getUsageTemplateFunc() + err := fn(c.OutOrStderr(), c) + if err != nil { + c.PrintErrln(err) + } + return err + } +} + +// getUsageTemplateFunc returns the usage template function for the command +// going up the command tree if necessary. +func (c *Command) getUsageTemplateFunc() func(w io.Writer, data interface{}) error { + if c.usageTemplate != nil { + return c.usageTemplate.fn + } + + if c.HasParent() { + return c.parent.getUsageTemplateFunc() + } + return defaultUsageFunc +} + +// Usage puts out the usage for the command. +// Used when a user provides invalid input. +// Can be defined by user by overriding UsageFunc. +func (c *Command) Usage() error { + return c.UsageFunc()(c) +} + +// HelpFunc returns either the function set by SetHelpFunc for this command +// or a parent, or it returns a function with default help behavior. +func (c *Command) HelpFunc() func(*Command, []string) { + if c.helpFunc != nil { + return c.helpFunc + } + if c.HasParent() { + return c.Parent().HelpFunc() + } + return func(c *Command, a []string) { + c.mergePersistentFlags() + fn := c.getHelpTemplateFunc() + // The help should be sent to stdout + // See https://github.com/spf13/cobra/issues/1002 + err := fn(c.OutOrStdout(), c) + if err != nil { + c.PrintErrln(err) + } + } +} + +// getHelpTemplateFunc returns the help template function for the command +// going up the command tree if necessary. +func (c *Command) getHelpTemplateFunc() func(w io.Writer, data interface{}) error { + if c.helpTemplate != nil { + return c.helpTemplate.fn + } + + if c.HasParent() { + return c.parent.getHelpTemplateFunc() + } + + return defaultHelpFunc +} + +// Help puts out the help for the command. +// Used when a user calls help [command]. +// Can be defined by user by overriding HelpFunc. +func (c *Command) Help() error { + c.HelpFunc()(c, []string{}) + return nil +} + +// UsageString returns usage string. +func (c *Command) UsageString() string { + // Storing normal writers + tmpOutput := c.outWriter + tmpErr := c.errWriter + + bb := new(bytes.Buffer) + c.outWriter = bb + c.errWriter = bb + + CheckErr(c.Usage()) + + // Setting things back to normal + c.outWriter = tmpOutput + c.errWriter = tmpErr + + return bb.String() +} + +// FlagErrorFunc returns either the function set by SetFlagErrorFunc for this +// command or a parent, or it returns a function which returns the original +// error. +func (c *Command) FlagErrorFunc() (f func(*Command, error) error) { + if c.flagErrorFunc != nil { + return c.flagErrorFunc + } + + if c.HasParent() { + return c.parent.FlagErrorFunc() + } + return func(c *Command, err error) error { + return err + } +} + +var minUsagePadding = 25 + +// UsagePadding return padding for the usage. +func (c *Command) UsagePadding() int { + if c.parent == nil || minUsagePadding > c.parent.commandsMaxUseLen { + return minUsagePadding + } + return c.parent.commandsMaxUseLen +} + +var minCommandPathPadding = 11 + +// CommandPathPadding return padding for the command path. +func (c *Command) CommandPathPadding() int { + if c.parent == nil || minCommandPathPadding > c.parent.commandsMaxCommandPathLen { + return minCommandPathPadding + } + return c.parent.commandsMaxCommandPathLen +} + +var minNamePadding = 11 + +// NamePadding returns padding for the name. +func (c *Command) NamePadding() int { + if c.parent == nil || minNamePadding > c.parent.commandsMaxNameLen { + return minNamePadding + } + return c.parent.commandsMaxNameLen +} + +// UsageTemplate returns usage template for the command. +// This function is kept for backwards-compatibility reasons. +func (c *Command) UsageTemplate() string { + if c.usageTemplate != nil { + return c.usageTemplate.tmpl + } + + if c.HasParent() { + return c.parent.UsageTemplate() + } + return defaultUsageTemplate +} + +// HelpTemplate return help template for the command. +// This function is kept for backwards-compatibility reasons. +func (c *Command) HelpTemplate() string { + if c.helpTemplate != nil { + return c.helpTemplate.tmpl + } + + if c.HasParent() { + return c.parent.HelpTemplate() + } + return defaultHelpTemplate +} + +// VersionTemplate return version template for the command. +// This function is kept for backwards-compatibility reasons. +func (c *Command) VersionTemplate() string { + if c.versionTemplate != nil { + return c.versionTemplate.tmpl + } + + if c.HasParent() { + return c.parent.VersionTemplate() + } + return defaultVersionTemplate +} + +// getVersionTemplateFunc returns the version template function for the command +// going up the command tree if necessary. +func (c *Command) getVersionTemplateFunc() func(w io.Writer, data interface{}) error { + if c.versionTemplate != nil { + return c.versionTemplate.fn + } + + if c.HasParent() { + return c.parent.getVersionTemplateFunc() + } + return defaultVersionFunc +} + +// ErrPrefix return error message prefix for the command +func (c *Command) ErrPrefix() string { + if c.errPrefix != "" { + return c.errPrefix + } + + if c.HasParent() { + return c.parent.ErrPrefix() + } + return "Error:" +} + +func hasNoOptDefVal(name string, fs *flag.FlagSet) bool { + flag := fs.Lookup(name) + if flag == nil { + return false + } + return flag.NoOptDefVal != "" +} + +func shortHasNoOptDefVal(name string, fs *flag.FlagSet) bool { + if len(name) == 0 { + return false + } + + flag := fs.ShorthandLookup(name[:1]) + if flag == nil { + return false + } + return flag.NoOptDefVal != "" +} + +func stripFlags(args []string, c *Command) []string { + if len(args) == 0 { + return args + } + c.mergePersistentFlags() + + commands := []string{} + flags := c.Flags() + +Loop: + for len(args) > 0 { + s := args[0] + args = args[1:] + switch { + case s == "--": + // "--" terminates the flags + break Loop + case strings.HasPrefix(s, "--") && !strings.Contains(s, "=") && !hasNoOptDefVal(s[2:], flags): + // If '--flag arg' then + // delete arg from args. + fallthrough // (do the same as below) + case strings.HasPrefix(s, "-") && !strings.Contains(s, "=") && len(s) == 2 && !shortHasNoOptDefVal(s[1:], flags): + // If '-f arg' then + // delete 'arg' from args or break the loop if len(args) <= 1. + if len(args) <= 1 { + break Loop + } else { + args = args[1:] + continue + } + case s != "" && !strings.HasPrefix(s, "-"): + commands = append(commands, s) + } + } + + return commands +} + +// argsMinusFirstX removes only the first x from args. Otherwise, commands that look like +// openshift admin policy add-role-to-user admin my-user, lose the admin argument (arg[4]). +// Special care needs to be taken not to remove a flag value. +func (c *Command) argsMinusFirstX(args []string, x string) []string { + if len(args) == 0 { + return args + } + c.mergePersistentFlags() + flags := c.Flags() + +Loop: + for pos := 0; pos < len(args); pos++ { + s := args[pos] + switch { + case s == "--": + // -- means we have reached the end of the parseable args. Break out of the loop now. + break Loop + case strings.HasPrefix(s, "--") && !strings.Contains(s, "=") && !hasNoOptDefVal(s[2:], flags): + fallthrough + case strings.HasPrefix(s, "-") && !strings.Contains(s, "=") && len(s) == 2 && !shortHasNoOptDefVal(s[1:], flags): + // This is a flag without a default value, and an equal sign is not used. Increment pos in order to skip + // over the next arg, because that is the value of this flag. + pos++ + continue + case !strings.HasPrefix(s, "-"): + // This is not a flag or a flag value. Check to see if it matches what we're looking for, and if so, + // return the args, excluding the one at this position. + if s == x { + ret := make([]string, 0, len(args)-1) + ret = append(ret, args[:pos]...) + ret = append(ret, args[pos+1:]...) + return ret + } + } + } + return args +} + +func isFlagArg(arg string) bool { + return ((len(arg) >= 3 && arg[0:2] == "--") || + (len(arg) >= 2 && arg[0] == '-' && arg[1] != '-')) +} + +// Find the target command given the args and command tree +// Meant to be run on the highest node. Only searches down. +func (c *Command) Find(args []string) (*Command, []string, error) { + var innerfind func(*Command, []string) (*Command, []string) + + innerfind = func(c *Command, innerArgs []string) (*Command, []string) { + argsWOflags := stripFlags(innerArgs, c) + if len(argsWOflags) == 0 { + return c, innerArgs + } + nextSubCmd := argsWOflags[0] + + cmd := c.findNext(nextSubCmd) + if cmd != nil { + return innerfind(cmd, c.argsMinusFirstX(innerArgs, nextSubCmd)) + } + return c, innerArgs + } + + commandFound, a := innerfind(c, args) + if commandFound.Args == nil { + return commandFound, a, legacyArgs(commandFound, stripFlags(a, commandFound)) + } + return commandFound, a, nil +} + +func (c *Command) findSuggestions(arg string) string { + if c.DisableSuggestions { + return "" + } + if c.SuggestionsMinimumDistance <= 0 { + c.SuggestionsMinimumDistance = 2 + } + var sb strings.Builder + if suggestions := c.SuggestionsFor(arg); len(suggestions) > 0 { + sb.WriteString("\n\nDid you mean this?\n") + for _, s := range suggestions { + _, _ = fmt.Fprintf(&sb, "\t%v\n", s) + } + } + return sb.String() +} + +func (c *Command) findNext(next string) *Command { + matches := make([]*Command, 0) + for _, cmd := range c.commands { + if commandNameMatches(cmd.Name(), next) || cmd.HasAlias(next) { + cmd.commandCalledAs.name = next + return cmd + } + if EnablePrefixMatching && cmd.hasNameOrAliasPrefix(next) { + matches = append(matches, cmd) + } + } + + if len(matches) == 1 { + // Temporarily disable gosec G602, which produces a false positive. + // See https://github.com/securego/gosec/issues/1005. + return matches[0] // #nosec G602 + } + + return nil +} + +// Traverse the command tree to find the command, and parse args for +// each parent. +func (c *Command) Traverse(args []string) (*Command, []string, error) { + flags := []string{} + inFlag := false + + for i, arg := range args { + switch { + // A long flag with a space separated value + case strings.HasPrefix(arg, "--") && !strings.Contains(arg, "="): + // TODO: this isn't quite right, we should really check ahead for 'true' or 'false' + inFlag = !hasNoOptDefVal(arg[2:], c.Flags()) + flags = append(flags, arg) + continue + // A short flag with a space separated value + case strings.HasPrefix(arg, "-") && !strings.Contains(arg, "=") && len(arg) == 2 && !shortHasNoOptDefVal(arg[1:], c.Flags()): + inFlag = true + flags = append(flags, arg) + continue + // The value for a flag + case inFlag: + inFlag = false + flags = append(flags, arg) + continue + // A flag without a value, or with an `=` separated value + case isFlagArg(arg): + flags = append(flags, arg) + continue + } + + cmd := c.findNext(arg) + if cmd == nil { + return c, args, nil + } + + if err := c.ParseFlags(flags); err != nil { + return nil, args, err + } + return cmd.Traverse(args[i+1:]) + } + return c, args, nil +} + +// SuggestionsFor provides suggestions for the typedName. +func (c *Command) SuggestionsFor(typedName string) []string { + suggestions := []string{} + for _, cmd := range c.commands { + if cmd.IsAvailableCommand() { + levenshteinDistance := ld(typedName, cmd.Name(), true) + suggestByLevenshtein := levenshteinDistance <= c.SuggestionsMinimumDistance + suggestByPrefix := strings.HasPrefix(strings.ToLower(cmd.Name()), strings.ToLower(typedName)) + if suggestByLevenshtein || suggestByPrefix { + suggestions = append(suggestions, cmd.Name()) + } + for _, explicitSuggestion := range cmd.SuggestFor { + if strings.EqualFold(typedName, explicitSuggestion) { + suggestions = append(suggestions, cmd.Name()) + } + } + } + } + return suggestions +} + +// VisitParents visits all parents of the command and invokes fn on each parent. +func (c *Command) VisitParents(fn func(*Command)) { + if c.HasParent() { + fn(c.Parent()) + c.Parent().VisitParents(fn) + } +} + +// Root finds root command. +func (c *Command) Root() *Command { + if c.HasParent() { + return c.Parent().Root() + } + return c +} + +// ArgsLenAtDash will return the length of c.Flags().Args at the moment +// when a -- was found during args parsing. +func (c *Command) ArgsLenAtDash() int { + return c.Flags().ArgsLenAtDash() +} + +func (c *Command) execute(a []string) (err error) { + if c == nil { + return fmt.Errorf("called Execute() on a nil Command") + } + + if len(c.Deprecated) > 0 { + c.Printf("Command %q is deprecated, %s\n", c.Name(), c.Deprecated) + } + + // initialize help and version flag at the last point possible to allow for user + // overriding + c.InitDefaultHelpFlag() + c.InitDefaultVersionFlag() + + err = c.ParseFlags(a) + if err != nil { + return c.FlagErrorFunc()(c, err) + } + + // If help is called, regardless of other flags, return we want help. + // Also say we need help if the command isn't runnable. + helpVal, err := c.Flags().GetBool(helpFlagName) + if err != nil { + // should be impossible to get here as we always declare a help + // flag in InitDefaultHelpFlag() + c.Println("\"help\" flag declared as non-bool. Please correct your code") + return err + } + + if helpVal { + return flag.ErrHelp + } + + // for back-compat, only add version flag behavior if version is defined + if c.Version != "" { + versionVal, err := c.Flags().GetBool("version") + if err != nil { + c.Println("\"version\" flag declared as non-bool. Please correct your code") + return err + } + if versionVal { + fn := c.getVersionTemplateFunc() + err := fn(c.OutOrStdout(), c) + if err != nil { + c.Println(err) + } + return err + } + } + + if !c.Runnable() { + return flag.ErrHelp + } + + c.preRun() + + defer c.postRun() + + argWoFlags := c.Flags().Args() + if c.DisableFlagParsing { + argWoFlags = a + } + + if err := c.ValidateArgs(argWoFlags); err != nil { + return err + } + + parents := make([]*Command, 0, 5) + for p := c; p != nil; p = p.Parent() { + if EnableTraverseRunHooks { + // When EnableTraverseRunHooks is set: + // - Execute all persistent pre-runs from the root parent till this command. + // - Execute all persistent post-runs from this command till the root parent. + parents = append([]*Command{p}, parents...) + } else { + // Otherwise, execute only the first found persistent hook. + parents = append(parents, p) + } + } + for _, p := range parents { + if p.PersistentPreRunE != nil { + if err := p.PersistentPreRunE(c, argWoFlags); err != nil { + return err + } + if !EnableTraverseRunHooks { + break + } + } else if p.PersistentPreRun != nil { + p.PersistentPreRun(c, argWoFlags) + if !EnableTraverseRunHooks { + break + } + } + } + if c.PreRunE != nil { + if err := c.PreRunE(c, argWoFlags); err != nil { + return err + } + } else if c.PreRun != nil { + c.PreRun(c, argWoFlags) + } + + if err := c.ValidateRequiredFlags(); err != nil { + return err + } + if err := c.ValidateFlagGroups(); err != nil { + return err + } + + if c.RunE != nil { + if err := c.RunE(c, argWoFlags); err != nil { + return err + } + } else { + c.Run(c, argWoFlags) + } + if c.PostRunE != nil { + if err := c.PostRunE(c, argWoFlags); err != nil { + return err + } + } else if c.PostRun != nil { + c.PostRun(c, argWoFlags) + } + for p := c; p != nil; p = p.Parent() { + if p.PersistentPostRunE != nil { + if err := p.PersistentPostRunE(c, argWoFlags); err != nil { + return err + } + if !EnableTraverseRunHooks { + break + } + } else if p.PersistentPostRun != nil { + p.PersistentPostRun(c, argWoFlags) + if !EnableTraverseRunHooks { + break + } + } + } + + return nil +} + +func (c *Command) preRun() { + for _, x := range initializers { + x() + } +} + +func (c *Command) postRun() { + for _, x := range finalizers { + x() + } +} + +// ExecuteContext is the same as Execute(), but sets the ctx on the command. +// Retrieve ctx by calling cmd.Context() inside your *Run lifecycle or ValidArgs +// functions. +func (c *Command) ExecuteContext(ctx context.Context) error { + c.ctx = ctx + return c.Execute() +} + +// Execute uses the args (os.Args[1:] by default) +// and run through the command tree finding appropriate matches +// for commands and then corresponding flags. +func (c *Command) Execute() error { + _, err := c.ExecuteC() + return err +} + +// ExecuteContextC is the same as ExecuteC(), but sets the ctx on the command. +// Retrieve ctx by calling cmd.Context() inside your *Run lifecycle or ValidArgs +// functions. +func (c *Command) ExecuteContextC(ctx context.Context) (*Command, error) { + c.ctx = ctx + return c.ExecuteC() +} + +// ExecuteC executes the command. +func (c *Command) ExecuteC() (cmd *Command, err error) { + if c.ctx == nil { + c.ctx = context.Background() + } + + // Regardless of what command execute is called on, run on Root only + if c.HasParent() { + return c.Root().ExecuteC() + } + + // windows hook + if preExecHookFn != nil { + preExecHookFn(c) + } + + // initialize help at the last point to allow for user overriding + c.InitDefaultHelpCmd() + + args := c.args + + // Workaround FAIL with "go test -v" or "cobra.test -test.v", see #155 + if c.args == nil && filepath.Base(os.Args[0]) != "cobra.test" { + args = os.Args[1:] + } + + // initialize the __complete command to be used for shell completion + c.initCompleteCmd(args) + + // initialize the default completion command + c.InitDefaultCompletionCmd(args...) + + // Now that all commands have been created, let's make sure all groups + // are properly created also + c.checkCommandGroups() + + var flags []string + if c.TraverseChildren { + cmd, flags, err = c.Traverse(args) + } else { + cmd, flags, err = c.Find(args) + } + if err != nil { + // If found parse to a subcommand and then failed, talk about the subcommand + if cmd != nil { + c = cmd + } + if !c.SilenceErrors { + c.PrintErrln(c.ErrPrefix(), err.Error()) + c.PrintErrf("Run '%v --help' for usage.\n", c.CommandPath()) + } + return c, err + } + + cmd.commandCalledAs.called = true + if cmd.commandCalledAs.name == "" { + cmd.commandCalledAs.name = cmd.Name() + } + + // We have to pass global context to children command + // if context is present on the parent command. + if cmd.ctx == nil { + cmd.ctx = c.ctx + } + + err = cmd.execute(flags) + if err != nil { + // Always show help if requested, even if SilenceErrors is in + // effect + if errors.Is(err, flag.ErrHelp) { + cmd.HelpFunc()(cmd, args) + return cmd, nil + } + + // If root command has SilenceErrors flagged, + // all subcommands should respect it + if !cmd.SilenceErrors && !c.SilenceErrors { + c.PrintErrln(cmd.ErrPrefix(), err.Error()) + } + + // If root command has SilenceUsage flagged, + // all subcommands should respect it + if !cmd.SilenceUsage && !c.SilenceUsage { + c.Println(cmd.UsageString()) + } + } + return cmd, err +} + +func (c *Command) ValidateArgs(args []string) error { + if c.Args == nil { + return ArbitraryArgs(c, args) + } + return c.Args(c, args) +} + +// ValidateRequiredFlags validates all required flags are present and returns an error otherwise +func (c *Command) ValidateRequiredFlags() error { + if c.DisableFlagParsing { + return nil + } + + flags := c.Flags() + missingFlagNames := []string{} + flags.VisitAll(func(pflag *flag.Flag) { + requiredAnnotation, found := pflag.Annotations[BashCompOneRequiredFlag] + if !found { + return + } + if (requiredAnnotation[0] == "true") && !pflag.Changed { + missingFlagNames = append(missingFlagNames, pflag.Name) + } + }) + + if len(missingFlagNames) > 0 { + return fmt.Errorf(`required flag(s) "%s" not set`, strings.Join(missingFlagNames, `", "`)) + } + return nil +} + +// checkCommandGroups checks if a command has been added to a group that does not exists. +// If so, we panic because it indicates a coding error that should be corrected. +func (c *Command) checkCommandGroups() { + for _, sub := range c.commands { + // if Group is not defined let the developer know right away + if sub.GroupID != "" && !c.ContainsGroup(sub.GroupID) { + panic(fmt.Sprintf("group id '%s' is not defined for subcommand '%s'", sub.GroupID, sub.CommandPath())) + } + + sub.checkCommandGroups() + } +} + +// InitDefaultHelpFlag adds default help flag to c. +// It is called automatically by executing the c or by calling help and usage. +// If c already has help flag, it will do nothing. +func (c *Command) InitDefaultHelpFlag() { + c.mergePersistentFlags() + if c.Flags().Lookup(helpFlagName) == nil { + usage := "help for " + name := c.DisplayName() + if name == "" { + usage += "this command" + } else { + usage += name + } + c.Flags().BoolP(helpFlagName, "h", false, usage) + _ = c.Flags().SetAnnotation(helpFlagName, FlagSetByCobraAnnotation, []string{"true"}) + } +} + +// InitDefaultVersionFlag adds default version flag to c. +// It is called automatically by executing the c. +// If c already has a version flag, it will do nothing. +// If c.Version is empty, it will do nothing. +func (c *Command) InitDefaultVersionFlag() { + if c.Version == "" { + return + } + + c.mergePersistentFlags() + if c.Flags().Lookup("version") == nil { + usage := "version for " + if c.Name() == "" { + usage += "this command" + } else { + usage += c.DisplayName() + } + if c.Flags().ShorthandLookup("v") == nil { + c.Flags().BoolP("version", "v", false, usage) + } else { + c.Flags().Bool("version", false, usage) + } + _ = c.Flags().SetAnnotation("version", FlagSetByCobraAnnotation, []string{"true"}) + } +} + +// InitDefaultHelpCmd adds default help command to c. +// It is called automatically by executing the c or by calling help and usage. +// If c already has help command or c has no subcommands, it will do nothing. +func (c *Command) InitDefaultHelpCmd() { + if !c.HasSubCommands() { + return + } + + if c.helpCommand == nil { + c.helpCommand = &Command{ + Use: "help [command]", + Short: "Help about any command", + Long: `Help provides help for any command in the application. +Simply type ` + c.DisplayName() + ` help [path to command] for full details.`, + ValidArgsFunction: func(c *Command, args []string, toComplete string) ([]Completion, ShellCompDirective) { + var completions []Completion + cmd, _, e := c.Root().Find(args) + if e != nil { + return nil, ShellCompDirectiveNoFileComp + } + if cmd == nil { + // Root help command. + cmd = c.Root() + } + for _, subCmd := range cmd.Commands() { + if subCmd.IsAvailableCommand() || subCmd == cmd.helpCommand { + if strings.HasPrefix(subCmd.Name(), toComplete) { + completions = append(completions, CompletionWithDesc(subCmd.Name(), subCmd.Short)) + } + } + } + return completions, ShellCompDirectiveNoFileComp + }, + Run: func(c *Command, args []string) { + cmd, _, e := c.Root().Find(args) + if cmd == nil || e != nil { + c.Printf("Unknown help topic %#q\n", args) + CheckErr(c.Root().Usage()) + } else { + cmd.InitDefaultHelpFlag() // make possible 'help' flag to be shown + cmd.InitDefaultVersionFlag() // make possible 'version' flag to be shown + CheckErr(cmd.Help()) + } + }, + GroupID: c.helpCommandGroupID, + } + } + c.RemoveCommand(c.helpCommand) + c.AddCommand(c.helpCommand) +} + +// ResetCommands delete parent, subcommand and help command from c. +func (c *Command) ResetCommands() { + c.parent = nil + c.commands = nil + c.helpCommand = nil + c.parentsPflags = nil +} + +// Sorts commands by their names. +type commandSorterByName []*Command + +func (c commandSorterByName) Len() int { return len(c) } +func (c commandSorterByName) Swap(i, j int) { c[i], c[j] = c[j], c[i] } +func (c commandSorterByName) Less(i, j int) bool { return c[i].Name() < c[j].Name() } + +// Commands returns a sorted slice of child commands. +func (c *Command) Commands() []*Command { + // do not sort commands if it already sorted or sorting was disabled + if EnableCommandSorting && !c.commandsAreSorted { + sort.Sort(commandSorterByName(c.commands)) + c.commandsAreSorted = true + } + return c.commands +} + +// AddCommand adds one or more commands to this parent command. +func (c *Command) AddCommand(cmds ...*Command) { + for i, x := range cmds { + if cmds[i] == c { + panic("Command can't be a child of itself") + } + cmds[i].parent = c + // update max lengths + usageLen := len(x.Use) + if usageLen > c.commandsMaxUseLen { + c.commandsMaxUseLen = usageLen + } + commandPathLen := len(x.CommandPath()) + if commandPathLen > c.commandsMaxCommandPathLen { + c.commandsMaxCommandPathLen = commandPathLen + } + nameLen := len(x.Name()) + if nameLen > c.commandsMaxNameLen { + c.commandsMaxNameLen = nameLen + } + // If global normalization function exists, update all children + if c.globNormFunc != nil { + x.SetGlobalNormalizationFunc(c.globNormFunc) + } + c.commands = append(c.commands, x) + c.commandsAreSorted = false + } +} + +// Groups returns a slice of child command groups. +func (c *Command) Groups() []*Group { + return c.commandgroups +} + +// AllChildCommandsHaveGroup returns if all subcommands are assigned to a group +func (c *Command) AllChildCommandsHaveGroup() bool { + for _, sub := range c.commands { + if (sub.IsAvailableCommand() || sub == c.helpCommand) && sub.GroupID == "" { + return false + } + } + return true +} + +// ContainsGroup return if groupID exists in the list of command groups. +func (c *Command) ContainsGroup(groupID string) bool { + for _, x := range c.commandgroups { + if x.ID == groupID { + return true + } + } + return false +} + +// AddGroup adds one or more command groups to this parent command. +func (c *Command) AddGroup(groups ...*Group) { + c.commandgroups = append(c.commandgroups, groups...) +} + +// RemoveCommand removes one or more commands from a parent command. +func (c *Command) RemoveCommand(cmds ...*Command) { + commands := []*Command{} +main: + for _, command := range c.commands { + for _, cmd := range cmds { + if command == cmd { + command.parent = nil + continue main + } + } + commands = append(commands, command) + } + c.commands = commands + // recompute all lengths + c.commandsMaxUseLen = 0 + c.commandsMaxCommandPathLen = 0 + c.commandsMaxNameLen = 0 + for _, command := range c.commands { + usageLen := len(command.Use) + if usageLen > c.commandsMaxUseLen { + c.commandsMaxUseLen = usageLen + } + commandPathLen := len(command.CommandPath()) + if commandPathLen > c.commandsMaxCommandPathLen { + c.commandsMaxCommandPathLen = commandPathLen + } + nameLen := len(command.Name()) + if nameLen > c.commandsMaxNameLen { + c.commandsMaxNameLen = nameLen + } + } +} + +// Print is a convenience method to Print to the defined output, fallback to Stderr if not set. +func (c *Command) Print(i ...interface{}) { + fmt.Fprint(c.OutOrStderr(), i...) +} + +// Println is a convenience method to Println to the defined output, fallback to Stderr if not set. +func (c *Command) Println(i ...interface{}) { + c.Print(fmt.Sprintln(i...)) +} + +// Printf is a convenience method to Printf to the defined output, fallback to Stderr if not set. +func (c *Command) Printf(format string, i ...interface{}) { + c.Print(fmt.Sprintf(format, i...)) +} + +// PrintErr is a convenience method to Print to the defined Err output, fallback to Stderr if not set. +func (c *Command) PrintErr(i ...interface{}) { + fmt.Fprint(c.ErrOrStderr(), i...) +} + +// PrintErrln is a convenience method to Println to the defined Err output, fallback to Stderr if not set. +func (c *Command) PrintErrln(i ...interface{}) { + c.PrintErr(fmt.Sprintln(i...)) +} + +// PrintErrf is a convenience method to Printf to the defined Err output, fallback to Stderr if not set. +func (c *Command) PrintErrf(format string, i ...interface{}) { + c.PrintErr(fmt.Sprintf(format, i...)) +} + +// CommandPath returns the full path to this command. +func (c *Command) CommandPath() string { + if c.HasParent() { + return c.Parent().CommandPath() + " " + c.Name() + } + return c.DisplayName() +} + +// DisplayName returns the name to display in help text. Returns command Name() +// If CommandDisplayNameAnnoation is not set +func (c *Command) DisplayName() string { + if displayName, ok := c.Annotations[CommandDisplayNameAnnotation]; ok { + return displayName + } + return c.Name() +} + +// UseLine puts out the full usage for a given command (including parents). +func (c *Command) UseLine() string { + var useline string + use := strings.Replace(c.Use, c.Name(), c.DisplayName(), 1) + if c.HasParent() { + useline = c.parent.CommandPath() + " " + use + } else { + useline = use + } + if c.DisableFlagsInUseLine { + return useline + } + if c.HasAvailableFlags() && !strings.Contains(useline, "[flags]") { + useline += " [flags]" + } + return useline +} + +// DebugFlags used to determine which flags have been assigned to which commands +// and which persist. +func (c *Command) DebugFlags() { + c.Println("DebugFlags called on", c.Name()) + var debugflags func(*Command) + + debugflags = func(x *Command) { + if x.HasFlags() || x.HasPersistentFlags() { + c.Println(x.Name()) + } + if x.HasFlags() { + x.flags.VisitAll(func(f *flag.Flag) { + if x.HasPersistentFlags() && x.persistentFlag(f.Name) != nil { + c.Println(" -"+f.Shorthand+",", "--"+f.Name, "["+f.DefValue+"]", "", f.Value, " [LP]") + } else { + c.Println(" -"+f.Shorthand+",", "--"+f.Name, "["+f.DefValue+"]", "", f.Value, " [L]") + } + }) + } + if x.HasPersistentFlags() { + x.pflags.VisitAll(func(f *flag.Flag) { + if x.HasFlags() { + if x.flags.Lookup(f.Name) == nil { + c.Println(" -"+f.Shorthand+",", "--"+f.Name, "["+f.DefValue+"]", "", f.Value, " [P]") + } + } else { + c.Println(" -"+f.Shorthand+",", "--"+f.Name, "["+f.DefValue+"]", "", f.Value, " [P]") + } + }) + } + c.Println(x.flagErrorBuf) + if x.HasSubCommands() { + for _, y := range x.commands { + debugflags(y) + } + } + } + + debugflags(c) +} + +// Name returns the command's name: the first word in the use line. +func (c *Command) Name() string { + name := c.Use + i := strings.Index(name, " ") + if i >= 0 { + name = name[:i] + } + return name +} + +// HasAlias determines if a given string is an alias of the command. +func (c *Command) HasAlias(s string) bool { + for _, a := range c.Aliases { + if commandNameMatches(a, s) { + return true + } + } + return false +} + +// CalledAs returns the command name or alias that was used to invoke +// this command or an empty string if the command has not been called. +func (c *Command) CalledAs() string { + if c.commandCalledAs.called { + return c.commandCalledAs.name + } + return "" +} + +// hasNameOrAliasPrefix returns true if the Name or any of aliases start +// with prefix +func (c *Command) hasNameOrAliasPrefix(prefix string) bool { + if strings.HasPrefix(c.Name(), prefix) { + c.commandCalledAs.name = c.Name() + return true + } + for _, alias := range c.Aliases { + if strings.HasPrefix(alias, prefix) { + c.commandCalledAs.name = alias + return true + } + } + return false +} + +// NameAndAliases returns a list of the command name and all aliases +func (c *Command) NameAndAliases() string { + return strings.Join(append([]string{c.Name()}, c.Aliases...), ", ") +} + +// HasExample determines if the command has example. +func (c *Command) HasExample() bool { + return len(c.Example) > 0 +} + +// Runnable determines if the command is itself runnable. +func (c *Command) Runnable() bool { + return c.Run != nil || c.RunE != nil +} + +// HasSubCommands determines if the command has children commands. +func (c *Command) HasSubCommands() bool { + return len(c.commands) > 0 +} + +// IsAvailableCommand determines if a command is available as a non-help command +// (this includes all non deprecated/hidden commands). +func (c *Command) IsAvailableCommand() bool { + if len(c.Deprecated) != 0 || c.Hidden { + return false + } + + if c.HasParent() && c.Parent().helpCommand == c { + return false + } + + if c.Runnable() || c.HasAvailableSubCommands() { + return true + } + + return false +} + +// IsAdditionalHelpTopicCommand determines if a command is an additional +// help topic command; additional help topic command is determined by the +// fact that it is NOT runnable/hidden/deprecated, and has no sub commands that +// are runnable/hidden/deprecated. +// Concrete example: https://github.com/spf13/cobra/issues/393#issuecomment-282741924. +func (c *Command) IsAdditionalHelpTopicCommand() bool { + // if a command is runnable, deprecated, or hidden it is not a 'help' command + if c.Runnable() || len(c.Deprecated) != 0 || c.Hidden { + return false + } + + // if any non-help sub commands are found, the command is not a 'help' command + for _, sub := range c.commands { + if !sub.IsAdditionalHelpTopicCommand() { + return false + } + } + + // the command either has no sub commands, or no non-help sub commands + return true +} + +// HasHelpSubCommands determines if a command has any available 'help' sub commands +// that need to be shown in the usage/help default template under 'additional help +// topics'. +func (c *Command) HasHelpSubCommands() bool { + // return true on the first found available 'help' sub command + for _, sub := range c.commands { + if sub.IsAdditionalHelpTopicCommand() { + return true + } + } + + // the command either has no sub commands, or no available 'help' sub commands + return false +} + +// HasAvailableSubCommands determines if a command has available sub commands that +// need to be shown in the usage/help default template under 'available commands'. +func (c *Command) HasAvailableSubCommands() bool { + // return true on the first found available (non deprecated/help/hidden) + // sub command + for _, sub := range c.commands { + if sub.IsAvailableCommand() { + return true + } + } + + // the command either has no sub commands, or no available (non deprecated/help/hidden) + // sub commands + return false +} + +// HasParent determines if the command is a child command. +func (c *Command) HasParent() bool { + return c.parent != nil +} + +// GlobalNormalizationFunc returns the global normalization function or nil if it doesn't exist. +func (c *Command) GlobalNormalizationFunc() func(f *flag.FlagSet, name string) flag.NormalizedName { + return c.globNormFunc +} + +// Flags returns the complete FlagSet that applies +// to this command (local and persistent declared here and by all parents). +func (c *Command) Flags() *flag.FlagSet { + if c.flags == nil { + c.flags = flag.NewFlagSet(c.DisplayName(), flag.ContinueOnError) + if c.flagErrorBuf == nil { + c.flagErrorBuf = new(bytes.Buffer) + } + c.flags.SetOutput(c.flagErrorBuf) + } + + return c.flags +} + +// LocalNonPersistentFlags are flags specific to this command which will NOT persist to subcommands. +// This function does not modify the flags of the current command, it's purpose is to return the current state. +func (c *Command) LocalNonPersistentFlags() *flag.FlagSet { + persistentFlags := c.PersistentFlags() + + out := flag.NewFlagSet(c.DisplayName(), flag.ContinueOnError) + c.LocalFlags().VisitAll(func(f *flag.Flag) { + if persistentFlags.Lookup(f.Name) == nil { + out.AddFlag(f) + } + }) + return out +} + +// LocalFlags returns the local FlagSet specifically set in the current command. +// This function does not modify the flags of the current command, it's purpose is to return the current state. +func (c *Command) LocalFlags() *flag.FlagSet { + c.mergePersistentFlags() + + if c.lflags == nil { + c.lflags = flag.NewFlagSet(c.DisplayName(), flag.ContinueOnError) + if c.flagErrorBuf == nil { + c.flagErrorBuf = new(bytes.Buffer) + } + c.lflags.SetOutput(c.flagErrorBuf) + } + c.lflags.SortFlags = c.Flags().SortFlags + if c.globNormFunc != nil { + c.lflags.SetNormalizeFunc(c.globNormFunc) + } + + addToLocal := func(f *flag.Flag) { + // Add the flag if it is not a parent PFlag, or it shadows a parent PFlag + if c.lflags.Lookup(f.Name) == nil && f != c.parentsPflags.Lookup(f.Name) { + c.lflags.AddFlag(f) + } + } + c.Flags().VisitAll(addToLocal) + c.PersistentFlags().VisitAll(addToLocal) + return c.lflags +} + +// InheritedFlags returns all flags which were inherited from parent commands. +// This function does not modify the flags of the current command, it's purpose is to return the current state. +func (c *Command) InheritedFlags() *flag.FlagSet { + c.mergePersistentFlags() + + if c.iflags == nil { + c.iflags = flag.NewFlagSet(c.DisplayName(), flag.ContinueOnError) + if c.flagErrorBuf == nil { + c.flagErrorBuf = new(bytes.Buffer) + } + c.iflags.SetOutput(c.flagErrorBuf) + } + + local := c.LocalFlags() + if c.globNormFunc != nil { + c.iflags.SetNormalizeFunc(c.globNormFunc) + } + + c.parentsPflags.VisitAll(func(f *flag.Flag) { + if c.iflags.Lookup(f.Name) == nil && local.Lookup(f.Name) == nil { + c.iflags.AddFlag(f) + } + }) + return c.iflags +} + +// NonInheritedFlags returns all flags which were not inherited from parent commands. +// This function does not modify the flags of the current command, it's purpose is to return the current state. +func (c *Command) NonInheritedFlags() *flag.FlagSet { + return c.LocalFlags() +} + +// PersistentFlags returns the persistent FlagSet specifically set in the current command. +func (c *Command) PersistentFlags() *flag.FlagSet { + if c.pflags == nil { + c.pflags = flag.NewFlagSet(c.DisplayName(), flag.ContinueOnError) + if c.flagErrorBuf == nil { + c.flagErrorBuf = new(bytes.Buffer) + } + c.pflags.SetOutput(c.flagErrorBuf) + } + return c.pflags +} + +// ResetFlags deletes all flags from command. +func (c *Command) ResetFlags() { + c.flagErrorBuf = new(bytes.Buffer) + c.flagErrorBuf.Reset() + c.flags = flag.NewFlagSet(c.DisplayName(), flag.ContinueOnError) + c.flags.SetOutput(c.flagErrorBuf) + c.pflags = flag.NewFlagSet(c.DisplayName(), flag.ContinueOnError) + c.pflags.SetOutput(c.flagErrorBuf) + + c.lflags = nil + c.iflags = nil + c.parentsPflags = nil +} + +// HasFlags checks if the command contains any flags (local plus persistent from the entire structure). +func (c *Command) HasFlags() bool { + return c.Flags().HasFlags() +} + +// HasPersistentFlags checks if the command contains persistent flags. +func (c *Command) HasPersistentFlags() bool { + return c.PersistentFlags().HasFlags() +} + +// HasLocalFlags checks if the command has flags specifically declared locally. +func (c *Command) HasLocalFlags() bool { + return c.LocalFlags().HasFlags() +} + +// HasInheritedFlags checks if the command has flags inherited from its parent command. +func (c *Command) HasInheritedFlags() bool { + return c.InheritedFlags().HasFlags() +} + +// HasAvailableFlags checks if the command contains any flags (local plus persistent from the entire +// structure) which are not hidden or deprecated. +func (c *Command) HasAvailableFlags() bool { + return c.Flags().HasAvailableFlags() +} + +// HasAvailablePersistentFlags checks if the command contains persistent flags which are not hidden or deprecated. +func (c *Command) HasAvailablePersistentFlags() bool { + return c.PersistentFlags().HasAvailableFlags() +} + +// HasAvailableLocalFlags checks if the command has flags specifically declared locally which are not hidden +// or deprecated. +func (c *Command) HasAvailableLocalFlags() bool { + return c.LocalFlags().HasAvailableFlags() +} + +// HasAvailableInheritedFlags checks if the command has flags inherited from its parent command which are +// not hidden or deprecated. +func (c *Command) HasAvailableInheritedFlags() bool { + return c.InheritedFlags().HasAvailableFlags() +} + +// Flag climbs up the command tree looking for matching flag. +func (c *Command) Flag(name string) (flag *flag.Flag) { + flag = c.Flags().Lookup(name) + + if flag == nil { + flag = c.persistentFlag(name) + } + + return +} + +// Recursively find matching persistent flag. +func (c *Command) persistentFlag(name string) (flag *flag.Flag) { + if c.HasPersistentFlags() { + flag = c.PersistentFlags().Lookup(name) + } + + if flag == nil { + c.updateParentsPflags() + flag = c.parentsPflags.Lookup(name) + } + return +} + +// ParseFlags parses persistent flag tree and local flags. +func (c *Command) ParseFlags(args []string) error { + if c.DisableFlagParsing { + return nil + } + + if c.flagErrorBuf == nil { + c.flagErrorBuf = new(bytes.Buffer) + } + beforeErrorBufLen := c.flagErrorBuf.Len() + c.mergePersistentFlags() + + // do it here after merging all flags and just before parse + c.Flags().ParseErrorsWhitelist = flag.ParseErrorsWhitelist(c.FParseErrWhitelist) + + err := c.Flags().Parse(args) + // Print warnings if they occurred (e.g. deprecated flag messages). + if c.flagErrorBuf.Len()-beforeErrorBufLen > 0 && err == nil { + c.Print(c.flagErrorBuf.String()) + } + + return err +} + +// Parent returns a commands parent command. +func (c *Command) Parent() *Command { + return c.parent +} + +// mergePersistentFlags merges c.PersistentFlags() to c.Flags() +// and adds missing persistent flags of all parents. +func (c *Command) mergePersistentFlags() { + c.updateParentsPflags() + c.Flags().AddFlagSet(c.PersistentFlags()) + c.Flags().AddFlagSet(c.parentsPflags) +} + +// updateParentsPflags updates c.parentsPflags by adding +// new persistent flags of all parents. +// If c.parentsPflags == nil, it makes new. +func (c *Command) updateParentsPflags() { + if c.parentsPflags == nil { + c.parentsPflags = flag.NewFlagSet(c.DisplayName(), flag.ContinueOnError) + c.parentsPflags.SetOutput(c.flagErrorBuf) + c.parentsPflags.SortFlags = false + } + + if c.globNormFunc != nil { + c.parentsPflags.SetNormalizeFunc(c.globNormFunc) + } + + c.Root().PersistentFlags().AddFlagSet(flag.CommandLine) + + c.VisitParents(func(parent *Command) { + c.parentsPflags.AddFlagSet(parent.PersistentFlags()) + }) +} + +// commandNameMatches checks if two command names are equal +// taking into account case sensitivity according to +// EnableCaseInsensitive global configuration. +func commandNameMatches(s string, t string) bool { + if EnableCaseInsensitive { + return strings.EqualFold(s, t) + } + + return s == t +} + +// tmplFunc holds a template and a function that will execute said template. +type tmplFunc struct { + tmpl string + fn func(io.Writer, interface{}) error +} + +var defaultUsageTemplate = `Usage:{{if .Runnable}} + {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} + {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} + +Aliases: + {{.NameAndAliases}}{{end}}{{if .HasExample}} + +Examples: +{{.Example}}{{end}}{{if .HasAvailableSubCommands}}{{$cmds := .Commands}}{{if eq (len .Groups) 0}} + +Available Commands:{{range $cmds}}{{if (or .IsAvailableCommand (eq .Name "help"))}} + {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{else}}{{range $group := .Groups}} + +{{.Title}}{{range $cmds}}{{if (and (eq .GroupID $group.ID) (or .IsAvailableCommand (eq .Name "help")))}} + {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if not .AllChildCommandsHaveGroup}} + +Additional Commands:{{range $cmds}}{{if (and (eq .GroupID "") (or .IsAvailableCommand (eq .Name "help")))}} + {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} + +Flags: +{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} + +Global Flags: +{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} + +Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} + {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} + +Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}} +` + +// defaultUsageFunc is equivalent to executing defaultUsageTemplate. The two should be changed in sync. +func defaultUsageFunc(w io.Writer, in interface{}) error { + c := in.(*Command) + fmt.Fprint(w, "Usage:") + if c.Runnable() { + fmt.Fprintf(w, "\n %s", c.UseLine()) + } + if c.HasAvailableSubCommands() { + fmt.Fprintf(w, "\n %s [command]", c.CommandPath()) + } + if len(c.Aliases) > 0 { + fmt.Fprintf(w, "\n\nAliases:\n") + fmt.Fprintf(w, " %s", c.NameAndAliases()) + } + if c.HasExample() { + fmt.Fprintf(w, "\n\nExamples:\n") + fmt.Fprintf(w, "%s", c.Example) + } + if c.HasAvailableSubCommands() { + cmds := c.Commands() + if len(c.Groups()) == 0 { + fmt.Fprintf(w, "\n\nAvailable Commands:") + for _, subcmd := range cmds { + if subcmd.IsAvailableCommand() || subcmd.Name() == helpCommandName { + fmt.Fprintf(w, "\n %s %s", rpad(subcmd.Name(), subcmd.NamePadding()), subcmd.Short) + } + } + } else { + for _, group := range c.Groups() { + fmt.Fprintf(w, "\n\n%s", group.Title) + for _, subcmd := range cmds { + if subcmd.GroupID == group.ID && (subcmd.IsAvailableCommand() || subcmd.Name() == helpCommandName) { + fmt.Fprintf(w, "\n %s %s", rpad(subcmd.Name(), subcmd.NamePadding()), subcmd.Short) + } + } + } + if !c.AllChildCommandsHaveGroup() { + fmt.Fprintf(w, "\n\nAdditional Commands:") + for _, subcmd := range cmds { + if subcmd.GroupID == "" && (subcmd.IsAvailableCommand() || subcmd.Name() == helpCommandName) { + fmt.Fprintf(w, "\n %s %s", rpad(subcmd.Name(), subcmd.NamePadding()), subcmd.Short) + } + } + } + } + } + if c.HasAvailableLocalFlags() { + fmt.Fprintf(w, "\n\nFlags:\n") + fmt.Fprint(w, trimRightSpace(c.LocalFlags().FlagUsages())) + } + if c.HasAvailableInheritedFlags() { + fmt.Fprintf(w, "\n\nGlobal Flags:\n") + fmt.Fprint(w, trimRightSpace(c.InheritedFlags().FlagUsages())) + } + if c.HasHelpSubCommands() { + fmt.Fprintf(w, "\n\nAdditional help topcis:") + for _, subcmd := range c.Commands() { + if subcmd.IsAdditionalHelpTopicCommand() { + fmt.Fprintf(w, "\n %s %s", rpad(subcmd.CommandPath(), subcmd.CommandPathPadding()), subcmd.Short) + } + } + } + if c.HasAvailableSubCommands() { + fmt.Fprintf(w, "\n\nUse \"%s [command] --help\" for more information about a command.", c.CommandPath()) + } + fmt.Fprintln(w) + return nil +} + +var defaultHelpTemplate = `{{with (or .Long .Short)}}{{. | trimTrailingWhitespaces}} + +{{end}}{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}}` + +// defaultHelpFunc is equivalent to executing defaultHelpTemplate. The two should be changed in sync. +func defaultHelpFunc(w io.Writer, in interface{}) error { + c := in.(*Command) + usage := c.Long + if usage == "" { + usage = c.Short + } + usage = trimRightSpace(usage) + if usage != "" { + fmt.Fprintln(w, usage) + fmt.Fprintln(w) + } + if c.Runnable() || c.HasSubCommands() { + fmt.Fprint(w, c.UsageString()) + } + return nil +} + +var defaultVersionTemplate = `{{with .DisplayName}}{{printf "%s " .}}{{end}}{{printf "version %s" .Version}} +` + +// defaultVersionFunc is equivalent to executing defaultVersionTemplate. The two should be changed in sync. +func defaultVersionFunc(w io.Writer, in interface{}) error { + c := in.(*Command) + _, err := fmt.Fprintf(w, "%s version %s\n", c.DisplayName(), c.Version) + return err +} diff --git a/go-controller/vendor/github.com/spf13/cobra/command_notwin.go b/go-controller/vendor/github.com/spf13/cobra/command_notwin.go new file mode 100644 index 0000000000..307f0c127f --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/command_notwin.go @@ -0,0 +1,20 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows +// +build !windows + +package cobra + +var preExecHookFn func(*Command) diff --git a/go-controller/vendor/github.com/spf13/cobra/command_win.go b/go-controller/vendor/github.com/spf13/cobra/command_win.go new file mode 100644 index 0000000000..adbef395c2 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/command_win.go @@ -0,0 +1,41 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows +// +build windows + +package cobra + +import ( + "fmt" + "os" + "time" + + "github.com/inconshreveable/mousetrap" +) + +var preExecHookFn = preExecHook + +func preExecHook(c *Command) { + if MousetrapHelpText != "" && mousetrap.StartedByExplorer() { + c.Print(MousetrapHelpText) + if MousetrapDisplayDuration > 0 { + time.Sleep(MousetrapDisplayDuration) + } else { + c.Println("Press return to continue...") + fmt.Scanln() + } + os.Exit(1) + } +} diff --git a/go-controller/vendor/github.com/spf13/cobra/completions.go b/go-controller/vendor/github.com/spf13/cobra/completions.go new file mode 100644 index 0000000000..a1752f7631 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/completions.go @@ -0,0 +1,1005 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "fmt" + "os" + "regexp" + "strconv" + "strings" + "sync" + + "github.com/spf13/pflag" +) + +const ( + // ShellCompRequestCmd is the name of the hidden command that is used to request + // completion results from the program. It is used by the shell completion scripts. + ShellCompRequestCmd = "__complete" + // ShellCompNoDescRequestCmd is the name of the hidden command that is used to request + // completion results without their description. It is used by the shell completion scripts. + ShellCompNoDescRequestCmd = "__completeNoDesc" +) + +// Global map of flag completion functions. Make sure to use flagCompletionMutex before you try to read and write from it. +var flagCompletionFunctions = map[*pflag.Flag]CompletionFunc{} + +// lock for reading and writing from flagCompletionFunctions +var flagCompletionMutex = &sync.RWMutex{} + +// ShellCompDirective is a bit map representing the different behaviors the shell +// can be instructed to have once completions have been provided. +type ShellCompDirective int + +type flagCompError struct { + subCommand string + flagName string +} + +func (e *flagCompError) Error() string { + return "Subcommand '" + e.subCommand + "' does not support flag '" + e.flagName + "'" +} + +const ( + // ShellCompDirectiveError indicates an error occurred and completions should be ignored. + ShellCompDirectiveError ShellCompDirective = 1 << iota + + // ShellCompDirectiveNoSpace indicates that the shell should not add a space + // after the completion even if there is a single completion provided. + ShellCompDirectiveNoSpace + + // ShellCompDirectiveNoFileComp indicates that the shell should not provide + // file completion even when no completion is provided. + ShellCompDirectiveNoFileComp + + // ShellCompDirectiveFilterFileExt indicates that the provided completions + // should be used as file extension filters. + // For flags, using Command.MarkFlagFilename() and Command.MarkPersistentFlagFilename() + // is a shortcut to using this directive explicitly. The BashCompFilenameExt + // annotation can also be used to obtain the same behavior for flags. + ShellCompDirectiveFilterFileExt + + // ShellCompDirectiveFilterDirs indicates that only directory names should + // be provided in file completion. To request directory names within another + // directory, the returned completions should specify the directory within + // which to search. The BashCompSubdirsInDir annotation can be used to + // obtain the same behavior but only for flags. + ShellCompDirectiveFilterDirs + + // ShellCompDirectiveKeepOrder indicates that the shell should preserve the order + // in which the completions are provided + ShellCompDirectiveKeepOrder + + // =========================================================================== + + // All directives using iota should be above this one. + // For internal use. + shellCompDirectiveMaxValue + + // ShellCompDirectiveDefault indicates to let the shell perform its default + // behavior after completions have been provided. + // This one must be last to avoid messing up the iota count. + ShellCompDirectiveDefault ShellCompDirective = 0 +) + +const ( + // Constants for the completion command + compCmdName = "completion" + compCmdNoDescFlagName = "no-descriptions" + compCmdNoDescFlagDesc = "disable completion descriptions" + compCmdNoDescFlagDefault = false +) + +// CompletionOptions are the options to control shell completion +type CompletionOptions struct { + // DisableDefaultCmd prevents Cobra from creating a default 'completion' command + DisableDefaultCmd bool + // DisableNoDescFlag prevents Cobra from creating the '--no-descriptions' flag + // for shells that support completion descriptions + DisableNoDescFlag bool + // DisableDescriptions turns off all completion descriptions for shells + // that support them + DisableDescriptions bool + // HiddenDefaultCmd makes the default 'completion' command hidden + HiddenDefaultCmd bool +} + +// Completion is a string that can be used for completions +// +// two formats are supported: +// - the completion choice +// - the completion choice with a textual description (separated by a TAB). +// +// [CompletionWithDesc] can be used to create a completion string with a textual description. +// +// Note: Go type alias is used to provide a more descriptive name in the documentation, but any string can be used. +type Completion = string + +// CompletionFunc is a function that provides completion results. +type CompletionFunc = func(cmd *Command, args []string, toComplete string) ([]Completion, ShellCompDirective) + +// CompletionWithDesc returns a [Completion] with a description by using the TAB delimited format. +func CompletionWithDesc(choice string, description string) Completion { + return choice + "\t" + description +} + +// NoFileCompletions can be used to disable file completion for commands that should +// not trigger file completions. +// +// This method satisfies [CompletionFunc]. +// It can be used with [Command.RegisterFlagCompletionFunc] and for [Command.ValidArgsFunction]. +func NoFileCompletions(cmd *Command, args []string, toComplete string) ([]Completion, ShellCompDirective) { + return nil, ShellCompDirectiveNoFileComp +} + +// FixedCompletions can be used to create a completion function which always +// returns the same results. +// +// This method returns a function that satisfies [CompletionFunc] +// It can be used with [Command.RegisterFlagCompletionFunc] and for [Command.ValidArgsFunction]. +func FixedCompletions(choices []Completion, directive ShellCompDirective) CompletionFunc { + return func(cmd *Command, args []string, toComplete string) ([]Completion, ShellCompDirective) { + return choices, directive + } +} + +// RegisterFlagCompletionFunc should be called to register a function to provide completion for a flag. +// +// You can use pre-defined completion functions such as [FixedCompletions] or [NoFileCompletions], +// or you can define your own. +func (c *Command) RegisterFlagCompletionFunc(flagName string, f CompletionFunc) error { + flag := c.Flag(flagName) + if flag == nil { + return fmt.Errorf("RegisterFlagCompletionFunc: flag '%s' does not exist", flagName) + } + flagCompletionMutex.Lock() + defer flagCompletionMutex.Unlock() + + if _, exists := flagCompletionFunctions[flag]; exists { + return fmt.Errorf("RegisterFlagCompletionFunc: flag '%s' already registered", flagName) + } + flagCompletionFunctions[flag] = f + return nil +} + +// GetFlagCompletionFunc returns the completion function for the given flag of the command, if available. +func (c *Command) GetFlagCompletionFunc(flagName string) (CompletionFunc, bool) { + flag := c.Flag(flagName) + if flag == nil { + return nil, false + } + + flagCompletionMutex.RLock() + defer flagCompletionMutex.RUnlock() + + completionFunc, exists := flagCompletionFunctions[flag] + return completionFunc, exists +} + +// Returns a string listing the different directive enabled in the specified parameter +func (d ShellCompDirective) string() string { + var directives []string + if d&ShellCompDirectiveError != 0 { + directives = append(directives, "ShellCompDirectiveError") + } + if d&ShellCompDirectiveNoSpace != 0 { + directives = append(directives, "ShellCompDirectiveNoSpace") + } + if d&ShellCompDirectiveNoFileComp != 0 { + directives = append(directives, "ShellCompDirectiveNoFileComp") + } + if d&ShellCompDirectiveFilterFileExt != 0 { + directives = append(directives, "ShellCompDirectiveFilterFileExt") + } + if d&ShellCompDirectiveFilterDirs != 0 { + directives = append(directives, "ShellCompDirectiveFilterDirs") + } + if d&ShellCompDirectiveKeepOrder != 0 { + directives = append(directives, "ShellCompDirectiveKeepOrder") + } + if len(directives) == 0 { + directives = append(directives, "ShellCompDirectiveDefault") + } + + if d >= shellCompDirectiveMaxValue { + return fmt.Sprintf("ERROR: unexpected ShellCompDirective value: %d", d) + } + return strings.Join(directives, ", ") +} + +// initCompleteCmd adds a special hidden command that can be used to request custom completions. +func (c *Command) initCompleteCmd(args []string) { + completeCmd := &Command{ + Use: fmt.Sprintf("%s [command-line]", ShellCompRequestCmd), + Aliases: []string{ShellCompNoDescRequestCmd}, + DisableFlagsInUseLine: true, + Hidden: true, + DisableFlagParsing: true, + Args: MinimumNArgs(1), + Short: "Request shell completion choices for the specified command-line", + Long: fmt.Sprintf("%[2]s is a special command that is used by the shell completion logic\n%[1]s", + "to request completion choices for the specified command-line.", ShellCompRequestCmd), + Run: func(cmd *Command, args []string) { + finalCmd, completions, directive, err := cmd.getCompletions(args) + if err != nil { + CompErrorln(err.Error()) + // Keep going for multiple reasons: + // 1- There could be some valid completions even though there was an error + // 2- Even without completions, we need to print the directive + } + + noDescriptions := cmd.CalledAs() == ShellCompNoDescRequestCmd + if !noDescriptions { + if doDescriptions, err := strconv.ParseBool(getEnvConfig(cmd, configEnvVarSuffixDescriptions)); err == nil { + noDescriptions = !doDescriptions + } + } + noActiveHelp := GetActiveHelpConfig(finalCmd) == activeHelpGlobalDisable + out := finalCmd.OutOrStdout() + for _, comp := range completions { + if noActiveHelp && strings.HasPrefix(comp, activeHelpMarker) { + // Remove all activeHelp entries if it's disabled. + continue + } + if noDescriptions { + // Remove any description that may be included following a tab character. + comp = strings.SplitN(comp, "\t", 2)[0] + } + + // Make sure we only write the first line to the output. + // This is needed if a description contains a linebreak. + // Otherwise the shell scripts will interpret the other lines as new flags + // and could therefore provide a wrong completion. + comp = strings.SplitN(comp, "\n", 2)[0] + + // Finally trim the completion. This is especially important to get rid + // of a trailing tab when there are no description following it. + // For example, a sub-command without a description should not be completed + // with a tab at the end (or else zsh will show a -- following it + // although there is no description). + comp = strings.TrimSpace(comp) + + // Print each possible completion to the output for the completion script to consume. + fmt.Fprintln(out, comp) + } + + // As the last printout, print the completion directive for the completion script to parse. + // The directive integer must be that last character following a single colon (:). + // The completion script expects : + fmt.Fprintf(out, ":%d\n", directive) + + // Print some helpful info to stderr for the user to understand. + // Output from stderr must be ignored by the completion script. + fmt.Fprintf(finalCmd.ErrOrStderr(), "Completion ended with directive: %s\n", directive.string()) + }, + } + c.AddCommand(completeCmd) + subCmd, _, err := c.Find(args) + if err != nil || subCmd.Name() != ShellCompRequestCmd { + // Only create this special command if it is actually being called. + // This reduces possible side-effects of creating such a command; + // for example, having this command would cause problems to a + // cobra program that only consists of the root command, since this + // command would cause the root command to suddenly have a subcommand. + c.RemoveCommand(completeCmd) + } +} + +// SliceValue is a reduced version of [pflag.SliceValue]. It is used to detect +// flags that accept multiple values and therefore can provide completion +// multiple times. +type SliceValue interface { + // GetSlice returns the flag value list as an array of strings. + GetSlice() []string +} + +func (c *Command) getCompletions(args []string) (*Command, []Completion, ShellCompDirective, error) { + // The last argument, which is not completely typed by the user, + // should not be part of the list of arguments + toComplete := args[len(args)-1] + trimmedArgs := args[:len(args)-1] + + var finalCmd *Command + var finalArgs []string + var err error + // Find the real command for which completion must be performed + // check if we need to traverse here to parse local flags on parent commands + if c.Root().TraverseChildren { + finalCmd, finalArgs, err = c.Root().Traverse(trimmedArgs) + } else { + // For Root commands that don't specify any value for their Args fields, when we call + // Find(), if those Root commands don't have any sub-commands, they will accept arguments. + // However, because we have added the __complete sub-command in the current code path, the + // call to Find() -> legacyArgs() will return an error if there are any arguments. + // To avoid this, we first remove the __complete command to get back to having no sub-commands. + rootCmd := c.Root() + if len(rootCmd.Commands()) == 1 { + rootCmd.RemoveCommand(c) + } + + finalCmd, finalArgs, err = rootCmd.Find(trimmedArgs) + } + if err != nil { + // Unable to find the real command. E.g., someInvalidCmd + return c, []Completion{}, ShellCompDirectiveDefault, fmt.Errorf("unable to find a command for arguments: %v", trimmedArgs) + } + finalCmd.ctx = c.ctx + + // These flags are normally added when `execute()` is called on `finalCmd`, + // however, when doing completion, we don't call `finalCmd.execute()`. + // Let's add the --help and --version flag ourselves but only if the finalCmd + // has not disabled flag parsing; if flag parsing is disabled, it is up to the + // finalCmd itself to handle the completion of *all* flags. + if !finalCmd.DisableFlagParsing { + finalCmd.InitDefaultHelpFlag() + finalCmd.InitDefaultVersionFlag() + } + + // Check if we are doing flag value completion before parsing the flags. + // This is important because if we are completing a flag value, we need to also + // remove the flag name argument from the list of finalArgs or else the parsing + // could fail due to an invalid value (incomplete) for the flag. + flag, finalArgs, toComplete, flagErr := checkIfFlagCompletion(finalCmd, finalArgs, toComplete) + + // Check if interspersed is false or -- was set on a previous arg. + // This works by counting the arguments. Normally -- is not counted as arg but + // if -- was already set or interspersed is false and there is already one arg then + // the extra added -- is counted as arg. + flagCompletion := true + _ = finalCmd.ParseFlags(append(finalArgs, "--")) + newArgCount := finalCmd.Flags().NArg() + + // Parse the flags early so we can check if required flags are set + if err = finalCmd.ParseFlags(finalArgs); err != nil { + return finalCmd, []Completion{}, ShellCompDirectiveDefault, fmt.Errorf("Error while parsing flags from args %v: %s", finalArgs, err.Error()) + } + + realArgCount := finalCmd.Flags().NArg() + if newArgCount > realArgCount { + // don't do flag completion (see above) + flagCompletion = false + } + // Error while attempting to parse flags + if flagErr != nil { + // If error type is flagCompError and we don't want flagCompletion we should ignore the error + if _, ok := flagErr.(*flagCompError); !(ok && !flagCompletion) { + return finalCmd, []Completion{}, ShellCompDirectiveDefault, flagErr + } + } + + // Look for the --help or --version flags. If they are present, + // there should be no further completions. + if helpOrVersionFlagPresent(finalCmd) { + return finalCmd, []Completion{}, ShellCompDirectiveNoFileComp, nil + } + + // We only remove the flags from the arguments if DisableFlagParsing is not set. + // This is important for commands which have requested to do their own flag completion. + if !finalCmd.DisableFlagParsing { + finalArgs = finalCmd.Flags().Args() + } + + if flag != nil && flagCompletion { + // Check if we are completing a flag value subject to annotations + if validExts, present := flag.Annotations[BashCompFilenameExt]; present { + if len(validExts) != 0 { + // File completion filtered by extensions + return finalCmd, validExts, ShellCompDirectiveFilterFileExt, nil + } + + // The annotation requests simple file completion. There is no reason to do + // that since it is the default behavior anyway. Let's ignore this annotation + // in case the program also registered a completion function for this flag. + // Even though it is a mistake on the program's side, let's be nice when we can. + } + + if subDir, present := flag.Annotations[BashCompSubdirsInDir]; present { + if len(subDir) == 1 { + // Directory completion from within a directory + return finalCmd, subDir, ShellCompDirectiveFilterDirs, nil + } + // Directory completion + return finalCmd, []Completion{}, ShellCompDirectiveFilterDirs, nil + } + } + + var completions []Completion + var directive ShellCompDirective + + // Enforce flag groups before doing flag completions + finalCmd.enforceFlagGroupsForCompletion() + + // Note that we want to perform flagname completion even if finalCmd.DisableFlagParsing==true; + // doing this allows for completion of persistent flag names even for commands that disable flag parsing. + // + // When doing completion of a flag name, as soon as an argument starts with + // a '-' we know it is a flag. We cannot use isFlagArg() here as it requires + // the flag name to be complete + if flag == nil && len(toComplete) > 0 && toComplete[0] == '-' && !strings.Contains(toComplete, "=") && flagCompletion { + // First check for required flags + completions = completeRequireFlags(finalCmd, toComplete) + + // If we have not found any required flags, only then can we show regular flags + if len(completions) == 0 { + doCompleteFlags := func(flag *pflag.Flag) { + _, acceptsMultiple := flag.Value.(SliceValue) + acceptsMultiple = acceptsMultiple || + strings.Contains(flag.Value.Type(), "Slice") || + strings.Contains(flag.Value.Type(), "Array") || + strings.HasPrefix(flag.Value.Type(), "stringTo") + + if !flag.Changed || acceptsMultiple { + // If the flag is not already present, or if it can be specified multiple times (Array, Slice, or stringTo) + // we suggest it as a completion + completions = append(completions, getFlagNameCompletions(flag, toComplete)...) + } + } + + // We cannot use finalCmd.Flags() because we may not have called ParsedFlags() for commands + // that have set DisableFlagParsing; it is ParseFlags() that merges the inherited and + // non-inherited flags. + finalCmd.InheritedFlags().VisitAll(func(flag *pflag.Flag) { + doCompleteFlags(flag) + }) + // Try to complete non-inherited flags even if DisableFlagParsing==true. + // This allows programs to tell Cobra about flags for completion even + // if the actual parsing of flags is not done by Cobra. + // For instance, Helm uses this to provide flag name completion for + // some of its plugins. + finalCmd.NonInheritedFlags().VisitAll(func(flag *pflag.Flag) { + doCompleteFlags(flag) + }) + } + + directive = ShellCompDirectiveNoFileComp + if len(completions) == 1 && strings.HasSuffix(completions[0], "=") { + // If there is a single completion, the shell usually adds a space + // after the completion. We don't want that if the flag ends with an = + directive = ShellCompDirectiveNoSpace + } + + if !finalCmd.DisableFlagParsing { + // If DisableFlagParsing==false, we have completed the flags as known by Cobra; + // we can return what we found. + // If DisableFlagParsing==true, Cobra may not be aware of all flags, so we + // let the logic continue to see if ValidArgsFunction needs to be called. + return finalCmd, completions, directive, nil + } + } else { + directive = ShellCompDirectiveDefault + if flag == nil { + foundLocalNonPersistentFlag := false + // If TraverseChildren is true on the root command we don't check for + // local flags because we can use a local flag on a parent command + if !finalCmd.Root().TraverseChildren { + // Check if there are any local, non-persistent flags on the command-line + localNonPersistentFlags := finalCmd.LocalNonPersistentFlags() + finalCmd.NonInheritedFlags().VisitAll(func(flag *pflag.Flag) { + if localNonPersistentFlags.Lookup(flag.Name) != nil && flag.Changed { + foundLocalNonPersistentFlag = true + } + }) + } + + // Complete subcommand names, including the help command + if len(finalArgs) == 0 && !foundLocalNonPersistentFlag { + // We only complete sub-commands if: + // - there are no arguments on the command-line and + // - there are no local, non-persistent flags on the command-line or TraverseChildren is true + for _, subCmd := range finalCmd.Commands() { + if subCmd.IsAvailableCommand() || subCmd == finalCmd.helpCommand { + if strings.HasPrefix(subCmd.Name(), toComplete) { + completions = append(completions, CompletionWithDesc(subCmd.Name(), subCmd.Short)) + } + directive = ShellCompDirectiveNoFileComp + } + } + } + + // Complete required flags even without the '-' prefix + completions = append(completions, completeRequireFlags(finalCmd, toComplete)...) + + // Always complete ValidArgs, even if we are completing a subcommand name. + // This is for commands that have both subcommands and ValidArgs. + if len(finalCmd.ValidArgs) > 0 { + if len(finalArgs) == 0 { + // ValidArgs are only for the first argument + for _, validArg := range finalCmd.ValidArgs { + if strings.HasPrefix(validArg, toComplete) { + completions = append(completions, validArg) + } + } + directive = ShellCompDirectiveNoFileComp + + // If no completions were found within commands or ValidArgs, + // see if there are any ArgAliases that should be completed. + if len(completions) == 0 { + for _, argAlias := range finalCmd.ArgAliases { + if strings.HasPrefix(argAlias, toComplete) { + completions = append(completions, argAlias) + } + } + } + } + + // If there are ValidArgs specified (even if they don't match), we stop completion. + // Only one of ValidArgs or ValidArgsFunction can be used for a single command. + return finalCmd, completions, directive, nil + } + + // Let the logic continue so as to add any ValidArgsFunction completions, + // even if we already found sub-commands. + // This is for commands that have subcommands but also specify a ValidArgsFunction. + } + } + + // Find the completion function for the flag or command + var completionFn CompletionFunc + if flag != nil && flagCompletion { + flagCompletionMutex.RLock() + completionFn = flagCompletionFunctions[flag] + flagCompletionMutex.RUnlock() + } else { + completionFn = finalCmd.ValidArgsFunction + } + if completionFn != nil { + // Go custom completion defined for this flag or command. + // Call the registered completion function to get the completions. + var comps []Completion + comps, directive = completionFn(finalCmd, finalArgs, toComplete) + completions = append(completions, comps...) + } + + return finalCmd, completions, directive, nil +} + +func helpOrVersionFlagPresent(cmd *Command) bool { + if versionFlag := cmd.Flags().Lookup("version"); versionFlag != nil && + len(versionFlag.Annotations[FlagSetByCobraAnnotation]) > 0 && versionFlag.Changed { + return true + } + if helpFlag := cmd.Flags().Lookup(helpFlagName); helpFlag != nil && + len(helpFlag.Annotations[FlagSetByCobraAnnotation]) > 0 && helpFlag.Changed { + return true + } + return false +} + +func getFlagNameCompletions(flag *pflag.Flag, toComplete string) []Completion { + if nonCompletableFlag(flag) { + return []Completion{} + } + + var completions []Completion + flagName := "--" + flag.Name + if strings.HasPrefix(flagName, toComplete) { + // Flag without the = + completions = append(completions, CompletionWithDesc(flagName, flag.Usage)) + + // Why suggest both long forms: --flag and --flag= ? + // This forces the user to *always* have to type either an = or a space after the flag name. + // Let's be nice and avoid making users have to do that. + // Since boolean flags and shortname flags don't show the = form, let's go that route and never show it. + // The = form will still work, we just won't suggest it. + // This also makes the list of suggested flags shorter as we avoid all the = forms. + // + // if len(flag.NoOptDefVal) == 0 { + // // Flag requires a value, so it can be suffixed with = + // flagName += "=" + // completions = append(completions, CompletionWithDesc(flagName, flag.Usage)) + // } + } + + flagName = "-" + flag.Shorthand + if len(flag.Shorthand) > 0 && strings.HasPrefix(flagName, toComplete) { + completions = append(completions, CompletionWithDesc(flagName, flag.Usage)) + } + + return completions +} + +func completeRequireFlags(finalCmd *Command, toComplete string) []Completion { + var completions []Completion + + doCompleteRequiredFlags := func(flag *pflag.Flag) { + if _, present := flag.Annotations[BashCompOneRequiredFlag]; present { + if !flag.Changed { + // If the flag is not already present, we suggest it as a completion + completions = append(completions, getFlagNameCompletions(flag, toComplete)...) + } + } + } + + // We cannot use finalCmd.Flags() because we may not have called ParsedFlags() for commands + // that have set DisableFlagParsing; it is ParseFlags() that merges the inherited and + // non-inherited flags. + finalCmd.InheritedFlags().VisitAll(func(flag *pflag.Flag) { + doCompleteRequiredFlags(flag) + }) + finalCmd.NonInheritedFlags().VisitAll(func(flag *pflag.Flag) { + doCompleteRequiredFlags(flag) + }) + + return completions +} + +func checkIfFlagCompletion(finalCmd *Command, args []string, lastArg string) (*pflag.Flag, []string, string, error) { + if finalCmd.DisableFlagParsing { + // We only do flag completion if we are allowed to parse flags + // This is important for commands which have requested to do their own flag completion. + return nil, args, lastArg, nil + } + + var flagName string + trimmedArgs := args + flagWithEqual := false + orgLastArg := lastArg + + // When doing completion of a flag name, as soon as an argument starts with + // a '-' we know it is a flag. We cannot use isFlagArg() here as that function + // requires the flag name to be complete + if len(lastArg) > 0 && lastArg[0] == '-' { + if index := strings.Index(lastArg, "="); index >= 0 { + // Flag with an = + if strings.HasPrefix(lastArg[:index], "--") { + // Flag has full name + flagName = lastArg[2:index] + } else { + // Flag is shorthand + // We have to get the last shorthand flag name + // e.g. `-asd` => d to provide the correct completion + // https://github.com/spf13/cobra/issues/1257 + flagName = lastArg[index-1 : index] + } + lastArg = lastArg[index+1:] + flagWithEqual = true + } else { + // Normal flag completion + return nil, args, lastArg, nil + } + } + + if len(flagName) == 0 { + if len(args) > 0 { + prevArg := args[len(args)-1] + if isFlagArg(prevArg) { + // Only consider the case where the flag does not contain an =. + // If the flag contains an = it means it has already been fully processed, + // so we don't need to deal with it here. + if index := strings.Index(prevArg, "="); index < 0 { + if strings.HasPrefix(prevArg, "--") { + // Flag has full name + flagName = prevArg[2:] + } else { + // Flag is shorthand + // We have to get the last shorthand flag name + // e.g. `-asd` => d to provide the correct completion + // https://github.com/spf13/cobra/issues/1257 + flagName = prevArg[len(prevArg)-1:] + } + // Remove the uncompleted flag or else there could be an error created + // for an invalid value for that flag + trimmedArgs = args[:len(args)-1] + } + } + } + } + + if len(flagName) == 0 { + // Not doing flag completion + return nil, trimmedArgs, lastArg, nil + } + + flag := findFlag(finalCmd, flagName) + if flag == nil { + // Flag not supported by this command, the interspersed option might be set so return the original args + return nil, args, orgLastArg, &flagCompError{subCommand: finalCmd.Name(), flagName: flagName} + } + + if !flagWithEqual { + if len(flag.NoOptDefVal) != 0 { + // We had assumed dealing with a two-word flag but the flag is a boolean flag. + // In that case, there is no value following it, so we are not really doing flag completion. + // Reset everything to do noun completion. + trimmedArgs = args + flag = nil + } + } + + return flag, trimmedArgs, lastArg, nil +} + +// InitDefaultCompletionCmd adds a default 'completion' command to c. +// This function will do nothing if any of the following is true: +// 1- the feature has been explicitly disabled by the program, +// 2- c has no subcommands (to avoid creating one), +// 3- c already has a 'completion' command provided by the program. +func (c *Command) InitDefaultCompletionCmd(args ...string) { + if c.CompletionOptions.DisableDefaultCmd { + return + } + + for _, cmd := range c.commands { + if cmd.Name() == compCmdName || cmd.HasAlias(compCmdName) { + // A completion command is already available + return + } + } + + haveNoDescFlag := !c.CompletionOptions.DisableNoDescFlag && !c.CompletionOptions.DisableDescriptions + + // Special case to know if there are sub-commands or not. + hasSubCommands := false + for _, cmd := range c.commands { + if cmd.Name() != ShellCompRequestCmd && cmd.Name() != helpCommandName { + // We found a real sub-command (not 'help' or '__complete') + hasSubCommands = true + break + } + } + + completionCmd := &Command{ + Use: compCmdName, + Short: "Generate the autocompletion script for the specified shell", + Long: fmt.Sprintf(`Generate the autocompletion script for %[1]s for the specified shell. +See each sub-command's help for details on how to use the generated script. +`, c.Root().Name()), + Args: NoArgs, + ValidArgsFunction: NoFileCompletions, + Hidden: c.CompletionOptions.HiddenDefaultCmd, + GroupID: c.completionCommandGroupID, + } + c.AddCommand(completionCmd) + + if !hasSubCommands { + // If the 'completion' command will be the only sub-command, + // we only create it if it is actually being called. + // This avoids breaking programs that would suddenly find themselves with + // a subcommand, which would prevent them from accepting arguments. + // We also create the 'completion' command if the user is triggering + // shell completion for it (prog __complete completion '') + subCmd, cmdArgs, err := c.Find(args) + if err != nil || subCmd.Name() != compCmdName && + !(subCmd.Name() == ShellCompRequestCmd && len(cmdArgs) > 1 && cmdArgs[0] == compCmdName) { + // The completion command is not being called or being completed so we remove it. + c.RemoveCommand(completionCmd) + return + } + } + + out := c.OutOrStdout() + noDesc := c.CompletionOptions.DisableDescriptions + shortDesc := "Generate the autocompletion script for %s" + bash := &Command{ + Use: "bash", + Short: fmt.Sprintf(shortDesc, "bash"), + Long: fmt.Sprintf(`Generate the autocompletion script for the bash shell. + +This script depends on the 'bash-completion' package. +If it is not installed already, you can install it via your OS's package manager. + +To load completions in your current shell session: + + source <(%[1]s completion bash) + +To load completions for every new session, execute once: + +#### Linux: + + %[1]s completion bash > /etc/bash_completion.d/%[1]s + +#### macOS: + + %[1]s completion bash > $(brew --prefix)/etc/bash_completion.d/%[1]s + +You will need to start a new shell for this setup to take effect. +`, c.Root().Name()), + Args: NoArgs, + DisableFlagsInUseLine: true, + ValidArgsFunction: NoFileCompletions, + RunE: func(cmd *Command, args []string) error { + return cmd.Root().GenBashCompletionV2(out, !noDesc) + }, + } + if haveNoDescFlag { + bash.Flags().BoolVar(&noDesc, compCmdNoDescFlagName, compCmdNoDescFlagDefault, compCmdNoDescFlagDesc) + } + + zsh := &Command{ + Use: "zsh", + Short: fmt.Sprintf(shortDesc, "zsh"), + Long: fmt.Sprintf(`Generate the autocompletion script for the zsh shell. + +If shell completion is not already enabled in your environment you will need +to enable it. You can execute the following once: + + echo "autoload -U compinit; compinit" >> ~/.zshrc + +To load completions in your current shell session: + + source <(%[1]s completion zsh) + +To load completions for every new session, execute once: + +#### Linux: + + %[1]s completion zsh > "${fpath[1]}/_%[1]s" + +#### macOS: + + %[1]s completion zsh > $(brew --prefix)/share/zsh/site-functions/_%[1]s + +You will need to start a new shell for this setup to take effect. +`, c.Root().Name()), + Args: NoArgs, + ValidArgsFunction: NoFileCompletions, + RunE: func(cmd *Command, args []string) error { + if noDesc { + return cmd.Root().GenZshCompletionNoDesc(out) + } + return cmd.Root().GenZshCompletion(out) + }, + } + if haveNoDescFlag { + zsh.Flags().BoolVar(&noDesc, compCmdNoDescFlagName, compCmdNoDescFlagDefault, compCmdNoDescFlagDesc) + } + + fish := &Command{ + Use: "fish", + Short: fmt.Sprintf(shortDesc, "fish"), + Long: fmt.Sprintf(`Generate the autocompletion script for the fish shell. + +To load completions in your current shell session: + + %[1]s completion fish | source + +To load completions for every new session, execute once: + + %[1]s completion fish > ~/.config/fish/completions/%[1]s.fish + +You will need to start a new shell for this setup to take effect. +`, c.Root().Name()), + Args: NoArgs, + ValidArgsFunction: NoFileCompletions, + RunE: func(cmd *Command, args []string) error { + return cmd.Root().GenFishCompletion(out, !noDesc) + }, + } + if haveNoDescFlag { + fish.Flags().BoolVar(&noDesc, compCmdNoDescFlagName, compCmdNoDescFlagDefault, compCmdNoDescFlagDesc) + } + + powershell := &Command{ + Use: "powershell", + Short: fmt.Sprintf(shortDesc, "powershell"), + Long: fmt.Sprintf(`Generate the autocompletion script for powershell. + +To load completions in your current shell session: + + %[1]s completion powershell | Out-String | Invoke-Expression + +To load completions for every new session, add the output of the above command +to your powershell profile. +`, c.Root().Name()), + Args: NoArgs, + ValidArgsFunction: NoFileCompletions, + RunE: func(cmd *Command, args []string) error { + if noDesc { + return cmd.Root().GenPowerShellCompletion(out) + } + return cmd.Root().GenPowerShellCompletionWithDesc(out) + + }, + } + if haveNoDescFlag { + powershell.Flags().BoolVar(&noDesc, compCmdNoDescFlagName, compCmdNoDescFlagDefault, compCmdNoDescFlagDesc) + } + + completionCmd.AddCommand(bash, zsh, fish, powershell) +} + +func findFlag(cmd *Command, name string) *pflag.Flag { + flagSet := cmd.Flags() + if len(name) == 1 { + // First convert the short flag into a long flag + // as the cmd.Flag() search only accepts long flags + if short := flagSet.ShorthandLookup(name); short != nil { + name = short.Name + } else { + set := cmd.InheritedFlags() + if short = set.ShorthandLookup(name); short != nil { + name = short.Name + } else { + return nil + } + } + } + return cmd.Flag(name) +} + +// CompDebug prints the specified string to the same file as where the +// completion script prints its logs. +// Note that completion printouts should never be on stdout as they would +// be wrongly interpreted as actual completion choices by the completion script. +func CompDebug(msg string, printToStdErr bool) { + msg = fmt.Sprintf("[Debug] %s", msg) + + // Such logs are only printed when the user has set the environment + // variable BASH_COMP_DEBUG_FILE to the path of some file to be used. + if path := os.Getenv("BASH_COMP_DEBUG_FILE"); path != "" { + f, err := os.OpenFile(path, + os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err == nil { + defer f.Close() + WriteStringAndCheck(f, msg) + } + } + + if printToStdErr { + // Must print to stderr for this not to be read by the completion script. + fmt.Fprint(os.Stderr, msg) + } +} + +// CompDebugln prints the specified string with a newline at the end +// to the same file as where the completion script prints its logs. +// Such logs are only printed when the user has set the environment +// variable BASH_COMP_DEBUG_FILE to the path of some file to be used. +func CompDebugln(msg string, printToStdErr bool) { + CompDebug(fmt.Sprintf("%s\n", msg), printToStdErr) +} + +// CompError prints the specified completion message to stderr. +func CompError(msg string) { + msg = fmt.Sprintf("[Error] %s", msg) + CompDebug(msg, true) +} + +// CompErrorln prints the specified completion message to stderr with a newline at the end. +func CompErrorln(msg string) { + CompError(fmt.Sprintf("%s\n", msg)) +} + +// These values should not be changed: users will be using them explicitly. +const ( + configEnvVarGlobalPrefix = "COBRA" + configEnvVarSuffixDescriptions = "COMPLETION_DESCRIPTIONS" +) + +var configEnvVarPrefixSubstRegexp = regexp.MustCompile(`[^A-Z0-9_]`) + +// configEnvVar returns the name of the program-specific configuration environment +// variable. It has the format _ where is the name of the +// root command in upper case, with all non-ASCII-alphanumeric characters replaced by `_`. +func configEnvVar(name, suffix string) string { + // This format should not be changed: users will be using it explicitly. + v := strings.ToUpper(fmt.Sprintf("%s_%s", name, suffix)) + v = configEnvVarPrefixSubstRegexp.ReplaceAllString(v, "_") + return v +} + +// getEnvConfig returns the value of the configuration environment variable +// _ where is the name of the root command in upper +// case, with all non-ASCII-alphanumeric characters replaced by `_`. +// If the value is empty or not set, the value of the environment variable +// COBRA_ is returned instead. +func getEnvConfig(cmd *Command, suffix string) string { + v := os.Getenv(configEnvVar(cmd.Root().Name(), suffix)) + if v == "" { + v = os.Getenv(configEnvVar(configEnvVarGlobalPrefix, suffix)) + } + return v +} diff --git a/go-controller/vendor/github.com/spf13/cobra/fish_completions.go b/go-controller/vendor/github.com/spf13/cobra/fish_completions.go new file mode 100644 index 0000000000..12d61b6911 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/fish_completions.go @@ -0,0 +1,292 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" +) + +func genFishComp(buf io.StringWriter, name string, includeDesc bool) { + // Variables should not contain a '-' or ':' character + nameForVar := name + nameForVar = strings.ReplaceAll(nameForVar, "-", "_") + nameForVar = strings.ReplaceAll(nameForVar, ":", "_") + + compCmd := ShellCompRequestCmd + if !includeDesc { + compCmd = ShellCompNoDescRequestCmd + } + WriteStringAndCheck(buf, fmt.Sprintf("# fish completion for %-36s -*- shell-script -*-\n", name)) + WriteStringAndCheck(buf, fmt.Sprintf(` +function __%[1]s_debug + set -l file "$BASH_COMP_DEBUG_FILE" + if test -n "$file" + echo "$argv" >> $file + end +end + +function __%[1]s_perform_completion + __%[1]s_debug "Starting __%[1]s_perform_completion" + + # Extract all args except the last one + set -l args (commandline -opc) + # Extract the last arg and escape it in case it is a space + set -l lastArg (string escape -- (commandline -ct)) + + __%[1]s_debug "args: $args" + __%[1]s_debug "last arg: $lastArg" + + # Disable ActiveHelp which is not supported for fish shell + set -l requestComp "%[10]s=0 $args[1] %[3]s $args[2..-1] $lastArg" + + __%[1]s_debug "Calling $requestComp" + set -l results (eval $requestComp 2> /dev/null) + + # Some programs may output extra empty lines after the directive. + # Let's ignore them or else it will break completion. + # Ref: https://github.com/spf13/cobra/issues/1279 + for line in $results[-1..1] + if test (string trim -- $line) = "" + # Found an empty line, remove it + set results $results[1..-2] + else + # Found non-empty line, we have our proper output + break + end + end + + set -l comps $results[1..-2] + set -l directiveLine $results[-1] + + # For Fish, when completing a flag with an = (e.g., -n=) + # completions must be prefixed with the flag + set -l flagPrefix (string match -r -- '-.*=' "$lastArg") + + __%[1]s_debug "Comps: $comps" + __%[1]s_debug "DirectiveLine: $directiveLine" + __%[1]s_debug "flagPrefix: $flagPrefix" + + for comp in $comps + printf "%%s%%s\n" "$flagPrefix" "$comp" + end + + printf "%%s\n" "$directiveLine" +end + +# this function limits calls to __%[1]s_perform_completion, by caching the result behind $__%[1]s_perform_completion_once_result +function __%[1]s_perform_completion_once + __%[1]s_debug "Starting __%[1]s_perform_completion_once" + + if test -n "$__%[1]s_perform_completion_once_result" + __%[1]s_debug "Seems like a valid result already exists, skipping __%[1]s_perform_completion" + return 0 + end + + set --global __%[1]s_perform_completion_once_result (__%[1]s_perform_completion) + if test -z "$__%[1]s_perform_completion_once_result" + __%[1]s_debug "No completions, probably due to a failure" + return 1 + end + + __%[1]s_debug "Performed completions and set __%[1]s_perform_completion_once_result" + return 0 +end + +# this function is used to clear the $__%[1]s_perform_completion_once_result variable after completions are run +function __%[1]s_clear_perform_completion_once_result + __%[1]s_debug "" + __%[1]s_debug "========= clearing previously set __%[1]s_perform_completion_once_result variable ==========" + set --erase __%[1]s_perform_completion_once_result + __%[1]s_debug "Successfully erased the variable __%[1]s_perform_completion_once_result" +end + +function __%[1]s_requires_order_preservation + __%[1]s_debug "" + __%[1]s_debug "========= checking if order preservation is required ==========" + + __%[1]s_perform_completion_once + if test -z "$__%[1]s_perform_completion_once_result" + __%[1]s_debug "Error determining if order preservation is required" + return 1 + end + + set -l directive (string sub --start 2 $__%[1]s_perform_completion_once_result[-1]) + __%[1]s_debug "Directive is: $directive" + + set -l shellCompDirectiveKeepOrder %[9]d + set -l keeporder (math (math --scale 0 $directive / $shellCompDirectiveKeepOrder) %% 2) + __%[1]s_debug "Keeporder is: $keeporder" + + if test $keeporder -ne 0 + __%[1]s_debug "This does require order preservation" + return 0 + end + + __%[1]s_debug "This doesn't require order preservation" + return 1 +end + + +# This function does two things: +# - Obtain the completions and store them in the global __%[1]s_comp_results +# - Return false if file completion should be performed +function __%[1]s_prepare_completions + __%[1]s_debug "" + __%[1]s_debug "========= starting completion logic ==========" + + # Start fresh + set --erase __%[1]s_comp_results + + __%[1]s_perform_completion_once + __%[1]s_debug "Completion results: $__%[1]s_perform_completion_once_result" + + if test -z "$__%[1]s_perform_completion_once_result" + __%[1]s_debug "No completion, probably due to a failure" + # Might as well do file completion, in case it helps + return 1 + end + + set -l directive (string sub --start 2 $__%[1]s_perform_completion_once_result[-1]) + set --global __%[1]s_comp_results $__%[1]s_perform_completion_once_result[1..-2] + + __%[1]s_debug "Completions are: $__%[1]s_comp_results" + __%[1]s_debug "Directive is: $directive" + + set -l shellCompDirectiveError %[4]d + set -l shellCompDirectiveNoSpace %[5]d + set -l shellCompDirectiveNoFileComp %[6]d + set -l shellCompDirectiveFilterFileExt %[7]d + set -l shellCompDirectiveFilterDirs %[8]d + + if test -z "$directive" + set directive 0 + end + + set -l compErr (math (math --scale 0 $directive / $shellCompDirectiveError) %% 2) + if test $compErr -eq 1 + __%[1]s_debug "Received error directive: aborting." + # Might as well do file completion, in case it helps + return 1 + end + + set -l filefilter (math (math --scale 0 $directive / $shellCompDirectiveFilterFileExt) %% 2) + set -l dirfilter (math (math --scale 0 $directive / $shellCompDirectiveFilterDirs) %% 2) + if test $filefilter -eq 1; or test $dirfilter -eq 1 + __%[1]s_debug "File extension filtering or directory filtering not supported" + # Do full file completion instead + return 1 + end + + set -l nospace (math (math --scale 0 $directive / $shellCompDirectiveNoSpace) %% 2) + set -l nofiles (math (math --scale 0 $directive / $shellCompDirectiveNoFileComp) %% 2) + + __%[1]s_debug "nospace: $nospace, nofiles: $nofiles" + + # If we want to prevent a space, or if file completion is NOT disabled, + # we need to count the number of valid completions. + # To do so, we will filter on prefix as the completions we have received + # may not already be filtered so as to allow fish to match on different + # criteria than the prefix. + if test $nospace -ne 0; or test $nofiles -eq 0 + set -l prefix (commandline -t | string escape --style=regex) + __%[1]s_debug "prefix: $prefix" + + set -l completions (string match -r -- "^$prefix.*" $__%[1]s_comp_results) + set --global __%[1]s_comp_results $completions + __%[1]s_debug "Filtered completions are: $__%[1]s_comp_results" + + # Important not to quote the variable for count to work + set -l numComps (count $__%[1]s_comp_results) + __%[1]s_debug "numComps: $numComps" + + if test $numComps -eq 1; and test $nospace -ne 0 + # We must first split on \t to get rid of the descriptions to be + # able to check what the actual completion will be. + # We don't need descriptions anyway since there is only a single + # real completion which the shell will expand immediately. + set -l split (string split --max 1 \t $__%[1]s_comp_results[1]) + + # Fish won't add a space if the completion ends with any + # of the following characters: @=/:., + set -l lastChar (string sub -s -1 -- $split) + if not string match -r -q "[@=/:.,]" -- "$lastChar" + # In other cases, to support the "nospace" directive we trick the shell + # by outputting an extra, longer completion. + __%[1]s_debug "Adding second completion to perform nospace directive" + set --global __%[1]s_comp_results $split[1] $split[1]. + __%[1]s_debug "Completions are now: $__%[1]s_comp_results" + end + end + + if test $numComps -eq 0; and test $nofiles -eq 0 + # To be consistent with bash and zsh, we only trigger file + # completion when there are no other completions + __%[1]s_debug "Requesting file completion" + return 1 + end + end + + return 0 +end + +# Since Fish completions are only loaded once the user triggers them, we trigger them ourselves +# so we can properly delete any completions provided by another script. +# Only do this if the program can be found, or else fish may print some errors; besides, +# the existing completions will only be loaded if the program can be found. +if type -q "%[2]s" + # The space after the program name is essential to trigger completion for the program + # and not completion of the program name itself. + # Also, we use '> /dev/null 2>&1' since '&>' is not supported in older versions of fish. + complete --do-complete "%[2]s " > /dev/null 2>&1 +end + +# Remove any pre-existing completions for the program since we will be handling all of them. +complete -c %[2]s -e + +# this will get called after the two calls below and clear the $__%[1]s_perform_completion_once_result global +complete -c %[2]s -n '__%[1]s_clear_perform_completion_once_result' +# The call to __%[1]s_prepare_completions will setup __%[1]s_comp_results +# which provides the program's completion choices. +# If this doesn't require order preservation, we don't use the -k flag +complete -c %[2]s -n 'not __%[1]s_requires_order_preservation && __%[1]s_prepare_completions' -f -a '$__%[1]s_comp_results' +# otherwise we use the -k flag +complete -k -c %[2]s -n '__%[1]s_requires_order_preservation && __%[1]s_prepare_completions' -f -a '$__%[1]s_comp_results' +`, nameForVar, name, compCmd, + ShellCompDirectiveError, ShellCompDirectiveNoSpace, ShellCompDirectiveNoFileComp, + ShellCompDirectiveFilterFileExt, ShellCompDirectiveFilterDirs, ShellCompDirectiveKeepOrder, activeHelpEnvVar(name))) +} + +// GenFishCompletion generates fish completion file and writes to the passed writer. +func (c *Command) GenFishCompletion(w io.Writer, includeDesc bool) error { + buf := new(bytes.Buffer) + genFishComp(buf, c.Name(), includeDesc) + _, err := buf.WriteTo(w) + return err +} + +// GenFishCompletionFile generates fish completion file. +func (c *Command) GenFishCompletionFile(filename string, includeDesc bool) error { + outFile, err := os.Create(filename) + if err != nil { + return err + } + defer outFile.Close() + + return c.GenFishCompletion(outFile, includeDesc) +} diff --git a/go-controller/vendor/github.com/spf13/cobra/flag_groups.go b/go-controller/vendor/github.com/spf13/cobra/flag_groups.go new file mode 100644 index 0000000000..560612fd33 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/flag_groups.go @@ -0,0 +1,290 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "fmt" + "sort" + "strings" + + flag "github.com/spf13/pflag" +) + +const ( + requiredAsGroupAnnotation = "cobra_annotation_required_if_others_set" + oneRequiredAnnotation = "cobra_annotation_one_required" + mutuallyExclusiveAnnotation = "cobra_annotation_mutually_exclusive" +) + +// MarkFlagsRequiredTogether marks the given flags with annotations so that Cobra errors +// if the command is invoked with a subset (but not all) of the given flags. +func (c *Command) MarkFlagsRequiredTogether(flagNames ...string) { + c.mergePersistentFlags() + for _, v := range flagNames { + f := c.Flags().Lookup(v) + if f == nil { + panic(fmt.Sprintf("Failed to find flag %q and mark it as being required in a flag group", v)) + } + if err := c.Flags().SetAnnotation(v, requiredAsGroupAnnotation, append(f.Annotations[requiredAsGroupAnnotation], strings.Join(flagNames, " "))); err != nil { + // Only errs if the flag isn't found. + panic(err) + } + } +} + +// MarkFlagsOneRequired marks the given flags with annotations so that Cobra errors +// if the command is invoked without at least one flag from the given set of flags. +func (c *Command) MarkFlagsOneRequired(flagNames ...string) { + c.mergePersistentFlags() + for _, v := range flagNames { + f := c.Flags().Lookup(v) + if f == nil { + panic(fmt.Sprintf("Failed to find flag %q and mark it as being in a one-required flag group", v)) + } + if err := c.Flags().SetAnnotation(v, oneRequiredAnnotation, append(f.Annotations[oneRequiredAnnotation], strings.Join(flagNames, " "))); err != nil { + // Only errs if the flag isn't found. + panic(err) + } + } +} + +// MarkFlagsMutuallyExclusive marks the given flags with annotations so that Cobra errors +// if the command is invoked with more than one flag from the given set of flags. +func (c *Command) MarkFlagsMutuallyExclusive(flagNames ...string) { + c.mergePersistentFlags() + for _, v := range flagNames { + f := c.Flags().Lookup(v) + if f == nil { + panic(fmt.Sprintf("Failed to find flag %q and mark it as being in a mutually exclusive flag group", v)) + } + // Each time this is called is a single new entry; this allows it to be a member of multiple groups if needed. + if err := c.Flags().SetAnnotation(v, mutuallyExclusiveAnnotation, append(f.Annotations[mutuallyExclusiveAnnotation], strings.Join(flagNames, " "))); err != nil { + panic(err) + } + } +} + +// ValidateFlagGroups validates the mutuallyExclusive/oneRequired/requiredAsGroup logic and returns the +// first error encountered. +func (c *Command) ValidateFlagGroups() error { + if c.DisableFlagParsing { + return nil + } + + flags := c.Flags() + + // groupStatus format is the list of flags as a unique ID, + // then a map of each flag name and whether it is set or not. + groupStatus := map[string]map[string]bool{} + oneRequiredGroupStatus := map[string]map[string]bool{} + mutuallyExclusiveGroupStatus := map[string]map[string]bool{} + flags.VisitAll(func(pflag *flag.Flag) { + processFlagForGroupAnnotation(flags, pflag, requiredAsGroupAnnotation, groupStatus) + processFlagForGroupAnnotation(flags, pflag, oneRequiredAnnotation, oneRequiredGroupStatus) + processFlagForGroupAnnotation(flags, pflag, mutuallyExclusiveAnnotation, mutuallyExclusiveGroupStatus) + }) + + if err := validateRequiredFlagGroups(groupStatus); err != nil { + return err + } + if err := validateOneRequiredFlagGroups(oneRequiredGroupStatus); err != nil { + return err + } + if err := validateExclusiveFlagGroups(mutuallyExclusiveGroupStatus); err != nil { + return err + } + return nil +} + +func hasAllFlags(fs *flag.FlagSet, flagnames ...string) bool { + for _, fname := range flagnames { + f := fs.Lookup(fname) + if f == nil { + return false + } + } + return true +} + +func processFlagForGroupAnnotation(flags *flag.FlagSet, pflag *flag.Flag, annotation string, groupStatus map[string]map[string]bool) { + groupInfo, found := pflag.Annotations[annotation] + if found { + for _, group := range groupInfo { + if groupStatus[group] == nil { + flagnames := strings.Split(group, " ") + + // Only consider this flag group at all if all the flags are defined. + if !hasAllFlags(flags, flagnames...) { + continue + } + + groupStatus[group] = make(map[string]bool, len(flagnames)) + for _, name := range flagnames { + groupStatus[group][name] = false + } + } + + groupStatus[group][pflag.Name] = pflag.Changed + } + } +} + +func validateRequiredFlagGroups(data map[string]map[string]bool) error { + keys := sortedKeys(data) + for _, flagList := range keys { + flagnameAndStatus := data[flagList] + + unset := []string{} + for flagname, isSet := range flagnameAndStatus { + if !isSet { + unset = append(unset, flagname) + } + } + if len(unset) == len(flagnameAndStatus) || len(unset) == 0 { + continue + } + + // Sort values, so they can be tested/scripted against consistently. + sort.Strings(unset) + return fmt.Errorf("if any flags in the group [%v] are set they must all be set; missing %v", flagList, unset) + } + + return nil +} + +func validateOneRequiredFlagGroups(data map[string]map[string]bool) error { + keys := sortedKeys(data) + for _, flagList := range keys { + flagnameAndStatus := data[flagList] + var set []string + for flagname, isSet := range flagnameAndStatus { + if isSet { + set = append(set, flagname) + } + } + if len(set) >= 1 { + continue + } + + // Sort values, so they can be tested/scripted against consistently. + sort.Strings(set) + return fmt.Errorf("at least one of the flags in the group [%v] is required", flagList) + } + return nil +} + +func validateExclusiveFlagGroups(data map[string]map[string]bool) error { + keys := sortedKeys(data) + for _, flagList := range keys { + flagnameAndStatus := data[flagList] + var set []string + for flagname, isSet := range flagnameAndStatus { + if isSet { + set = append(set, flagname) + } + } + if len(set) == 0 || len(set) == 1 { + continue + } + + // Sort values, so they can be tested/scripted against consistently. + sort.Strings(set) + return fmt.Errorf("if any flags in the group [%v] are set none of the others can be; %v were all set", flagList, set) + } + return nil +} + +func sortedKeys(m map[string]map[string]bool) []string { + keys := make([]string, len(m)) + i := 0 + for k := range m { + keys[i] = k + i++ + } + sort.Strings(keys) + return keys +} + +// enforceFlagGroupsForCompletion will do the following: +// - when a flag in a group is present, other flags in the group will be marked required +// - when none of the flags in a one-required group are present, all flags in the group will be marked required +// - when a flag in a mutually exclusive group is present, other flags in the group will be marked as hidden +// This allows the standard completion logic to behave appropriately for flag groups +func (c *Command) enforceFlagGroupsForCompletion() { + if c.DisableFlagParsing { + return + } + + flags := c.Flags() + groupStatus := map[string]map[string]bool{} + oneRequiredGroupStatus := map[string]map[string]bool{} + mutuallyExclusiveGroupStatus := map[string]map[string]bool{} + c.Flags().VisitAll(func(pflag *flag.Flag) { + processFlagForGroupAnnotation(flags, pflag, requiredAsGroupAnnotation, groupStatus) + processFlagForGroupAnnotation(flags, pflag, oneRequiredAnnotation, oneRequiredGroupStatus) + processFlagForGroupAnnotation(flags, pflag, mutuallyExclusiveAnnotation, mutuallyExclusiveGroupStatus) + }) + + // If a flag that is part of a group is present, we make all the other flags + // of that group required so that the shell completion suggests them automatically + for flagList, flagnameAndStatus := range groupStatus { + for _, isSet := range flagnameAndStatus { + if isSet { + // One of the flags of the group is set, mark the other ones as required + for _, fName := range strings.Split(flagList, " ") { + _ = c.MarkFlagRequired(fName) + } + } + } + } + + // If none of the flags of a one-required group are present, we make all the flags + // of that group required so that the shell completion suggests them automatically + for flagList, flagnameAndStatus := range oneRequiredGroupStatus { + isSet := false + + for _, isSet = range flagnameAndStatus { + if isSet { + break + } + } + + // None of the flags of the group are set, mark all flags in the group + // as required + if !isSet { + for _, fName := range strings.Split(flagList, " ") { + _ = c.MarkFlagRequired(fName) + } + } + } + + // If a flag that is mutually exclusive to others is present, we hide the other + // flags of that group so the shell completion does not suggest them + for flagList, flagnameAndStatus := range mutuallyExclusiveGroupStatus { + for flagName, isSet := range flagnameAndStatus { + if isSet { + // One of the flags of the mutually exclusive group is set, mark the other ones as hidden + // Don't mark the flag that is already set as hidden because it may be an + // array or slice flag and therefore must continue being suggested + for _, fName := range strings.Split(flagList, " ") { + if fName != flagName { + flag := c.Flags().Lookup(fName) + flag.Hidden = true + } + } + } + } + } +} diff --git a/go-controller/vendor/github.com/spf13/cobra/powershell_completions.go b/go-controller/vendor/github.com/spf13/cobra/powershell_completions.go new file mode 100644 index 0000000000..746dcb92e3 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/powershell_completions.go @@ -0,0 +1,350 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The generated scripts require PowerShell v5.0+ (which comes Windows 10, but +// can be downloaded separately for windows 7 or 8.1). + +package cobra + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" +) + +func genPowerShellComp(buf io.StringWriter, name string, includeDesc bool) { + // Variables should not contain a '-' or ':' character + nameForVar := name + nameForVar = strings.ReplaceAll(nameForVar, "-", "_") + nameForVar = strings.ReplaceAll(nameForVar, ":", "_") + + compCmd := ShellCompRequestCmd + if !includeDesc { + compCmd = ShellCompNoDescRequestCmd + } + WriteStringAndCheck(buf, fmt.Sprintf(`# powershell completion for %-36[1]s -*- shell-script -*- + +function __%[1]s_debug { + if ($env:BASH_COMP_DEBUG_FILE) { + "$args" | Out-File -Append -FilePath "$env:BASH_COMP_DEBUG_FILE" + } +} + +filter __%[1]s_escapeStringWithSpecialChars { +`+" $_ -replace '\\s|#|@|\\$|;|,|''|\\{|\\}|\\(|\\)|\"|`|\\||<|>|&','`$&'"+` +} + +[scriptblock]${__%[2]sCompleterBlock} = { + param( + $WordToComplete, + $CommandAst, + $CursorPosition + ) + + # Get the current command line and convert into a string + $Command = $CommandAst.CommandElements + $Command = "$Command" + + __%[1]s_debug "" + __%[1]s_debug "========= starting completion logic ==========" + __%[1]s_debug "WordToComplete: $WordToComplete Command: $Command CursorPosition: $CursorPosition" + + # The user could have moved the cursor backwards on the command-line. + # We need to trigger completion from the $CursorPosition location, so we need + # to truncate the command-line ($Command) up to the $CursorPosition location. + # Make sure the $Command is longer then the $CursorPosition before we truncate. + # This happens because the $Command does not include the last space. + if ($Command.Length -gt $CursorPosition) { + $Command=$Command.Substring(0,$CursorPosition) + } + __%[1]s_debug "Truncated command: $Command" + + $ShellCompDirectiveError=%[4]d + $ShellCompDirectiveNoSpace=%[5]d + $ShellCompDirectiveNoFileComp=%[6]d + $ShellCompDirectiveFilterFileExt=%[7]d + $ShellCompDirectiveFilterDirs=%[8]d + $ShellCompDirectiveKeepOrder=%[9]d + + # Prepare the command to request completions for the program. + # Split the command at the first space to separate the program and arguments. + $Program,$Arguments = $Command.Split(" ",2) + + $RequestComp="$Program %[3]s $Arguments" + __%[1]s_debug "RequestComp: $RequestComp" + + # we cannot use $WordToComplete because it + # has the wrong values if the cursor was moved + # so use the last argument + if ($WordToComplete -ne "" ) { + $WordToComplete = $Arguments.Split(" ")[-1] + } + __%[1]s_debug "New WordToComplete: $WordToComplete" + + + # Check for flag with equal sign + $IsEqualFlag = ($WordToComplete -Like "--*=*" ) + if ( $IsEqualFlag ) { + __%[1]s_debug "Completing equal sign flag" + # Remove the flag part + $Flag,$WordToComplete = $WordToComplete.Split("=",2) + } + + if ( $WordToComplete -eq "" -And ( -Not $IsEqualFlag )) { + # If the last parameter is complete (there is a space following it) + # We add an extra empty parameter so we can indicate this to the go method. + __%[1]s_debug "Adding extra empty parameter" + # PowerShell 7.2+ changed the way how the arguments are passed to executables, + # so for pre-7.2 or when Legacy argument passing is enabled we need to use +`+" # `\"`\" to pass an empty argument, a \"\" or '' does not work!!!"+` + if ($PSVersionTable.PsVersion -lt [version]'7.2.0' -or + ($PSVersionTable.PsVersion -lt [version]'7.3.0' -and -not [ExperimentalFeature]::IsEnabled("PSNativeCommandArgumentPassing")) -or + (($PSVersionTable.PsVersion -ge [version]'7.3.0' -or [ExperimentalFeature]::IsEnabled("PSNativeCommandArgumentPassing")) -and + $PSNativeCommandArgumentPassing -eq 'Legacy')) { +`+" $RequestComp=\"$RequestComp\" + ' `\"`\"'"+` + } else { + $RequestComp="$RequestComp" + ' ""' + } + } + + __%[1]s_debug "Calling $RequestComp" + # First disable ActiveHelp which is not supported for Powershell + ${env:%[10]s}=0 + + #call the command store the output in $out and redirect stderr and stdout to null + # $Out is an array contains each line per element + Invoke-Expression -OutVariable out "$RequestComp" 2>&1 | Out-Null + + # get directive from last line + [int]$Directive = $Out[-1].TrimStart(':') + if ($Directive -eq "") { + # There is no directive specified + $Directive = 0 + } + __%[1]s_debug "The completion directive is: $Directive" + + # remove directive (last element) from out + $Out = $Out | Where-Object { $_ -ne $Out[-1] } + __%[1]s_debug "The completions are: $Out" + + if (($Directive -band $ShellCompDirectiveError) -ne 0 ) { + # Error code. No completion. + __%[1]s_debug "Received error from custom completion go code" + return + } + + $Longest = 0 + [Array]$Values = $Out | ForEach-Object { + #Split the output in name and description +`+" $Name, $Description = $_.Split(\"`t\",2)"+` + __%[1]s_debug "Name: $Name Description: $Description" + + # Look for the longest completion so that we can format things nicely + if ($Longest -lt $Name.Length) { + $Longest = $Name.Length + } + + # Set the description to a one space string if there is none set. + # This is needed because the CompletionResult does not accept an empty string as argument + if (-Not $Description) { + $Description = " " + } + New-Object -TypeName PSCustomObject -Property @{ + Name = "$Name" + Description = "$Description" + } + } + + + $Space = " " + if (($Directive -band $ShellCompDirectiveNoSpace) -ne 0 ) { + # remove the space here + __%[1]s_debug "ShellCompDirectiveNoSpace is called" + $Space = "" + } + + if ((($Directive -band $ShellCompDirectiveFilterFileExt) -ne 0 ) -or + (($Directive -band $ShellCompDirectiveFilterDirs) -ne 0 )) { + __%[1]s_debug "ShellCompDirectiveFilterFileExt ShellCompDirectiveFilterDirs are not supported" + + # return here to prevent the completion of the extensions + return + } + + $Values = $Values | Where-Object { + # filter the result + $_.Name -like "$WordToComplete*" + + # Join the flag back if we have an equal sign flag + if ( $IsEqualFlag ) { + __%[1]s_debug "Join the equal sign flag back to the completion value" + $_.Name = $Flag + "=" + $_.Name + } + } + + # we sort the values in ascending order by name if keep order isn't passed + if (($Directive -band $ShellCompDirectiveKeepOrder) -eq 0 ) { + $Values = $Values | Sort-Object -Property Name + } + + if (($Directive -band $ShellCompDirectiveNoFileComp) -ne 0 ) { + __%[1]s_debug "ShellCompDirectiveNoFileComp is called" + + if ($Values.Length -eq 0) { + # Just print an empty string here so the + # shell does not start to complete paths. + # We cannot use CompletionResult here because + # it does not accept an empty string as argument. + "" + return + } + } + + # Get the current mode + $Mode = (Get-PSReadLineKeyHandler | Where-Object {$_.Key -eq "Tab" }).Function + __%[1]s_debug "Mode: $Mode" + + $Values | ForEach-Object { + + # store temporary because switch will overwrite $_ + $comp = $_ + + # PowerShell supports three different completion modes + # - TabCompleteNext (default windows style - on each key press the next option is displayed) + # - Complete (works like bash) + # - MenuComplete (works like zsh) + # You set the mode with Set-PSReadLineKeyHandler -Key Tab -Function + + # CompletionResult Arguments: + # 1) CompletionText text to be used as the auto completion result + # 2) ListItemText text to be displayed in the suggestion list + # 3) ResultType type of completion result + # 4) ToolTip text for the tooltip with details about the object + + switch ($Mode) { + + # bash like + "Complete" { + + if ($Values.Length -eq 1) { + __%[1]s_debug "Only one completion left" + + # insert space after value + $CompletionText = $($comp.Name | __%[1]s_escapeStringWithSpecialChars) + $Space + if ($ExecutionContext.SessionState.LanguageMode -eq "FullLanguage"){ + [System.Management.Automation.CompletionResult]::new($CompletionText, "$($comp.Name)", 'ParameterValue', "$($comp.Description)") + } else { + $CompletionText + } + + } else { + # Add the proper number of spaces to align the descriptions + while($comp.Name.Length -lt $Longest) { + $comp.Name = $comp.Name + " " + } + + # Check for empty description and only add parentheses if needed + if ($($comp.Description) -eq " " ) { + $Description = "" + } else { + $Description = " ($($comp.Description))" + } + + $CompletionText = "$($comp.Name)$Description" + if ($ExecutionContext.SessionState.LanguageMode -eq "FullLanguage"){ + [System.Management.Automation.CompletionResult]::new($CompletionText, "$($comp.Name)$Description", 'ParameterValue', "$($comp.Description)") + } else { + $CompletionText + } + } + } + + # zsh like + "MenuComplete" { + # insert space after value + # MenuComplete will automatically show the ToolTip of + # the highlighted value at the bottom of the suggestions. + + $CompletionText = $($comp.Name | __%[1]s_escapeStringWithSpecialChars) + $Space + if ($ExecutionContext.SessionState.LanguageMode -eq "FullLanguage"){ + [System.Management.Automation.CompletionResult]::new($CompletionText, "$($comp.Name)", 'ParameterValue', "$($comp.Description)") + } else { + $CompletionText + } + } + + # TabCompleteNext and in case we get something unknown + Default { + # Like MenuComplete but we don't want to add a space here because + # the user need to press space anyway to get the completion. + # Description will not be shown because that's not possible with TabCompleteNext + + $CompletionText = $($comp.Name | __%[1]s_escapeStringWithSpecialChars) + if ($ExecutionContext.SessionState.LanguageMode -eq "FullLanguage"){ + [System.Management.Automation.CompletionResult]::new($CompletionText, "$($comp.Name)", 'ParameterValue', "$($comp.Description)") + } else { + $CompletionText + } + } + } + + } +} + +Register-ArgumentCompleter -CommandName '%[1]s' -ScriptBlock ${__%[2]sCompleterBlock} +`, name, nameForVar, compCmd, + ShellCompDirectiveError, ShellCompDirectiveNoSpace, ShellCompDirectiveNoFileComp, + ShellCompDirectiveFilterFileExt, ShellCompDirectiveFilterDirs, ShellCompDirectiveKeepOrder, activeHelpEnvVar(name))) +} + +func (c *Command) genPowerShellCompletion(w io.Writer, includeDesc bool) error { + buf := new(bytes.Buffer) + genPowerShellComp(buf, c.Name(), includeDesc) + _, err := buf.WriteTo(w) + return err +} + +func (c *Command) genPowerShellCompletionFile(filename string, includeDesc bool) error { + outFile, err := os.Create(filename) + if err != nil { + return err + } + defer outFile.Close() + + return c.genPowerShellCompletion(outFile, includeDesc) +} + +// GenPowerShellCompletionFile generates powershell completion file without descriptions. +func (c *Command) GenPowerShellCompletionFile(filename string) error { + return c.genPowerShellCompletionFile(filename, false) +} + +// GenPowerShellCompletion generates powershell completion file without descriptions +// and writes it to the passed writer. +func (c *Command) GenPowerShellCompletion(w io.Writer) error { + return c.genPowerShellCompletion(w, false) +} + +// GenPowerShellCompletionFileWithDesc generates powershell completion file with descriptions. +func (c *Command) GenPowerShellCompletionFileWithDesc(filename string) error { + return c.genPowerShellCompletionFile(filename, true) +} + +// GenPowerShellCompletionWithDesc generates powershell completion file with descriptions +// and writes it to the passed writer. +func (c *Command) GenPowerShellCompletionWithDesc(w io.Writer) error { + return c.genPowerShellCompletion(w, true) +} diff --git a/go-controller/vendor/github.com/spf13/cobra/shell_completions.go b/go-controller/vendor/github.com/spf13/cobra/shell_completions.go new file mode 100644 index 0000000000..b035742d39 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/shell_completions.go @@ -0,0 +1,98 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "github.com/spf13/pflag" +) + +// MarkFlagRequired instructs the various shell completion implementations to +// prioritize the named flag when performing completion, +// and causes your command to report an error if invoked without the flag. +func (c *Command) MarkFlagRequired(name string) error { + return MarkFlagRequired(c.Flags(), name) +} + +// MarkPersistentFlagRequired instructs the various shell completion implementations to +// prioritize the named persistent flag when performing completion, +// and causes your command to report an error if invoked without the flag. +func (c *Command) MarkPersistentFlagRequired(name string) error { + return MarkFlagRequired(c.PersistentFlags(), name) +} + +// MarkFlagRequired instructs the various shell completion implementations to +// prioritize the named flag when performing completion, +// and causes your command to report an error if invoked without the flag. +func MarkFlagRequired(flags *pflag.FlagSet, name string) error { + return flags.SetAnnotation(name, BashCompOneRequiredFlag, []string{"true"}) +} + +// MarkFlagFilename instructs the various shell completion implementations to +// limit completions for the named flag to the specified file extensions. +func (c *Command) MarkFlagFilename(name string, extensions ...string) error { + return MarkFlagFilename(c.Flags(), name, extensions...) +} + +// MarkFlagCustom adds the BashCompCustom annotation to the named flag, if it exists. +// The bash completion script will call the bash function f for the flag. +// +// This will only work for bash completion. +// It is recommended to instead use c.RegisterFlagCompletionFunc(...) which allows +// to register a Go function which will work across all shells. +func (c *Command) MarkFlagCustom(name string, f string) error { + return MarkFlagCustom(c.Flags(), name, f) +} + +// MarkPersistentFlagFilename instructs the various shell completion +// implementations to limit completions for the named persistent flag to the +// specified file extensions. +func (c *Command) MarkPersistentFlagFilename(name string, extensions ...string) error { + return MarkFlagFilename(c.PersistentFlags(), name, extensions...) +} + +// MarkFlagFilename instructs the various shell completion implementations to +// limit completions for the named flag to the specified file extensions. +func MarkFlagFilename(flags *pflag.FlagSet, name string, extensions ...string) error { + return flags.SetAnnotation(name, BashCompFilenameExt, extensions) +} + +// MarkFlagCustom adds the BashCompCustom annotation to the named flag, if it exists. +// The bash completion script will call the bash function f for the flag. +// +// This will only work for bash completion. +// It is recommended to instead use c.RegisterFlagCompletionFunc(...) which allows +// to register a Go function which will work across all shells. +func MarkFlagCustom(flags *pflag.FlagSet, name string, f string) error { + return flags.SetAnnotation(name, BashCompCustom, []string{f}) +} + +// MarkFlagDirname instructs the various shell completion implementations to +// limit completions for the named flag to directory names. +func (c *Command) MarkFlagDirname(name string) error { + return MarkFlagDirname(c.Flags(), name) +} + +// MarkPersistentFlagDirname instructs the various shell completion +// implementations to limit completions for the named persistent flag to +// directory names. +func (c *Command) MarkPersistentFlagDirname(name string) error { + return MarkFlagDirname(c.PersistentFlags(), name) +} + +// MarkFlagDirname instructs the various shell completion implementations to +// limit completions for the named flag to directory names. +func MarkFlagDirname(flags *pflag.FlagSet, name string) error { + return flags.SetAnnotation(name, BashCompSubdirsInDir, []string{}) +} diff --git a/go-controller/vendor/github.com/spf13/cobra/zsh_completions.go b/go-controller/vendor/github.com/spf13/cobra/zsh_completions.go new file mode 100644 index 0000000000..1856e4c7f6 --- /dev/null +++ b/go-controller/vendor/github.com/spf13/cobra/zsh_completions.go @@ -0,0 +1,308 @@ +// Copyright 2013-2023 The Cobra Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cobra + +import ( + "bytes" + "fmt" + "io" + "os" +) + +// GenZshCompletionFile generates zsh completion file including descriptions. +func (c *Command) GenZshCompletionFile(filename string) error { + return c.genZshCompletionFile(filename, true) +} + +// GenZshCompletion generates zsh completion file including descriptions +// and writes it to the passed writer. +func (c *Command) GenZshCompletion(w io.Writer) error { + return c.genZshCompletion(w, true) +} + +// GenZshCompletionFileNoDesc generates zsh completion file without descriptions. +func (c *Command) GenZshCompletionFileNoDesc(filename string) error { + return c.genZshCompletionFile(filename, false) +} + +// GenZshCompletionNoDesc generates zsh completion file without descriptions +// and writes it to the passed writer. +func (c *Command) GenZshCompletionNoDesc(w io.Writer) error { + return c.genZshCompletion(w, false) +} + +// MarkZshCompPositionalArgumentFile only worked for zsh and its behavior was +// not consistent with Bash completion. It has therefore been disabled. +// Instead, when no other completion is specified, file completion is done by +// default for every argument. One can disable file completion on a per-argument +// basis by using ValidArgsFunction and ShellCompDirectiveNoFileComp. +// To achieve file extension filtering, one can use ValidArgsFunction and +// ShellCompDirectiveFilterFileExt. +// +// Deprecated +func (c *Command) MarkZshCompPositionalArgumentFile(argPosition int, patterns ...string) error { + return nil +} + +// MarkZshCompPositionalArgumentWords only worked for zsh. It has therefore +// been disabled. +// To achieve the same behavior across all shells, one can use +// ValidArgs (for the first argument only) or ValidArgsFunction for +// any argument (can include the first one also). +// +// Deprecated +func (c *Command) MarkZshCompPositionalArgumentWords(argPosition int, words ...string) error { + return nil +} + +func (c *Command) genZshCompletionFile(filename string, includeDesc bool) error { + outFile, err := os.Create(filename) + if err != nil { + return err + } + defer outFile.Close() + + return c.genZshCompletion(outFile, includeDesc) +} + +func (c *Command) genZshCompletion(w io.Writer, includeDesc bool) error { + buf := new(bytes.Buffer) + genZshComp(buf, c.Name(), includeDesc) + _, err := buf.WriteTo(w) + return err +} + +func genZshComp(buf io.StringWriter, name string, includeDesc bool) { + compCmd := ShellCompRequestCmd + if !includeDesc { + compCmd = ShellCompNoDescRequestCmd + } + WriteStringAndCheck(buf, fmt.Sprintf(`#compdef %[1]s +compdef _%[1]s %[1]s + +# zsh completion for %-36[1]s -*- shell-script -*- + +__%[1]s_debug() +{ + local file="$BASH_COMP_DEBUG_FILE" + if [[ -n ${file} ]]; then + echo "$*" >> "${file}" + fi +} + +_%[1]s() +{ + local shellCompDirectiveError=%[3]d + local shellCompDirectiveNoSpace=%[4]d + local shellCompDirectiveNoFileComp=%[5]d + local shellCompDirectiveFilterFileExt=%[6]d + local shellCompDirectiveFilterDirs=%[7]d + local shellCompDirectiveKeepOrder=%[8]d + + local lastParam lastChar flagPrefix requestComp out directive comp lastComp noSpace keepOrder + local -a completions + + __%[1]s_debug "\n========= starting completion logic ==========" + __%[1]s_debug "CURRENT: ${CURRENT}, words[*]: ${words[*]}" + + # The user could have moved the cursor backwards on the command-line. + # We need to trigger completion from the $CURRENT location, so we need + # to truncate the command-line ($words) up to the $CURRENT location. + # (We cannot use $CURSOR as its value does not work when a command is an alias.) + words=("${=words[1,CURRENT]}") + __%[1]s_debug "Truncated words[*]: ${words[*]}," + + lastParam=${words[-1]} + lastChar=${lastParam[-1]} + __%[1]s_debug "lastParam: ${lastParam}, lastChar: ${lastChar}" + + # For zsh, when completing a flag with an = (e.g., %[1]s -n=) + # completions must be prefixed with the flag + setopt local_options BASH_REMATCH + if [[ "${lastParam}" =~ '-.*=' ]]; then + # We are dealing with a flag with an = + flagPrefix="-P ${BASH_REMATCH}" + fi + + # Prepare the command to obtain completions + requestComp="${words[1]} %[2]s ${words[2,-1]}" + if [ "${lastChar}" = "" ]; then + # If the last parameter is complete (there is a space following it) + # We add an extra empty parameter so we can indicate this to the go completion code. + __%[1]s_debug "Adding extra empty parameter" + requestComp="${requestComp} \"\"" + fi + + __%[1]s_debug "About to call: eval ${requestComp}" + + # Use eval to handle any environment variables and such + out=$(eval ${requestComp} 2>/dev/null) + __%[1]s_debug "completion output: ${out}" + + # Extract the directive integer following a : from the last line + local lastLine + while IFS='\n' read -r line; do + lastLine=${line} + done < <(printf "%%s\n" "${out[@]}") + __%[1]s_debug "last line: ${lastLine}" + + if [ "${lastLine[1]}" = : ]; then + directive=${lastLine[2,-1]} + # Remove the directive including the : and the newline + local suffix + (( suffix=${#lastLine}+2)) + out=${out[1,-$suffix]} + else + # There is no directive specified. Leave $out as is. + __%[1]s_debug "No directive found. Setting do default" + directive=0 + fi + + __%[1]s_debug "directive: ${directive}" + __%[1]s_debug "completions: ${out}" + __%[1]s_debug "flagPrefix: ${flagPrefix}" + + if [ $((directive & shellCompDirectiveError)) -ne 0 ]; then + __%[1]s_debug "Completion received error. Ignoring completions." + return + fi + + local activeHelpMarker="%[9]s" + local endIndex=${#activeHelpMarker} + local startIndex=$((${#activeHelpMarker}+1)) + local hasActiveHelp=0 + while IFS='\n' read -r comp; do + # Check if this is an activeHelp statement (i.e., prefixed with $activeHelpMarker) + if [ "${comp[1,$endIndex]}" = "$activeHelpMarker" ];then + __%[1]s_debug "ActiveHelp found: $comp" + comp="${comp[$startIndex,-1]}" + if [ -n "$comp" ]; then + compadd -x "${comp}" + __%[1]s_debug "ActiveHelp will need delimiter" + hasActiveHelp=1 + fi + + continue + fi + + if [ -n "$comp" ]; then + # If requested, completions are returned with a description. + # The description is preceded by a TAB character. + # For zsh's _describe, we need to use a : instead of a TAB. + # We first need to escape any : as part of the completion itself. + comp=${comp//:/\\:} + + local tab="$(printf '\t')" + comp=${comp//$tab/:} + + __%[1]s_debug "Adding completion: ${comp}" + completions+=${comp} + lastComp=$comp + fi + done < <(printf "%%s\n" "${out[@]}") + + # Add a delimiter after the activeHelp statements, but only if: + # - there are completions following the activeHelp statements, or + # - file completion will be performed (so there will be choices after the activeHelp) + if [ $hasActiveHelp -eq 1 ]; then + if [ ${#completions} -ne 0 ] || [ $((directive & shellCompDirectiveNoFileComp)) -eq 0 ]; then + __%[1]s_debug "Adding activeHelp delimiter" + compadd -x "--" + hasActiveHelp=0 + fi + fi + + if [ $((directive & shellCompDirectiveNoSpace)) -ne 0 ]; then + __%[1]s_debug "Activating nospace." + noSpace="-S ''" + fi + + if [ $((directive & shellCompDirectiveKeepOrder)) -ne 0 ]; then + __%[1]s_debug "Activating keep order." + keepOrder="-V" + fi + + if [ $((directive & shellCompDirectiveFilterFileExt)) -ne 0 ]; then + # File extension filtering + local filteringCmd + filteringCmd='_files' + for filter in ${completions[@]}; do + if [ ${filter[1]} != '*' ]; then + # zsh requires a glob pattern to do file filtering + filter="\*.$filter" + fi + filteringCmd+=" -g $filter" + done + filteringCmd+=" ${flagPrefix}" + + __%[1]s_debug "File filtering command: $filteringCmd" + _arguments '*:filename:'"$filteringCmd" + elif [ $((directive & shellCompDirectiveFilterDirs)) -ne 0 ]; then + # File completion for directories only + local subdir + subdir="${completions[1]}" + if [ -n "$subdir" ]; then + __%[1]s_debug "Listing directories in $subdir" + pushd "${subdir}" >/dev/null 2>&1 + else + __%[1]s_debug "Listing directories in ." + fi + + local result + _arguments '*:dirname:_files -/'" ${flagPrefix}" + result=$? + if [ -n "$subdir" ]; then + popd >/dev/null 2>&1 + fi + return $result + else + __%[1]s_debug "Calling _describe" + if eval _describe $keepOrder "completions" completions $flagPrefix $noSpace; then + __%[1]s_debug "_describe found some completions" + + # Return the success of having called _describe + return 0 + else + __%[1]s_debug "_describe did not find completions." + __%[1]s_debug "Checking if we should do file completion." + if [ $((directive & shellCompDirectiveNoFileComp)) -ne 0 ]; then + __%[1]s_debug "deactivating file completion" + + # We must return an error code here to let zsh know that there were no + # completions found by _describe; this is what will trigger other + # matching algorithms to attempt to find completions. + # For example zsh can match letters in the middle of words. + return 1 + else + # Perform file completion + __%[1]s_debug "Activating file completion" + + # We must return the result of this command, so it must be the + # last command, or else we must store its result to return it. + _arguments '*:filename:_files'" ${flagPrefix}" + fi + fi + fi +} + +# don't run the completion function when being source-ed or eval-ed +if [ "$funcstack[1]" = "_%[1]s" ]; then + _%[1]s +fi +`, name, compCmd, + ShellCompDirectiveError, ShellCompDirectiveNoSpace, ShellCompDirectiveNoFileComp, + ShellCompDirectiveFilterFileExt, ShellCompDirectiveFilterDirs, ShellCompDirectiveKeepOrder, + activeHelpMarker)) +} diff --git a/go-controller/vendor/golang.org/x/oauth2/oauth2.go b/go-controller/vendor/golang.org/x/oauth2/oauth2.go index 74f052aa9f..eacdd7fd93 100644 --- a/go-controller/vendor/golang.org/x/oauth2/oauth2.go +++ b/go-controller/vendor/golang.org/x/oauth2/oauth2.go @@ -288,7 +288,7 @@ func (tf *tokenRefresher) Token() (*Token, error) { if tf.refreshToken != tk.RefreshToken { tf.refreshToken = tk.RefreshToken } - return tk, err + return tk, nil } // reuseTokenSource is a TokenSource that holds a single token in memory @@ -356,11 +356,15 @@ func NewClient(ctx context.Context, src TokenSource) *http.Client { if src == nil { return internal.ContextClient(ctx) } + cc := internal.ContextClient(ctx) return &http.Client{ Transport: &Transport{ - Base: internal.ContextClient(ctx).Transport, + Base: cc.Transport, Source: ReuseTokenSource(nil, src), }, + CheckRedirect: cc.CheckRedirect, + Jar: cc.Jar, + Timeout: cc.Timeout, } } diff --git a/go-controller/vendor/golang.org/x/oauth2/token.go b/go-controller/vendor/golang.org/x/oauth2/token.go index 109997d77c..8c31136c40 100644 --- a/go-controller/vendor/golang.org/x/oauth2/token.go +++ b/go-controller/vendor/golang.org/x/oauth2/token.go @@ -169,7 +169,7 @@ func tokenFromInternal(t *internal.Token) *Token { // retrieveToken takes a *Config and uses that to retrieve an *internal.Token. // This token is then mapped from *internal.Token into an *oauth2.Token which is returned along -// with an error.. +// with an error. func retrieveToken(ctx context.Context, c *Config, v url.Values) (*Token, error) { tk, err := internal.RetrieveToken(ctx, c.ClientID, c.ClientSecret, c.Endpoint.TokenURL, v, internal.AuthStyle(c.Endpoint.AuthStyle), c.authStyleCache.Get()) if err != nil { diff --git a/go-controller/vendor/golang.org/x/time/rate/rate.go b/go-controller/vendor/golang.org/x/time/rate/rate.go index 93a798ab63..794b2e32bf 100644 --- a/go-controller/vendor/golang.org/x/time/rate/rate.go +++ b/go-controller/vendor/golang.org/x/time/rate/rate.go @@ -85,7 +85,7 @@ func (lim *Limiter) Burst() int { // TokensAt returns the number of tokens available at time t. func (lim *Limiter) TokensAt(t time.Time) float64 { lim.mu.Lock() - _, tokens := lim.advance(t) // does not mutate lim + tokens := lim.advance(t) // does not mutate lim lim.mu.Unlock() return tokens } @@ -186,7 +186,7 @@ func (r *Reservation) CancelAt(t time.Time) { return } // advance time to now - t, tokens := r.lim.advance(t) + tokens := r.lim.advance(t) // calculate new number of tokens tokens += restoreTokens if burst := float64(r.lim.burst); tokens > burst { @@ -307,7 +307,7 @@ func (lim *Limiter) SetLimitAt(t time.Time, newLimit Limit) { lim.mu.Lock() defer lim.mu.Unlock() - t, tokens := lim.advance(t) + tokens := lim.advance(t) lim.last = t lim.tokens = tokens @@ -324,7 +324,7 @@ func (lim *Limiter) SetBurstAt(t time.Time, newBurst int) { lim.mu.Lock() defer lim.mu.Unlock() - t, tokens := lim.advance(t) + tokens := lim.advance(t) lim.last = t lim.tokens = tokens @@ -347,7 +347,7 @@ func (lim *Limiter) reserveN(t time.Time, n int, maxFutureReserve time.Duration) } } - t, tokens := lim.advance(t) + tokens := lim.advance(t) // Calculate the remaining number of tokens resulting from the request. tokens -= float64(n) @@ -380,10 +380,11 @@ func (lim *Limiter) reserveN(t time.Time, n int, maxFutureReserve time.Duration) return r } -// advance calculates and returns an updated state for lim resulting from the passage of time. +// advance calculates and returns an updated number of tokens for lim +// resulting from the passage of time. // lim is not changed. // advance requires that lim.mu is held. -func (lim *Limiter) advance(t time.Time) (newT time.Time, newTokens float64) { +func (lim *Limiter) advance(t time.Time) (newTokens float64) { last := lim.last if t.Before(last) { last = t @@ -396,7 +397,7 @@ func (lim *Limiter) advance(t time.Time) (newT time.Time, newTokens float64) { if burst := float64(lim.burst); tokens > burst { tokens = burst } - return t, tokens + return tokens } // durationFromTokens is a unit conversion function from the number of tokens to the duration @@ -405,8 +406,15 @@ func (limit Limit) durationFromTokens(tokens float64) time.Duration { if limit <= 0 { return InfDuration } - seconds := tokens / float64(limit) - return time.Duration(float64(time.Second) * seconds) + + duration := (tokens / float64(limit)) * float64(time.Second) + + // Cap the duration to the maximum representable int64 value, to avoid overflow. + if duration > float64(math.MaxInt64) { + return InfDuration + } + + return time.Duration(duration) } // tokensFromDuration is a unit conversion function from a time duration to the number of tokens diff --git a/go-controller/vendor/gomodules.xyz/jsonpatch/v2/jsonpatch.go b/go-controller/vendor/gomodules.xyz/jsonpatch/v2/jsonpatch.go index 0d7823b3cd..d88162ff57 100644 --- a/go-controller/vendor/gomodules.xyz/jsonpatch/v2/jsonpatch.go +++ b/go-controller/vendor/gomodules.xyz/jsonpatch/v2/jsonpatch.go @@ -70,12 +70,14 @@ func CreatePatch(a, b []byte) ([]Operation, error) { } var aI interface{} var bI interface{} - err := json.Unmarshal(a, &aI) - if err != nil { + aDec := json.NewDecoder(bytes.NewReader(a)) + aDec.UseNumber() + if err := aDec.Decode(&aI); err != nil { return nil, errBadJSONDoc } - err = json.Unmarshal(b, &bI) - if err != nil { + bDec := json.NewDecoder(bytes.NewReader(b)) + bDec.UseNumber() + if err := bDec.Decode(&bI); err != nil { return nil, errBadJSONDoc } return handleValues(aI, bI, "", []Operation{}) @@ -94,6 +96,11 @@ func matchesValue(av, bv interface{}) bool { if ok && bt == at { return true } + case json.Number: + bt, ok := bv.(json.Number) + if ok && bt == at { + return true + } case float64: bt, ok := bv.(float64) if ok && bt == at { @@ -212,7 +219,7 @@ func handleValues(av, bv interface{}, p string, patch []Operation) ([]Operation, if err != nil { return nil, err } - case string, float64, bool: + case string, float64, bool, json.Number: if !matchesValue(av, bv) { patch = append(patch, NewOperation("replace", p, bv)) } diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/ciphersuites_flag.go b/go-controller/vendor/k8s.io/component-base/cli/flag/ciphersuites_flag.go new file mode 100644 index 0000000000..11adc26831 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/ciphersuites_flag.go @@ -0,0 +1,147 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "crypto/tls" + "fmt" + + "k8s.io/apimachinery/pkg/util/sets" +) + +var ( + // ciphers maps strings into tls package cipher constants in + // https://golang.org/pkg/crypto/tls/#pkg-constants + ciphers = map[string]uint16{} + insecureCiphers = map[string]uint16{} +) + +func init() { + for _, suite := range tls.CipherSuites() { + ciphers[suite.Name] = suite.ID + } + // keep legacy names for backward compatibility + ciphers["TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"] = tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 + ciphers["TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305"] = tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 + + for _, suite := range tls.InsecureCipherSuites() { + insecureCiphers[suite.Name] = suite.ID + } +} + +// InsecureTLSCiphers returns the cipher suites implemented by crypto/tls which have +// security issues. +func InsecureTLSCiphers() map[string]uint16 { + cipherKeys := make(map[string]uint16, len(insecureCiphers)) + for k, v := range insecureCiphers { + cipherKeys[k] = v + } + return cipherKeys +} + +// InsecureTLSCipherNames returns a list of cipher suite names implemented by crypto/tls +// which have security issues. +func InsecureTLSCipherNames() []string { + cipherKeys := sets.NewString() + for key := range insecureCiphers { + cipherKeys.Insert(key) + } + return cipherKeys.List() +} + +// PreferredTLSCipherNames returns a list of cipher suite names implemented by crypto/tls. +func PreferredTLSCipherNames() []string { + cipherKeys := sets.NewString() + for key := range ciphers { + cipherKeys.Insert(key) + } + return cipherKeys.List() +} + +func allCiphers() map[string]uint16 { + acceptedCiphers := make(map[string]uint16, len(ciphers)+len(insecureCiphers)) + for k, v := range ciphers { + acceptedCiphers[k] = v + } + for k, v := range insecureCiphers { + acceptedCiphers[k] = v + } + return acceptedCiphers +} + +// TLSCipherPossibleValues returns all acceptable cipher suite names. +// This is a combination of both InsecureTLSCipherNames() and PreferredTLSCipherNames(). +func TLSCipherPossibleValues() []string { + cipherKeys := sets.NewString() + acceptedCiphers := allCiphers() + for key := range acceptedCiphers { + cipherKeys.Insert(key) + } + return cipherKeys.List() +} + +// TLSCipherSuites returns a list of cipher suite IDs from the cipher suite names passed. +func TLSCipherSuites(cipherNames []string) ([]uint16, error) { + if len(cipherNames) == 0 { + return nil, nil + } + ciphersIntSlice := make([]uint16, 0) + possibleCiphers := allCiphers() + for _, cipher := range cipherNames { + intValue, ok := possibleCiphers[cipher] + if !ok { + return nil, fmt.Errorf("Cipher suite %s not supported or doesn't exist", cipher) + } + ciphersIntSlice = append(ciphersIntSlice, intValue) + } + return ciphersIntSlice, nil +} + +var versions = map[string]uint16{ + "VersionTLS10": tls.VersionTLS10, + "VersionTLS11": tls.VersionTLS11, + "VersionTLS12": tls.VersionTLS12, + "VersionTLS13": tls.VersionTLS13, +} + +// TLSPossibleVersions returns all acceptable values for TLS Version. +func TLSPossibleVersions() []string { + versionsKeys := sets.NewString() + for key := range versions { + versionsKeys.Insert(key) + } + return versionsKeys.List() +} + +// TLSVersion returns the TLS Version ID for the version name passed. +func TLSVersion(versionName string) (uint16, error) { + if len(versionName) == 0 { + return DefaultTLSVersion(), nil + } + if version, ok := versions[versionName]; ok { + return version, nil + } + return 0, fmt.Errorf("unknown tls version %q", versionName) +} + +// DefaultTLSVersion defines the default TLS Version. +func DefaultTLSVersion() uint16 { + // Can't use SSLv3 because of POODLE and BEAST + // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher + // Can't use TLSv1.1 because of RC4 cipher usage + return tls.VersionTLS12 +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go b/go-controller/vendor/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go new file mode 100644 index 0000000000..728fa520b6 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go @@ -0,0 +1,114 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "sort" + "strings" +) + +// ColonSeparatedMultimapStringString supports setting a map[string][]string from an encoding +// that separates keys from values with ':' and separates key-value pairs with ','. +// A key can be repeated multiple times, in which case the values are appended to a +// slice of strings associated with that key. Items in the list associated with a given +// key will appear in the order provided. +// For example: `a:hello,b:again,c:world,b:beautiful` results in `{"a": ["hello"], "b": ["again", "beautiful"], "c": ["world"]}` +// The first call to Set will clear the map before adding entries; subsequent calls will simply append to the map. +// This makes it possible to override default values with a command-line option rather than appending to defaults, +// while still allowing the distribution of key-value pairs across multiple flag invocations. +// For example: `--flag "a:hello" --flag "b:again" --flag "b:beautiful" --flag "c:world"` results in `{"a": ["hello"], "b": ["again", "beautiful"], "c": ["world"]}` +type ColonSeparatedMultimapStringString struct { + Multimap *map[string][]string + initialized bool // set to true after the first Set call + allowDefaultEmptyKey bool +} + +// NewColonSeparatedMultimapStringString takes a pointer to a map[string][]string and returns the +// ColonSeparatedMultimapStringString flag parsing shim for that map. +func NewColonSeparatedMultimapStringString(m *map[string][]string) *ColonSeparatedMultimapStringString { + return &ColonSeparatedMultimapStringString{Multimap: m} +} + +// NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey takes a pointer to a map[string][]string and returns the +// ColonSeparatedMultimapStringString flag parsing shim for that map. It allows default empty key with no colon in the flag. +func NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(m *map[string][]string) *ColonSeparatedMultimapStringString { + return &ColonSeparatedMultimapStringString{Multimap: m, allowDefaultEmptyKey: true} +} + +// Set implements github.com/spf13/pflag.Value +func (m *ColonSeparatedMultimapStringString) Set(value string) error { + if m.Multimap == nil { + return fmt.Errorf("no target (nil pointer to map[string][]string)") + } + if !m.initialized || *m.Multimap == nil { + // clear default values, or allocate if no existing map + *m.Multimap = make(map[string][]string) + m.initialized = true + } + for _, pair := range strings.Split(value, ",") { + if len(pair) == 0 { + continue + } + kv := strings.SplitN(pair, ":", 2) + var k, v string + if m.allowDefaultEmptyKey && len(kv) == 1 { + v = strings.TrimSpace(kv[0]) + } else { + if len(kv) != 2 { + return fmt.Errorf("malformed pair, expect string:string") + } + k = strings.TrimSpace(kv[0]) + v = strings.TrimSpace(kv[1]) + } + (*m.Multimap)[k] = append((*m.Multimap)[k], v) + } + return nil +} + +// String implements github.com/spf13/pflag.Value +func (m *ColonSeparatedMultimapStringString) String() string { + type kv struct { + k string + v string + } + kvs := make([]kv, 0, len(*m.Multimap)) + for k, vs := range *m.Multimap { + for i := range vs { + kvs = append(kvs, kv{k: k, v: vs[i]}) + } + } + // stable sort by keys, order of values should be preserved + sort.SliceStable(kvs, func(i, j int) bool { + return kvs[i].k < kvs[j].k + }) + pairs := make([]string, 0, len(kvs)) + for i := range kvs { + pairs = append(pairs, fmt.Sprintf("%s:%s", kvs[i].k, kvs[i].v)) + } + return strings.Join(pairs, ",") +} + +// Type implements github.com/spf13/pflag.Value +func (m *ColonSeparatedMultimapStringString) Type() string { + return "colonSeparatedMultimapStringString" +} + +// Empty implements OmitEmpty +func (m *ColonSeparatedMultimapStringString) Empty() bool { + return len(*m.Multimap) == 0 +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/configuration_map.go b/go-controller/vendor/k8s.io/component-base/cli/flag/configuration_map.go new file mode 100644 index 0000000000..911b05ec6c --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/configuration_map.go @@ -0,0 +1,53 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "sort" + "strings" +) + +type ConfigurationMap map[string]string + +func (m *ConfigurationMap) String() string { + pairs := []string{} + for k, v := range *m { + pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) + } + sort.Strings(pairs) + return strings.Join(pairs, ",") +} + +func (m *ConfigurationMap) Set(value string) error { + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + arr := strings.SplitN(s, "=", 2) + if len(arr) == 2 { + (*m)[strings.TrimSpace(arr[0])] = strings.TrimSpace(arr[1]) + } else { + (*m)[strings.TrimSpace(arr[0])] = "" + } + } + return nil +} + +func (*ConfigurationMap) Type() string { + return "mapStringString" +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/flags.go b/go-controller/vendor/k8s.io/component-base/cli/flag/flags.go new file mode 100644 index 0000000000..8d4a59ce96 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/flags.go @@ -0,0 +1,66 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + goflag "flag" + "strings" + + "github.com/spf13/pflag" + "k8s.io/klog/v2" +) + +var underscoreWarnings = make(map[string]struct{}) + +// WordSepNormalizeFunc changes all flags that contain "_" separators +func WordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { + if strings.Contains(name, "_") { + return pflag.NormalizedName(strings.Replace(name, "_", "-", -1)) + } + return pflag.NormalizedName(name) +} + +// WarnWordSepNormalizeFunc changes and warns for flags that contain "_" separators +func WarnWordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { + if strings.Contains(name, "_") { + nname := strings.Replace(name, "_", "-", -1) + if _, alreadyWarned := underscoreWarnings[name]; !alreadyWarned { + klog.Warningf("using an underscore in a flag name is not supported. %s has been converted to %s.", name, nname) + underscoreWarnings[name] = struct{}{} + } + + return pflag.NormalizedName(nname) + } + return pflag.NormalizedName(name) +} + +// InitFlags normalizes, parses, then logs the command line flags +func InitFlags() { + pflag.CommandLine.SetNormalizeFunc(WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + pflag.Parse() + pflag.VisitAll(func(flag *pflag.Flag) { + klog.V(2).Infof("FLAG: --%s=%q", flag.Name, flag.Value) + }) +} + +// PrintFlags logs the flags in the flagset +func PrintFlags(flags *pflag.FlagSet) { + flags.VisitAll(func(flag *pflag.Flag) { + klog.V(1).Infof("FLAG: --%s=%q", flag.Name, flag.Value) + }) +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/langle_separated_map_string_string.go b/go-controller/vendor/k8s.io/component-base/cli/flag/langle_separated_map_string_string.go new file mode 100644 index 0000000000..bf8dbfb9bf --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/langle_separated_map_string_string.go @@ -0,0 +1,82 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "sort" + "strings" +) + +// LangleSeparatedMapStringString can be set from the command line with the format `--flag "string 0 { + s = s + ":" + strings.Join(nkc.Names, ",") + } + return s +} + +func (nkc *NamedCertKey) Set(value string) error { + cs := strings.SplitN(value, ":", 2) + var keycert string + if len(cs) == 2 { + var names string + keycert, names = strings.TrimSpace(cs[0]), strings.TrimSpace(cs[1]) + if names == "" { + return errors.New("empty names list is not allowed") + } + nkc.Names = nil + for _, name := range strings.Split(names, ",") { + nkc.Names = append(nkc.Names, strings.TrimSpace(name)) + } + } else { + nkc.Names = nil + keycert = strings.TrimSpace(cs[0]) + } + cs = strings.Split(keycert, ",") + if len(cs) != 2 { + return errors.New("expected comma separated certificate and key file paths") + } + nkc.CertFile = strings.TrimSpace(cs[0]) + nkc.KeyFile = strings.TrimSpace(cs[1]) + return nil +} + +func (*NamedCertKey) Type() string { + return "namedCertKey" +} + +// NamedCertKeyArray is a flag value parsing NamedCertKeys, each passed with its own +// flag instance (in contrast to comma separated slices). +type NamedCertKeyArray struct { + value *[]NamedCertKey + changed bool +} + +var _ flag.Value = &NamedCertKeyArray{} + +// NewNamedKeyCertArray creates a new NamedCertKeyArray with the internal value +// pointing to p. +func NewNamedCertKeyArray(p *[]NamedCertKey) *NamedCertKeyArray { + return &NamedCertKeyArray{ + value: p, + } +} + +func (a *NamedCertKeyArray) Set(val string) error { + nkc := NamedCertKey{} + err := nkc.Set(val) + if err != nil { + return err + } + if !a.changed { + *a.value = []NamedCertKey{nkc} + a.changed = true + } else { + *a.value = append(*a.value, nkc) + } + return nil +} + +func (a *NamedCertKeyArray) Type() string { + return "namedCertKey" +} + +func (a *NamedCertKeyArray) String() string { + nkcs := make([]string, 0, len(*a.value)) + for i := range *a.value { + nkcs = append(nkcs, (*a.value)[i].String()) + } + return "[" + strings.Join(nkcs, ";") + "]" +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/noop.go b/go-controller/vendor/k8s.io/component-base/cli/flag/noop.go new file mode 100644 index 0000000000..03f7f14c0b --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/noop.go @@ -0,0 +1,41 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + goflag "flag" + "github.com/spf13/pflag" +) + +// NoOp implements goflag.Value and plfag.Value, +// but has a noop Set implementation +type NoOp struct{} + +var _ goflag.Value = NoOp{} +var _ pflag.Value = NoOp{} + +func (NoOp) String() string { + return "" +} + +func (NoOp) Set(val string) error { + return nil +} + +func (NoOp) Type() string { + return "NoOp" +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/omitempty.go b/go-controller/vendor/k8s.io/component-base/cli/flag/omitempty.go new file mode 100644 index 0000000000..c354754ea7 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/omitempty.go @@ -0,0 +1,24 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +// OmitEmpty is an interface for flags to report whether their underlying value +// is "empty." If a flag implements OmitEmpty and returns true for a call to Empty(), +// it is assumed that flag may be omitted from the command line. +type OmitEmpty interface { + Empty() bool +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/sectioned.go b/go-controller/vendor/k8s.io/component-base/cli/flag/sectioned.go new file mode 100644 index 0000000000..2357428761 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/sectioned.go @@ -0,0 +1,105 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "bytes" + "fmt" + "io" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +const ( + usageFmt = "Usage:\n %s\n" +) + +// NamedFlagSets stores named flag sets in the order of calling FlagSet. +type NamedFlagSets struct { + // Order is an ordered list of flag set names. + Order []string + // FlagSets stores the flag sets by name. + FlagSets map[string]*pflag.FlagSet + // NormalizeNameFunc is the normalize function which used to initialize FlagSets created by NamedFlagSets. + NormalizeNameFunc func(f *pflag.FlagSet, name string) pflag.NormalizedName +} + +// FlagSet returns the flag set with the given name and adds it to the +// ordered name list if it is not in there yet. +func (nfs *NamedFlagSets) FlagSet(name string) *pflag.FlagSet { + if nfs.FlagSets == nil { + nfs.FlagSets = map[string]*pflag.FlagSet{} + } + if _, ok := nfs.FlagSets[name]; !ok { + flagSet := pflag.NewFlagSet(name, pflag.ExitOnError) + flagSet.SetNormalizeFunc(pflag.CommandLine.GetNormalizeFunc()) + if nfs.NormalizeNameFunc != nil { + flagSet.SetNormalizeFunc(nfs.NormalizeNameFunc) + } + nfs.FlagSets[name] = flagSet + nfs.Order = append(nfs.Order, name) + } + return nfs.FlagSets[name] +} + +// PrintSections prints the given names flag sets in sections, with the maximal given column number. +// If cols is zero, lines are not wrapped. +func PrintSections(w io.Writer, fss NamedFlagSets, cols int) { + for _, name := range fss.Order { + fs := fss.FlagSets[name] + if !fs.HasFlags() { + continue + } + + wideFS := pflag.NewFlagSet("", pflag.ExitOnError) + wideFS.AddFlagSet(fs) + + var zzz string + if cols > 24 { + zzz = strings.Repeat("z", cols-24) + wideFS.Int(zzz, 0, strings.Repeat("z", cols-24)) + } + + var buf bytes.Buffer + fmt.Fprintf(&buf, "\n%s flags:\n\n%s", strings.ToUpper(name[:1])+name[1:], wideFS.FlagUsagesWrapped(cols)) + + if cols > 24 { + i := strings.Index(buf.String(), zzz) + lines := strings.Split(buf.String()[:i], "\n") + fmt.Fprint(w, strings.Join(lines[:len(lines)-1], "\n")) + fmt.Fprintln(w) + } else { + fmt.Fprint(w, buf.String()) + } + } +} + +// SetUsageAndHelpFunc set both usage and help function. +// Print the flag sets we need instead of all of them. +func SetUsageAndHelpFunc(cmd *cobra.Command, fss NamedFlagSets, cols int) { + cmd.SetUsageFunc(func(cmd *cobra.Command) error { + fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) + PrintSections(cmd.OutOrStderr(), fss, cols) + return nil + }) + cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { + fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) + PrintSections(cmd.OutOrStdout(), fss, cols) + }) +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/string_flag.go b/go-controller/vendor/k8s.io/component-base/cli/flag/string_flag.go new file mode 100644 index 0000000000..331bdb66e2 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/string_flag.go @@ -0,0 +1,56 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +// StringFlag is a string flag compatible with flags and pflags that keeps track of whether it had a value supplied or not. +type StringFlag struct { + // If Set has been invoked this value is true + provided bool + // The exact value provided on the flag + value string +} + +func NewStringFlag(defaultVal string) StringFlag { + return StringFlag{value: defaultVal} +} + +func (f *StringFlag) Default(value string) { + f.value = value +} + +func (f StringFlag) String() string { + return f.value +} + +func (f StringFlag) Value() string { + return f.value +} + +func (f *StringFlag) Set(value string) error { + f.value = value + f.provided = true + + return nil +} + +func (f StringFlag) Provided() bool { + return f.provided +} + +func (f *StringFlag) Type() string { + return "string" +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/string_slice_flag.go b/go-controller/vendor/k8s.io/component-base/cli/flag/string_slice_flag.go new file mode 100644 index 0000000000..ad0d07d758 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/string_slice_flag.go @@ -0,0 +1,62 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + goflag "flag" + "fmt" + "strings" + + "github.com/spf13/pflag" +) + +// StringSlice implements goflag.Value and plfag.Value, +// and allows set to be invoked repeatedly to accumulate values. +type StringSlice struct { + value *[]string + changed bool +} + +func NewStringSlice(s *[]string) *StringSlice { + return &StringSlice{value: s} +} + +var _ goflag.Value = &StringSlice{} +var _ pflag.Value = &StringSlice{} + +func (s *StringSlice) String() string { + if s == nil || s.value == nil { + return "" + } + return strings.Join(*s.value, " ") +} + +func (s *StringSlice) Set(val string) error { + if s.value == nil { + return fmt.Errorf("no target (nil pointer to []string)") + } + if !s.changed { + *s.value = make([]string, 0) + } + *s.value = append(*s.value, val) + s.changed = true + return nil +} + +func (StringSlice) Type() string { + return "sliceString" +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/tracker_flag.go b/go-controller/vendor/k8s.io/component-base/cli/flag/tracker_flag.go new file mode 100644 index 0000000000..a7f6efed3e --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/tracker_flag.go @@ -0,0 +1,82 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "github.com/spf13/pflag" +) + +// TrackerValue wraps a non-boolean value and stores true in the provided boolean when it is set. +type TrackerValue struct { + value pflag.Value + provided *bool +} + +// BoolTrackerValue wraps a boolean value and stores true in the provided boolean when it is set. +type BoolTrackerValue struct { + boolValue + provided *bool +} + +type boolValue interface { + pflag.Value + IsBoolFlag() bool +} + +var _ pflag.Value = &TrackerValue{} +var _ boolValue = &BoolTrackerValue{} + +// NewTracker returns a Value wrapping the given value which stores true in the provided boolean when it is set. +func NewTracker(value pflag.Value, provided *bool) pflag.Value { + if value == nil { + panic("value must not be nil") + } + + if provided == nil { + panic("provided boolean must not be nil") + } + + if boolValue, ok := value.(boolValue); ok { + return &BoolTrackerValue{boolValue: boolValue, provided: provided} + } + return &TrackerValue{value: value, provided: provided} +} + +func (f *TrackerValue) String() string { + return f.value.String() +} + +func (f *TrackerValue) Set(value string) error { + err := f.value.Set(value) + if err == nil { + *f.provided = true + } + return err +} + +func (f *TrackerValue) Type() string { + return f.value.Type() +} + +func (f *BoolTrackerValue) Set(value string) error { + err := f.boolValue.Set(value) + if err == nil { + *f.provided = true + } + + return err +} diff --git a/go-controller/vendor/k8s.io/component-base/cli/flag/tristate.go b/go-controller/vendor/k8s.io/component-base/cli/flag/tristate.go new file mode 100644 index 0000000000..cf16376bf9 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/cli/flag/tristate.go @@ -0,0 +1,83 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "strconv" +) + +// Tristate is a flag compatible with flags and pflags that +// keeps track of whether it had a value supplied or not. +type Tristate int + +const ( + Unset Tristate = iota // 0 + True + False +) + +func (f *Tristate) Default(value bool) { + *f = triFromBool(value) +} + +func (f Tristate) String() string { + b := boolFromTri(f) + return fmt.Sprintf("%t", b) +} + +func (f Tristate) Value() bool { + b := boolFromTri(f) + return b +} + +func (f *Tristate) Set(value string) error { + boolVal, err := strconv.ParseBool(value) + if err != nil { + return err + } + + *f = triFromBool(boolVal) + return nil +} + +func (f Tristate) Provided() bool { + if f != Unset { + return true + } + return false +} + +func (f *Tristate) Type() string { + return "tristate" +} + +func boolFromTri(t Tristate) bool { + if t == True { + return true + } else { + return false + } +} + +func triFromBool(b bool) Tristate { + if b { + return True + } else { + return False + } +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/doc.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/doc.go new file mode 100644 index 0000000000..dee5335fa8 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/doc.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package + +// Package v1 contains the configuration API for logging. +// +// The intention is to only have a single version of this API, potentially with +// new fields added over time in a backwards-compatible manner. Fields for +// alpha or beta features are allowed as long as they are defined so that not +// changing the defaults leaves those features disabled. +// +// The "v1" package name is just a reminder that API compatibility rules apply, +// not an indication of the stability of all features covered by it. + +// The LoggingAlphaOptions and LoggingBetaOptions feature gates control whether +// these unstable features can get enabled. This can be used to ensure that +// command invocations do not accidentally rely on unstable features. +package v1 diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/kube_features.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/kube_features.go new file mode 100644 index 0000000000..fe23f57931 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/kube_features.go @@ -0,0 +1,82 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" +) + +const ( + // owner: @pohly + // kep: https://kep.k8s.io/3077 + // alpha: v1.24 + // beta: v1.30 + // + // Enables looking up a logger from a context.Context instead of using + // the global fallback logger and manipulating the logger that is + // used by a call chain. + ContextualLogging featuregate.Feature = "ContextualLogging" + + // Allow fine-tuning of experimental, alpha-quality logging options. + // + // Per https://groups.google.com/g/kubernetes-sig-architecture/c/Nxsc7pfe5rw/m/vF2djJh0BAAJ + // we want to avoid a proliferation of feature gates. This feature gate: + // - will guard *a group* of logging options whose quality level is alpha. + // - will never graduate to beta or stable. + // + // IMPORTANT: Unlike typical feature gates, LoggingAlphaOptions is NOT affected by + // emulation version changes. Its behavior remains constant regardless of the + // emulation version being used. + LoggingAlphaOptions featuregate.Feature = "LoggingAlphaOptions" + + // Allow fine-tuning of experimental, beta-quality logging options. + // + // Per https://groups.google.com/g/kubernetes-sig-architecture/c/Nxsc7pfe5rw/m/vF2djJh0BAAJ + // we want to avoid a proliferation of feature gates. This feature gate: + // - will guard *a group* of logging options whose quality level is beta. + // - is thus *introduced* as beta + // - will never graduate to stable. + // + // IMPORTANT: Unlike typical feature gates, LoggingBetaOptions is NOT affected by + // emulation version changes. Its behavior remains constant regardless of the + // emulation version being used. + LoggingBetaOptions featuregate.Feature = "LoggingBetaOptions" + + // Stable logging options. Always enabled. + LoggingStableOptions featuregate.Feature = "LoggingStableOptions" +) + +func featureGates() map[featuregate.Feature]featuregate.VersionedSpecs { + return map[featuregate.Feature]featuregate.VersionedSpecs{ + ContextualLogging: { + {Version: version.MustParse("1.24"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, + }, + LoggingAlphaOptions: { + {Version: version.MustParse("1.24"), Default: false, PreRelease: featuregate.Alpha}, + }, + LoggingBetaOptions: { + {Version: version.MustParse("1.24"), Default: true, PreRelease: featuregate.Beta}, + }, + } +} + +// AddFeatureGates adds all feature gates used by this package. +func AddFeatureGates(mutableFeatureGate featuregate.MutableVersionedFeatureGate) error { + return mutableFeatureGate.AddVersioned(featureGates()) +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/options.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/options.go new file mode 100644 index 0000000000..4c8a0d2c53 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/options.go @@ -0,0 +1,449 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "errors" + "flag" + "fmt" + "io" + "math" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/spf13/pflag" + + "k8s.io/klog/v2" + "k8s.io/klog/v2/textlogger" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/validation/field" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" + "k8s.io/component-base/logs/internal/setverbositylevel" + "k8s.io/component-base/logs/klogflags" +) + +const ( + // LogFlushFreqDefault is the default for the corresponding command line + // parameter. + LogFlushFreqDefault = 5 * time.Second +) + +const ( + // LogFlushFreqFlagName is the name of the command line parameter. + // Depending on how flags get added, it is either a stand-alone + // value (logs.AddFlags) or part of LoggingConfiguration. + LogFlushFreqFlagName = "log-flush-frequency" +) + +// NewLoggingConfiguration returns a struct holding the default logging configuration. +func NewLoggingConfiguration() *LoggingConfiguration { + c := LoggingConfiguration{} + SetRecommendedLoggingConfiguration(&c) + return &c +} + +// Applying configurations multiple times is not safe unless it's guaranteed that there +// are no goroutines which might call logging functions. The default for ValidateAndApply +// and ValidateAndApplyWithOptions is to return an error when called more than once. +// Binaries and unit tests can override that behavior. +var ReapplyHandling = ReapplyHandlingError + +type ReapplyHandlingType int + +const ( + // ReapplyHandlingError is the default: calling ValidateAndApply or + // ValidateAndApplyWithOptions again returns an error. + ReapplyHandlingError ReapplyHandlingType = iota + // ReapplyHandlingIgnoreUnchanged silently ignores any additional calls of + // ValidateAndApply or ValidateAndApplyWithOptions if the configuration + // is unchanged, otherwise they return an error. + ReapplyHandlingIgnoreUnchanged +) + +// ValidateAndApply combines validation and application of the logging configuration. +// This should be invoked as early as possible because then the rest of the program +// startup (including validation of other options) will already run with the final +// logging configuration. +// +// The optional FeatureGate controls logging features. If nil, the default for +// these features is used. +// +// Logging options must be applied as early as possible during the program +// startup. Some changes are global and cannot be done safely when there are +// already goroutines running. +func ValidateAndApply(c *LoggingConfiguration, featureGate featuregate.FeatureGate) error { + return validateAndApply(c, nil, featureGate, nil) +} + +// ValidateAndApplyWithOptions is a variant of ValidateAndApply which accepts +// additional options beyond those that can be configured through the API. This +// is meant for testing. +// +// Logging options must be applied as early as possible during the program +// startup. Some changes are global and cannot be done safely when there are +// already goroutines running. +func ValidateAndApplyWithOptions(c *LoggingConfiguration, options *LoggingOptions, featureGate featuregate.FeatureGate) error { + return validateAndApply(c, options, featureGate, nil) +} + +// +k8s:deepcopy-gen=false + +// LoggingOptions can be used with ValidateAndApplyWithOptions to override +// certain global defaults. +type LoggingOptions struct { + // ErrorStream can be used to override the os.Stderr default. + ErrorStream io.Writer + + // InfoStream can be used to override the os.Stdout default. + InfoStream io.Writer +} + +// ValidateAndApplyAsField is a variant of ValidateAndApply that should be used +// when the LoggingConfiguration is embedded in some larger configuration +// structure. +func ValidateAndApplyAsField(c *LoggingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) error { + return validateAndApply(c, nil, featureGate, fldPath) +} + +func validateAndApply(c *LoggingConfiguration, options *LoggingOptions, featureGate featuregate.FeatureGate, fldPath *field.Path) error { + errs := Validate(c, featureGate, fldPath) + if len(errs) > 0 { + return errs.ToAggregate() + } + return apply(c, options, featureGate) +} + +// Validate can be used to check for invalid settings without applying them. +// Most binaries should validate and apply the logging configuration as soon +// as possible via ValidateAndApply. The field path is optional: nil +// can be passed when the struct is not embedded in some larger struct. +func Validate(c *LoggingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) field.ErrorList { + errs := field.ErrorList{} + if c.Format != DefaultLogFormat { + // WordSepNormalizeFunc is just a guess. Commands should use it, + // but we cannot know for sure. + allFlags := unsupportedLoggingFlags(cliflag.WordSepNormalizeFunc) + for _, f := range allFlags { + if f.DefValue != f.Value.String() { + errs = append(errs, field.Invalid(fldPath.Child("format"), c.Format, fmt.Sprintf("Non-default format doesn't honor flag: %s", f.Name))) + } + } + } + format, err := logRegistry.get(c.Format) + if err != nil { + errs = append(errs, field.Invalid(fldPath.Child("format"), c.Format, "Unsupported log format")) + } else if format != nil { + if format.feature != LoggingStableOptions { + enabled := featureGates()[format.feature][len(featureGates()[format.feature])-1].Default + if featureGate != nil { + enabled = featureGate.Enabled(format.feature) + } + if !enabled { + errs = append(errs, field.Forbidden(fldPath.Child("format"), fmt.Sprintf("Log format %s is disabled, see %s feature", c.Format, format.feature))) + } + } + } + + // The type in our struct is uint32, but klog only accepts positive int32. + if c.Verbosity > math.MaxInt32 { + errs = append(errs, field.Invalid(fldPath.Child("verbosity"), c.Verbosity, fmt.Sprintf("Must be <= %d", math.MaxInt32))) + } + vmoduleFldPath := fldPath.Child("vmodule") + if len(c.VModule) > 0 && c.Format != "" && c.Format != "text" { + errs = append(errs, field.Forbidden(vmoduleFldPath, "Only supported for text log format")) + } + for i, item := range c.VModule { + if item.FilePattern == "" { + errs = append(errs, field.Required(vmoduleFldPath.Index(i), "File pattern must not be empty")) + } + if strings.ContainsAny(item.FilePattern, "=,") { + errs = append(errs, field.Invalid(vmoduleFldPath.Index(i), item.FilePattern, "File pattern must not contain equal sign or comma")) + } + if item.Verbosity > math.MaxInt32 { + errs = append(errs, field.Invalid(vmoduleFldPath.Index(i), item.Verbosity, fmt.Sprintf("Must be <= %d", math.MaxInt32))) + } + } + + errs = append(errs, validateFormatOptions(c, featureGate, fldPath.Child("options"))...) + return errs +} + +func validateFormatOptions(c *LoggingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) field.ErrorList { + errs := field.ErrorList{} + errs = append(errs, validateTextOptions(c, featureGate, fldPath.Child("text"))...) + errs = append(errs, validateJSONOptions(c, featureGate, fldPath.Child("json"))...) + return errs +} + +func validateTextOptions(c *LoggingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) field.ErrorList { + errs := field.ErrorList{} + if gate := LoggingAlphaOptions; c.Options.Text.SplitStream && !featureEnabled(featureGate, gate) { + errs = append(errs, field.Forbidden(fldPath.Child("splitStream"), fmt.Sprintf("Feature %s is disabled", gate))) + } + if gate := LoggingAlphaOptions; c.Options.Text.InfoBufferSize.Value() != 0 && !featureEnabled(featureGate, gate) { + errs = append(errs, field.Forbidden(fldPath.Child("infoBufferSize"), fmt.Sprintf("Feature %s is disabled", gate))) + } + return errs +} + +func validateJSONOptions(c *LoggingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) field.ErrorList { + errs := field.ErrorList{} + if gate := LoggingAlphaOptions; c.Options.JSON.SplitStream && !featureEnabled(featureGate, gate) { + errs = append(errs, field.Forbidden(fldPath.Child("splitStream"), fmt.Sprintf("Feature %s is disabled", gate))) + } + if gate := LoggingAlphaOptions; c.Options.JSON.InfoBufferSize.Value() != 0 && !featureEnabled(featureGate, gate) { + errs = append(errs, field.Forbidden(fldPath.Child("infoBufferSize"), fmt.Sprintf("Feature %s is disabled", gate))) + } + return errs +} + +func featureEnabled(featureGate featuregate.FeatureGate, feature featuregate.Feature) bool { + enabled := false + if featureGate != nil { + enabled = featureGate.Enabled(feature) + } + return enabled +} + +func apply(c *LoggingConfiguration, options *LoggingOptions, featureGate featuregate.FeatureGate) error { + p := ¶meters{ + C: c, + Options: options, + ContextualLoggingEnabled: true, + } + if featureGate != nil { + p.ContextualLoggingEnabled = featureGate.Enabled(ContextualLogging) + } + + oldP := applyParameters.Load() + if oldP != nil { + switch ReapplyHandling { + case ReapplyHandlingError: + return errors.New("logging configuration was already applied earlier, changing it is not allowed") + case ReapplyHandlingIgnoreUnchanged: + if diff := diff.Diff(oldP, p); diff != "" { + return fmt.Errorf("the logging configuration should not be changed after setting it once (- old setting, + new setting):\n%s", diff) + } + return nil + default: + return fmt.Errorf("invalid value %d for ReapplyHandling", ReapplyHandling) + } + } + applyParameters.Store(p) + + // if log format not exists, use nil loggr + format, _ := logRegistry.get(c.Format) + if format.factory == nil { + klog.ClearLogger() + } else { + if options == nil { + options = &LoggingOptions{ + ErrorStream: os.Stderr, + InfoStream: os.Stdout, + } + } + log, control := format.factory.Create(*c, *options) + if control.SetVerbosityLevel != nil { + setverbositylevel.Mutex.Lock() + defer setverbositylevel.Mutex.Unlock() + setverbositylevel.Callbacks = append(setverbositylevel.Callbacks, control.SetVerbosityLevel) + } + opts := []klog.LoggerOption{ + klog.ContextualLogger(p.ContextualLoggingEnabled), + klog.FlushLogger(control.Flush), + } + if writer, ok := log.GetSink().(textlogger.KlogBufferWriter); ok { + opts = append(opts, klog.WriteKlogBuffer(writer.WriteKlogBuffer)) + } + klog.SetLoggerWithOptions(log, opts...) + } + if err := loggingFlags.Lookup("v").Value.Set(VerbosityLevelPflag(&c.Verbosity).String()); err != nil { + return fmt.Errorf("internal error while setting klog verbosity: %v", err) + } + if err := loggingFlags.Lookup("vmodule").Value.Set(VModuleConfigurationPflag(&c.VModule).String()); err != nil { + return fmt.Errorf("internal error while setting klog vmodule: %v", err) + } + setSlogDefaultLogger() + klog.StartFlushDaemon(c.FlushFrequency.Duration.Duration) + klog.EnableContextualLogging(p.ContextualLoggingEnabled) + return nil +} + +type parameters struct { + C *LoggingConfiguration + Options *LoggingOptions + ContextualLoggingEnabled bool +} + +var applyParameters atomic.Pointer[parameters] + +// ResetForTest restores the default settings. This is not thread-safe and should only +// be used when there are no goroutines running. The intended users are unit +// tests in other packages. +func ResetForTest(featureGate featuregate.FeatureGate) error { + oldP := applyParameters.Load() + if oldP == nil { + // Nothing to do. + return nil + } + + // This makes it possible to call apply again without triggering errors. + applyParameters.Store(nil) + + // Restore defaults. Shouldn't fail, but check anyway. + config := NewLoggingConfiguration() + if err := ValidateAndApply(config, featureGate); err != nil { + return fmt.Errorf("apply default configuration: %v", err) + } + + // And again... + applyParameters.Store(nil) + + return nil +} + +// AddFlags adds command line flags for the configuration. +func AddFlags(c *LoggingConfiguration, fs *pflag.FlagSet) { + addFlags(c, fs) +} + +// AddGoFlags is a variant of AddFlags for a standard FlagSet. +func AddGoFlags(c *LoggingConfiguration, fs *flag.FlagSet) { + addFlags(c, goFlagSet{FlagSet: fs}) +} + +// flagSet is the interface implemented by pflag.FlagSet, with +// just those methods defined which are needed by addFlags. +type flagSet interface { + BoolVar(p *bool, name string, value bool, usage string) + DurationVar(p *time.Duration, name string, value time.Duration, usage string) + StringVar(p *string, name string, value string, usage string) + Var(value pflag.Value, name string, usage string) + VarP(value pflag.Value, name, shorthand, usage string) +} + +// goFlagSet implements flagSet for a stdlib flag.FlagSet. +type goFlagSet struct { + *flag.FlagSet +} + +func (fs goFlagSet) Var(value pflag.Value, name string, usage string) { + fs.FlagSet.Var(value, name, usage) +} + +func (fs goFlagSet) VarP(value pflag.Value, name, shorthand, usage string) { + // Ignore shorthand, it's not needed and not supported. + fs.FlagSet.Var(value, name, usage) +} + +// addFlags can be used with both flag.FlagSet and pflag.FlagSet. The internal +// interface definition avoids duplicating this code. +func addFlags(c *LoggingConfiguration, fs flagSet) { + formats := logRegistry.list() + fs.StringVar(&c.Format, "logging-format", c.Format, fmt.Sprintf("Sets the log format. Permitted formats: %s.", formats)) + // No new log formats should be added after generation is of flag options + logRegistry.freeze() + + fs.DurationVar(&c.FlushFrequency.Duration.Duration, LogFlushFreqFlagName, c.FlushFrequency.Duration.Duration, "Maximum number of seconds between log flushes") + fs.VarP(VerbosityLevelPflag(&c.Verbosity), "v", "v", "number for the log level verbosity") + fs.Var(VModuleConfigurationPflag(&c.VModule), "vmodule", "comma-separated list of pattern=N settings for file-filtered logging (only works for text log format)") + + fs.BoolVar(&c.Options.Text.SplitStream, "log-text-split-stream", false, "[Alpha] In text format, write error messages to stderr and info messages to stdout. The default is to write a single stream to stdout. Enable the LoggingAlphaOptions feature gate to use this.") + fs.Var(&c.Options.Text.InfoBufferSize, "log-text-info-buffer-size", "[Alpha] In text format with split output streams, the info messages can be buffered for a while to increase performance. The default value of zero bytes disables buffering. The size can be specified as number of bytes (512), multiples of 1000 (1K), multiples of 1024 (2Ki), or powers of those (3M, 4G, 5Mi, 6Gi). Enable the LoggingAlphaOptions feature gate to use this.") + + // JSON options. We only register them if "json" is a valid format. The + // config file API however always has them. + if _, err := logRegistry.get("json"); err == nil { + fs.BoolVar(&c.Options.JSON.SplitStream, "log-json-split-stream", false, "[Alpha] In JSON format, write error messages to stderr and info messages to stdout. The default is to write a single stream to stdout. Enable the LoggingAlphaOptions feature gate to use this.") + fs.Var(&c.Options.JSON.InfoBufferSize, "log-json-info-buffer-size", "[Alpha] In JSON format with split output streams, the info messages can be buffered for a while to increase performance. The default value of zero bytes disables buffering. The size can be specified as number of bytes (512), multiples of 1000 (1K), multiples of 1024 (2Ki), or powers of those (3M, 4G, 5Mi, 6Gi). Enable the LoggingAlphaOptions feature gate to use this.") + } +} + +// SetRecommendedLoggingConfiguration sets the default logging configuration +// for fields that are unset. +// +// Consumers who embed LoggingConfiguration in their own configuration structs +// may set custom defaults and then should call this function to add the +// global defaults. +func SetRecommendedLoggingConfiguration(c *LoggingConfiguration) { + if c.Format == "" { + c.Format = "text" + } + if c.FlushFrequency.Duration.Duration == 0 { + c.FlushFrequency.Duration.Duration = LogFlushFreqDefault + c.FlushFrequency.SerializeAsString = true + } + setRecommendedOutputRouting(&c.Options.Text.OutputRoutingOptions) + setRecommendedOutputRouting(&c.Options.JSON.OutputRoutingOptions) +} + +func setRecommendedOutputRouting(o *OutputRoutingOptions) { + var empty resource.QuantityValue + if o.InfoBufferSize == empty { + o.InfoBufferSize = resource.QuantityValue{ + // This is similar, but not quite the same as a default + // constructed instance. + Quantity: *resource.NewQuantity(0, resource.DecimalSI), + } + // This sets the unexported Quantity.s which will be compared + // by reflect.DeepEqual in some tests. + _ = o.InfoBufferSize.String() + } +} + +// loggingFlags captures the state of the logging flags, in particular their default value +// before flag parsing. It is used by unsupportedLoggingFlags. +var loggingFlags pflag.FlagSet + +func init() { + var fs flag.FlagSet + klogflags.Init(&fs) + loggingFlags.AddGoFlagSet(&fs) +} + +// List of logs (k8s.io/klog + k8s.io/component-base/logs) flags supported by all logging formats +var supportedLogsFlags = map[string]struct{}{ + "v": {}, +} + +// unsupportedLoggingFlags lists unsupported logging flags. The normalize +// function is optional. +func unsupportedLoggingFlags(normalizeFunc func(f *pflag.FlagSet, name string) pflag.NormalizedName) []*pflag.Flag { + // k8s.io/component-base/logs and klog flags + pfs := &pflag.FlagSet{} + loggingFlags.VisitAll(func(flag *pflag.Flag) { + if _, found := supportedLogsFlags[flag.Name]; !found { + // Normalization changes flag.Name, so make a copy. + clone := *flag + pfs.AddFlag(&clone) + } + }) + + // Apply normalization. + pfs.SetNormalizeFunc(normalizeFunc) + + var allFlags []*pflag.Flag + pfs.VisitAll(func(flag *pflag.Flag) { + allFlags = append(allFlags, flag) + }) + return allFlags +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/options_no_slog.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/options_no_slog.go new file mode 100644 index 0000000000..816948e0a6 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/options_no_slog.go @@ -0,0 +1,24 @@ +//go:build !go1.21 +// +build !go1.21 + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +func setSlogDefaultLogger() { + // Do nothing when build with Go < 1.21. +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/options_slog.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/options_slog.go new file mode 100644 index 0000000000..31765e644d --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/options_slog.go @@ -0,0 +1,37 @@ +//go:build go1.21 +// +build go1.21 + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "log/slog" + + "github.com/go-logr/logr" + "k8s.io/klog/v2" +) + +// setSlogDefaultLogger sets the global slog default logger to the same default +// that klog currently uses. +func setSlogDefaultLogger() { + // klog.Background() always returns a valid logr.Logger, regardless of + // how logging was configured. We just need to turn it into a + // slog.Handler. SetDefault then needs a slog.Logger. + handler := logr.ToSlogHandler(klog.Background()) + slog.SetDefault(slog.New(handler)) +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/pflags.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/pflags.go new file mode 100644 index 0000000000..b74e132a72 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/pflags.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + "strconv" + "strings" + + "github.com/spf13/pflag" +) + +// VModuleConfigurationPflag implements the pflag.Value interface for a +// VModuleConfiguration. The value pointer must not be nil. +func VModuleConfigurationPflag(value *VModuleConfiguration) pflag.Value { + return vmoduleConfigurationPFlag{value} +} + +type vmoduleConfigurationPFlag struct { + value *VModuleConfiguration +} + +// String returns the -vmodule parameter (comma-separated list of pattern=N). +func (wrapper vmoduleConfigurationPFlag) String() string { + if wrapper.value == nil { + return "" + } + var patterns []string + for _, item := range *wrapper.value { + patterns = append(patterns, fmt.Sprintf("%s=%d", item.FilePattern, item.Verbosity)) + } + return strings.Join(patterns, ",") +} + +// Set parses the -vmodule parameter (comma-separated list of pattern=N). +func (wrapper vmoduleConfigurationPFlag) Set(value string) error { + // This code mirrors https://github.com/kubernetes/klog/blob/9ad246211af1ed84621ee94a26fcce0038b69cd1/klog.go#L287-L313 + + for _, pat := range strings.Split(value, ",") { + if len(pat) == 0 { + // Empty strings such as from a trailing comma can be ignored. + continue + } + patLev := strings.Split(pat, "=") + if len(patLev) != 2 || len(patLev[0]) == 0 || len(patLev[1]) == 0 { + return fmt.Errorf("%q does not have the pattern=N format", pat) + } + pattern := patLev[0] + // 31 instead of 32 to ensure that it also fits into int32. + v, err := strconv.ParseUint(patLev[1], 10, 31) + if err != nil { + return fmt.Errorf("parsing verbosity in %q: %v", pat, err) + } + *wrapper.value = append(*wrapper.value, VModuleItem{FilePattern: pattern, Verbosity: VerbosityLevel(v)}) + } + return nil +} + +func (wrapper vmoduleConfigurationPFlag) Type() string { + return "pattern=N,..." +} + +// VerbosityLevelPflag implements the pflag.Value interface for a verbosity +// level value. +func VerbosityLevelPflag(value *VerbosityLevel) pflag.Value { + return verbosityLevelPflag{value} +} + +type verbosityLevelPflag struct { + value *VerbosityLevel +} + +func (wrapper verbosityLevelPflag) String() string { + if wrapper.value == nil { + return "0" + } + return strconv.FormatInt(int64(*wrapper.value), 10) +} + +func (wrapper verbosityLevelPflag) Get() interface{} { + if wrapper.value == nil { + return VerbosityLevel(0) + } + return *wrapper.value +} + +func (wrapper verbosityLevelPflag) Set(value string) error { + // Limited to int32 for compatibility with klog. + v, err := strconv.ParseUint(value, 10, 31) + if err != nil { + return err + } + *wrapper.value = VerbosityLevel(v) + return nil +} + +func (wrapper verbosityLevelPflag) Type() string { + return "Level" +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/registry.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/registry.go new file mode 100644 index 0000000000..f16c9ce6f1 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/registry.go @@ -0,0 +1,135 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + "sort" + "strings" + "sync" + + "github.com/go-logr/logr" + + "k8s.io/component-base/featuregate" +) + +var logRegistry = newLogFormatRegistry() + +// logFormatRegistry stores factories for all supported logging formats. +type logFormatRegistry struct { + mutex sync.Mutex + registry map[string]logFormat + frozen bool +} + +type logFormat struct { + factory LogFormatFactory + feature featuregate.Feature +} + +// +k8s:deepcopy-gen=false + +// RuntimeControl provides operations that aren't available through the normal +// Logger or LogSink API. +type RuntimeControl struct { + // Flush ensures that all in-memory data is written. + // May be nil. + Flush func() + + // SetVerbosityLevel changes the level for all Logger instances + // derived from the initial one. May be nil. + // + // The parameter is intentionally a plain uint32 instead of + // VerbosityLevel to enable implementations that don't need to import + // the API (helps avoid circular dependencies). + SetVerbosityLevel func(v uint32) error +} + +// LogFormatFactory provides support for a certain additional, +// non-default log format. +type LogFormatFactory interface { + // Create returns a logger with the requested configuration. + Create(c LoggingConfiguration, o LoggingOptions) (logr.Logger, RuntimeControl) +} + +// RegisterLogFormat registers support for a new logging format. This must be called +// before using any of the methods in LoggingConfiguration. The feature must +// be one of those defined in this package (typically LoggingAlphaOptions, +// LoggingBetaOptions or LoggingStableOptions). +func RegisterLogFormat(name string, factory LogFormatFactory, feature featuregate.Feature) error { + return logRegistry.register(name, logFormat{factory, feature}) +} + +func newLogFormatRegistry() *logFormatRegistry { + registry := &logFormatRegistry{ + registry: make(map[string]logFormat), + frozen: false, + } + _ = registry.register(DefaultLogFormat, logFormat{factory: textFactory{}, feature: LoggingStableOptions}) + return registry +} + +// register adds a new log format. It's an error to modify an existing one. +func (lfr *logFormatRegistry) register(name string, format logFormat) error { + lfr.mutex.Lock() + defer lfr.mutex.Unlock() + if lfr.frozen { + return fmt.Errorf("log format registry is frozen, unable to register log format %s", name) + } + if _, ok := lfr.registry[name]; ok { + return fmt.Errorf("log format: %s already exists", name) + } + if _, ok := featureGates()[format.feature]; !ok && format.feature != LoggingStableOptions { + return fmt.Errorf("log format %s: unsupported feature gate %s", name, format.feature) + } + lfr.registry[name] = format + return nil +} + +// get specified log format factory +func (lfr *logFormatRegistry) get(name string) (*logFormat, error) { + lfr.mutex.Lock() + defer lfr.mutex.Unlock() + format, ok := lfr.registry[name] + if !ok { + return nil, fmt.Errorf("log format: %s does not exists", name) + } + return &format, nil +} + +// list names of registered log formats, including feature gates (sorted) +func (lfr *logFormatRegistry) list() string { + lfr.mutex.Lock() + defer lfr.mutex.Unlock() + formats := make([]string, 0, len(lfr.registry)) + for name, format := range lfr.registry { + item := fmt.Sprintf(`"%s"`, name) + if format.feature != LoggingStableOptions { + item += fmt.Sprintf(" (gated by %s)", format.feature) + } + formats = append(formats, item) + } + sort.Strings(formats) + return strings.Join(formats, ", ") +} + +// freeze prevents further modifications of the registered log formats. +func (lfr *logFormatRegistry) freeze() { + lfr.mutex.Lock() + defer lfr.mutex.Unlock() + lfr.frozen = true +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/text.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/text.go new file mode 100644 index 0000000000..2983d7d920 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/text.go @@ -0,0 +1,142 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "bufio" + "fmt" + "io" + "sync" + + "github.com/go-logr/logr" + + "k8s.io/component-base/featuregate" + "k8s.io/klog/v2/textlogger" +) + +// textFactory produces klog text logger instances. +type textFactory struct{} + +var _ LogFormatFactory = textFactory{} + +func (f textFactory) Feature() featuregate.Feature { + return LoggingStableOptions +} + +func (f textFactory) Create(c LoggingConfiguration, o LoggingOptions) (logr.Logger, RuntimeControl) { + output := o.ErrorStream + var flush func() + if c.Options.Text.SplitStream { + r := &klogMsgRouter{ + info: o.InfoStream, + error: o.ErrorStream, + } + size := c.Options.Text.InfoBufferSize.Value() + if size > 0 { + // Prevent integer overflow. + if size > 2*1024*1024*1024 { + size = 2 * 1024 * 1024 * 1024 + } + info := newBufferedWriter(r.info, int(size)) + flush = info.Flush + r.info = info + } + output = r + } + + options := []textlogger.ConfigOption{ + textlogger.Verbosity(int(c.Verbosity)), + textlogger.Output(output), + } + loggerConfig := textlogger.NewConfig(options...) + + // This should never fail, we produce a valid string here. + _ = loggerConfig.VModule().Set(VModuleConfigurationPflag(&c.VModule).String()) + + return textlogger.NewLogger(loggerConfig), + RuntimeControl{ + SetVerbosityLevel: func(v uint32) error { + return loggerConfig.Verbosity().Set(fmt.Sprintf("%d", v)) + }, + Flush: flush, + } +} + +type klogMsgRouter struct { + info, error io.Writer +} + +var _ io.Writer = &klogMsgRouter{} + +// Write redirects the message into either the info or error +// stream, depending on its type as indicated in text format +// by the first byte. +func (r *klogMsgRouter) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + + if p[0] == 'I' { + return r.info.Write(p) + } + return r.error.Write(p) +} + +// bufferedWriter is an io.Writer that buffers writes in-memory before +// flushing them to a wrapped io.Writer after reaching some limit +// or getting flushed. +type bufferedWriter struct { + mu sync.Mutex + writer *bufio.Writer + out io.Writer +} + +func newBufferedWriter(out io.Writer, size int) *bufferedWriter { + return &bufferedWriter{ + writer: bufio.NewWriterSize(out, size), + out: out, + } +} + +func (b *bufferedWriter) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + + // To avoid partial writes into the underlying writer, we ensure that + // the entire new data fits into the buffer or flush first. + if len(p) > b.writer.Available() && b.writer.Buffered() > 0 { + if err := b.writer.Flush(); err != nil { + return 0, err + } + } + + // If it still doesn't fit, then we bypass the now empty buffer + // and write directly. + if len(p) > b.writer.Available() { + return b.out.Write(p) + } + + // This goes into the buffer. + return b.writer.Write(p) +} + +func (b *bufferedWriter) Flush() { + b.mu.Lock() + defer b.mu.Unlock() + + _ = b.writer.Flush() +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/types.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/types.go new file mode 100644 index 0000000000..603ccb4740 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/types.go @@ -0,0 +1,146 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "encoding/json" + "fmt" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Supported output formats. +const ( + // DefaultLogFormat is the traditional klog output format. + DefaultLogFormat = "text" + + // JSONLogFormat emits each log message as a JSON struct. + JSONLogFormat = "json" +) + +// The alpha or beta level of structs is the highest stability level of any field +// inside it. Feature gates will get checked during LoggingConfiguration.ValidateAndApply. + +// LoggingConfiguration contains logging options. +type LoggingConfiguration struct { + // Format Flag specifies the structure of log messages. + // default value of format is `text` + Format string `json:"format,omitempty"` + // Maximum time between log flushes. + // If a string, parsed as a duration (i.e. "1s") + // If an int, the maximum number of nanoseconds (i.e. 1s = 1000000000). + // Ignored if the selected logging backend writes log messages without buffering. + FlushFrequency TimeOrMetaDuration `json:"flushFrequency"` + // Verbosity is the threshold that determines which log messages are + // logged. Default is zero which logs only the most important + // messages. Higher values enable additional messages. Error messages + // are always logged. + Verbosity VerbosityLevel `json:"verbosity"` + // VModule overrides the verbosity threshold for individual files. + // Only supported for "text" log format. + VModule VModuleConfiguration `json:"vmodule,omitempty"` + // [Alpha] Options holds additional parameters that are specific + // to the different logging formats. Only the options for the selected + // format get used, but all of them get validated. + // Only available when the LoggingAlphaOptions feature gate is enabled. + Options FormatOptions `json:"options,omitempty"` +} + +// TimeOrMetaDuration is present only for backwards compatibility for the +// flushFrequency field, and new fields should use metav1.Duration. +type TimeOrMetaDuration struct { + // Duration holds the duration + Duration metav1.Duration + // SerializeAsString controls whether the value is serialized as a string or an integer + SerializeAsString bool `json:"-"` +} + +func (t TimeOrMetaDuration) MarshalJSON() ([]byte, error) { + if t.SerializeAsString { + return t.Duration.MarshalJSON() + } else { + // Marshal as integer for backwards compatibility + return json.Marshal(t.Duration.Duration) + } +} + +func (t *TimeOrMetaDuration) UnmarshalJSON(b []byte) error { + if len(b) > 0 && b[0] == '"' { + // string values unmarshal as metav1.Duration + t.SerializeAsString = true + return json.Unmarshal(b, &t.Duration) + } + t.SerializeAsString = false + if err := json.Unmarshal(b, &t.Duration.Duration); err != nil { + return fmt.Errorf("invalid duration %q: %w", string(b), err) + } + return nil +} + +// FormatOptions contains options for the different logging formats. +type FormatOptions struct { + // [Alpha] Text contains options for logging format "text". + // Only available when the LoggingAlphaOptions feature gate is enabled. + Text TextOptions `json:"text,omitempty"` + // [Alpha] JSON contains options for logging format "json". + // Only available when the LoggingAlphaOptions feature gate is enabled. + JSON JSONOptions `json:"json,omitempty"` +} + +// TextOptions contains options for logging format "text". +type TextOptions struct { + OutputRoutingOptions `json:",inline"` +} + +// JSONOptions contains options for logging format "json". +type JSONOptions struct { + OutputRoutingOptions `json:",inline"` +} + +// OutputRoutingOptions contains options that are supported by both "text" and "json". +type OutputRoutingOptions struct { + // [Alpha] SplitStream redirects error messages to stderr while + // info messages go to stdout, with buffering. The default is to write + // both to stdout, without buffering. Only available when + // the LoggingAlphaOptions feature gate is enabled. + SplitStream bool `json:"splitStream,omitempty"` + // [Alpha] InfoBufferSize sets the size of the info stream when + // using split streams. The default is zero, which disables buffering. + // Only available when the LoggingAlphaOptions feature gate is enabled. + InfoBufferSize resource.QuantityValue `json:"infoBufferSize,omitempty"` +} + +// VModuleConfiguration is a collection of individual file names or patterns +// and the corresponding verbosity threshold. +type VModuleConfiguration []VModuleItem + +// VModuleItem defines verbosity for one or more files which match a certain +// glob pattern. +type VModuleItem struct { + // FilePattern is a base file name (i.e. minus the ".go" suffix and + // directory) or a "glob" pattern for such a name. It must not contain + // comma and equal signs because those are separators for the + // corresponding klog command line argument. + FilePattern string `json:"filePattern"` + // Verbosity is the threshold for log messages emitted inside files + // that match the pattern. + Verbosity VerbosityLevel `json:"verbosity"` +} + +// VerbosityLevel represents a klog or logr verbosity threshold. +type VerbosityLevel uint32 diff --git a/go-controller/vendor/k8s.io/component-base/logs/api/v1/zz_generated.deepcopy.go b/go-controller/vendor/k8s.io/component-base/logs/api/v1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..0317c80202 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/api/v1/zz_generated.deepcopy.go @@ -0,0 +1,167 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FormatOptions) DeepCopyInto(out *FormatOptions) { + *out = *in + in.Text.DeepCopyInto(&out.Text) + in.JSON.DeepCopyInto(&out.JSON) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FormatOptions. +func (in *FormatOptions) DeepCopy() *FormatOptions { + if in == nil { + return nil + } + out := new(FormatOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JSONOptions) DeepCopyInto(out *JSONOptions) { + *out = *in + in.OutputRoutingOptions.DeepCopyInto(&out.OutputRoutingOptions) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JSONOptions. +func (in *JSONOptions) DeepCopy() *JSONOptions { + if in == nil { + return nil + } + out := new(JSONOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoggingConfiguration) DeepCopyInto(out *LoggingConfiguration) { + *out = *in + out.FlushFrequency = in.FlushFrequency + if in.VModule != nil { + in, out := &in.VModule, &out.VModule + *out = make(VModuleConfiguration, len(*in)) + copy(*out, *in) + } + in.Options.DeepCopyInto(&out.Options) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoggingConfiguration. +func (in *LoggingConfiguration) DeepCopy() *LoggingConfiguration { + if in == nil { + return nil + } + out := new(LoggingConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OutputRoutingOptions) DeepCopyInto(out *OutputRoutingOptions) { + *out = *in + in.InfoBufferSize.DeepCopyInto(&out.InfoBufferSize) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OutputRoutingOptions. +func (in *OutputRoutingOptions) DeepCopy() *OutputRoutingOptions { + if in == nil { + return nil + } + out := new(OutputRoutingOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TextOptions) DeepCopyInto(out *TextOptions) { + *out = *in + in.OutputRoutingOptions.DeepCopyInto(&out.OutputRoutingOptions) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TextOptions. +func (in *TextOptions) DeepCopy() *TextOptions { + if in == nil { + return nil + } + out := new(TextOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TimeOrMetaDuration) DeepCopyInto(out *TimeOrMetaDuration) { + *out = *in + out.Duration = in.Duration + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeOrMetaDuration. +func (in *TimeOrMetaDuration) DeepCopy() *TimeOrMetaDuration { + if in == nil { + return nil + } + out := new(TimeOrMetaDuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in VModuleConfiguration) DeepCopyInto(out *VModuleConfiguration) { + { + in := &in + *out = make(VModuleConfiguration, len(*in)) + copy(*out, *in) + return + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VModuleConfiguration. +func (in VModuleConfiguration) DeepCopy() VModuleConfiguration { + if in == nil { + return nil + } + out := new(VModuleConfiguration) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VModuleItem) DeepCopyInto(out *VModuleItem) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VModuleItem. +func (in *VModuleItem) DeepCopy() *VModuleItem { + if in == nil { + return nil + } + out := new(VModuleItem) + in.DeepCopyInto(out) + return out +} diff --git a/go-controller/vendor/k8s.io/component-base/logs/internal/setverbositylevel/setverbositylevel.go b/go-controller/vendor/k8s.io/component-base/logs/internal/setverbositylevel/setverbositylevel.go new file mode 100644 index 0000000000..c643bae9bc --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/internal/setverbositylevel/setverbositylevel.go @@ -0,0 +1,34 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package setverbositylevel stores callbacks that will be invoked by logs.GlogLevel. +// +// This is a separate package to avoid a dependency from +// k8s.io/component-base/logs (uses the callbacks) to +// k8s.io/component-base/logs/api/v1 (adds them). Not all users of the logs +// package also use the API. +package setverbositylevel + +import ( + "sync" +) + +var ( + // Mutex controls access to the callbacks. + Mutex sync.Mutex + + Callbacks []func(v uint32) error +) diff --git a/go-controller/vendor/k8s.io/component-base/logs/klogflags/klogflags.go b/go-controller/vendor/k8s.io/component-base/logs/klogflags/klogflags.go new file mode 100644 index 0000000000..6c8284fa7c --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/logs/klogflags/klogflags.go @@ -0,0 +1,41 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package klogflags + +import ( + "flag" + + "k8s.io/klog/v2" +) + +// Init is a replacement for klog.InitFlags which only adds those flags +// that are still supported for Kubernetes components (i.e. -v and -vmodule). +// See +// https://github.com/kubernetes/enhancements/tree/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components. +func Init(fs *flag.FlagSet) { + var allFlags flag.FlagSet + klog.InitFlags(&allFlags) + if fs == nil { + fs = flag.CommandLine + } + allFlags.VisitAll(func(f *flag.Flag) { + switch f.Name { + case "v", "vmodule": + fs.Var(f.Value, f.Name, f.Usage) + } + }) +} diff --git a/go-controller/vendor/k8s.io/component-base/tracing/api/v1/config.go b/go-controller/vendor/k8s.io/component-base/tracing/api/v1/config.go new file mode 100644 index 0000000000..ae9bbbfc09 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/tracing/api/v1/config.go @@ -0,0 +1,88 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + "net/url" + "strings" + + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/component-base/featuregate" +) + +var ( + maxSamplingRatePerMillion = int32(1000000) +) + +// ValidateTracingConfiguration validates the tracing configuration +func ValidateTracingConfiguration(traceConfig *TracingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if traceConfig == nil { + return allErrs + } + if traceConfig.SamplingRatePerMillion != nil { + allErrs = append(allErrs, validateSamplingRate(*traceConfig.SamplingRatePerMillion, fldPath.Child("samplingRatePerMillion"))...) + } + if traceConfig.Endpoint != nil { + allErrs = append(allErrs, validateEndpoint(*traceConfig.Endpoint, fldPath.Child("endpoint"))...) + } + return allErrs +} + +func validateSamplingRate(rate int32, fldPath *field.Path) field.ErrorList { + errs := field.ErrorList{} + if rate < 0 { + errs = append(errs, field.Invalid( + fldPath, rate, + "sampling rate must be positive", + )) + } + if rate > maxSamplingRatePerMillion { + errs = append(errs, field.Invalid( + fldPath, rate, + "sampling rate per million must be less than or equal to one million", + )) + } + return errs +} + +func validateEndpoint(endpoint string, fldPath *field.Path) field.ErrorList { + errs := field.ErrorList{} + if !strings.Contains(endpoint, "//") { + endpoint = "dns://" + endpoint + } + url, err := url.Parse(endpoint) + if err != nil { + errs = append(errs, field.Invalid( + fldPath, endpoint, + err.Error(), + )) + return errs + } + switch url.Scheme { + case "dns": + case "unix": + case "unix-abstract": + default: + errs = append(errs, field.Invalid( + fldPath, endpoint, + fmt.Sprintf("unsupported scheme: %v. Options are none, dns, unix, or unix-abstract. See https://github.com/grpc/grpc/blob/master/doc/naming.md", url.Scheme), + )) + } + return errs +} diff --git a/go-controller/vendor/k8s.io/component-base/tracing/api/v1/doc.go b/go-controller/vendor/k8s.io/component-base/tracing/api/v1/doc.go new file mode 100644 index 0000000000..48e6e20f3f --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/tracing/api/v1/doc.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package + +// Package v1 contains the configuration API for tracing. +// +// The intention is to only have a single version of this API, potentially with +// new fields added over time in a backwards-compatible manner. Fields for +// alpha or beta features are allowed as long as they are defined so that not +// changing the defaults leaves those features disabled. +// +// The "v1" package name is just a reminder that API compatibility rules apply, +// not an indication of the stability of all features covered by it. + +package v1 diff --git a/go-controller/vendor/k8s.io/component-base/tracing/api/v1/types.go b/go-controller/vendor/k8s.io/component-base/tracing/api/v1/types.go new file mode 100644 index 0000000000..a59d564050 --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/tracing/api/v1/types.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// TracingConfiguration provides versioned configuration for OpenTelemetry tracing clients. +type TracingConfiguration struct { + // Endpoint of the collector this component will report traces to. + // The connection is insecure, and does not currently support TLS. + // Recommended is unset, and endpoint is the otlp grpc default, localhost:4317. + // +optional + Endpoint *string `json:"endpoint,omitempty"` + + // SamplingRatePerMillion is the number of samples to collect per million spans. + // Recommended is unset. If unset, sampler respects its parent span's sampling + // rate, but otherwise never samples. + // +optional + SamplingRatePerMillion *int32 `json:"samplingRatePerMillion,omitempty"` +} diff --git a/go-controller/vendor/k8s.io/component-base/tracing/api/v1/zz_generated.deepcopy.go b/go-controller/vendor/k8s.io/component-base/tracing/api/v1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..2afc68117b --- /dev/null +++ b/go-controller/vendor/k8s.io/component-base/tracing/api/v1/zz_generated.deepcopy.go @@ -0,0 +1,48 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TracingConfiguration) DeepCopyInto(out *TracingConfiguration) { + *out = *in + if in.Endpoint != nil { + in, out := &in.Endpoint, &out.Endpoint + *out = new(string) + **out = **in + } + if in.SamplingRatePerMillion != nil { + in, out := &in.SamplingRatePerMillion, &out.SamplingRatePerMillion + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TracingConfiguration. +func (in *TracingConfiguration) DeepCopy() *TracingConfiguration { + if in == nil { + return nil + } + out := new(TracingConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/go-controller/vendor/k8s.io/kubelet/config/v1beta1/doc.go b/go-controller/vendor/k8s.io/kubelet/config/v1beta1/doc.go new file mode 100644 index 0000000000..0edffa4b69 --- /dev/null +++ b/go-controller/vendor/k8s.io/kubelet/config/v1beta1/doc.go @@ -0,0 +1,21 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +k8s:openapi-gen=true +// +groupName=kubelet.config.k8s.io + +package v1beta1 diff --git a/go-controller/vendor/k8s.io/kubelet/config/v1beta1/register.go b/go-controller/vendor/k8s.io/kubelet/config/v1beta1/register.go new file mode 100644 index 0000000000..22bdfbb4cb --- /dev/null +++ b/go-controller/vendor/k8s.io/kubelet/config/v1beta1/register.go @@ -0,0 +1,45 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name used in this package +const GroupName = "kubelet.config.k8s.io" + +// SchemeGroupVersion is group version used to register these objects +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"} + +var ( + // SchemeBuilder is the scheme builder with scheme init functions to run for this API package + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + // AddToScheme is a global function that registers this API group & version to a scheme + AddToScheme = SchemeBuilder.AddToScheme +) + +// addKnownTypes registers known types to the given scheme +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &KubeletConfiguration{}, + &SerializedNodeConfigSource{}, + &CredentialProviderConfig{}, + ) + return nil +} diff --git a/go-controller/vendor/k8s.io/kubelet/config/v1beta1/types.go b/go-controller/vendor/k8s.io/kubelet/config/v1beta1/types.go new file mode 100644 index 0000000000..610737c87e --- /dev/null +++ b/go-controller/vendor/k8s.io/kubelet/config/v1beta1/types.go @@ -0,0 +1,1137 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + logsapi "k8s.io/component-base/logs/api/v1" + tracingapi "k8s.io/component-base/tracing/api/v1" +) + +// HairpinMode denotes how the kubelet should configure networking to handle +// hairpin packets. +type HairpinMode string + +// Enum settings for different ways to handle hairpin packets. +const ( + // Set the hairpin flag on the veth of containers in the respective + // container runtime. + HairpinVeth = "hairpin-veth" + // Make the container bridge promiscuous. This will force it to accept + // hairpin packets, even if the flag isn't set on ports of the bridge. + PromiscuousBridge = "promiscuous-bridge" + // Neither of the above. If the kubelet is started in this hairpin mode + // and kube-proxy is running in iptables mode, hairpin packets will be + // dropped by the container bridge. + HairpinNone = "none" +) + +// ResourceChangeDetectionStrategy denotes a mode in which internal +// managers (secret, configmap) are discovering object changes. +type ResourceChangeDetectionStrategy string + +// Enum settings for different strategies of kubelet managers. +const ( + // GetChangeDetectionStrategy is a mode in which kubelet fetches + // necessary objects directly from apiserver. + GetChangeDetectionStrategy ResourceChangeDetectionStrategy = "Get" + // TTLCacheChangeDetectionStrategy is a mode in which kubelet uses + // ttl cache for object directly fetched from apiserver. + TTLCacheChangeDetectionStrategy ResourceChangeDetectionStrategy = "Cache" + // WatchChangeDetectionStrategy is a mode in which kubelet uses + // watches to observe changes to objects that are in its interest. + WatchChangeDetectionStrategy ResourceChangeDetectionStrategy = "Watch" + // RestrictedTopologyManagerPolicy is a mode in which kubelet only allows + // pods with optimal NUMA node alignment for requested resources + RestrictedTopologyManagerPolicy = "restricted" + // BestEffortTopologyManagerPolicy is a mode in which kubelet will favour + // pods with NUMA alignment of CPU and device resources. + BestEffortTopologyManagerPolicy = "best-effort" + // NoneTopologyManagerPolicy is a mode in which kubelet has no knowledge + // of NUMA alignment of a pod's CPU and device resources. + NoneTopologyManagerPolicy = "none" + // SingleNumaNodeTopologyManagerPolicy is a mode in which kubelet only allows + // pods with a single NUMA alignment of CPU and device resources. + SingleNumaNodeTopologyManagerPolicy = "single-numa-node" + // ContainerTopologyManagerScope represents that + // topology policy is applied on a per-container basis. + ContainerTopologyManagerScope = "container" + // PodTopologyManagerScope represents that + // topology policy is applied on a per-pod basis. + PodTopologyManagerScope = "pod" + // NoneMemoryManagerPolicy is a memory manager none policy, under the none policy + // the memory manager will not pin containers memory of guaranteed pods + NoneMemoryManagerPolicy = "None" + // StaticMemoryManagerPolicy is a memory manager static policy, under the static policy + // the memory manager will try to pin containers memory of guaranteed pods to the smallest + // possible sub-set of NUMA nodes + StaticMemoryManagerPolicy = "Static" +) + +// ImagePullCredentialsVerificationPolicy is an enum for the policy that is enforced +// when pod is requesting an image that appears on the system +type ImagePullCredentialsVerificationPolicy string + +const ( + // NeverVerify will never require credential verification for images that + // already exist on the node + NeverVerify ImagePullCredentialsVerificationPolicy = "NeverVerify" + // NeverVerifyPreloadedImages does not require credential verification for images + // pulled outside the kubelet process + NeverVerifyPreloadedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyPreloadedImages" + // NeverVerifyAllowlistedImages does not require credential verification for + // a list of images that were pulled outside the kubelet process + NeverVerifyAllowlistedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyAllowlistedImages" + // AlwaysVerify requires credential verification for accessing any image on the + // node irregardless how it was pulled + AlwaysVerify ImagePullCredentialsVerificationPolicy = "AlwaysVerify" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// KubeletConfiguration contains the configuration for the Kubelet +type KubeletConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // enableServer enables Kubelet's secured server. + // Note: Kubelet's insecure port is controlled by the readOnlyPort option. + // Default: true + EnableServer *bool `json:"enableServer,omitempty"` + // staticPodPath is the path to the directory containing local (static) pods to + // run, or the path to a single static pod file. + // Default: "" + // +optional + StaticPodPath string `json:"staticPodPath,omitempty"` + // podLogsDir is a custom root directory path kubelet will use to place pod's log files. + // Default: "/var/log/pods/" + // Note: it is not recommended to use the temp folder as a log directory as it may cause + // unexpected behavior in many places. + // +optional + PodLogsDir string `json:"podLogsDir,omitempty"` + // syncFrequency is the max period between synchronizing running + // containers and config. + // Default: "1m" + // +optional + SyncFrequency metav1.Duration `json:"syncFrequency,omitempty"` + // fileCheckFrequency is the duration between checking config files for + // new data. + // Default: "20s" + // +optional + FileCheckFrequency metav1.Duration `json:"fileCheckFrequency,omitempty"` + // httpCheckFrequency is the duration between checking http for new data. + // Default: "20s" + // +optional + HTTPCheckFrequency metav1.Duration `json:"httpCheckFrequency,omitempty"` + // staticPodURL is the URL for accessing static pods to run. + // Default: "" + // +optional + StaticPodURL string `json:"staticPodURL,omitempty"` + // staticPodURLHeader is a map of slices with HTTP headers to use when accessing the podURL. + // Default: nil + // +optional + StaticPodURLHeader map[string][]string `json:"staticPodURLHeader,omitempty"` + // address is the IP address for the Kubelet to serve on (set to 0.0.0.0 + // for all interfaces). + // Default: "0.0.0.0" + // +optional + Address string `json:"address,omitempty"` + // port is the port for the Kubelet to serve on. + // The port number must be between 1 and 65535, inclusive. + // Default: 10250 + // +optional + Port int32 `json:"port,omitempty"` + // readOnlyPort is the read-only port for the Kubelet to serve on with + // no authentication/authorization. + // The port number must be between 1 and 65535, inclusive. + // Setting this field to 0 disables the read-only service. + // Default: 0 (disabled) + // +optional + ReadOnlyPort int32 `json:"readOnlyPort,omitempty"` + // tlsCertFile is the file containing x509 Certificate for HTTPS. (CA cert, + // if any, concatenated after server cert). If tlsCertFile and + // tlsPrivateKeyFile are not provided, a self-signed certificate + // and key are generated for the public address and saved to the directory + // passed to the Kubelet's --cert-dir flag. + // Default: "" + // +optional + TLSCertFile string `json:"tlsCertFile,omitempty"` + // tlsPrivateKeyFile is the file containing x509 private key matching tlsCertFile. + // Default: "" + // +optional + TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty"` + // tlsCipherSuites is the list of allowed cipher suites for the server. + // Note that TLS 1.3 ciphersuites are not configurable. + // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). + // Default: nil + // +optional + TLSCipherSuites []string `json:"tlsCipherSuites,omitempty"` + // tlsMinVersion is the minimum TLS version supported. + // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). + // Default: "" + // +optional + TLSMinVersion string `json:"tlsMinVersion,omitempty"` + // rotateCertificates enables client certificate rotation. The Kubelet will request a + // new certificate from the certificates.k8s.io API. This requires an approver to approve the + // certificate signing requests. + // Default: false + // +optional + RotateCertificates bool `json:"rotateCertificates,omitempty"` + // serverTLSBootstrap enables server certificate bootstrap. Instead of self + // signing a serving certificate, the Kubelet will request a certificate from + // the 'certificates.k8s.io' API. This requires an approver to approve the + // certificate signing requests (CSR). The RotateKubeletServerCertificate feature + // must be enabled when setting this field. + // Default: false + // +optional + ServerTLSBootstrap bool `json:"serverTLSBootstrap,omitempty"` + // authentication specifies how requests to the Kubelet's server are authenticated. + // Defaults: + // anonymous: + // enabled: false + // webhook: + // enabled: true + // cacheTTL: "2m" + // +optional + Authentication KubeletAuthentication `json:"authentication"` + // authorization specifies how requests to the Kubelet's server are authorized. + // Defaults: + // mode: Webhook + // webhook: + // cacheAuthorizedTTL: "5m" + // cacheUnauthorizedTTL: "30s" + // +optional + Authorization KubeletAuthorization `json:"authorization"` + // registryPullQPS is the limit of registry pulls per second. + // The value must not be a negative number. + // Setting it to 0 means no limit. + // Default: 5 + // +optional + RegistryPullQPS *int32 `json:"registryPullQPS,omitempty"` + // registryBurst is the maximum size of bursty pulls, temporarily allows + // pulls to burst to this number, while still not exceeding registryPullQPS. + // The value must not be a negative number. + // Only used if registryPullQPS is greater than 0. + // Default: 10 + // +optional + RegistryBurst int32 `json:"registryBurst,omitempty"` + // imagePullCredentialsVerificationPolicy determines how credentials should be + // verified when pod requests an image that is already present on the node: + // - NeverVerify + // - anyone on a node can use any image present on the node + // - NeverVerifyPreloadedImages + // - images that were pulled to the node by something else than the kubelet + // can be used without reverifying pull credentials + // - NeverVerifyAllowlistedImages + // - like "NeverVerifyPreloadedImages" but only node images from + // `preloadedImagesVerificationAllowlist` don't require reverification + // - AlwaysVerify + // - all images require credential reverification + // +optional + ImagePullCredentialsVerificationPolicy ImagePullCredentialsVerificationPolicy `json:"imagePullCredentialsVerificationPolicy,omitempty"` + // preloadedImagesVerificationAllowlist specifies a list of images that are + // exempted from credential reverification for the "NeverVerifyAllowlistedImages" + // `imagePullCredentialsVerificationPolicy`. + // The list accepts a full path segment wildcard suffix "/*". + // Only use image specs without an image tag or digest. + // +optional + // +listType=set + PreloadedImagesVerificationAllowlist []string `json:"preloadedImagesVerificationAllowlist,omitempty"` + // eventRecordQPS is the maximum event creations per second. If 0, there + // is no limit enforced. The value cannot be a negative number. + // Default: 50 + // +optional + EventRecordQPS *int32 `json:"eventRecordQPS,omitempty"` + // eventBurst is the maximum size of a burst of event creations, temporarily + // allows event creations to burst to this number, while still not exceeding + // eventRecordQPS. This field canot be a negative number and it is only used + // when eventRecordQPS > 0. + // Default: 100 + // +optional + EventBurst int32 `json:"eventBurst,omitempty"` + // enableDebuggingHandlers enables server endpoints for log access + // and local running of containers and commands, including the exec, + // attach, logs, and portforward features. + // Default: true + // +optional + EnableDebuggingHandlers *bool `json:"enableDebuggingHandlers,omitempty"` + // enableContentionProfiling enables block profiling, if enableDebuggingHandlers is true. + // Default: false + // +optional + EnableContentionProfiling bool `json:"enableContentionProfiling,omitempty"` + // healthzPort is the port of the localhost healthz endpoint (set to 0 to disable). + // A valid number is between 1 and 65535. + // Default: 10248 + // +optional + HealthzPort *int32 `json:"healthzPort,omitempty"` + // healthzBindAddress is the IP address for the healthz server to serve on. + // Default: "127.0.0.1" + // +optional + HealthzBindAddress string `json:"healthzBindAddress,omitempty"` + // oomScoreAdj is The oom-score-adj value for kubelet process. Values + // must be within the range [-1000, 1000]. + // Default: -999 + // +optional + OOMScoreAdj *int32 `json:"oomScoreAdj,omitempty"` + // clusterDomain is the DNS domain for this cluster. If set, kubelet will + // configure all containers to search this domain in addition to the + // host's search domains. + // Default: "" + // +optional + ClusterDomain string `json:"clusterDomain,omitempty"` + // clusterDNS is a list of IP addresses for the cluster DNS server. If set, + // kubelet will configure all containers to use this for DNS resolution + // instead of the host's DNS servers. + // Default: nil + // +optional + ClusterDNS []string `json:"clusterDNS,omitempty"` + // streamingConnectionIdleTimeout is the maximum time a streaming connection + // can be idle before the connection is automatically closed. + // Deprecated: no longer has any effect. + // Default: "4h" + // +optional + StreamingConnectionIdleTimeout metav1.Duration `json:"streamingConnectionIdleTimeout,omitempty"` + // nodeStatusUpdateFrequency is the frequency that kubelet computes node + // status. If node lease feature is not enabled, it is also the frequency that + // kubelet posts node status to master. + // Note: When node lease feature is not enabled, be cautious when changing the + // constant, it must work with nodeMonitorGracePeriod in nodecontroller. + // Default: "10s" + // +optional + NodeStatusUpdateFrequency metav1.Duration `json:"nodeStatusUpdateFrequency,omitempty"` + // nodeStatusReportFrequency is the frequency that kubelet posts node + // status to master if node status does not change. Kubelet will ignore this + // frequency and post node status immediately if any change is detected. It is + // only used when node lease feature is enabled. nodeStatusReportFrequency's + // default value is 5m. But if nodeStatusUpdateFrequency is set explicitly, + // nodeStatusReportFrequency's default value will be set to + // nodeStatusUpdateFrequency for backward compatibility. + // Default: "5m" + // +optional + NodeStatusReportFrequency metav1.Duration `json:"nodeStatusReportFrequency,omitempty"` + // nodeLeaseDurationSeconds is the duration the Kubelet will set on its corresponding Lease. + // NodeLease provides an indicator of node health by having the Kubelet create and + // periodically renew a lease, named after the node, in the kube-node-lease namespace. + // If the lease expires, the node can be considered unhealthy. + // The lease is currently renewed every 10s, per KEP-0009. In the future, the lease renewal + // interval may be set based on the lease duration. + // The field value must be greater than 0. + // Default: 40 + // +optional + NodeLeaseDurationSeconds int32 `json:"nodeLeaseDurationSeconds,omitempty"` + // imageMinimumGCAge is the minimum age for an unused image before it is + // garbage collected. + // Default: "2m" + // +optional + ImageMinimumGCAge metav1.Duration `json:"imageMinimumGCAge,omitempty"` + // imageMaximumGCAge is the maximum age an image can be unused before it is garbage collected. + // The default of this field is "0s", which disables this field--meaning images won't be garbage + // collected based on being unused for too long. + // Default: "0s" (disabled) + // +optional + ImageMaximumGCAge metav1.Duration `json:"imageMaximumGCAge,omitempty"` + // imageGCHighThresholdPercent is the percent of disk usage after which + // image garbage collection is always run. The percent is calculated by + // dividing this field value by 100, so this field must be between 0 and + // 100, inclusive. When specified, the value must be greater than + // imageGCLowThresholdPercent. + // Default: 85 + // +optional + ImageGCHighThresholdPercent *int32 `json:"imageGCHighThresholdPercent,omitempty"` + // imageGCLowThresholdPercent is the percent of disk usage before which + // image garbage collection is never run. Lowest disk usage to garbage + // collect to. The percent is calculated by dividing this field value by 100, + // so the field value must be between 0 and 100, inclusive. When specified, the + // value must be less than imageGCHighThresholdPercent. + // Default: 80 + // +optional + ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent,omitempty"` + // volumeStatsAggPeriod is the frequency for calculating and caching volume + // disk usage for all pods. + // Default: "1m" + // +optional + VolumeStatsAggPeriod metav1.Duration `json:"volumeStatsAggPeriod,omitempty"` + // kubeletCgroups is the absolute name of cgroups to isolate the kubelet in + // Default: "" + // +optional + KubeletCgroups string `json:"kubeletCgroups,omitempty"` + // systemCgroups is absolute name of cgroups in which to place + // all non-kernel processes that are not already in a container. Empty + // for no container. Rolling back the flag requires a reboot. + // The cgroupRoot must be specified if this field is not empty. + // Default: "" + // +optional + SystemCgroups string `json:"systemCgroups,omitempty"` + // cgroupRoot is the root cgroup to use for pods. This is handled by the + // container runtime on a best effort basis. + // +optional + CgroupRoot string `json:"cgroupRoot,omitempty"` + // cgroupsPerQOS enable QoS based CGroup hierarchy: top level CGroups for QoS classes + // and all Burstable and BestEffort Pods are brought up under their specific top level + // QoS CGroup. + // Default: true + // +optional + CgroupsPerQOS *bool `json:"cgroupsPerQOS,omitempty"` + // cgroupDriver is the driver kubelet uses to manipulate CGroups on the host (cgroupfs + // or systemd). + // Default: "cgroupfs" + // +optional + CgroupDriver string `json:"cgroupDriver,omitempty"` + // cpuManagerPolicy is the name of the policy to use. + // Default: "None" + // +optional + CPUManagerPolicy string `json:"cpuManagerPolicy,omitempty"` + // singleProcessOOMKill, if true, will prevent the `memory.oom.group` flag from being set for container + // cgroups in cgroups v2. This causes processes in the container to be OOM killed individually instead of as + // a group. It means that if true, the behavior aligns with the behavior of cgroups v1. + // The default value is determined automatically when you don't specify. + // On non-linux such as windows, only null / absent is allowed. + // On cgroup v1 linux, only null / absent and true are allowed. + // On cgroup v2 linux, null / absent, true and false are allowed. The default value is false. + // +optional + SingleProcessOOMKill *bool `json:"singleProcessOOMKill,omitempty"` + // cpuManagerPolicyOptions is a set of key=value which allows to set extra options + // to fine tune the behaviour of the cpu manager policies. + // Default: nil + // +optional + CPUManagerPolicyOptions map[string]string `json:"cpuManagerPolicyOptions,omitempty"` + // cpuManagerReconcilePeriod is the reconciliation period for the CPU Manager. + // Default: "10s" + // +optional + CPUManagerReconcilePeriod metav1.Duration `json:"cpuManagerReconcilePeriod,omitempty"` + // memoryManagerPolicy is the name of the policy to use by memory manager. + // Requires the MemoryManager feature gate to be enabled. + // Default: "none" + // +optional + MemoryManagerPolicy string `json:"memoryManagerPolicy,omitempty"` + // topologyManagerPolicy is the name of the topology manager policy to use. + // Valid values include: + // + // - `restricted`: kubelet only allows pods with optimal NUMA node alignment for + // requested resources; + // - `best-effort`: kubelet will favor pods with NUMA alignment of CPU and device + // resources; + // - `none`: kubelet has no knowledge of NUMA alignment of a pod's CPU and device resources. + // - `single-numa-node`: kubelet only allows pods with a single NUMA alignment + // of CPU and device resources. + // + // Default: "none" + // +optional + TopologyManagerPolicy string `json:"topologyManagerPolicy,omitempty"` + // topologyManagerScope represents the scope of topology hint generation + // that topology manager requests and hint providers generate. Valid values include: + // + // - `container`: topology policy is applied on a per-container basis. + // - `pod`: topology policy is applied on a per-pod basis. + // + // Default: "container" + // +optional + TopologyManagerScope string `json:"topologyManagerScope,omitempty"` + // TopologyManagerPolicyOptions is a set of key=value which allows to set extra options + // to fine tune the behaviour of the topology manager policies. + // Requires both the "TopologyManager" and "TopologyManagerPolicyOptions" feature gates to be enabled. + // Default: nil + // +optional + TopologyManagerPolicyOptions map[string]string `json:"topologyManagerPolicyOptions,omitempty"` + // qosReserved is a set of resource name to percentage pairs that specify + // the minimum percentage of a resource reserved for exclusive use by the + // guaranteed QoS tier. + // Currently supported resources: "memory" + // Requires the QOSReserved feature gate to be enabled. + // Default: nil + // +optional + QOSReserved map[string]string `json:"qosReserved,omitempty"` + // runtimeRequestTimeout is the timeout for all runtime requests except long running + // requests - pull, logs, exec and attach. + // Default: "2m" + // +optional + RuntimeRequestTimeout metav1.Duration `json:"runtimeRequestTimeout,omitempty"` + // hairpinMode specifies how the Kubelet should configure the container + // bridge for hairpin packets. + // Setting this flag allows endpoints in a Service to loadbalance back to + // themselves if they should try to access their own Service. Values: + // + // - "promiscuous-bridge": make the container bridge promiscuous. + // - "hairpin-veth": set the hairpin flag on container veth interfaces. + // - "none": do nothing. + // + // Generally, one must set `--hairpin-mode=hairpin-veth to` achieve hairpin NAT, + // because promiscuous-bridge assumes the existence of a container bridge named cbr0. + // Default: "promiscuous-bridge" + // +optional + HairpinMode string `json:"hairpinMode,omitempty"` + // maxPods is the maximum number of Pods that can run on this Kubelet. + // The value must be a non-negative integer. + // Default: 110 + // +optional + MaxPods int32 `json:"maxPods,omitempty"` + // podCIDR is the CIDR to use for pod IP addresses, only used in standalone mode. + // In cluster mode, this is obtained from the control plane. + // Default: "" + // +optional + PodCIDR string `json:"podCIDR,omitempty"` + // podPidsLimit is the maximum number of PIDs in any pod. + // Default: -1 + // +optional + PodPidsLimit *int64 `json:"podPidsLimit,omitempty"` + // resolvConf is the resolver configuration file used as the basis + // for the container DNS resolution configuration. + // If set to the empty string, will override the default and effectively disable DNS lookups. + // Default: "/etc/resolv.conf" + // +optional + ResolverConfig *string `json:"resolvConf,omitempty"` + // runOnce causes the Kubelet to check the API server once for pods, + // run those in addition to the pods specified by static pod files, and exit. + // Default: false + // +optional + RunOnce bool `json:"runOnce,omitempty"` + // cpuCFSQuota enables CPU CFS quota enforcement for containers that + // specify CPU limits. + // Default: true + // +optional + CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` + // cpuCFSQuotaPeriod is the CPU CFS quota period value, `cpu.cfs_period_us`. + // The value must be between 1 ms and 1 second, inclusive. + // Requires the CustomCPUCFSQuotaPeriod feature gate to be enabled. + // Default: "100ms" + // +optional + CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty"` + // nodeStatusMaxImages caps the number of images reported in Node.status.images. + // The value must be greater than -2. + // Note: If -1 is specified, no cap will be applied. If 0 is specified, no image is returned. + // Default: 50 + // +optional + NodeStatusMaxImages *int32 `json:"nodeStatusMaxImages,omitempty"` + // maxOpenFiles is Number of files that can be opened by Kubelet process. + // The value must be a non-negative number. + // Default: 1000000 + // +optional + MaxOpenFiles int64 `json:"maxOpenFiles,omitempty"` + // contentType is contentType of requests sent to apiserver. + // Default: "application/vnd.kubernetes.protobuf" + // +optional + ContentType string `json:"contentType,omitempty"` + // kubeAPIQPS is the QPS to use while talking with kubernetes apiserver. + // Default: 50 + // +optional + KubeAPIQPS *int32 `json:"kubeAPIQPS,omitempty"` + // kubeAPIBurst is the burst to allow while talking with kubernetes API server. + // This field cannot be a negative number. + // Default: 100 + // +optional + KubeAPIBurst int32 `json:"kubeAPIBurst,omitempty"` + // serializeImagePulls when enabled, tells the Kubelet to pull images one + // at a time. We recommend *not* changing the default value on nodes that + // run docker daemon with version < 1.9 or an Aufs storage backend. + // Issue #10959 has more details. + // Default: true + // +optional + SerializeImagePulls *bool `json:"serializeImagePulls,omitempty"` + // MaxParallelImagePulls sets the maximum number of image pulls in parallel. + // This field cannot be set if SerializeImagePulls is true. + // Setting it to nil means no limit. + // Default: nil + // +optional + MaxParallelImagePulls *int32 `json:"maxParallelImagePulls,omitempty"` + // evictionHard is a map of signal names to quantities that defines hard eviction + // thresholds. For example: `{"memory.available": "300Mi"}`. + // To explicitly disable, pass a 0% or 100% threshold on an arbitrary resource. + // Default: + // memory.available: "100Mi" + // nodefs.available: "10%" + // nodefs.inodesFree: "5%" + // imagefs.available: "15%" + // +optional + EvictionHard map[string]string `json:"evictionHard,omitempty"` + // evictionSoft is a map of signal names to quantities that defines soft eviction thresholds. + // For example: `{"memory.available": "300Mi"}`. + // Default: nil + // +optional + EvictionSoft map[string]string `json:"evictionSoft,omitempty"` + // evictionSoftGracePeriod is a map of signal names to quantities that defines grace + // periods for each soft eviction signal. For example: `{"memory.available": "30s"}`. + // Default: nil + // +optional + EvictionSoftGracePeriod map[string]string `json:"evictionSoftGracePeriod,omitempty"` + // evictionPressureTransitionPeriod is the duration for which the kubelet has to wait + // before transitioning out of an eviction pressure condition. + // A duration of 0s will be converted to the default value of 5m + // Default: "5m" + // +optional + EvictionPressureTransitionPeriod metav1.Duration `json:"evictionPressureTransitionPeriod,omitempty"` + // evictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use + // when terminating pods in response to a soft eviction threshold being met. This value + // effectively caps the Pod's terminationGracePeriodSeconds value during soft evictions. + // Default: 0 + // +optional + EvictionMaxPodGracePeriod int32 `json:"evictionMaxPodGracePeriod,omitempty"` + // evictionMinimumReclaim is a map of signal names to quantities that defines minimum reclaims, + // which describe the minimum amount of a given resource the kubelet will reclaim when + // performing a pod eviction while that resource is under pressure. + // For example: `{"imagefs.available": "2Gi"}`. + // Default: nil + // +optional + EvictionMinimumReclaim map[string]string `json:"evictionMinimumReclaim,omitempty"` + // mergeDefaultEvictionSettings indicates that defaults for the evictionHard, evictionSoft, evictionSoftGracePeriod, and evictionMinimumReclaim + // fields should be merged into values specified for those fields in this configuration. + // Signals specified in this configuration take precedence. + // Signals not specified in this configuration inherit their defaults. + // If false, and if any signal is specified in this configuration then other signals that + // are not specified in this configuration will be set to 0. + // It applies to merging the fields for which the default exists, and currently only evictionHard has default values. + // Default: false + // +optional + MergeDefaultEvictionSettings *bool `json:"mergeDefaultEvictionSettings,omitempty"` + // podsPerCore is the maximum number of pods per core. Cannot exceed maxPods. + // The value must be a non-negative integer. + // If 0, there is no limit on the number of Pods. + // Default: 0 + // +optional + PodsPerCore int32 `json:"podsPerCore,omitempty"` + // enableControllerAttachDetach enables the Attach/Detach controller to + // manage attachment/detachment of volumes scheduled to this node, and + // disables kubelet from executing any attach/detach operations. + // Note: attaching/detaching CSI volumes is not supported by the kubelet, + // so this option needs to be true for that use case. + // Default: true + // +optional + EnableControllerAttachDetach *bool `json:"enableControllerAttachDetach,omitempty"` + // protectKernelDefaults, if true, causes the Kubelet to error if kernel + // flags are not as it expects. Otherwise the Kubelet will attempt to modify + // kernel flags to match its expectation. + // Default: false + // +optional + ProtectKernelDefaults bool `json:"protectKernelDefaults,omitempty"` + // makeIPTablesUtilChains, if true, causes the Kubelet to create the + // KUBE-IPTABLES-HINT chain in iptables as a hint to other components about the + // configuration of iptables on the system. + // Default: true + // +optional + MakeIPTablesUtilChains *bool `json:"makeIPTablesUtilChains,omitempty"` + // iptablesMasqueradeBit formerly controlled the creation of the KUBE-MARK-MASQ + // chain. + // Deprecated: no longer has any effect. + // Default: 14 + // +optional + IPTablesMasqueradeBit *int32 `json:"iptablesMasqueradeBit,omitempty"` + // iptablesDropBit formerly controlled the creation of the KUBE-MARK-DROP chain. + // Deprecated: no longer has any effect. + // Default: 15 + // +optional + IPTablesDropBit *int32 `json:"iptablesDropBit,omitempty"` + // featureGates is a map of feature names to bools that enable or disable experimental + // features. This field modifies piecemeal the built-in default values from + // "k8s.io/kubernetes/pkg/features/kube_features.go". + // Default: nil + // +optional + FeatureGates map[string]bool `json:"featureGates,omitempty"` + // failSwapOn tells the Kubelet to fail to start if swap is enabled on the node. + // Default: true + // +optional + FailSwapOn *bool `json:"failSwapOn,omitempty"` + // memorySwap configures swap memory available to container workloads. + // +featureGate=NodeSwap + // +optional + MemorySwap MemorySwapConfiguration `json:"memorySwap,omitempty"` + // containerLogMaxSize is a quantity defining the maximum size of the container log + // file before it is rotated. For example: "5Mi" or "256Ki". + // Default: "10Mi" + // +optional + ContainerLogMaxSize string `json:"containerLogMaxSize,omitempty"` + // containerLogMaxFiles specifies the maximum number of container log files that can + // be present for a container. + // Default: 5 + // +optional + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + + // ContainerLogMaxWorkers specifies the maximum number of concurrent workers to spawn + // for performing the log rotate operations. Set this count to 1 for disabling the + // concurrent log rotation workflows + // Default: 1 + // +optional + ContainerLogMaxWorkers *int32 `json:"containerLogMaxWorkers,omitempty"` + + // ContainerLogMonitorInterval specifies the duration at which the container logs are monitored + // for performing the log rotate operation. This defaults to 10 * time.Seconds. But can be + // customized to a smaller value based on the log generation rate and the size required to be + // rotated against + // Default: 10s + // +optional + ContainerLogMonitorInterval *metav1.Duration `json:"containerLogMonitorInterval,omitempty"` + // configMapAndSecretChangeDetectionStrategy is a mode in which ConfigMap and Secret + // managers are running. Valid values include: + // + // - `Get`: kubelet fetches necessary objects directly from the API server; + // - `Cache`: kubelet uses TTL cache for object fetched from the API server; + // - `Watch`: kubelet uses watches to observe changes to objects that are in its interest. + // + // Default: "Watch" + // +optional + ConfigMapAndSecretChangeDetectionStrategy ResourceChangeDetectionStrategy `json:"configMapAndSecretChangeDetectionStrategy,omitempty"` + + /* the following fields are meant for Node Allocatable */ + + // systemReserved is a set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) + // pairs that describe resources reserved for non-kubernetes components. + // Currently only cpu and memory are supported. + // See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources for more detail. + // Default: nil + // +optional + SystemReserved map[string]string `json:"systemReserved,omitempty"` + // kubeReserved is a set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) pairs + // that describe resources reserved for kubernetes system components. + // Currently cpu, memory and local storage for root file system are supported. + // See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources + // for more details. + // Default: nil + // +optional + KubeReserved map[string]string `json:"kubeReserved,omitempty"` + // The reservedSystemCPUs option specifies the CPU list reserved for the host + // level system threads and kubernetes related threads. This provide a "static" + // CPU list rather than the "dynamic" list by systemReserved and kubeReserved. + // This option does not support systemReservedCgroup or kubeReservedCgroup. + ReservedSystemCPUs string `json:"reservedSystemCPUs,omitempty"` + // showHiddenMetricsForVersion is the previous version for which you want to show + // hidden metrics. + // Only the previous minor version is meaningful, other values will not be allowed. + // The format is `.`, e.g.: `1.16`. + // The purpose of this format is make sure you have the opportunity to notice + // if the next release hides additional metrics, rather than being surprised + // when they are permanently removed in the release after that. + // Default: "" + // +optional + ShowHiddenMetricsForVersion string `json:"showHiddenMetricsForVersion,omitempty"` + // systemReservedCgroup helps the kubelet identify absolute name of top level CGroup used + // to enforce `systemReserved` compute resource reservation for OS system daemons. + // Refer to [Node Allocatable](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable) + // doc for more information. + // Default: "" + // +optional + SystemReservedCgroup string `json:"systemReservedCgroup,omitempty"` + // kubeReservedCgroup helps the kubelet identify absolute name of top level CGroup used + // to enforce `KubeReserved` compute resource reservation for Kubernetes node system daemons. + // Refer to [Node Allocatable](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable) + // doc for more information. + // Default: "" + // +optional + KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` + // This flag specifies the various Node Allocatable enforcements that Kubelet needs to perform. + // This flag accepts a list of options. Acceptable options are `none`, `pods`, + // `system-reserved` and `kube-reserved`. + // If `none` is specified, no other options may be specified. + // When `system-reserved` is in the list, systemReservedCgroup must be specified. + // When `kube-reserved` is in the list, kubeReservedCgroup must be specified. + // This field is supported only when `cgroupsPerQOS` is set to true. + // Refer to [Node Allocatable](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable) + // for more information. + // Default: ["pods"] + // +optional + EnforceNodeAllocatable []string `json:"enforceNodeAllocatable,omitempty"` + // A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in `*`). + // Unsafe sysctl groups are `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, + // and `net.*`. For example: "`kernel.msg*,net.ipv4.route.min_pmtu`" + // Default: [] + // +optional + AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"` + // volumePluginDir is the full path of the directory in which to search + // for additional third party volume plugins. + // Default: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" + // +optional + VolumePluginDir string `json:"volumePluginDir,omitempty"` + // providerID, if set, sets the unique ID of the instance that an external + // provider (i.e. cloudprovider) can use to identify a specific node. + // Default: "" + // +optional + ProviderID string `json:"providerID,omitempty"` + // kernelMemcgNotification, if set, instructs the kubelet to integrate with the + // kernel memcg notification for determining if memory eviction thresholds are + // exceeded rather than polling. + // Default: false + // +optional + KernelMemcgNotification bool `json:"kernelMemcgNotification,omitempty"` + // logging specifies the options of logging. + // Refer to [Logs Options](https://github.com/kubernetes/component-base/blob/master/logs/options.go) + // for more information. + // Default: + // Format: text + // + optional + Logging logsapi.LoggingConfiguration `json:"logging,omitempty"` + // enableSystemLogHandler enables system logs via web interface host:port/logs/ + // Default: true + // +optional + EnableSystemLogHandler *bool `json:"enableSystemLogHandler,omitempty"` + // enableSystemLogQuery enables the node log query feature on the /logs endpoint. + // EnableSystemLogHandler has to be enabled in addition for this feature to work. + // Enabling this feature has security implications. The recommendation is to enable it on a need basis for debugging + // purposes and disabling otherwise. + // Default: false + // +featureGate=NodeLogQuery + // +optional + EnableSystemLogQuery *bool `json:"enableSystemLogQuery,omitempty"` + // shutdownGracePeriod specifies the total duration that the node should delay the + // shutdown and total grace period for pod termination during a node shutdown. + // Default: "0s" + // +featureGate=GracefulNodeShutdown + // +optional + ShutdownGracePeriod metav1.Duration `json:"shutdownGracePeriod,omitempty"` + // shutdownGracePeriodCriticalPods specifies the duration used to terminate critical + // pods during a node shutdown. This should be less than shutdownGracePeriod. + // For example, if shutdownGracePeriod=30s, and shutdownGracePeriodCriticalPods=10s, + // during a node shutdown the first 20 seconds would be reserved for gracefully + // terminating normal pods, and the last 10 seconds would be reserved for terminating + // critical pods. + // Default: "0s" + // +featureGate=GracefulNodeShutdown + // +optional + ShutdownGracePeriodCriticalPods metav1.Duration `json:"shutdownGracePeriodCriticalPods,omitempty"` + // shutdownGracePeriodByPodPriority specifies the shutdown grace period for Pods based + // on their associated priority class value. + // When a shutdown request is received, the Kubelet will initiate shutdown on all pods + // running on the node with a grace period that depends on the priority of the pod, + // and then wait for all pods to exit. + // Each entry in the array represents the graceful shutdown time a pod with a priority + // class value that lies in the range of that value and the next higher entry in the + // list when the node is shutting down. + // For example, to allow critical pods 10s to shutdown, priority>=10000 pods 20s to + // shutdown, and all remaining pods 30s to shutdown. + // + // shutdownGracePeriodByPodPriority: + // - priority: 2000000000 + // shutdownGracePeriodSeconds: 10 + // - priority: 10000 + // shutdownGracePeriodSeconds: 20 + // - priority: 0 + // shutdownGracePeriodSeconds: 30 + // + // The time the Kubelet will wait before exiting will at most be the maximum of all + // shutdownGracePeriodSeconds for each priority class range represented on the node. + // When all pods have exited or reached their grace periods, the Kubelet will release + // the shutdown inhibit lock. + // Requires the GracefulNodeShutdown feature gate to be enabled. + // This configuration must be empty if either ShutdownGracePeriod or ShutdownGracePeriodCriticalPods is set. + // Default: nil + // +featureGate=GracefulNodeShutdownBasedOnPodPriority + // +optional + ShutdownGracePeriodByPodPriority []ShutdownGracePeriodByPodPriority `json:"shutdownGracePeriodByPodPriority,omitempty"` + // CrashLoopBackOff contains config to modify node-level parameters for + // container restart behavior + // +featureGate=KubeletCrashLoopBackOffMax + // +optional + CrashLoopBackOff CrashLoopBackOffConfig `json:"crashLoopBackOff,omitempty"` + // reservedMemory specifies a comma-separated list of memory reservations for NUMA nodes. + // The parameter makes sense only in the context of the memory manager feature. + // The memory manager will not allocate reserved memory for container workloads. + // For example, if you have a NUMA0 with 10Gi of memory and the reservedMemory was + // specified to reserve 1Gi of memory at NUMA0, the memory manager will assume that + // only 9Gi is available for allocation. + // You can specify a different amount of NUMA node and memory types. + // You can omit this parameter at all, but you should be aware that the amount of + // reserved memory from all NUMA nodes should be equal to the amount of memory specified + // by the [node allocatable](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable). + // If at least one node allocatable parameter has a non-zero value, you will need + // to specify at least one NUMA node. + // Also, avoid specifying: + // + // 1. Duplicates, the same NUMA node, and memory type, but with a different value. + // 2. zero limits for any memory type. + // 3. NUMAs nodes IDs that do not exist under the machine. + // 4. memory types except for memory and hugepages- + // + // Default: nil + // +optional + ReservedMemory []MemoryReservation `json:"reservedMemory,omitempty"` + // enableProfilingHandler enables profiling via web interface host:port/debug/pprof/ + // Default: true + // +optional + EnableProfilingHandler *bool `json:"enableProfilingHandler,omitempty"` + // enableDebugFlagsHandler enables flags endpoint via web interface host:port/debug/flags/v + // Default: true + // +optional + EnableDebugFlagsHandler *bool `json:"enableDebugFlagsHandler,omitempty"` + // SeccompDefault enables the use of `RuntimeDefault` as the default seccomp profile for all workloads. + // Default: false + // +optional + SeccompDefault *bool `json:"seccompDefault,omitempty"` + // MemoryThrottlingFactor specifies the factor multiplied by the memory limit or node allocatable memory + // when setting the cgroupv2 memory.high value to enforce MemoryQoS. + // Decreasing this factor will set lower high limit for container cgroups and put heavier reclaim pressure + // while increasing will put less reclaim pressure. + // See https://kep.k8s.io/2570 for more details. + // Default: 0.9 + // +featureGate=MemoryQoS + // +optional + MemoryThrottlingFactor *float64 `json:"memoryThrottlingFactor,omitempty"` + // registerWithTaints are an array of taints to add to a node object when + // the kubelet registers itself. This only takes effect when registerNode + // is true and upon the initial registration of the node. + // Default: nil + // +optional + RegisterWithTaints []v1.Taint `json:"registerWithTaints,omitempty"` + // registerNode enables automatic registration with the apiserver. + // Default: true + // +optional + RegisterNode *bool `json:"registerNode,omitempty"` + // Tracing specifies the versioned configuration for OpenTelemetry tracing clients. + // See https://kep.k8s.io/2832 for more details. + // Default: nil + // +optional + Tracing *tracingapi.TracingConfiguration `json:"tracing,omitempty"` + + // LocalStorageCapacityIsolation enables local ephemeral storage isolation feature. The default setting is true. + // This feature allows users to set request/limit for container's ephemeral storage and manage it in a similar way + // as cpu and memory. It also allows setting sizeLimit for emptyDir volume, which will trigger pod eviction if disk + // usage from the volume exceeds the limit. + // This feature depends on the capability of detecting correct root file system disk usage. For certain systems, + // such as kind rootless, if this capability cannot be supported, the feature LocalStorageCapacityIsolation should be + // disabled. Once disabled, user should not set request/limit for container's ephemeral storage, or sizeLimit for emptyDir. + // Default: true + // +optional + LocalStorageCapacityIsolation *bool `json:"localStorageCapacityIsolation,omitempty"` + + // ContainerRuntimeEndpoint is the endpoint of container runtime. + // Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime' + ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint"` + + // ImageServiceEndpoint is the endpoint of container image service. + // Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'. + // If not specified, the value in containerRuntimeEndpoint is used. + // +optional + ImageServiceEndpoint string `json:"imageServiceEndpoint,omitempty"` + + // FailCgroupV1 prevents the kubelet from starting on hosts + // that use cgroup v1. By default, this is set to 'false', meaning + // the kubelet is allowed to start on cgroup v1 hosts unless this + // option is explicitly enabled. + // Default: false + // +optional + FailCgroupV1 *bool `json:"failCgroupV1,omitempty"` + + // UserNamespaces contains User Namespace configurations. + // +featureGate=UserNamespacesSupport + // +optional + UserNamespaces *UserNamespaces `json:"userNamespaces,omitempty"` +} + +type KubeletAuthorizationMode string + +const ( + // KubeletAuthorizationModeAlwaysAllow authorizes all authenticated requests + KubeletAuthorizationModeAlwaysAllow KubeletAuthorizationMode = "AlwaysAllow" + // KubeletAuthorizationModeWebhook uses the SubjectAccessReview API to determine authorization + KubeletAuthorizationModeWebhook KubeletAuthorizationMode = "Webhook" +) + +type KubeletAuthorization struct { + // mode is the authorization mode to apply to requests to the kubelet server. + // Valid values are `AlwaysAllow` and `Webhook`. + // Webhook mode uses the SubjectAccessReview API to determine authorization. + // +optional + Mode KubeletAuthorizationMode `json:"mode,omitempty"` + + // webhook contains settings related to Webhook authorization. + // +optional + Webhook KubeletWebhookAuthorization `json:"webhook"` +} + +type KubeletWebhookAuthorization struct { + // cacheAuthorizedTTL is the duration to cache 'authorized' responses from the + // webhook authorizer. + // +optional + CacheAuthorizedTTL metav1.Duration `json:"cacheAuthorizedTTL,omitempty"` + // cacheUnauthorizedTTL is the duration to cache 'unauthorized' responses from + // the webhook authorizer. + // +optional + CacheUnauthorizedTTL metav1.Duration `json:"cacheUnauthorizedTTL,omitempty"` +} + +type KubeletAuthentication struct { + // x509 contains settings related to x509 client certificate authentication. + // +optional + X509 KubeletX509Authentication `json:"x509"` + // webhook contains settings related to webhook bearer token authentication. + // +optional + Webhook KubeletWebhookAuthentication `json:"webhook"` + // anonymous contains settings related to anonymous authentication. + // +optional + Anonymous KubeletAnonymousAuthentication `json:"anonymous"` +} + +type KubeletX509Authentication struct { + // clientCAFile is the path to a PEM-encoded certificate bundle. If set, any request + // presenting a client certificate signed by one of the authorities in the bundle + // is authenticated with a username corresponding to the CommonName, + // and groups corresponding to the Organization in the client certificate. + // +optional + ClientCAFile string `json:"clientCAFile,omitempty"` +} + +type KubeletWebhookAuthentication struct { + // enabled allows bearer token authentication backed by the + // tokenreviews.authentication.k8s.io API. + // +optional + Enabled *bool `json:"enabled,omitempty"` + // cacheTTL enables caching of authentication results + // +optional + CacheTTL metav1.Duration `json:"cacheTTL,omitempty"` +} + +type KubeletAnonymousAuthentication struct { + // enabled allows anonymous requests to the kubelet server. + // Requests that are not rejected by another authentication method are treated as + // anonymous requests. + // Anonymous requests have a username of `system:anonymous`, and a group name of + // `system:unauthenticated`. + // +optional + Enabled *bool `json:"enabled,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// SerializedNodeConfigSource allows us to serialize v1.NodeConfigSource. +// This type is used internally by the Kubelet for tracking checkpointed dynamic configs. +// It exists in the kubeletconfig API group because it is classified as a versioned input to the Kubelet. +type SerializedNodeConfigSource struct { + metav1.TypeMeta `json:",inline"` + // source is the source that we are serializing. + // +optional + Source v1.NodeConfigSource `json:"source,omitempty" protobuf:"bytes,1,opt,name=source"` +} + +// MemoryReservation specifies the memory reservation of different types for each NUMA node +type MemoryReservation struct { + NumaNode int32 `json:"numaNode"` + Limits v1.ResourceList `json:"limits"` +} + +// ShutdownGracePeriodByPodPriority specifies the shutdown grace period for Pods based on their associated priority class value +type ShutdownGracePeriodByPodPriority struct { + // priority is the priority value associated with the shutdown grace period + Priority int32 `json:"priority"` + // shutdownGracePeriodSeconds is the shutdown grace period in seconds + ShutdownGracePeriodSeconds int64 `json:"shutdownGracePeriodSeconds"` +} + +type MemorySwapConfiguration struct { + // swapBehavior configures swap memory available to container workloads. May be one of + // "", "NoSwap": workloads can not use swap, default option. + // "LimitedSwap": workload swap usage is limited. The swap limit is proportionate to the container's memory request. + // +featureGate=NodeSwap + // +optional + SwapBehavior string `json:"swapBehavior,omitempty"` +} + +type CrashLoopBackOffConfig struct { + // maxContainerRestartPeriod is the maximum duration the backoff delay can accrue + // to for container restarts, minimum 1 second, maximum 300 seconds. If not set, + // defaults to the internal crashloopbackoff maximum (300s). + // +featureGate=KubeletCrashLoopBackOffMax + // +optional + MaxContainerRestartPeriod *metav1.Duration `json:"maxContainerRestartPeriod,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CredentialProviderConfig is the configuration containing information about +// each exec credential provider. Kubelet reads this configuration from disk and enables +// each provider as specified by the CredentialProvider type. +type CredentialProviderConfig struct { + metav1.TypeMeta `json:",inline"` + + // providers is a list of credential provider plugins that will be enabled by the kubelet. + // Multiple providers may match against a single image, in which case credentials + // from all providers will be returned to the kubelet. If multiple providers are called + // for a single image, the results are combined. If providers return overlapping + // auth keys, the value from the provider earlier in this list is attempted first. + Providers []CredentialProvider `json:"providers"` +} + +// CredentialProvider represents an exec plugin to be invoked by the kubelet. The plugin is only +// invoked when an image being pulled matches the images handled by the plugin (see matchImages). +type CredentialProvider struct { + // name is the required name of the credential provider. It must match the name of the + // provider executable as seen by the kubelet. The executable must be in the kubelet's + // bin directory (set by the --image-credential-provider-bin-dir flag). + // Required to be unique across all providers. + Name string `json:"name"` + + // matchImages is a required list of strings used to match against images in order to + // determine if this provider should be invoked. If one of the strings matches the + // requested image from the kubelet, the plugin will be invoked and given a chance + // to provide credentials. Images are expected to contain the registry domain + // and URL path. + // + // Each entry in matchImages is a pattern which can optionally contain a port and a path. + // Globs can be used in the domain, but not in the port or the path. Globs are supported + // as subdomains like '*.k8s.io' or 'k8s.*.io', and top-level-domains such as 'k8s.*'. + // Matching partial subdomains like 'app*.k8s.io' is also supported. Each glob can only match + // a single subdomain segment, so *.io does not match *.k8s.io. + // + // A match exists between an image and a matchImage when all of the below are true: + // - Both contain the same number of domain parts and each part matches. + // - The URL path of an imageMatch must be a prefix of the target image URL path. + // - If the imageMatch contains a port, then the port must match in the image as well. + // + // Example values of matchImages: + // - 123456789.dkr.ecr.us-east-1.amazonaws.com + // - *.azurecr.io + // - gcr.io + // - *.*.registry.io + // - registry.io:8080/path + MatchImages []string `json:"matchImages"` + + // defaultCacheDuration is the default duration the plugin will cache credentials in-memory + // if a cache duration is not provided in the plugin response. This field is required. + DefaultCacheDuration *metav1.Duration `json:"defaultCacheDuration"` + + // Required input version of the exec CredentialProviderRequest. The returned CredentialProviderResponse + // MUST use the same encoding version as the input. Current supported values are: + // - credentialprovider.kubelet.k8s.io/v1beta1 + APIVersion string `json:"apiVersion"` + + // Arguments to pass to the command when executing it. + // +optional + Args []string `json:"args,omitempty"` + + // Env defines additional environment variables to expose to the process. These + // are unioned with the host's environment, as well as variables client-go uses + // to pass argument to the plugin. + // +optional + Env []ExecEnvVar `json:"env,omitempty"` +} + +// ExecEnvVar is used for setting environment variables when executing an exec-based +// credential plugin. +type ExecEnvVar struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// UserNamespaces contains User Namespace configurations. +type UserNamespaces struct { + // IDsPerPod is the mapping length of UIDs and GIDs. + // The length must be a multiple of 65536, and must be less than 1<<32. + // On non-linux such as windows, only null / absent is allowed. + // + // Changing the value may require recreating all containers on the node. + // + // Default: 65536 + // +featureGate=UserNamespacesSupport + // +optional + IDsPerPod *int64 `json:"idsPerPod,omitempty"` +} diff --git a/go-controller/vendor/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go b/go-controller/vendor/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..ac0ba7096b --- /dev/null +++ b/go-controller/vendor/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,712 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1beta1 + +import ( + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + apiv1 "k8s.io/component-base/tracing/api/v1" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CrashLoopBackOffConfig) DeepCopyInto(out *CrashLoopBackOffConfig) { + *out = *in + if in.MaxContainerRestartPeriod != nil { + in, out := &in.MaxContainerRestartPeriod, &out.MaxContainerRestartPeriod + *out = new(v1.Duration) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CrashLoopBackOffConfig. +func (in *CrashLoopBackOffConfig) DeepCopy() *CrashLoopBackOffConfig { + if in == nil { + return nil + } + out := new(CrashLoopBackOffConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialProvider) DeepCopyInto(out *CredentialProvider) { + *out = *in + if in.MatchImages != nil { + in, out := &in.MatchImages, &out.MatchImages + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DefaultCacheDuration != nil { + in, out := &in.DefaultCacheDuration, &out.DefaultCacheDuration + *out = new(v1.Duration) + **out = **in + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]ExecEnvVar, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialProvider. +func (in *CredentialProvider) DeepCopy() *CredentialProvider { + if in == nil { + return nil + } + out := new(CredentialProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialProviderConfig) DeepCopyInto(out *CredentialProviderConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Providers != nil { + in, out := &in.Providers, &out.Providers + *out = make([]CredentialProvider, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialProviderConfig. +func (in *CredentialProviderConfig) DeepCopy() *CredentialProviderConfig { + if in == nil { + return nil + } + out := new(CredentialProviderConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CredentialProviderConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecEnvVar) DeepCopyInto(out *ExecEnvVar) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecEnvVar. +func (in *ExecEnvVar) DeepCopy() *ExecEnvVar { + if in == nil { + return nil + } + out := new(ExecEnvVar) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletAnonymousAuthentication) DeepCopyInto(out *KubeletAnonymousAuthentication) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAnonymousAuthentication. +func (in *KubeletAnonymousAuthentication) DeepCopy() *KubeletAnonymousAuthentication { + if in == nil { + return nil + } + out := new(KubeletAnonymousAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletAuthentication) DeepCopyInto(out *KubeletAuthentication) { + *out = *in + out.X509 = in.X509 + in.Webhook.DeepCopyInto(&out.Webhook) + in.Anonymous.DeepCopyInto(&out.Anonymous) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAuthentication. +func (in *KubeletAuthentication) DeepCopy() *KubeletAuthentication { + if in == nil { + return nil + } + out := new(KubeletAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletAuthorization) DeepCopyInto(out *KubeletAuthorization) { + *out = *in + out.Webhook = in.Webhook + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAuthorization. +func (in *KubeletAuthorization) DeepCopy() *KubeletAuthorization { + if in == nil { + return nil + } + out := new(KubeletAuthorization) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.EnableServer != nil { + in, out := &in.EnableServer, &out.EnableServer + *out = new(bool) + **out = **in + } + out.SyncFrequency = in.SyncFrequency + out.FileCheckFrequency = in.FileCheckFrequency + out.HTTPCheckFrequency = in.HTTPCheckFrequency + if in.StaticPodURLHeader != nil { + in, out := &in.StaticPodURLHeader, &out.StaticPodURLHeader + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.TLSCipherSuites != nil { + in, out := &in.TLSCipherSuites, &out.TLSCipherSuites + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Authentication.DeepCopyInto(&out.Authentication) + out.Authorization = in.Authorization + if in.RegistryPullQPS != nil { + in, out := &in.RegistryPullQPS, &out.RegistryPullQPS + *out = new(int32) + **out = **in + } + if in.PreloadedImagesVerificationAllowlist != nil { + in, out := &in.PreloadedImagesVerificationAllowlist, &out.PreloadedImagesVerificationAllowlist + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.EventRecordQPS != nil { + in, out := &in.EventRecordQPS, &out.EventRecordQPS + *out = new(int32) + **out = **in + } + if in.EnableDebuggingHandlers != nil { + in, out := &in.EnableDebuggingHandlers, &out.EnableDebuggingHandlers + *out = new(bool) + **out = **in + } + if in.HealthzPort != nil { + in, out := &in.HealthzPort, &out.HealthzPort + *out = new(int32) + **out = **in + } + if in.OOMScoreAdj != nil { + in, out := &in.OOMScoreAdj, &out.OOMScoreAdj + *out = new(int32) + **out = **in + } + if in.ClusterDNS != nil { + in, out := &in.ClusterDNS, &out.ClusterDNS + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout + out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + out.NodeStatusReportFrequency = in.NodeStatusReportFrequency + out.ImageMinimumGCAge = in.ImageMinimumGCAge + out.ImageMaximumGCAge = in.ImageMaximumGCAge + if in.ImageGCHighThresholdPercent != nil { + in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent + *out = new(int32) + **out = **in + } + if in.ImageGCLowThresholdPercent != nil { + in, out := &in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent + *out = new(int32) + **out = **in + } + out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod + if in.CgroupsPerQOS != nil { + in, out := &in.CgroupsPerQOS, &out.CgroupsPerQOS + *out = new(bool) + **out = **in + } + if in.SingleProcessOOMKill != nil { + in, out := &in.SingleProcessOOMKill, &out.SingleProcessOOMKill + *out = new(bool) + **out = **in + } + if in.CPUManagerPolicyOptions != nil { + in, out := &in.CPUManagerPolicyOptions, &out.CPUManagerPolicyOptions + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.CPUManagerReconcilePeriod = in.CPUManagerReconcilePeriod + if in.TopologyManagerPolicyOptions != nil { + in, out := &in.TopologyManagerPolicyOptions, &out.TopologyManagerPolicyOptions + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.QOSReserved != nil { + in, out := &in.QOSReserved, &out.QOSReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.RuntimeRequestTimeout = in.RuntimeRequestTimeout + if in.PodPidsLimit != nil { + in, out := &in.PodPidsLimit, &out.PodPidsLimit + *out = new(int64) + **out = **in + } + if in.ResolverConfig != nil { + in, out := &in.ResolverConfig, &out.ResolverConfig + *out = new(string) + **out = **in + } + if in.CPUCFSQuota != nil { + in, out := &in.CPUCFSQuota, &out.CPUCFSQuota + *out = new(bool) + **out = **in + } + if in.CPUCFSQuotaPeriod != nil { + in, out := &in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod + *out = new(v1.Duration) + **out = **in + } + if in.NodeStatusMaxImages != nil { + in, out := &in.NodeStatusMaxImages, &out.NodeStatusMaxImages + *out = new(int32) + **out = **in + } + if in.KubeAPIQPS != nil { + in, out := &in.KubeAPIQPS, &out.KubeAPIQPS + *out = new(int32) + **out = **in + } + if in.SerializeImagePulls != nil { + in, out := &in.SerializeImagePulls, &out.SerializeImagePulls + *out = new(bool) + **out = **in + } + if in.MaxParallelImagePulls != nil { + in, out := &in.MaxParallelImagePulls, &out.MaxParallelImagePulls + *out = new(int32) + **out = **in + } + if in.EvictionHard != nil { + in, out := &in.EvictionHard, &out.EvictionHard + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoft != nil { + in, out := &in.EvictionSoft, &out.EvictionSoft + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoftGracePeriod != nil { + in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod + if in.EvictionMinimumReclaim != nil { + in, out := &in.EvictionMinimumReclaim, &out.EvictionMinimumReclaim + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MergeDefaultEvictionSettings != nil { + in, out := &in.MergeDefaultEvictionSettings, &out.MergeDefaultEvictionSettings + *out = new(bool) + **out = **in + } + if in.EnableControllerAttachDetach != nil { + in, out := &in.EnableControllerAttachDetach, &out.EnableControllerAttachDetach + *out = new(bool) + **out = **in + } + if in.MakeIPTablesUtilChains != nil { + in, out := &in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains + *out = new(bool) + **out = **in + } + if in.IPTablesMasqueradeBit != nil { + in, out := &in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit + *out = new(int32) + **out = **in + } + if in.IPTablesDropBit != nil { + in, out := &in.IPTablesDropBit, &out.IPTablesDropBit + *out = new(int32) + **out = **in + } + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.FailSwapOn != nil { + in, out := &in.FailSwapOn, &out.FailSwapOn + *out = new(bool) + **out = **in + } + out.MemorySwap = in.MemorySwap + if in.ContainerLogMaxFiles != nil { + in, out := &in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles + *out = new(int32) + **out = **in + } + if in.ContainerLogMaxWorkers != nil { + in, out := &in.ContainerLogMaxWorkers, &out.ContainerLogMaxWorkers + *out = new(int32) + **out = **in + } + if in.ContainerLogMonitorInterval != nil { + in, out := &in.ContainerLogMonitorInterval, &out.ContainerLogMonitorInterval + *out = new(v1.Duration) + **out = **in + } + if in.SystemReserved != nil { + in, out := &in.SystemReserved, &out.SystemReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.KubeReserved != nil { + in, out := &in.KubeReserved, &out.KubeReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EnforceNodeAllocatable != nil { + in, out := &in.EnforceNodeAllocatable, &out.EnforceNodeAllocatable + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Logging.DeepCopyInto(&out.Logging) + if in.EnableSystemLogHandler != nil { + in, out := &in.EnableSystemLogHandler, &out.EnableSystemLogHandler + *out = new(bool) + **out = **in + } + if in.EnableSystemLogQuery != nil { + in, out := &in.EnableSystemLogQuery, &out.EnableSystemLogQuery + *out = new(bool) + **out = **in + } + out.ShutdownGracePeriod = in.ShutdownGracePeriod + out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods + if in.ShutdownGracePeriodByPodPriority != nil { + in, out := &in.ShutdownGracePeriodByPodPriority, &out.ShutdownGracePeriodByPodPriority + *out = make([]ShutdownGracePeriodByPodPriority, len(*in)) + copy(*out, *in) + } + in.CrashLoopBackOff.DeepCopyInto(&out.CrashLoopBackOff) + if in.ReservedMemory != nil { + in, out := &in.ReservedMemory, &out.ReservedMemory + *out = make([]MemoryReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.EnableProfilingHandler != nil { + in, out := &in.EnableProfilingHandler, &out.EnableProfilingHandler + *out = new(bool) + **out = **in + } + if in.EnableDebugFlagsHandler != nil { + in, out := &in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler + *out = new(bool) + **out = **in + } + if in.SeccompDefault != nil { + in, out := &in.SeccompDefault, &out.SeccompDefault + *out = new(bool) + **out = **in + } + if in.MemoryThrottlingFactor != nil { + in, out := &in.MemoryThrottlingFactor, &out.MemoryThrottlingFactor + *out = new(float64) + **out = **in + } + if in.RegisterWithTaints != nil { + in, out := &in.RegisterWithTaints, &out.RegisterWithTaints + *out = make([]corev1.Taint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.RegisterNode != nil { + in, out := &in.RegisterNode, &out.RegisterNode + *out = new(bool) + **out = **in + } + if in.Tracing != nil { + in, out := &in.Tracing, &out.Tracing + *out = new(apiv1.TracingConfiguration) + (*in).DeepCopyInto(*out) + } + if in.LocalStorageCapacityIsolation != nil { + in, out := &in.LocalStorageCapacityIsolation, &out.LocalStorageCapacityIsolation + *out = new(bool) + **out = **in + } + if in.FailCgroupV1 != nil { + in, out := &in.FailCgroupV1, &out.FailCgroupV1 + *out = new(bool) + **out = **in + } + if in.UserNamespaces != nil { + in, out := &in.UserNamespaces, &out.UserNamespaces + *out = new(UserNamespaces) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. +func (in *KubeletConfiguration) DeepCopy() *KubeletConfiguration { + if in == nil { + return nil + } + out := new(KubeletConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KubeletConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletWebhookAuthentication) DeepCopyInto(out *KubeletWebhookAuthentication) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + out.CacheTTL = in.CacheTTL + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletWebhookAuthentication. +func (in *KubeletWebhookAuthentication) DeepCopy() *KubeletWebhookAuthentication { + if in == nil { + return nil + } + out := new(KubeletWebhookAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletWebhookAuthorization) DeepCopyInto(out *KubeletWebhookAuthorization) { + *out = *in + out.CacheAuthorizedTTL = in.CacheAuthorizedTTL + out.CacheUnauthorizedTTL = in.CacheUnauthorizedTTL + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletWebhookAuthorization. +func (in *KubeletWebhookAuthorization) DeepCopy() *KubeletWebhookAuthorization { + if in == nil { + return nil + } + out := new(KubeletWebhookAuthorization) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletX509Authentication) DeepCopyInto(out *KubeletX509Authentication) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletX509Authentication. +func (in *KubeletX509Authentication) DeepCopy() *KubeletX509Authentication { + if in == nil { + return nil + } + out := new(KubeletX509Authentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemoryReservation) DeepCopyInto(out *MemoryReservation) { + *out = *in + if in.Limits != nil { + in, out := &in.Limits, &out.Limits + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemoryReservation. +func (in *MemoryReservation) DeepCopy() *MemoryReservation { + if in == nil { + return nil + } + out := new(MemoryReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemorySwapConfiguration) DeepCopyInto(out *MemorySwapConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemorySwapConfiguration. +func (in *MemorySwapConfiguration) DeepCopy() *MemorySwapConfiguration { + if in == nil { + return nil + } + out := new(MemorySwapConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SerializedNodeConfigSource) DeepCopyInto(out *SerializedNodeConfigSource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.Source.DeepCopyInto(&out.Source) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SerializedNodeConfigSource. +func (in *SerializedNodeConfigSource) DeepCopy() *SerializedNodeConfigSource { + if in == nil { + return nil + } + out := new(SerializedNodeConfigSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SerializedNodeConfigSource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShutdownGracePeriodByPodPriority) DeepCopyInto(out *ShutdownGracePeriodByPodPriority) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShutdownGracePeriodByPodPriority. +func (in *ShutdownGracePeriodByPodPriority) DeepCopy() *ShutdownGracePeriodByPodPriority { + if in == nil { + return nil + } + out := new(ShutdownGracePeriodByPodPriority) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserNamespaces) DeepCopyInto(out *UserNamespaces) { + *out = *in + if in.IDsPerPod != nil { + in, out := &in.IDsPerPod, &out.IDsPerPod + *out = new(int64) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserNamespaces. +func (in *UserNamespaces) DeepCopy() *UserNamespaces { + if in == nil { + return nil + } + out := new(UserNamespaces) + in.DeepCopyInto(out) + return out +} diff --git a/go-controller/vendor/k8s.io/utils/cpuset/OWNERS b/go-controller/vendor/k8s.io/utils/cpuset/OWNERS new file mode 100644 index 0000000000..0ec2b08527 --- /dev/null +++ b/go-controller/vendor/k8s.io/utils/cpuset/OWNERS @@ -0,0 +1,8 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - dchen1107 + - derekwaynecarr + - ffromani + - klueska + - SergeyKanzhelev diff --git a/go-controller/vendor/k8s.io/utils/cpuset/cpuset.go b/go-controller/vendor/k8s.io/utils/cpuset/cpuset.go new file mode 100644 index 0000000000..52912d95bc --- /dev/null +++ b/go-controller/vendor/k8s.io/utils/cpuset/cpuset.go @@ -0,0 +1,256 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cpuset represents a collection of CPUs in a 'set' data structure. +// +// It can be used to represent core IDs, hyper thread siblings, CPU nodes, or processor IDs. +// +// The only special thing about this package is that +// methods are provided to convert back and forth from Linux 'list' syntax. +// See http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS for details. +// +// Future work can migrate this to use a 'set' library, and relax the dubious 'immutable' property. +// +// This package was originally developed in the 'kubernetes' repository. +package cpuset + +import ( + "bytes" + "fmt" + "reflect" + "sort" + "strconv" + "strings" +) + +// CPUSet is a thread-safe, immutable set-like data structure for CPU IDs. +type CPUSet struct { + elems map[int]struct{} +} + +// New returns a new CPUSet containing the supplied elements. +func New(cpus ...int) CPUSet { + s := CPUSet{ + elems: map[int]struct{}{}, + } + for _, c := range cpus { + s.add(c) + } + return s +} + +// add adds the supplied elements to the CPUSet. +// It is intended for internal use only, since it mutates the CPUSet. +func (s CPUSet) add(elems ...int) { + for _, elem := range elems { + s.elems[elem] = struct{}{} + } +} + +// Size returns the number of elements in this set. +func (s CPUSet) Size() int { + return len(s.elems) +} + +// IsEmpty returns true if there are zero elements in this set. +func (s CPUSet) IsEmpty() bool { + return s.Size() == 0 +} + +// Contains returns true if the supplied element is present in this set. +func (s CPUSet) Contains(cpu int) bool { + _, found := s.elems[cpu] + return found +} + +// Equals returns true if the supplied set contains exactly the same elements +// as this set (s IsSubsetOf s2 and s2 IsSubsetOf s). +func (s CPUSet) Equals(s2 CPUSet) bool { + return reflect.DeepEqual(s.elems, s2.elems) +} + +// filter returns a new CPU set that contains all of the elements from this +// set that match the supplied predicate, without mutating the source set. +func (s CPUSet) filter(predicate func(int) bool) CPUSet { + r := New() + for cpu := range s.elems { + if predicate(cpu) { + r.add(cpu) + } + } + return r +} + +// IsSubsetOf returns true if the supplied set contains all the elements +func (s CPUSet) IsSubsetOf(s2 CPUSet) bool { + result := true + for cpu := range s.elems { + if !s2.Contains(cpu) { + result = false + break + } + } + return result +} + +// Union returns a new CPU set that contains all of the elements from this +// set and all of the elements from the supplied sets, without mutating +// either source set. +func (s CPUSet) Union(s2 ...CPUSet) CPUSet { + r := New() + for cpu := range s.elems { + r.add(cpu) + } + for _, cs := range s2 { + for cpu := range cs.elems { + r.add(cpu) + } + } + return r +} + +// Intersection returns a new CPU set that contains all of the elements +// that are present in both this set and the supplied set, without mutating +// either source set. +func (s CPUSet) Intersection(s2 CPUSet) CPUSet { + return s.filter(func(cpu int) bool { return s2.Contains(cpu) }) +} + +// Difference returns a new CPU set that contains all of the elements that +// are present in this set and not the supplied set, without mutating either +// source set. +func (s CPUSet) Difference(s2 CPUSet) CPUSet { + return s.filter(func(cpu int) bool { return !s2.Contains(cpu) }) +} + +// List returns a slice of integers that contains all elements from +// this set. The list is sorted. +func (s CPUSet) List() []int { + result := s.UnsortedList() + sort.Ints(result) + return result +} + +// UnsortedList returns a slice of integers that contains all elements from +// this set. +func (s CPUSet) UnsortedList() []int { + result := make([]int, 0, len(s.elems)) + for cpu := range s.elems { + result = append(result, cpu) + } + return result +} + +// String returns a new string representation of the elements in this CPU set +// in canonical linux CPU list format. +// +// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS +func (s CPUSet) String() string { + if s.IsEmpty() { + return "" + } + + elems := s.List() + + type rng struct { + start int + end int + } + + ranges := []rng{{elems[0], elems[0]}} + + for i := 1; i < len(elems); i++ { + lastRange := &ranges[len(ranges)-1] + // if this element is adjacent to the high end of the last range + if elems[i] == lastRange.end+1 { + // then extend the last range to include this element + lastRange.end = elems[i] + continue + } + // otherwise, start a new range beginning with this element + ranges = append(ranges, rng{elems[i], elems[i]}) + } + + // construct string from ranges + var result bytes.Buffer + for _, r := range ranges { + if r.start == r.end { + result.WriteString(strconv.Itoa(r.start)) + } else { + result.WriteString(fmt.Sprintf("%d-%d", r.start, r.end)) + } + result.WriteString(",") + } + return strings.TrimRight(result.String(), ",") +} + +// Parse CPUSet constructs a new CPU set from a Linux CPU list formatted string. +// +// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS +func Parse(s string) (CPUSet, error) { + // Handle empty string. + if s == "" { + return New(), nil + } + + result := New() + + // Split CPU list string: + // "0-5,34,46-48" => ["0-5", "34", "46-48"] + ranges := strings.Split(s, ",") + + for _, r := range ranges { + boundaries := strings.SplitN(r, "-", 2) + if len(boundaries) == 1 { + // Handle ranges that consist of only one element like "34". + elem, err := strconv.Atoi(boundaries[0]) + if err != nil { + return New(), err + } + result.add(elem) + } else if len(boundaries) == 2 { + // Handle multi-element ranges like "0-5". + start, err := strconv.Atoi(boundaries[0]) + if err != nil { + return New(), err + } + end, err := strconv.Atoi(boundaries[1]) + if err != nil { + return New(), err + } + if start > end { + return New(), fmt.Errorf("invalid range %q (%d > %d)", r, start, end) + } + // start == end is acceptable (1-1 -> 1) + + // Add all elements to the result. + // e.g. "0-5", "46-48" => [0, 1, 2, 3, 4, 5, 46, 47, 48]. + for e := start; e <= end; e++ { + result.add(e) + } + } + } + return result, nil +} + +// Clone returns a copy of this CPU set. +func (s CPUSet) Clone() CPUSet { + r := New() + for elem := range s.elems { + r.add(elem) + } + return r +} diff --git a/go-controller/vendor/modules.txt b/go-controller/vendor/modules.txt index ae125bca5f..d4a181b484 100644 --- a/go-controller/vendor/modules.txt +++ b/go-controller/vendor/modules.txt @@ -126,14 +126,14 @@ github.com/go-logr/logr/testr # github.com/go-logr/stdr v1.2.2 ## explicit; go 1.16 github.com/go-logr/stdr -# github.com/go-openapi/jsonpointer v0.21.0 +# github.com/go-openapi/jsonpointer v0.21.1 ## explicit; go 1.20 github.com/go-openapi/jsonpointer # github.com/go-openapi/jsonreference v0.21.0 ## explicit; go 1.20 github.com/go-openapi/jsonreference github.com/go-openapi/jsonreference/internal -# github.com/go-openapi/swag v0.23.0 +# github.com/go-openapi/swag v0.23.1 ## explicit; go 1.20 github.com/go-openapi/swag # github.com/go-playground/locales v0.14.1 @@ -195,6 +195,9 @@ github.com/gorilla/mux # github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 ## explicit; go 1.20 github.com/gorilla/websocket +# github.com/inconshreveable/mousetrap v1.1.0 +## explicit; go 1.18 +github.com/inconshreveable/mousetrap # github.com/josharian/intern v1.0.0 ## explicit; go 1.5 github.com/josharian/intern @@ -268,8 +271,8 @@ github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/netlinkops ## explicit; go 1.18 github.com/leodido/go-urn github.com/leodido/go-urn/scim/schema -# github.com/mailru/easyjson v0.7.7 -## explicit; go 1.12 +# github.com/mailru/easyjson v0.9.0 +## explicit; go 1.20 github.com/mailru/easyjson/buffer github.com/mailru/easyjson/jlexer github.com/mailru/easyjson/jwriter @@ -288,8 +291,8 @@ github.com/mdlayher/packet # github.com/mdlayher/socket v0.2.1 ## explicit; go 1.17 github.com/mdlayher/socket -# github.com/metallb/frr-k8s v0.0.15 -## explicit; go 1.22.0 +# github.com/metallb/frr-k8s v0.0.21 +## explicit; go 1.24.0 github.com/metallb/frr-k8s/api/v1beta1 github.com/metallb/frr-k8s/pkg/client/clientset/versioned github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake @@ -437,15 +440,15 @@ github.com/prometheus/client_golang/prometheus/collectors github.com/prometheus/client_golang/prometheus/internal github.com/prometheus/client_golang/prometheus/promhttp github.com/prometheus/client_golang/prometheus/promhttp/internal -# github.com/prometheus/client_model v0.6.1 -## explicit; go 1.19 +# github.com/prometheus/client_model v0.6.2 +## explicit; go 1.22.0 github.com/prometheus/client_model/go -# github.com/prometheus/common v0.62.0 +# github.com/prometheus/common v0.63.0 ## explicit; go 1.21 github.com/prometheus/common/expfmt github.com/prometheus/common/model -# github.com/prometheus/procfs v0.15.1 -## explicit; go 1.20 +# github.com/prometheus/procfs v0.16.1 +## explicit; go 1.23.0 github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util @@ -463,6 +466,9 @@ github.com/sirupsen/logrus github.com/spf13/afero github.com/spf13/afero/internal/common github.com/spf13/afero/mem +# github.com/spf13/cobra v1.9.1 +## explicit; go 1.15 +github.com/spf13/cobra # github.com/spf13/pflag v1.0.6 ## explicit; go 1.12 github.com/spf13/pflag @@ -562,7 +568,7 @@ golang.org/x/net/ipv6 golang.org/x/net/proxy golang.org/x/net/trace golang.org/x/net/websocket -# golang.org/x/oauth2 v0.27.0 +# golang.org/x/oauth2 v0.29.0 ## explicit; go 1.23.0 golang.org/x/oauth2 golang.org/x/oauth2/internal @@ -602,15 +608,15 @@ golang.org/x/text/secure/bidirule golang.org/x/text/transform golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm -# golang.org/x/time v0.9.0 -## explicit; go 1.18 +# golang.org/x/time v0.11.0 +## explicit; go 1.23.0 golang.org/x/time/rate # golang.org/x/tools v0.38.0 ## explicit; go 1.24.0 golang.org/x/tools/cover golang.org/x/tools/go/ast/edge golang.org/x/tools/go/ast/inspector -# gomodules.xyz/jsonpatch/v2 v2.4.0 +# gomodules.xyz/jsonpatch/v2 v2.5.0 ## explicit; go 1.20 gomodules.xyz/jsonpatch/v2 # google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f @@ -1236,11 +1242,16 @@ k8s.io/client-go/util/testing k8s.io/client-go/util/workqueue # k8s.io/component-base v0.34.1 ## explicit; go 1.24.0 +k8s.io/component-base/cli/flag k8s.io/component-base/featuregate +k8s.io/component-base/logs/api/v1 +k8s.io/component-base/logs/internal/setverbositylevel +k8s.io/component-base/logs/klogflags k8s.io/component-base/metrics k8s.io/component-base/metrics/legacyregistry k8s.io/component-base/metrics/prometheus/feature k8s.io/component-base/metrics/prometheusextension +k8s.io/component-base/tracing/api/v1 k8s.io/component-base/version k8s.io/component-base/zpages/features # k8s.io/component-helpers v0.34.1 @@ -1273,6 +1284,7 @@ k8s.io/kube-openapi/pkg/util/proto k8s.io/kube-openapi/pkg/validation/spec # k8s.io/kubelet v0.34.1 ## explicit; go 1.24.0 +k8s.io/kubelet/config/v1beta1 k8s.io/kubelet/pkg/apis/podresources/v1 # k8s.io/kubernetes v1.34.1 ## explicit; go 1.24.0 @@ -1287,6 +1299,7 @@ k8s.io/kubernetes/pkg/util/iptables ## explicit; go 1.18 k8s.io/utils/buffer k8s.io/utils/clock +k8s.io/utils/cpuset k8s.io/utils/exec k8s.io/utils/exec/testing k8s.io/utils/internal/third_party/forked/golang/golang-lru diff --git a/helm/ovn-kubernetes/Chart.yaml b/helm/ovn-kubernetes/Chart.yaml index ec128571c7..9407621aff 100644 --- a/helm/ovn-kubernetes/Chart.yaml +++ b/helm/ovn-kubernetes/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: ovn-kubernetes description: Helm chart to deploy ovn-kubernetes cni type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" home: https://ovn-kubernetes.io/ icon: https://www.ovn.org/images/ovn-logo.png annotations: @@ -13,50 +13,54 @@ sources: - https://github.com/ovn-org/ovn dependencies: - name: ovn-ipsec - version: 1.1.0 + version: 1.2.0 tags: - ovn-ipsec - name: ovnkube-control-plane - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-control-plane - name: ovnkube-db - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-db - name: ovnkube-db-raft - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-db-raft - name: ovnkube-identity - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-identity - name: ovnkube-master - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-master - name: ovnkube-node - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-node - name: ovnkube-node-dpu - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-node-dpu - name: ovnkube-node-dpu-host - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-node-dpu-host - name: ovnkube-single-node-zone - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-single-node-zone - name: ovnkube-zone-controller - version: 1.1.0 + version: 1.2.0 tags: - ovnkube-zone-controller - name: ovs-node - version: 1.1.0 + version: 1.2.0 tags: - ovs-node + - name: ovnkube-single-node-zone-dpu + version: 1.2.0 + tags: + - ovnkube-single-node-zone-dpu diff --git a/helm/ovn-kubernetes/README.md b/helm/ovn-kubernetes/README.md index 95d2ef4cf9..d588e5ca45 100644 --- a/helm/ovn-kubernetes/README.md +++ b/helm/ovn-kubernetes/README.md @@ -2,7 +2,7 @@ ----------------------- -![Version: 1.1.0](https://img.shields.io/badge/Version-1.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.0](https://img.shields.io/badge/AppVersion-1.1.0-informational?style=flat-square) +![Version: 1.2.0](https://img.shields.io/badge/Version-1.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) **Homepage:** diff --git a/helm/ovn-kubernetes/charts/ovn-ipsec/Chart.yaml b/helm/ovn-kubernetes/charts/ovn-ipsec/Chart.yaml index c48adb4a0a..b413bb0b6e 100644 --- a/helm/ovn-kubernetes/charts/ovn-ipsec/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovn-ipsec/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovn-ipsec description: Helm chart to deploy ovn ipsec type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-control-plane/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-control-plane/Chart.yaml index 95a9c61770..ca423b464d 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-control-plane/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-control-plane/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-control-plane description: Helm chart to deploy ovnkube-cluster-manager (only if interconnect is enabled) type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/ovnkube-control-plane.yaml b/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/ovnkube-control-plane.yaml index c5affe4382..5698797434 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/ovnkube-control-plane.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/ovnkube-control-plane.yaml @@ -66,6 +66,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: cpu: 100m @@ -76,7 +79,7 @@ spec: value: "crash" {{ end -}} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: {{ default 4 .Values.logLevel | quote }} - name: OVNKUBE_LOGFILE_MAXSIZE @@ -130,12 +133,22 @@ spec: value: {{ hasKey .Values.global "enableMultiNetwork" | ternary .Values.global.enableMultiNetwork false | quote }} - name: OVN_NETWORK_SEGMENTATION_ENABLE value: {{ default "" .Values.global.enableNetworkSegmentation | quote }} + - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE + value: {{ hasKey .Values.global "enableRouteAdvertisements" | ternary .Values.global.enableRouteAdvertisements false | quote }} + - name: OVN_EVPN_ENABLE + value: {{ hasKey .Values.global "enableEVPN" | ternary .Values.global.enableEVPN false | quote }} - name: OVN_NETWORK_CONNECT_ENABLE value: {{ default "" .Values.global.enableNetworkConnect | quote }} - name: OVN_PRE_CONF_UDN_ADDR_ENABLE value: {{ default "" .Values.global.enablePreconfiguredUDNAddresses | quote }} + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: {{ hasKey .Values.global "enableDynamicUDNAllocation" | ternary .Values.global.enableDynamicUDNAllocation false | quote }} + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: {{ default "" .Values.global.dynamicUDNGracePeriod | quote }} - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: {{ default "strict" .Values.global.advertisedUDNIsolationMode | quote }} + - name: OVN_NO_OVERLAY_ENABLE + value: {{ default "false" .Values.global.enableNoOverlay | quote }} - name: OVN_HYBRID_OVERLAY_NET_CIDR value: {{ default "" .Values.global.hybridOverlayNetCidr | quote }} - name: OVN_DISABLE_SNAT_MULTIPLE_GWS @@ -192,5 +205,8 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + - name: ovnkube-config + configMap: + name: ovnkube-config tolerations: - operator: "Exists" diff --git a/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/rbac-ovnkube-cluster-manager.yaml b/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/rbac-ovnkube-cluster-manager.yaml index 4a62d3e661..45c801fa92 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/rbac-ovnkube-cluster-manager.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/rbac-ovnkube-cluster-manager.yaml @@ -76,6 +76,8 @@ rules: - networkqoses - userdefinednetworks - clusteruserdefinednetworks + - routeadvertisements + - vteps verbs: [ "get", "list", "watch" ] - apiGroups: ["k8s.ovn.org"] resources: @@ -87,6 +89,7 @@ rules: - clusteruserdefinednetworks - clusteruserdefinednetworks/status - clusteruserdefinednetworks/finalizers + - routeadvertisements/status verbs: [ "patch", "update" ] - apiGroups: [""] resources: @@ -127,3 +130,9 @@ rules: - dnsnameresolvers verbs: [ "create", "delete", "list", "patch", "update", "watch" ] {{- end }} + {{- if eq (hasKey .Values.global "enableRouteAdvertisements" | ternary .Values.global.enableRouteAdvertisements false) true }} + - apiGroups: ["frrk8s.metallb.io"] + resources: + - frrconfigurations + verbs: [ "create", "delete", "get", "list", "patch", "update", "watch" ] + {{- end }} diff --git a/helm/ovn-kubernetes/charts/ovnkube-db-raft/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-db-raft/Chart.yaml index d779bda07d..bc2455fdfc 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-db-raft/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-db-raft/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-db-raft description: Helm chart to build raft based ovnkube db type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-db-raft/templates/statefulset.yaml b/helm/ovn-kubernetes/charts/ovnkube-db-raft/templates/statefulset.yaml index a4c854b4de..c3e4207f32 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-db-raft/templates/statefulset.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-db-raft/templates/statefulset.yaml @@ -98,7 +98,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NB value: {{ default "-vconsole:info -vfile:info" .Values.nbLogLevel }} - name: K8S_APISERVER @@ -171,7 +171,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_SB value: {{ default "-vconsole:info -vfile:info" .Values.sbLogLevel }} - name: K8S_APISERVER @@ -241,7 +241,7 @@ spec: value: "crash" {{ end -}} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: {{ default 4 .Values.dbCheckerLogLevel | quote }} - name: OVNKUBE_LOGFILE_MAXSIZE diff --git a/helm/ovn-kubernetes/charts/ovnkube-db/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-db/Chart.yaml index 4b4e86accf..2b7c402741 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-db/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-db/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-db description: Helm chart for standalone ovnkube-db type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-db/templates/deployment.yaml b/helm/ovn-kubernetes/charts/ovnkube-db/templates/deployment.yaml index 4dd664db1d..e09b6a4c23 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-db/templates/deployment.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-db/templates/deployment.yaml @@ -82,7 +82,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NB value: {{ default "-vconsole:info -vfile:info" .Values.nbLogLevel }} - name: K8S_APISERVER @@ -146,7 +146,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_SB value: {{ default "-vconsole:info -vfile:info" .Values.sbLogLevel }} - name: K8S_APISERVER diff --git a/helm/ovn-kubernetes/charts/ovnkube-identity/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-identity/Chart.yaml index fc90ecb24d..673fa2116e 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-identity/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-identity/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-identity description: Helm chart to deploy ovnkube identity type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-identity/templates/ovnkube-identity.yaml b/helm/ovn-kubernetes/charts/ovnkube-identity/templates/ovnkube-identity.yaml index 896ba6fbb4..6e70329be7 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-identity/templates/ovnkube-identity.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-identity/templates/ovnkube-identity.yaml @@ -62,7 +62,7 @@ spec: value: "crash" {{ end -}} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: K8S_APISERVER valueFrom: configMapKeyRef: diff --git a/helm/ovn-kubernetes/charts/ovnkube-master/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-master/Chart.yaml index a5a2e9b744..fb6f501a18 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-master/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-master/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-master description: Helm chart to deploy ovnkube-master type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml b/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml index 8b0fee0ac3..df2a7a1d0f 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml @@ -74,13 +74,16 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: cpu: 100m memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NORTHD value: {{ default "-vconsole:info -vfile:info" .Values.northdLogLevel | quote }} - name: K8S_APISERVER @@ -147,6 +150,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true {{- if eq (hasKey .Values.global "enableCompactMode" | ternary .Values.global.enableCompactMode false) true }} # Common mounts # for the iptables wrapper @@ -180,7 +186,7 @@ spec: value: "crash" {{ end -}} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: {{ default 4 .Values.logLevel | quote }} - name: OVNKUBE_LOGFILE_MAXSIZE @@ -240,8 +246,18 @@ spec: value: {{ hasKey .Values.global "enableMultiNetwork" | ternary .Values.global.enableMultiNetwork false | quote }} - name: OVN_NETWORK_SEGMENTATION_ENABLE value: {{ default "" .Values.global.enableNetworkSegmentation | quote }} + - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE + value: {{ hasKey .Values.global "enableRouteAdvertisements" | ternary .Values.global.enableRouteAdvertisements false | quote }} + - name: OVN_EVPN_ENABLE + value: {{ hasKey .Values.global "enableEVPN" | ternary .Values.global.enableEVPN false | quote }} - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: {{ default "strict" .Values.global.advertisedUDNIsolationMode | quote }} + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: {{ hasKey .Values.global "enableDynamicUDNAllocation" | ternary .Values.global.enableDynamicUDNAllocation false | quote }} + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: {{ default "" .Values.global.dynamicUDNGracePeriod | quote }} + - name: OVN_NO_OVERLAY_ENABLE + value: {{ default "false" .Values.global.enableNoOverlay | quote }} - name: OVN_EGRESSSERVICE_ENABLE value: {{ default "" .Values.global.enableEgressService | quote }} - name: OVN_HYBRID_OVERLAY_NET_CIDR @@ -267,7 +283,7 @@ spec: - name: OVN_GATEWAY_MODE value: {{ default "shared" .Values.global.gatewayMode }} - name: OVN_GATEWAY_OPTS - value: {{ default "" .Values.global.gatewayOps | quote }} + value: {{ default "" .Values.global.gatewayOpts | quote }} - name: OVN_MULTICAST_ENABLE value: {{ default "" .Values.global.enableMulticast | quote }} - name: OVN_ACL_LOGGING_RATE_LIMIT @@ -312,6 +328,9 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + - name: ovnkube-config + configMap: + name: ovnkube-config {{- if eq (hasKey .Values.global "enableCompactMode" | ternary .Values.global.enableCompactMode false) true }} - name: host-slash hostPath: diff --git a/helm/ovn-kubernetes/charts/ovnkube-master/templates/rbac-ovnkube-master.yaml b/helm/ovn-kubernetes/charts/ovnkube-master/templates/rbac-ovnkube-master.yaml index 7474c69f8f..cb884a9bea 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-master/templates/rbac-ovnkube-master.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-master/templates/rbac-ovnkube-master.yaml @@ -87,6 +87,7 @@ rules: - userdefinednetworks - clusteruserdefinednetworks - networkqoses + - vteps verbs: [ "get", "list", "watch" ] - apiGroups: ["k8s.cni.cncf.io"] resources: diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/Chart.yaml index 4affd140d2..09e69d0836 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-node-dpu-host description: Helm chart to deploy ovnkube-node-dpu-host type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/templates/ovnkube-node-dpu-host.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/templates/ovnkube-node-dpu-host.yaml index 30d920689e..f009f7036f 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/templates/ovnkube-node-dpu-host.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/templates/ovnkube-node-dpu-host.yaml @@ -97,17 +97,32 @@ spec: # ovnkube-node dpu-host mounts - mountPath: /var/run/ovn name: var-run-ovn + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + {{- if .Values.global.enableNetworkSegmentation }} + - mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp + name: host-devinfo-dp + readOnly: true + {{- end }} resources: requests: cpu: 100m memory: 300Mi + {{- if and (.Values.global.enableNetworkSegmentation) (.Values.mgmtPortVFResourceName) (.Values.mgmtPortVFsCount) }} + {{ .Values.mgmtPortVFResourceName }}: {{ .Values.mgmtPortVFsCount }} + {{- end }} + limits: + {{- if and (.Values.global.enableNetworkSegmentation) (.Values.mgmtPortVFResourceName) (.Values.mgmtPortVFsCount) }} + {{ .Values.mgmtPortVFResourceName }}: {{ .Values.mgmtPortVFsCount }} + {{- end }} env: {{ if .Values.global.enableCoredumps -}} - name: GOTRACEBACK value: "crash" {{ end -}} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: {{ default 4 .Values.logLevel | quote }} - name: OVNKUBE_LOGFILE_MAXSIZE @@ -153,7 +168,7 @@ spec: - name: OVN_GATEWAY_MODE value: {{ default "shared" .Values.global.gatewayMode }} - name: OVN_GATEWAY_OPTS - value: {{ default "" .Values.global.gatewayOps | quote }} + value: {{ default "" .Values.global.gatewayOpts | quote }} - name: OVN_HYBRID_OVERLAY_ENABLE value: {{ default "" .Values.global.enableHybridOverlay | quote }} - name: OVN_ADMIN_NETWORK_POLICY_ENABLE @@ -206,8 +221,20 @@ spec: value: {{ hasKey .Values.global "enableNetworkQos" | ternary .Values.global.enableNetworkQos false | quote }} - name: OVNKUBE_NODE_MODE value: "dpu-host" + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: {{ hasKey .Values.global "enableDynamicUDNAllocation" | ternary .Values.global.enableDynamicUDNAllocation false | quote }} + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: {{ default "" .Values.global.dynamicUDNGracePeriod | quote }} + - name: OVN_NO_OVERLAY_ENABLE + value: {{ default "false" .Values.global.enableNoOverlay | quote }} - name: OVNKUBE_NODE_MGMT_PORT_NETDEV - value: {{ default "" .Values.global.nodeMgmtPortNetdev | quote }} + value: {{ default "" .Values.nodeMgmtPortNetdev | quote }} + - name: OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME + value: {{ default "" .Values.mgmtPortVFResourceName | quote }} + - name: OVN_ENABLE_INTERCONNECT + value: {{ default "false" .Values.global.enableInterconnect | quote }} + - name: OVN_NETWORK_SEGMENTATION_ENABLE + value: {{ default "" .Values.global.enableNetworkSegmentation | quote }} - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: @@ -259,5 +286,13 @@ spec: path: /run/systemd - name: var-run-ovn emptyDir: {} + - name: ovnkube-config + configMap: + name: ovnkube-config + {{- if .Values.global.enableNetworkSegmentation }} + - name: host-devinfo-dp + hostPath: + path: /var/run/k8s.cni.cncf.io/devinfo/dp + {{- end }} tolerations: - operator: "Exists" diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/values.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/values.yaml index 584127efca..914b06593e 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/values.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu-host/values.yaml @@ -2,4 +2,18 @@ logLevel: 4 logFileMaxSize: 100 logFileMaxBackups: 5 logFileMaxAge: 5 -ovnControllerLogLevel: 4 \ No newline at end of file +ovnControllerLogLevel: 4 + +# The netdevice or deviceplugin resourcename that specifies pool of devices +# that can be used for management ports has to be specified. +# mgmtPortVFResourceName will override nodeMgmtPortNetdev if both are specified + +# The net device to be used for management port +nodeMgmtPortNetdev: "" + +# The device plugin resource name that has allocated interfaces to be used for management ports +mgmtPortVFResourceName: "" + +# If using UDNs, the number of VFs required to handle management ports, which depends on +# the number of primary UDNs required should be specified. +mgmtPortVFsCount: 1 diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/Chart.yaml index 3ebd45680d..8d676af30a 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-node-dpu description: Helm chart to deploy ovnkube-node-dpu type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml index bbcc77ffc9..b1039f6d8f 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml @@ -108,6 +108,9 @@ spec: name: run-systemd subPath: private readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: cpu: 100m @@ -118,7 +121,7 @@ spec: value: "crash" {{ end -}} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: {{ default 4 .Values.logLevel | quote }} - name: OVNKUBE_LOGFILE_MAXSIZE @@ -164,7 +167,7 @@ spec: - name: OVN_GATEWAY_MODE value: {{ default "shared" .Values.global.gatewayMode }} - name: OVN_GATEWAY_OPTS - value: {{ default "" .Values.global.gatewayOps | quote }} + value: {{ default "" .Values.global.gatewayOpts | quote }} - name: OVN_HYBRID_OVERLAY_ENABLE value: {{ default "" .Values.global.enableHybridOverlay | quote }} - name: OVN_ADMIN_NETWORK_POLICY_ENABLE @@ -237,6 +240,12 @@ spec: value: {{ hasKey .Values.global "enableNetworkQos" | ternary .Values.global.enableNetworkQos false | quote }} - name: OVNKUBE_NODE_MODE value: "dpu" + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: {{ hasKey .Values.global "enableDynamicUDNAllocation" | ternary .Values.global.enableDynamicUDNAllocation false | quote }} + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: {{ default "" .Values.global.dynamicUDNGracePeriod | quote }} + - name: OVN_NO_OVERLAY_ENABLE + value: {{ default "false" .Values.global.enableNoOverlay | quote }} - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: @@ -286,7 +295,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_CONTROLLER value: {{ default "-vconsole:info" .Values.ovnControllerLogLevel | quote }} - name: K8S_APISERVER @@ -335,7 +344,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: K8S_NODE_IP valueFrom: fieldRef: @@ -393,5 +402,8 @@ spec: - name: run-systemd hostPath: path: /run/systemd + - name: ovnkube-config + configMap: + name: ovnkube-config tolerations: - operator: "Exists" diff --git a/helm/ovn-kubernetes/charts/ovnkube-node/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-node/Chart.yaml index 8b214a09db..f59b133891 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 name: ovnkube-node description: Subchart for ovnkube-node -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-node/templates/ovnkube-node.yaml b/helm/ovn-kubernetes/charts/ovnkube-node/templates/ovnkube-node.yaml index 633067a453..3a0b5162e4 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node/templates/ovnkube-node.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node/templates/ovnkube-node.yaml @@ -98,6 +98,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /etc/openvswitch/ name: host-etc-ovs readOnly: true @@ -118,7 +121,7 @@ spec: value: "crash" {{ end -}} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: {{ default 4 .Values.logLevel | quote }} - name: OVNKUBE_LOGFILE_MAXSIZE @@ -164,7 +167,7 @@ spec: - name: OVN_GATEWAY_MODE value: {{ default "shared" .Values.global.gatewayMode }} - name: OVN_GATEWAY_OPTS - value: {{ default "" .Values.global.gatewayOps | quote }} + value: {{ default "" .Values.global.gatewayOpts | quote }} - name: OVN_HYBRID_OVERLAY_ENABLE value: {{ default "" .Values.global.enableHybridOverlay | quote }} - name: OVN_ADMIN_NETWORK_POLICY_ENABLE @@ -231,12 +234,22 @@ spec: value: {{ hasKey .Values.global "enableMultiNetwork" | ternary .Values.global.enableMultiNetwork false | quote }} - name: OVN_NETWORK_SEGMENTATION_ENABLE value: {{ default "" .Values.global.enableNetworkSegmentation | quote }} + - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE + value: {{ hasKey .Values.global "enableRouteAdvertisements" | ternary .Values.global.enableRouteAdvertisements false | quote }} + - name: OVN_EVPN_ENABLE + value: {{ hasKey .Values.global "enableEVPN" | ternary .Values.global.enableEVPN false | quote }} - name: OVN_NETWORK_CONNECT_ENABLE value: {{ default "" .Values.global.enableNetworkConnect | quote }} - name: OVN_PRE_CONF_UDN_ADDR_ENABLE value: {{ default "" .Values.global.enablePreconfiguredUDNAddresses | quote }} + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: {{ hasKey .Values.global "enableDynamicUDNAllocation" | ternary .Values.global.enableDynamicUDNAllocation false | quote }} + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: {{ default "" .Values.global.dynamicUDNGracePeriod | quote }} - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: {{ default "strict" .Values.global.advertisedUDNIsolationMode | quote }} + - name: OVN_NO_OVERLAY_ENABLE + value: {{ default "false" .Values.global.enableNoOverlay | quote }} - name: OVN_ENABLE_INTERCONNECT value: {{ hasKey .Values.global "enableInterconnect" | ternary .Values.global.enableInterconnect false | quote }} - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY @@ -292,7 +305,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_CONTROLLER value: {{ default "-vconsole:info" .Values.ovnControllerLogLevel | quote }} - name: K8S_APISERVER @@ -341,7 +354,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: K8S_NODE_IP valueFrom: fieldRef: @@ -407,5 +420,8 @@ spec: - name: run-systemd hostPath: path: /run/systemd + - name: ovnkube-config + configMap: + name: ovnkube-config tolerations: - operator: "Exists" diff --git a/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/Chart.yaml new file mode 100644 index 0000000000..a956168562 --- /dev/null +++ b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: ovnkube-single-node-zone-dpu +description: Helm chart to deploy single node zone stack on DPUs +type: application +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/templates/ovnkube-single-node-zone-dpu.yaml b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/templates/ovnkube-single-node-zone-dpu.yaml new file mode 100644 index 0000000000..ac78168e4f --- /dev/null +++ b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/templates/ovnkube-single-node-zone-dpu.yaml @@ -0,0 +1,575 @@ +# ovnkube-node-dpu +# daemonset version 3 +# starts node daemons for single node zone ovn stack, each in a separate container on DPU +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovnkube-node-dpu + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This DaemonSet launches the ovn-kubernetes networking components on dpus in IC mode. +spec: + selector: + matchLabels: + app: ovnkube-node-dpu + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: ovnkube-node-dpu + name: ovnkube-node-dpu + component: network + type: infra + kubernetes.io/os: "linux" + ovn-db-pod: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + serviceAccountName: ovnkube-node + hostNetwork: true + dnsPolicy: Default + {{- if eq (hasKey .Values.global "unprivilegedMode" | ternary .Values.global.unprivilegedMode false) false }} + hostPID: true + {{- end }} + containers: + # nb-ovsdb - v3 + - name: nb-ovsdb + image: {{ include "getDPUImage" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.dpuImage.pullPolicy }} + command: ["/root/ovnkube.sh", "local-nb-ovsdb"] + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-etc-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_LOGLEVEL_NB + value: {{ default "-vconsole:info -vfile:info" .Values.nbLogLevel | quote }} + - name: OVN_NORTHD_BACKOFF_INTERVAL + value: {{ default "0" .Values.northdBackoffInterval | quote }} + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnnb-db"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of nb-ovsdb container + # sb-ovsdb - v3 + - name: sb-ovsdb + image: {{ include "getDPUImage" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.dpuImage.pullPolicy }} + command: ["/root/ovnkube.sh", "local-sb-ovsdb"] + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-etc-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_LOGLEVEL_SB + value: {{ default "-vconsole:info -vfile:info" .Values.sbLogLevel | quote }} + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: {{ include "isSslEnabled" . | quote }} + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnsb-db"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of sb-ovsdb container + # ovn-northd - v3 + - name: ovn-northd + image: {{ include "getDPUImage" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.dpuImage.pullPolicy }} + command: ["/root/ovnkube.sh", "run-ovn-northd"] + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Run directories where we need to be able to access sockets + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_LOGLEVEL_NORTHD + value: {{ default "-vconsole:info -vfile:info" .Values.northdLogLevel | quote }} + - name: OVN_SSL_ENABLE + value: {{ include "isSslEnabled" . | quote }} + - name: OVN_NORTH + value: "local" + - name: OVN_SOUTH + value: "local" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-northd"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of ovn-northd container + # ovnkube-controller + - name: ovnkube-controller + image: {{ include "getDPUImage" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.dpuImage.pullPolicy }} + command: ["/root/ovnkube.sh", "ovnkube-controller-with-node"] + securityContext: + runAsUser: 0 + {{- if eq (hasKey .Values.global "unprivilegedMode" | ternary .Values.global.unprivilegedMode false) false }} + privileged: true + {{- else }} + capabilities: + add: + - NET_ADMIN + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Common mounts + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true + - mountPath: /var/lib/kubelet + name: host-kubelet + readOnly: true + - mountPath: /host-kubernetes + name: host-kubeconfig + readOnly: true + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/ovn-kubernetes/ + name: host-var-log-ovnkube + # We mount our socket here + - mountPath: /var/run/ovn-kubernetes + name: host-var-run-ovn-kubernetes + # CNI related mounts which we take over + - mountPath: /opt/cni/bin + name: host-opt-cni-bin + - mountPath: /etc/cni/net.d + name: host-etc-cni-netd + - mountPath: /var/run/netns + name: host-netns + mountPropagation: Bidirectional + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + - mountPath: /etc/openvswitch/ + name: host-etc-ovs + readOnly: true + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + readOnly: true + - mountPath: /run/systemd/private + name: run-systemd + subPath: private + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_EGRESSSERVICE_ENABLE + value: {{ default "" .Values.global.enableEgressService | quote }} + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_LOGLEVEL + value: {{ default 4 .Values.ovnkubeNodeLogLevel | quote }} + - name: OVNKUBE_LOGFILE_MAXSIZE + value: {{ default 100 .Values.logFileMaxSize | quote }} + - name: OVNKUBE_LOGFILE_MAXBACKUPS + value: {{ default 5 .Values.logFileMaxBackups | quote }} + - name: OVNKUBE_LOGFILE_MAXAGE + value: {{ default 5 .Values.logFileMaxAge | quote }} + - name: OVNKUBE_LIBOVSDB_CLIENT_LOGFILE + value: {{ default "" .Values.libovsdbClientLogFile | quote }} + - name: OVNKUBE_CONFIG_DURATION_ENABLE + value: {{ default "" .Values.global.enableConfigDuration | quote }} + - name: OVNKUBE_METRICS_SCALE_ENABLE + value: {{ default "" .Values.global.enableMetricsScale | quote }} + - name: OVN_NET_CIDR + value: {{ default "" .Values.global.dpuHostClusterNetworkCIDR | quote }} + - name: OVN_SVC_CIDR + value: {{ default "" .Values.global.dpuHostClusterServiceCIDR | quote }} + - name: K8S_APISERVER + value: {{ default "" .Values.global.dpuHostClusterK8sAPIServer | quote }} + - name: K8S_TOKEN + value: {{ default "" .Values.global.dpuHostClusterK8sToken | quote }} + - name: K8S_CACERT_DATA + value: {{ default "" .Values.global.dpuHostClusterK8sCACertData | quote }} + - name: K8S_TOKEN_FILE + value: {{ default "" .Values.global.dpuHostClusterK8sTokenFile | quote }} + - name: K8S_CACERT + value: {{ default "" .Values.global.dpuHostClusterK8sCACert | quote }} + - name: OVN_MTU + value: {{ default "" .Values.global.mtu | quote }} + - name: OVN_GATEWAY_MODE + value: {{ default "shared" .Values.global.gatewayMode }} + - name: OVN_GATEWAY_OPTS + value: {{ default "" .Values.global.gatewayOpts | quote }} + - name: OVN_HYBRID_OVERLAY_ENABLE + value: {{ default "" .Values.global.enableHybridOverlay | quote }} + - name: OVN_ADMIN_NETWORK_POLICY_ENABLE + value: {{ default "" .Values.global.enableAdminNetworkPolicy | quote }} + - name: OVN_EGRESSIP_ENABLE + value: {{ default "" .Values.global.enableEgressIp | quote }} + - name: OVN_EGRESSIP_HEALTHCHECK_PORT + value: {{ default "" .Values.global.egressIpHealthCheckPort | quote }} + - name: OVN_EGRESSFIREWALL_ENABLE + value: {{ default "" .Values.global.enableEgressFirewall | quote }} + - name: OVN_EGRESSQOS_ENABLE + value: {{ default "" .Values.global.enableEgressQos | quote }} + - name: OVN_HYBRID_OVERLAY_NET_CIDR + value: {{ default "" .Values.global.hybridOverlayNetCidr | quote }} + - name: OVN_DISABLE_SNAT_MULTIPLE_GWS + value: {{ default "" .Values.global.disableSnatMultipleGws | quote }} + - name: OVN_DISABLE_FORWARDING + value: {{ default "" .Values.global.disableForwarding | quote }} + - name: OVN_ENCAP_PORT + value: {{ default 6081 .Values.global.encapPort | quote }} + - name: OVN_DISABLE_PKT_MTU_CHECK + value: {{ default "" .Values.global.disablePacketMtuCheck | quote }} + - name: OVN_NETFLOW_TARGETS + value: {{ default "" .Values.global.netFlowTargets | quote }} + - name: OVN_SFLOW_TARGETS + value: {{ default "" .Values.global.sflowTargets | quote }} + - name: OVN_IPFIX_TARGETS + value: {{ default "" .Values.global.ipfixTargets | quote }} + - name: OVN_IPFIX_SAMPLING + value: {{ default "" .Values.global.ipfixSampling | quote }} + - name: OVN_IPFIX_CACHE_MAX_FLOWS + value: {{ default "" .Values.global.ipfixCacheMaxFlows | quote }} + - name: OVN_IPFIX_CACHE_ACTIVE_TIMEOUT + value: {{ default "" .Values.global.ipfixCacheActiveTimeout | quote }} + - name: OVN_V4_JOIN_SUBNET + value: {{ default "" .Values.global.v4JoinSubnet | quote }} + - name: OVN_V6_JOIN_SUBNET + value: {{ default "" .Values.global.v6JoinSubnet | quote }} + - name: OVN_V4_MASQUERADE_SUBNET + value: {{ default "" .Values.global.v4MasqueradeSubnet | quote }} + - name: OVN_V6_MASQUERADE_SUBNET + value: {{ default "" .Values.global.v6MasqueradeSubnet | quote }} + - name: OVN_MULTICAST_ENABLE + value: {{ default "" .Values.global.enableMulticast | quote }} + - name: OVN_UNPRIVILEGED_MODE + value: {{ include "isUnprivilegedMode" . | quote }} + - name: OVN_EX_GW_NETWORK_INTERFACE + value: {{ default "" .Values.global.extGatewayNetworkInterface | quote }} + - name: OVN_SSL_ENABLE + value: {{ include "isSslEnabled" . | quote }} + - name: OVN_REMOTE_PROBE_INTERVAL + value: {{ default 100000 .Values.global.remoteProbeInterval | quote }} + - name: OVN_MONITOR_ALL + value: {{ hasKey .Values.global "monitorAll" | ternary .Values.global.monitorAll true | quote }} + - name: OVN_OFCTRL_WAIT_BEFORE_CLEAR + value: {{ default "" .Values.global.ofctrlWaitBeforeClear | quote }} + - name: OVN_ENABLE_LFLOW_CACHE + value: {{ hasKey .Values.global "enableLFlowCache" | ternary .Values.global.enableLFlowCache true | quote }} + - name: OVN_LFLOW_CACHE_LIMIT + value: {{ default "" .Values.global.lFlowCacheLimit | quote }} + - name: OVN_LFLOW_CACHE_LIMIT_KB + value: {{ default "" .Values.global.lFlowCacheLimitKb | quote }} + - name: OVN_MULTI_NETWORK_ENABLE + value: {{ hasKey .Values.global "enableMultiNetwork" | ternary .Values.global.enableMultiNetwork false | quote }} + - name: OVN_NETWORK_SEGMENTATION_ENABLE + value: {{ default "" .Values.global.enableNetworkSegmentation | quote }} + - name: OVN_NETWORK_CONNECT_ENABLE + value: {{ default "" .Values.global.enableNetworkConnect | quote }} + - name: OVN_PRE_CONF_UDN_ADDR_ENABLE + value: {{ default "" .Values.global.enablePreconfiguredUDNAddresses | quote }} + - name: OVN_ADVERTISED_UDN_ISOLATION_MODE + value: {{ default "strict" .Values.global.advertisedUDNIsolationMode | quote }} + - name: OVN_EMPTY_LB_EVENTS + value: {{ default "" .Values.global.emptyLbEvents | quote }} + - name: OVN_ACL_LOGGING_RATE_LIMIT + value: {{ default 20 .Values.global.aclLoggingRateLimit | quote }} + - name: OVN_NORTH + value: "local" + - name: OVN_SOUTH + value: "local" + - name: OVN_ENABLE_INTERCONNECT + value: {{ hasKey .Values.global "enableInterconnect" | ternary .Values.global.enableInterconnect false | quote }} + - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY + value: {{ hasKey .Values.global "enableMultiExternalGateway" | ternary .Values.global.enableMultiExternalGateway false | quote }} + - name: OVN_ENABLE_OVNKUBE_IDENTITY + value: {{ hasKey .Values.global "enableOvnKubeIdentity" | ternary .Values.global.enableOvnKubeIdentity false | quote }} + - name: OVN_ENABLE_SVC_TEMPLATE_SUPPORT + value: {{ hasKey .Values.global "enableSvcTemplate" | ternary .Values.global.enableSvcTemplate false | quote }} + - name: OVN_ENABLE_DNSNAMERESOLVER + value: {{ hasKey .Values.global "enableDNSNameResolver" | ternary .Values.global.enableDNSNameResolver false | quote }} + - name: OVN_OBSERV_ENABLE + value: {{ hasKey .Values.global "enableObservability" | ternary .Values.global.enableObservability false | quote }} + - name: OVN_NETWORK_QOS_ENABLE + value: {{ hasKey .Values.global "enableNetworkQos" | ternary .Values.global.enableNetworkQos false | quote }} + - name: OVN_NO_OVERLAY_ENABLE + value: {{ default "false" .Values.global.enableNoOverlay | quote }} + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + readinessProbe: + httpGet: + path: /metrics + port: {{ .Values.metricsPort }} + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + periodSeconds: 30 + # end of ovnkube-controller container + # ovn-controller + - name: ovn-controller + image: {{ include "getDPUImage" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.dpuImage.pullPolicy }} + command: ["/root/ovnkube.sh", "ovn-controller"] + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_LOGLEVEL_CONTROLLER + value: {{ default "-vconsole:info" .Values.ovnControllerLogLevel | quote }} + - name: OVN_SSL_ENABLE + value: {{ default "" .Values.global.enableSsl | quote }} + - name: OVN_NORTH + value: "local" + - name: OVN_SOUTH + value: "local" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-controller"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # ovs-metrics-exporter - v3 + - name: ovs-metrics-exporter + image: {{ include "getDPUImage" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.dpuImage.pullPolicy }} + command: ["/root/ovnkube.sh", "ovs-metrics"] + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.2.0" + - name: OVNKUBE_NODE_MODE + value: "dpu" + - name: OVN_NORTH + value: "local" + - name: OVN_SOUTH + value: "local" + # end of container + nodeSelector: + kubernetes.io/os: "linux" + k8s.ovn.org/dpu: "" + volumes: + # Common volumes + - name: host-var-run-dbus + hostPath: + path: /var/run/dbus + - name: host-kubelet + hostPath: + path: /var/lib/kubelet + - name: host-kubeconfig + hostPath: + path: /etc/kubernetes/ + - name: host-var-log-ovnkube + hostPath: + path: /var/log/ovn-kubernetes + - name: host-var-run-ovn-kubernetes + hostPath: + path: /var/run/ovn-kubernetes + - name: host-opt-cni-bin + hostPath: + path: /opt/cni/bin + - name: host-etc-cni-netd + hostPath: + path: /etc/cni/net.d + - name: host-slash + hostPath: + path: / + - name: host-netns + hostPath: + path: /var/run/netns + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-run-ovs + hostPath: + path: /var/run/openvswitch + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate + - name: host-etc-ovs + hostPath: + path: /etc/openvswitch + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: run-systemd + hostPath: + path: /run/systemd + - name: ovnkube-config + configMap: + name: ovnkube-config + tolerations: + - operator: "Exists" diff --git a/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/values.yaml b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/values.yaml new file mode 100644 index 0000000000..b7d2004c60 --- /dev/null +++ b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone-dpu/values.yaml @@ -0,0 +1,12 @@ +nbLogLevel: "-vconsole:info -vfile:info" +sbLogLevel: "-vconsole:info -vfile:info" +northdLogLevel: "-vconsole:info -vfile:info" +northdBackoffInterval: "0" +ovnkubeNodeLogLevel: 4 +ovnControllerLogLevel: "-vconsole:info" +logFileMaxSize: 100 +logFileMaxBackups: 5 +logFileMaxAge: 5 +libovsdbClientLogFile: "" +# -- TCP port serving metrics +metricsPort: 9476 diff --git a/helm/ovn-kubernetes/charts/ovnkube-single-node-zone/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone/Chart.yaml index b157b06e07..ad7c983f08 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-single-node-zone/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-single-node-zone description: Helm chart to deploy single node zone stack type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-single-node-zone/templates/ovnkube-single-node-zone.yaml b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone/templates/ovnkube-single-node-zone.yaml index 02e6aca267..c2503c0d1d 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-single-node-zone/templates/ovnkube-single-node-zone.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-single-node-zone/templates/ovnkube-single-node-zone.yaml @@ -64,6 +64,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /var/run/ovn/ name: host-var-run-ovs - mountPath: /var/run/openvswitch/ @@ -74,7 +77,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NB value: {{ default "-vconsole:info -vfile:info" .Values.nbLogLevel | quote }} - name: K8S_APISERVER @@ -126,6 +129,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /var/run/ovn/ name: host-var-run-ovs - mountPath: /var/run/openvswitch/ @@ -136,7 +142,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_SB value: {{ default "-vconsole:info -vfile:info" .Values.sbLogLevel | quote }} - name: K8S_APISERVER @@ -191,13 +197,16 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: cpu: 100m memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NORTHD value: {{ default "-vconsole:info -vfile:info" .Values.northdLogLevel | quote }} - name: K8S_APISERVER @@ -271,6 +280,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /etc/openvswitch/ name: host-etc-ovs readOnly: true @@ -296,7 +308,7 @@ spec: - name: OVN_EGRESSSERVICE_ENABLE value: {{ default "" .Values.global.enableEgressService | quote }} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: {{ default 4 .Values.ovnkubeNodeLogLevel | quote }} - name: OVNKUBE_LOGFILE_MAXSIZE @@ -419,12 +431,22 @@ spec: value: {{ hasKey .Values.global "enableMultiNetwork" | ternary .Values.global.enableMultiNetwork false | quote }} - name: OVN_NETWORK_SEGMENTATION_ENABLE value: {{ default "" .Values.global.enableNetworkSegmentation | quote }} + - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE + value: {{ hasKey .Values.global "enableRouteAdvertisements" | ternary .Values.global.enableRouteAdvertisements false | quote }} + - name: OVN_EVPN_ENABLE + value: {{ hasKey .Values.global "enableEVPN" | ternary .Values.global.enableEVPN false | quote }} - name: OVN_NETWORK_CONNECT_ENABLE value: {{ default "" .Values.global.enableNetworkConnect | quote }} - name: OVN_PRE_CONF_UDN_ADDR_ENABLE value: {{ default "" .Values.global.enablePreconfiguredUDNAddresses | quote }} + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: {{ hasKey .Values.global "enableDynamicUDNAllocation" | ternary .Values.global.enableDynamicUDNAllocation false | quote }} + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: {{ default "" .Values.global.dynamicUDNGracePeriod | quote }} - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: {{ default "strict" .Values.global.advertisedUDNIsolationMode | quote }} + - name: OVN_NO_OVERLAY_ENABLE + value: {{ default "false" .Values.global.enableNoOverlay | quote }} - name: OVNKUBE_NODE_MGMT_PORT_NETDEV value: {{ default "" .Values.global.nodeMgmtPortNetdev | quote }} - name: OVN_EMPTY_LB_EVENTS @@ -484,13 +506,16 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: cpu: 100m memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_CONTROLLER value: {{ default "-vconsole:info" .Values.ovnControllerLogLevel | quote }} - name: K8S_APISERVER @@ -539,7 +564,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: K8S_NODE_IP valueFrom: fieldRef: @@ -558,6 +583,8 @@ spec: - matchExpressions: - key: k8s.ovn.org/dpu-host operator: DoesNotExist + - key: k8s.ovn.org/dpu + operator: DoesNotExist volumes: # Common volumes - name: host-var-run-dbus @@ -600,6 +627,9 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + - name: ovnkube-config + configMap: + name: ovnkube-config - name: host-etc-ovs hostPath: path: /etc/openvswitch diff --git a/helm/ovn-kubernetes/charts/ovnkube-zone-controller/Chart.yaml b/helm/ovn-kubernetes/charts/ovnkube-zone-controller/Chart.yaml index 00ca7ce6f2..5ac637ca84 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-zone-controller/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-zone-controller/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovnkube-zone-controller description: Helm chart to deploy zone controller type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovnkube-zone-controller/templates/ovnkube-zone-controller.yaml b/helm/ovn-kubernetes/charts/ovnkube-zone-controller/templates/ovnkube-zone-controller.yaml index 137f361564..bd03a2518c 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-zone-controller/templates/ovnkube-zone-controller.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-zone-controller/templates/ovnkube-zone-controller.yaml @@ -69,6 +69,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /var/run/ovn/ name: host-var-run-ovs - mountPath: /var/run/openvswitch/ @@ -79,7 +82,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NB value: {{ default "-vconsole:info -vfile:info" .Values.nbLogLevel | quote }} - name: K8S_APISERVER @@ -131,6 +134,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true - mountPath: /var/run/ovn/ name: host-var-run-ovs - mountPath: /var/run/openvswitch/ @@ -141,7 +147,7 @@ spec: memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_SB value: {{ default "-vconsole:info -vfile:info" .Values.sbLogLevel | quote }} - name: K8S_APISERVER @@ -196,13 +202,16 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: cpu: 100m memory: 300Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVN_LOGLEVEL_NORTHD value: {{ default "-vconsole:info -vfile:info" .Values.northdLogLevel | quote }} - name: K8S_APISERVER @@ -249,6 +258,9 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true + - mountPath: /run/ovnkube-config + name: ovnkube-config + readOnly: true resources: requests: cpu: 100m @@ -259,7 +271,7 @@ spec: value: "crash" {{ end -}} - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" - name: OVNKUBE_LOGLEVEL value: {{ default 4 .Values.ovnkubeLocalLogLevel | quote }} - name: OVNKUBE_LOGFILE_MAXSIZE @@ -317,12 +329,18 @@ spec: value: {{ hasKey .Values.global "enableMultiNetwork" | ternary .Values.global.enableMultiNetwork false | quote }} - name: OVN_NETWORK_SEGMENTATION_ENABLE value: {{ default "" .Values.global.enableNetworkSegmentation | quote }} + - name: OVN_ROUTE_ADVERTISEMENTS_ENABLE + value: {{ hasKey .Values.global "enableRouteAdvertisements" | ternary .Values.global.enableRouteAdvertisements false | quote }} + - name: OVN_EVPN_ENABLE + value: {{ hasKey .Values.global "enableEVPN" | ternary .Values.global.enableEVPN false | quote }} - name: OVN_NETWORK_CONNECT_ENABLE value: {{ default "" .Values.global.enableNetworkConnect | quote }} - name: OVN_PRE_CONF_UDN_ADDR_ENABLE value: {{ default "" .Values.global.enablePreconfiguredUDNAddresses | quote }} - name: OVN_ADVERTISED_UDN_ISOLATION_MODE value: {{ default "strict" .Values.global.advertisedUDNIsolationMode | quote }} + - name: OVN_NO_OVERLAY_ENABLE + value: {{ default "false" .Values.global.enableNoOverlay | quote }} - name: OVN_HYBRID_OVERLAY_NET_CIDR value: {{ default "" .Values.global.hybridOverlayNetCidr | quote }} - name: OVN_DISABLE_SNAT_MULTIPLE_GWS @@ -360,6 +378,10 @@ spec: configMapKeyRef: name: ovn-config key: host_network_namespace + - name: OVN_DYNAMIC_UDN_ALLOCATION + value: {{ hasKey .Values.global "enableDynamicUDNAllocation" | ternary .Values.global.enableDynamicUDNAllocation false | quote }} + - name: OVN_DYNAMIC_UDN_GRACE_PERIOD + value: {{ default "" .Values.global.dynamicUDNGracePeriod | quote }} - name: OVN_NORTH value: "local" - name: OVN_SOUTH @@ -405,6 +427,9 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + - name: ovnkube-config + configMap: + name: ovnkube-config - name: host-var-lib-ovs hostPath: path: /var/lib/openvswitch diff --git a/helm/ovn-kubernetes/charts/ovs-node/Chart.yaml b/helm/ovn-kubernetes/charts/ovs-node/Chart.yaml index 1eb21bf1a9..ca202ed3b9 100644 --- a/helm/ovn-kubernetes/charts/ovs-node/Chart.yaml +++ b/helm/ovn-kubernetes/charts/ovs-node/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ovs-node description: Helm chart to deploy ovs type: application -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" diff --git a/helm/ovn-kubernetes/charts/ovs-node/templates/ovs-node.yaml b/helm/ovn-kubernetes/charts/ovs-node/templates/ovs-node.yaml index b71c7b0e0f..18855d51e6 100644 --- a/helm/ovn-kubernetes/charts/ovs-node/templates/ovs-node.yaml +++ b/helm/ovn-kubernetes/charts/ovs-node/templates/ovs-node.yaml @@ -84,12 +84,9 @@ spec: requests: cpu: 100m memory: 300Mi - limits: - cpu: 500m - memory: 500Mi env: - name: OVN_DAEMONSET_VERSION - value: "1.1.0" + value: "1.2.0" lifecycle: preStop: exec: diff --git a/helm/ovn-kubernetes/crds/k8s.ovn.org_vteps.yaml b/helm/ovn-kubernetes/crds/k8s.ovn.org_vteps.yaml new file mode 120000 index 0000000000..a59b403162 --- /dev/null +++ b/helm/ovn-kubernetes/crds/k8s.ovn.org_vteps.yaml @@ -0,0 +1 @@ +../../../dist/templates/k8s.ovn.org_vteps.yaml.j2 \ No newline at end of file diff --git a/helm/ovn-kubernetes/crds/monitoring.coreos.com_prometheusrule.yaml b/helm/ovn-kubernetes/crds/monitoring.coreos.com_prometheusrule.yaml deleted file mode 100644 index 16cedfe545..0000000000 --- a/helm/ovn-kubernetes/crds/monitoring.coreos.com_prometheusrule.yaml +++ /dev/null @@ -1,118 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: prometheusrules.monitoring.coreos.com -spec: - group: monitoring.coreos.com - names: - categories: - - prometheus-operator - kind: PrometheusRule - listKind: PrometheusRuleList - plural: prometheusrules - shortNames: - - promrule - singular: prometheusrule - scope: Namespaced - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - description: PrometheusRule defines recording and alerting rules for a Prometheus instance - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Specification of desired alerting rule definitions for Prometheus. - properties: - groups: - description: Content of Prometheus rule file - items: - description: RuleGroup is a list of sequentially evaluated recording - and alerting rules. - properties: - interval: - description: Interval determines how often rules in the group - are evaluated. - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - limit: - description: Limit the number of alerts an alerting rule and - series a recording rule can produce. Limit is supported starting - with Prometheus >= 2.31 and Thanos Ruler >= 0.24. - type: integer - name: - description: Name of the rule group. - minLength: 1 - type: string - partial_response_strategy: - description: 'PartialResponseStrategy is only used by ThanosRuler - and will be ignored by Prometheus instances. More info: https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response' - pattern: ^(?i)(abort|warn)?$ - type: string - rules: - description: List of alerting and recording rules. - items: - description: 'Rule describes an alerting or recording rule - See Prometheus documentation: [alerting](https://www.prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) - or [recording](https://www.prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules) - rule' - properties: - alert: - description: Name of the alert. Must be a valid label - value. Only one of `record` and `alert` must be set. - type: string - annotations: - additionalProperties: - type: string - description: Annotations to add to each alert. Only valid - for alerting rules. - type: object - expr: - anyOf: - - type: integer - - type: string - description: PromQL expression to evaluate. - x-kubernetes-int-or-string: true - for: - description: Alerts are considered firing once they have - been returned for this long. - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - labels: - additionalProperties: - type: string - description: Labels to add or overwrite. - type: object - record: - description: Name of the time series to output to. Must - be a valid metric name. Only one of `record` and `alert` - must be set. - type: string - required: - - expr - type: object - type: array - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - required: - - spec - type: object diff --git a/helm/ovn-kubernetes/crds/monitoring.coreos.com_servicemonitor.yaml b/helm/ovn-kubernetes/crds/monitoring.coreos.com_servicemonitor.yaml deleted file mode 100644 index d747e68f02..0000000000 --- a/helm/ovn-kubernetes/crds/monitoring.coreos.com_servicemonitor.yaml +++ /dev/null @@ -1,705 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: servicemonitors.monitoring.coreos.com -spec: - group: monitoring.coreos.com - names: - categories: - - prometheus-operator - kind: ServiceMonitor - listKind: ServiceMonitorList - plural: servicemonitors - shortNames: - - smon - singular: servicemonitor - scope: Namespaced - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - description: ServiceMonitor defines monitoring for a set of services. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Specification of desired Service selection for target discovery - by Prometheus. - properties: - attachMetadata: - description: Attaches node metadata to discovered targets. Requires - Prometheus v2.37.0 and above. - properties: - node: - description: When set to true, Prometheus must have permissions - to get Nodes. - type: boolean - type: object - endpoints: - description: A list of endpoints allowed as part of this ServiceMonitor. - items: - description: Endpoint defines a scrapeable endpoint serving Prometheus - metrics. - properties: - authorization: - description: Authorization section for this endpoint - properties: - credentials: - description: The secret's key that contains the credentials - of the request - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: - description: Set the authentication type. Defaults to Bearer, - Basic will cause an error - type: string - type: object - basicAuth: - description: 'BasicAuth allow an endpoint to authenticate over - basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints' - properties: - password: - description: The secret in the service monitor namespace - that contains the password for authentication. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - description: The secret in the service monitor namespace - that contains the username for authentication. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - bearerTokenFile: - description: File to read bearer token for scraping targets. - type: string - bearerTokenSecret: - description: Secret to mount to read bearer token for scraping - targets. The secret needs to be in the same namespace as the - service monitor and accessible by the Prometheus Operator. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - enableHttp2: - description: Whether to enable HTTP2. - type: boolean - filterRunning: - description: 'Drop pods that are not running. (Failed, Succeeded). - Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase' - type: boolean - followRedirects: - description: FollowRedirects configures whether scrape requests - follow HTTP 3xx redirects. - type: boolean - honorLabels: - description: HonorLabels chooses the metric's labels on collisions - with target labels. - type: boolean - honorTimestamps: - description: HonorTimestamps controls whether Prometheus respects - the timestamps present in scraped data. - type: boolean - interval: - description: Interval at which metrics should be scraped If - not specified Prometheus' global scrape interval is used. - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - metricRelabelings: - description: MetricRelabelConfigs to apply to samples before - ingestion. - items: - description: 'RelabelConfig allows dynamic rewriting of the - label set, being applied to samples before ingestion. It - defines ``-section of Prometheus - configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' - properties: - action: - default: replace - description: Action to perform based on regex matching. - Default is 'replace'. uppercase and lowercase actions - require Prometheus >= 2.36. - enum: - - replace - - Replace - - keep - - Keep - - drop - - Drop - - hashmod - - HashMod - - labelmap - - LabelMap - - labeldrop - - LabelDrop - - labelkeep - - LabelKeep - - lowercase - - Lowercase - - uppercase - - Uppercase - - keepequal - - KeepEqual - - dropequal - - DropEqual - type: string - modulus: - description: Modulus to take of the hash of the source - label values. - format: int64 - type: integer - regex: - description: Regular expression against which the extracted - value is matched. Default is '(.*)' - type: string - replacement: - description: Replacement value against which a regex replace - is performed if the regular expression matches. Regex - capture groups are available. Default is '$1' - type: string - separator: - description: Separator placed between concatenated source - label values. default is ';'. - type: string - sourceLabels: - description: The source labels select values from existing - labels. Their content is concatenated using the configured - separator and matched against the configured regular - expression for the replace, keep, and drop actions. - items: - description: LabelName is a valid Prometheus label name - which may only contain ASCII letters, numbers, as - well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ - type: string - type: array - targetLabel: - description: Label to which the resulting value is written - in a replace action. It is mandatory for replace actions. - Regex capture groups are available. - type: string - type: object - type: array - oauth2: - description: OAuth2 for the URL. Only valid in Prometheus versions - 2.27.0 and newer. - properties: - clientId: - description: The secret or configmap containing the OAuth2 - client id - properties: - configMap: - description: ConfigMap containing data to use for the - targets. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secret: - description: Secret containing data to use for the targets. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - clientSecret: - description: The secret containing the OAuth2 client secret - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - endpointParams: - additionalProperties: - type: string - description: Parameters to append to the token URL - type: object - scopes: - description: OAuth2 scopes used for the token request - items: - type: string - type: array - tokenUrl: - description: The URL to fetch the token from - minLength: 1 - type: string - required: - - clientId - - clientSecret - - tokenUrl - type: object - params: - additionalProperties: - items: - type: string - type: array - description: Optional HTTP URL parameters - type: object - path: - description: HTTP path to scrape for metrics. If empty, Prometheus - uses the default value (e.g. `/metrics`). - type: string - port: - description: Name of the service port this endpoint refers to. - Mutually exclusive with targetPort. - type: string - proxyUrl: - description: ProxyURL eg http://proxyserver:2195 Directs scrapes - to proxy through this endpoint. - type: string - relabelings: - description: 'RelabelConfigs to apply to samples before scraping. - Prometheus Operator automatically adds relabelings for a few - standard Kubernetes fields. The original scrape job''s name - is available via the `__tmp_prometheus_job_name` label. More - info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' - items: - description: 'RelabelConfig allows dynamic rewriting of the - label set, being applied to samples before ingestion. It - defines ``-section of Prometheus - configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' - properties: - action: - default: replace - description: Action to perform based on regex matching. - Default is 'replace'. uppercase and lowercase actions - require Prometheus >= 2.36. - enum: - - replace - - Replace - - keep - - Keep - - drop - - Drop - - hashmod - - HashMod - - labelmap - - LabelMap - - labeldrop - - LabelDrop - - labelkeep - - LabelKeep - - lowercase - - Lowercase - - uppercase - - Uppercase - - keepequal - - KeepEqual - - dropequal - - DropEqual - type: string - modulus: - description: Modulus to take of the hash of the source - label values. - format: int64 - type: integer - regex: - description: Regular expression against which the extracted - value is matched. Default is '(.*)' - type: string - replacement: - description: Replacement value against which a regex replace - is performed if the regular expression matches. Regex - capture groups are available. Default is '$1' - type: string - separator: - description: Separator placed between concatenated source - label values. default is ';'. - type: string - sourceLabels: - description: The source labels select values from existing - labels. Their content is concatenated using the configured - separator and matched against the configured regular - expression for the replace, keep, and drop actions. - items: - description: LabelName is a valid Prometheus label name - which may only contain ASCII letters, numbers, as - well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ - type: string - type: array - targetLabel: - description: Label to which the resulting value is written - in a replace action. It is mandatory for replace actions. - Regex capture groups are available. - type: string - type: object - type: array - scheme: - description: HTTP scheme to use for scraping. `http` and `https` - are the expected values unless you rewrite the `__scheme__` - label via relabeling. If empty, Prometheus uses the default - value `http`. - enum: - - http - - https - type: string - scrapeTimeout: - description: Timeout after which the scrape is ended If not - specified, the Prometheus global scrape timeout is used unless - it is less than `Interval` in which the latter is used. - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: Name or number of the target port of the Pod behind - the Service, the port must be specified with container port - property. Mutually exclusive with port. - x-kubernetes-int-or-string: true - tlsConfig: - description: TLS configuration to use when scraping the endpoint - properties: - ca: - description: Certificate authority used when verifying server - certificates. - properties: - configMap: - description: ConfigMap containing data to use for the - targets. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secret: - description: Secret containing data to use for the targets. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. - type: string - cert: - description: Client certificate to present when doing client-authentication. - properties: - configMap: - description: ConfigMap containing data to use for the - targets. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secret: - description: Secret containing data to use for the targets. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - certFile: - description: Path to the client cert file in the Prometheus - container for the targets. - type: string - insecureSkipVerify: - description: Disable target certificate validation. - type: boolean - keyFile: - description: Path to the client key file in the Prometheus - container for the targets. - type: string - keySecret: - description: Secret containing the client key file for the - targets. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - serverName: - description: Used to verify the hostname for the targets. - type: string - type: object - type: object - type: array - jobLabel: - description: "JobLabel selects the label from the associated Kubernetes - service which will be used as the `job` label for all metrics. \n - For example: If in `ServiceMonitor.spec.jobLabel: foo` and in `Service.metadata.labels.foo: - bar`, then the `job=\"bar\"` label is added to all metrics. \n If - the value of this field is empty or if the label doesn't exist for - the given Service, the `job` label of the metrics defaults to the - name of the Kubernetes Service." - type: string - labelLimit: - description: Per-scrape limit on number of labels that will be accepted - for a sample. Only valid in Prometheus versions 2.27.0 and newer. - format: int64 - type: integer - labelNameLengthLimit: - description: Per-scrape limit on length of labels name that will be - accepted for a sample. Only valid in Prometheus versions 2.27.0 - and newer. - format: int64 - type: integer - labelValueLengthLimit: - description: Per-scrape limit on length of labels value that will - be accepted for a sample. Only valid in Prometheus versions 2.27.0 - and newer. - format: int64 - type: integer - namespaceSelector: - description: Selector to select which namespaces the Kubernetes Endpoints - objects are discovered from. - properties: - any: - description: Boolean describing whether all namespaces are selected - in contrast to a list restricting them. - type: boolean - matchNames: - description: List of namespace names to select from. - items: - type: string - type: array - type: object - podTargetLabels: - description: PodTargetLabels transfers labels on the Kubernetes `Pod` - onto the created metrics. - items: - type: string - type: array - sampleLimit: - description: SampleLimit defines per-scrape limit on number of scraped - samples that will be accepted. - format: int64 - type: integer - selector: - description: Selector to select Endpoints objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - targetLabels: - description: TargetLabels transfers labels from the Kubernetes `Service` - onto the created metrics. - items: - type: string - type: array - targetLimit: - description: TargetLimit defines a limit on the number of scraped - targets that will be accepted. - format: int64 - type: integer - required: - - endpoints - - selector - type: object - required: - - spec - type: object diff --git a/helm/ovn-kubernetes/templates/ovn-setup.yaml b/helm/ovn-kubernetes/templates/ovn-setup.yaml index e9a5ef8981..b76ebf0929 100644 --- a/helm/ovn-kubernetes/templates/ovn-setup.yaml +++ b/helm/ovn-kubernetes/templates/ovn-setup.yaml @@ -50,6 +50,26 @@ data: mtu: {{ .Values.mtu | default 1500 | quote }} host_network_namespace: {{ $hostNetworkNamespace }} +--- +# ovnkube-config ConfigMap +# +# Configuration for ovnkube binaries +kind: ConfigMap +apiVersion: v1 +metadata: + name: ovnkube-config + namespace: ovn-kubernetes +data: + ovnkube.conf: | +{{- if .Values.global.enableNoOverlay }} + [default] + transport = no-overlay + + [no-overlay] + outbound-snat = disabled + routing = unmanaged +{{- end }} + {{- if or .Values.global.skipCallToK8s (eq (include "needNamespace" $hostNetworkNamespace) "true") }} --- # ovn-host-network-namespace.yaml @@ -64,6 +84,23 @@ metadata: name: {{ $hostNetworkNamespace }} {{- end }} +{{- if and (eq (hasKey .Values.global "enableRouteAdvertisements" | ternary .Values.global.enableRouteAdvertisements false) true) (eq (hasKey .Values.global "advertiseDefaultNetwork" | ternary .Values.global.advertiseDefaultNetwork false) true) }} +--- +apiVersion: k8s.ovn.org/v1 +kind: RouteAdvertisements +metadata: + name: default +spec: + networkSelectors: + - networkSelectionType: DefaultNetwork + nodeSelector: {} + frrConfigurationSelector: + matchLabels: + name: receive-all + advertisements: + - "PodNetwork" +{{- end }} + {{- if (and .Values.global.dockerConfigSecret .Values.global.dockerConfigSecret.create) }} --- apiVersion: v1 diff --git a/helm/ovn-kubernetes/templates/ovnkube-alerts.yaml b/helm/ovn-kubernetes/templates/ovnkube-alerts.yaml index 5907b73023..6153c57712 100644 --- a/helm/ovn-kubernetes/templates/ovnkube-alerts.yaml +++ b/helm/ovn-kubernetes/templates/ovnkube-alerts.yaml @@ -1,3 +1,8 @@ +{{- if and .Values.monitoring.enablePrometheusRule (not (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule")) }} + {{ fail "Prometheus Operator must be installed to enable PrometheusRule" }} +{{- end }} + +{{ if .Values.monitoring.enablePrometheusRule }} apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: @@ -387,4 +392,4 @@ spec: annotations: message: | {{`ovnkube clustermanager allocated IPv6 host subnets value for {{ $labels.network_name }} network should be same as the number of K8s nodes`}} - +{{ end }} diff --git a/helm/ovn-kubernetes/templates/ovnkube-monitor.yaml b/helm/ovn-kubernetes/templates/ovnkube-monitor.yaml index a6d9528592..1b834ad398 100644 --- a/helm/ovn-kubernetes/templates/ovnkube-monitor.yaml +++ b/helm/ovn-kubernetes/templates/ovnkube-monitor.yaml @@ -1,6 +1,10 @@ -# define ServiceMontior and Service resources for ovnkube-cluster-manager, +# define ServiceMonitor and Service resources for ovnkube-cluster-manager, # ovnkube-master (or ovnkube-controller), ovnkube-node and ovnkube-db (required for prometheus monitoring) +{{- if and .Values.monitoring.enableServiceMonitor (not (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor")) }} + {{ fail "Prometheus Operator must be installed to enable ServiceMonitor" }} +{{- end }} +{{ if .Values.monitoring.enableServiceMonitor }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -25,58 +29,83 @@ spec: matchLabels: k8s-app: ovnkube-master --- -apiVersion: v1 -kind: Service +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor metadata: labels: - k8s-app: ovnkube-master - name: ovnkube-master-prometheus-discovery + k8s-app: ovnkube-node + {{- with .Values.monitoring.commonServiceMonitorSelectorLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: monitor-ovnkube-node namespace: ovn-kubernetes spec: + endpoints: + - interval: 30s + port: ovnkube-node-metrics + path: /metrics + scheme: http + - interval: 30s + port: ovs-metrics + path: /metrics + scheme: http + - interval: 30s + port: ovn-metrics + path: /metrics + scheme: http + jobLabel: k8s-app + namespaceSelector: + matchNames: + - ovn-kubernetes selector: - name: ovnkube-master - type: ClusterIP - clusterIP: None - publishNotReadyAddresses: true - ports: - - name: http-metrics - port: 9409 - protocol: TCP - targetPort: 9409 + matchLabels: + k8s-app: ovnkube-node --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: - k8s-app: ovnkube-node + k8s-app: ovnkube-cluster-manager {{- with .Values.monitoring.commonServiceMonitorSelectorLabels }} {{- toYaml . | nindent 4 }} {{- end }} - name: monitor-ovnkube-node + name: monitor-ovnkube-cluster-manager namespace: ovn-kubernetes spec: endpoints: - - interval: 30s - port: ovnkube-node-metrics - path: /metrics - scheme: http - - interval: 30s - port: ovs-metrics - path: /metrics - scheme: http - - interval: 30s - port: ovn-metrics - path: /metrics - scheme: http + - interval: 30s + port: http-metrics + scheme: http + path: /metrics jobLabel: k8s-app namespaceSelector: matchNames: - - ovn-kubernetes + - ovn-kubernetes selector: matchLabels: - k8s-app: ovnkube-node + k8s-app: ovnkube-cluster-manager --- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: ovnkube-master + name: ovnkube-master-prometheus-discovery + namespace: ovn-kubernetes +spec: + selector: + name: ovnkube-master + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 9409 + protocol: TCP + targetPort: 9409 +--- + apiVersion: v1 kind: Service metadata: @@ -104,31 +133,6 @@ spec: protocol: TCP targetPort: 9310 --- - -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - k8s-app: ovnkube-cluster-manager - {{- with .Values.monitoring.commonServiceMonitorSelectorLabels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: monitor-ovnkube-cluster-manager - namespace: ovn-kubernetes -spec: - endpoints: - - interval: 30s - port: http-metrics - scheme: http - path: /metrics - jobLabel: k8s-app - namespaceSelector: - matchNames: - - ovn-kubernetes - selector: - matchLabels: - k8s-app: ovnkube-cluster-manager ---- apiVersion: v1 kind: Service metadata: @@ -148,3 +152,4 @@ spec: protocol: TCP targetPort: 9411 --- +{{ end }} diff --git a/helm/ovn-kubernetes/templates/rbac-ovnkube-node.yaml b/helm/ovn-kubernetes/templates/rbac-ovnkube-node.yaml index 7ad994a5a1..a2bec63d7e 100644 --- a/helm/ovn-kubernetes/templates/rbac-ovnkube-node.yaml +++ b/helm/ovn-kubernetes/templates/rbac-ovnkube-node.yaml @@ -162,6 +162,7 @@ rules: - egressfirewalls/status - adminpolicybasedexternalroutes/status - egressqoses/status + - routeadvertisements/status - networkqoses/status verbs: [ "patch", "update" ] - apiGroups: ["policy.networking.k8s.io"] @@ -184,7 +185,9 @@ rules: - adminpolicybasedexternalroutes - userdefinednetworks - clusteruserdefinednetworks + - routeadvertisements - networkqoses + - clusternetworkconnects verbs: [ "get", "list", "watch" ] {{- if eq (hasKey .Values.global "enableOvnKubeIdentity" | ternary .Values.global.enableOvnKubeIdentity true) true }} - apiGroups: ["certificates.k8s.io"] diff --git a/helm/ovn-kubernetes/values-multi-node-zone.yaml b/helm/ovn-kubernetes/values-multi-node-zone.yaml index 5afa6f6da5..d3b1c16755 100644 --- a/helm/ovn-kubernetes/values-multi-node-zone.yaml +++ b/helm/ovn-kubernetes/values-multi-node-zone.yaml @@ -76,6 +76,16 @@ global: enableMultiNetwork: false # -- Configure to use user defined networks (UDN) feature with ovn-kubernetes enableNetworkSegmentation: false + # -- Configure to use route advertisements feature with ovn-kubernetes + enableRouteAdvertisements: false + # -- Configure to use EVPN feature with ovn-kubernetes + enableEVPN: false + # -- Advertise default network on all nodes with a default RouteAdvertisements configuration + advertiseDefaultNetwork: false + # -- Pod network isolation between advertised UDN networks. (strict or loose) + advertisedUDNIsolationMode: "strict" + # -- Configure to enable no-overlay mode for the default network + enableNoOverlay: false # -- Configure to enable workloads with preconfigured network connect to user defined networks (UDN) with ovn-kubernetes enablePreconfiguredUDNAddresses: false # -- Configure to enable IPsec @@ -164,3 +174,7 @@ monitoring: # in defining that. commonServiceMonitorSelectorLabels: release: kube-prometheus-stack + # -- deploy ServiceMonitors for service discovery using the Prometheus Operator + enableServiceMonitor: false + # -- deploy PrometheusRules for specific metric collection using the Prometheus Operator + enablePrometheusRule: false diff --git a/helm/ovn-kubernetes/values-no-ic.yaml b/helm/ovn-kubernetes/values-no-ic.yaml index 0f54f5217b..1ed4e30a2d 100644 --- a/helm/ovn-kubernetes/values-no-ic.yaml +++ b/helm/ovn-kubernetes/values-no-ic.yaml @@ -8,6 +8,7 @@ tags: ovnkube-node-dpu: false ovnkube-node-dpu-host: false ovnkube-single-node-zone: false + ovnkube-single-node-zone-dpu: false ovnkube-zone-controller: false # -- Endpoint of Kubernetes api server @@ -70,6 +71,16 @@ global: enableMultiNetwork: false # -- Configure to use user defined networks (UDN) feature with ovn-kubernetes enableNetworkSegmentation: false + # -- Configure to use route advertisements feature with ovn-kubernetes + enableRouteAdvertisements: false + # -- Configure to use EVPN feature with ovn-kubernetes + enableEVPN: false + # -- Advertise default network on all nodes with a default RouteAdvertisements configuration + advertiseDefaultNetwork: false + # -- Pod network isolation between advertised UDN networks. (strict or loose) + advertisedUDNIsolationMode: "strict" + # -- Configure to enable no-overlay mode for the default network + enableNoOverlay: false # -- Configure to enable IPsec enableIpsec: false # -- Use SSL transport to NB/SB db and northd @@ -179,3 +190,7 @@ monitoring: # in defining that. commonServiceMonitorSelectorLabels: release: kube-prometheus-stack + # -- deploy ServiceMonitors for service discovery using the Prometheus Operator + enableServiceMonitor: false + # -- deploy PrometheusRules for specific metric collection using the Prometheus Operator + enablePrometheusRule: false diff --git a/helm/ovn-kubernetes/values-single-node-zone-dpu.yaml b/helm/ovn-kubernetes/values-single-node-zone-dpu.yaml new file mode 100644 index 0000000000..2105d1302b --- /dev/null +++ b/helm/ovn-kubernetes/values-single-node-zone-dpu.yaml @@ -0,0 +1,176 @@ +# Values for ovn-kubernetes with single-node zone interconnect for DPU cluster +# Requires: ovnkube-single-node-zone-dpu only + +# -- The following subcharts should be disabled +tags: + ovs-node: false + ovn-ipsec: false + ovnkube-db: false + ovnkube-db-raft: false + ovnkube-master: false + ovnkube-node: false + ovnkube-control-plane: false + ovnkube-node-dpu-host: false + ovnkube-node-dpu: false + ovnkube-single-node-zone: false + ovnkube-zone-controller: false + +# -- Whether or not call `lookup` Helm function, set it to `true` if you want to run `helm dry-run/template/lint` +skipCallToK8s: false + +global: + # -- The interface on nodes that will be used for external gateway network traffic + extGatewayNetworkInterface: "" + # -- GENEVE UDP port (default 6081) + encapPort: 6081 + # -- The gateway mode (shared or local), if not given, gateway functionality is disabled + gatewayMode: shared + # -- Optional extra gateway options + gatewayOpts: "" + # -- This allows ovnkube-node to run without SYS_ADMIN capability, by performing interface setup in the CNI plugin + unprivilegedMode: false + # -- The v4 join subnet used for assigning join switch IPv4 addresses + v4JoinSubnet: "100.64.0.0/16" + # -- The v4 masquerade subnet used for assigning masquerade IPv4 addresses + v4MasqueradeSubnet: "169.254.0.0/17" + # -- The v4 subnet for transit switches and routers + v4TransitSubnet: "100.88.0.0/16" + # -- The v6 join subnet used for assigning join switch IPv6 addresses + v6JoinSubnet: "fd98::/64" + # -- The v6 masquerade subnet used for assigning masquerade IPv6 addresses + v6MasqueradeSubnet: "fd69::/112" + # -- The v6 subnet for transit switches and routers + v6TransitSubnet: "fd97::/64" + # -- Whether or not enable ovnkube identity webhook + enableOvnKubeIdentity: false + # -- Whether or not to enable hybrid overlay functionality + enableHybridOverlay: "" + # -- A comma separated set of IP subnets and the associated hostsubnetlengths (eg, \"10.128.0.0/14/23,10.0.0.0/14/23\") to use with the extended hybrid network + hybridOverlayNetCidr: "" + # -- Whether or not to use Admin Network Policy CRD feature with ovn-kubernetes + enableAdminNetworkPolicy: false + # -- Configure to use EgressIP CRD feature with ovn-kubernetes + enableEgressIp: false + # -- Configure EgressIP node reachability using gRPC on this TCP port + egressIpHealthCheckPort: 9107 + # -- Configure to use EgressService CRD feature with ovn-kubernetes + enableEgressService: false + # -- Configure to use EgressFirewall CRD feature with ovn-kubernetes + enableEgressFirewall: false + # -- Configure to use EgressQoS CRD feature with ovn-kubernetes + enableEgressQos: false + # -- Enables network QoS support from/to pods + enableNetworkQos: false + # -- Enables multicast support between the pods within the same namespace + enableMulticast: "" + # -- Configure to use multiple NetworkAttachmentDefinition CRD feature with ovn-kubernetes + enableMultiNetwork: false + # -- Configure to use user defined networks (UDN) feature with ovn-kubernetes + enableNetworkSegmentation: false + # -- Configure to enable workloads with preconfigured network connect to user defined networks (UDN) with ovn-kubernetes + enablePreconfiguredUDNAddresses: false + # -- Configure to enable IPsec + enableIpsec: false + # -- Use SSL transport to NB/SB db and northd + enableSsl: false + # -- Configure to enable interconnecting multiple zones + # @default -- true + enableInterconnect: true + # -- Configure to use AdminPolicyBasedExternalRoute CRD feature with ovn-kubernetes + enableMultiExternalGateway: false + # -- Configure to use stateless network policy feature with ovn-kubernetes + enableStatelessNetworkPolicy: false + # -- Configure to use service template feature with ovn-kubernetes + enableSvcTemplate: false + # -- Enables metrics related to scaling + enableMetricsScale: "" + # -- Enables monitoring OVN-Kubernetes master and OVN configuration duration + enableConfigDuration: "" + # -- Indicates if ovn-controller should enable/disable the logical flow in-memory cache when processing Southbound database logical flow changes + # @default -- true + enableLFlowCache: true + # -- Maximum number of logical flow cache entries ovn-controller may create when the logical flow cache is enabled + # @default -- unlimited + lFlowCacheLimit: "" + # -- Maximum size of the logical flow cache (in KB) ovn-controller may create when the logical flow cache is enabled + lFlowCacheLimitKb: "" + # -- Configure to use the IPAMClaims CRD feature with ovn-kubernetes, thus granting persistent IPs across restarts / migration for KubeVirt VMs + enablePersistentIPs: false + # -- Configure to use DNSNameResolver feature with ovn-kubernetes + enableDNSNameResolver: false + # -- Whether to disable SNAT of egress traffic in namespaces annotated with routing-external-gws + disableSnatMultipleGws: "" + # -- Controls if forwarding is allowed on OVNK controlled interfaces + # @default -- false + disableForwarding: "" + # -- Disables adding openflow flows to check packets too large to be delivered to OVN due to pod MTU being lower than NIC MTU + disablePacketMtuCheck: "" + # -- The largest number of messages per second that gets logged before drop + # @default 20 + aclLoggingRateLimit: 20 + # -- If set, then load balancers do not get deleted when all backends are removed + emptyLbEvents: "" + # -- Port of north bound ovsdb + nbPort: 6641 + # -- Port of south bound ovsdb + sbPort: 6642 + # -- A comma separated set of NetFlow collectors to export flow data + netFlowTargets: "" + # -- A comma separated set of SFlow collectors to export flow data + sflowTargets: "" + # -- A comma separated set of IPFIX collectors to export flow data + ipfixTargets: "" + # -- Rate at which packets should be sampled and sent to each target collector + # @default 400 + ipfixSampling: "" + # -- Maximum number of IPFIX flow records that can be cached at a time + # @default 0, meaning disabled + ipfixCacheMaxFlows: "" + # -- Maximum period in seconds for which an IPFIX flow record is cached and aggregated before being sent + # @default 60 + ipfixCacheActiveTimeout: "" + # -- OVN remote probe interval in ms + # @default 100000 + remoteProbeInterval: 100000 + # -- Enable monitoring all data from SB DB instead of conditionally monitoring the data relevant to this node only + # @default true + monitorAll: true + # -- ovn-controller wait time in ms before clearing OpenFlow rules during start up + # @default 0 + ofctrlWaitBeforeClear: "0" + # -- Container images + # -- Image for DPUs + dpuImage: + # -- Image repository for ovn-kubernetes components + repository: ghcr.io/ovn-kubernetes/ovn-kubernetes/ovn-kube-ubuntu + # -- Specify image tag to run + tag: master + # -- Image pull policy + pullPolicy: IfNotPresent + # -- The name of secret used for pulling image. Use only if needed + imagePullSecretName: "" + # -- Endpoint of DPU Host cluster's Kubernetes api server + dpuHostClusterK8sAPIServer: https://172.25.0.2:6443 + # -- DPU Host cluster's Kubernetes Access Token + dpuHostClusterK8sToken: "" + # -- DPU Host cluster's Kubernetes Access Certs Data + dpuHostClusterK8sCACertData: "" + # -- DPU Host cluster's Kubernetes Access Token File + dpuHostClusterK8sTokenFile: "" + # -- DPU Host cluster's Kubernetes Access Certs File + dpuHostClusterK8sCACert: "" + # -- DPU Host cluster's Network CIDR + dpuHostClusterNetworkCIDR: 10.244.0.0/16/24 + # -- DPU Host cluster's Service CIDR + dpuHostClusterServiceCIDR: 10.96.0.0/16 + # -- MTU of network interface in a Kubernetes pod + mtu: 1400 + +# -- prometheus monitoring related fields +monitoring: + # -- specify the labels for serviceMonitors to be selected for target discovery. + # Prometheus operator defines what namespaces and what servicemonitors within these + # namespaces must be selected for target discovery. The fields defined below helps + # in defining that. + commonServiceMonitorSelectorLabels: + release: kube-prometheus-stack diff --git a/helm/ovn-kubernetes/values-single-node-zone.yaml b/helm/ovn-kubernetes/values-single-node-zone.yaml index c3ea4521d9..b9f4f2caf2 100644 --- a/helm/ovn-kubernetes/values-single-node-zone.yaml +++ b/helm/ovn-kubernetes/values-single-node-zone.yaml @@ -9,6 +9,7 @@ tags: ovnkube-master: false ovnkube-node: false ovnkube-node-dpu: false + ovnkube-single-node-zone-dpu: false ovnkube-node-dpu-host: false ovnkube-zone-controller: false @@ -76,6 +77,16 @@ global: enableMultiNetwork: false # -- Configure to use user defined networks (UDN) feature with ovn-kubernetes enableNetworkSegmentation: false + # -- Configure to use route advertisements feature with ovn-kubernetes + enableRouteAdvertisements: false + # -- Configure to use EVPN feature with ovn-kubernetes + enableEVPN: false + # -- Advertise default network on all nodes with a default RouteAdvertisements configuration + advertiseDefaultNetwork: false + # -- Pod network isolation between advertised UDN networks. (strict or loose) + advertisedUDNIsolationMode: "strict" + # -- Configure to enable no-overlay mode for the default network + enableNoOverlay: false # -- Configure to enable workloads with preconfigured network connect to user defined networks (UDN) with ovn-kubernetes enablePreconfiguredUDNAddresses: false # -- Configure to enable IPsec @@ -165,3 +176,7 @@ monitoring: # in defining that. commonServiceMonitorSelectorLabels: release: kube-prometheus-stack + # -- deploy ServiceMonitors for service discovery using the Prometheus Operator + enableServiceMonitor: false + # -- deploy PrometheusRules for specific metric collection using the Prometheus Operator + enablePrometheusRule: false diff --git a/helm/sdn-dashboard/Chart.yaml b/helm/sdn-dashboard/Chart.yaml index 2a5ebfcbc7..6411161174 100644 --- a/helm/sdn-dashboard/Chart.yaml +++ b/helm/sdn-dashboard/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: "v2" description: "A Helm chart for sdn-dashboard" name: "sdn-dashboard" -version: "1.1.0" +version: "1.2.0" home: https://www.ovn.org/ sources: - https://github.com/ovn-org/ovn-kubernetes -appVersion: "1.1.0" +appVersion: "1.2.0" diff --git a/mkdocs.yml b/mkdocs.yml index ed3172a7a1..3d09e08387 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -134,6 +134,7 @@ nav: - Usage Guide: features/network-qos-guide.md - LiveMigration: features/live-migration.md - HybridOverlay: features/hybrid-overlay.md + - OVS Dynamic CPU Affinity: features/ovs-dynamic-cpu-affinity.md - Hardware Acceleration: - OVS Acceleration with kernel datapath: features/hardware-offload/ovs-kernel.md - OVS Acceleration with DOCA datapath: features/hardware-offload/ovs-doca.md @@ -160,5 +161,6 @@ nav: - Connecting User Defined Networks: okeps/okep-5224-connecting-udns/okep-5224-connecting-udns.md - No-Overlay Mode: okeps/okep-5259-no-overlay.md - EVPN: okeps/okep-5088-evpn.md + - DPU Healthcheck: okeps/okep-5674-dpu-healthcheck.md - Blog: - blog/index.md diff --git a/openshift/cmd/annotate/main.go b/openshift/cmd/annotate/main.go new file mode 100644 index 0000000000..654928b8de --- /dev/null +++ b/openshift/cmd/annotate/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/ovn-org/ovn-kubernetes/openshift/test/annotate" +) + +func main() { + annotate.Run(annotate.LabelToTestNameMatchMaps, annotate.LabelToLabelMaps, func(name string) bool { return false }) +} diff --git a/openshift/cmd/ovn-kubernetes-tests-ext/labels.go b/openshift/cmd/ovn-kubernetes-tests-ext/labels.go new file mode 100644 index 0000000000..b6e7b94aa0 --- /dev/null +++ b/openshift/cmd/ovn-kubernetes-tests-ext/labels.go @@ -0,0 +1,24 @@ +package main + +import ( + "sort" + + "github.com/openshift-eng/openshift-tests-extension/pkg/util/sets" +) + +// getTestExtensionLabels returns labels that should be applied to all tests in this extension +func getTestExtensionLabels() []string { + return []string{"sig-network", "ovn-kubernetes-ote"} +} + +// generatePrependedLabelsStr generates labels that are prepended to a test name +func generatePrependedLabelsStr(labels sets.Set[string]) string { + labelList := labels.UnsortedList() + sort.Strings(labelList) + + var labelsStr = "" + for _, label := range labelList { + labelsStr += "[" + label + "]" + } + return labelsStr +} diff --git a/openshift/cmd/ovn-kubernetes-tests-ext/main.go b/openshift/cmd/ovn-kubernetes-tests-ext/main.go new file mode 100644 index 0000000000..62e1024c60 --- /dev/null +++ b/openshift/cmd/ovn-kubernetes-tests-ext/main.go @@ -0,0 +1,113 @@ +package main + +import ( + "os" + "strings" + + "github.com/ovn-org/ovn-kubernetes/openshift/test" + "github.com/ovn-org/ovn-kubernetes/openshift/test/generated" + // import ovn-kubernetes tests + _ "github.com/ovn-org/ovn-kubernetes/test/e2e" + + "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" + "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + "github.com/spf13/cobra" + + // ensure providers are initialised for configuring infra + _ "k8s.io/kubernetes/test/e2e/framework/providers/aws" + _ "k8s.io/kubernetes/test/e2e/framework/providers/azure" + _ "k8s.io/kubernetes/test/e2e/framework/providers/gce" + _ "k8s.io/kubernetes/test/e2e/framework/providers/kubemark" + _ "k8s.io/kubernetes/test/e2e/framework/providers/openstack" + _ "k8s.io/kubernetes/test/e2e/framework/providers/vsphere" + + // ensure that logging flags are part of the command line. + _ "k8s.io/component-base/logs/testinit" +) + +func loadBlockingTests() map[string]bool { + blockingTests := make(map[string]bool) + for _, testName := range test.BlockingTests { + blockingTests[testName] = true + } + return blockingTests +} + +func main() { + // Create our registry of openshift-tests extensions + extensionRegistry := extension.NewRegistry() + ovnTestsExtension := extension.NewExtension("openshift", "payload", "ovn-kubernetes") + // TODO: register test images using tests extension + // add ovn-kubernetes test suites into openshift suites + // by default, we treat all tests as parallel and only expose tests as Serial if the appropriate label is added - "Serial" + ovnTestsExtension.AddSuite(extension.Suite{ + Name: "ovn-kubernetes/conformance/serial", + Parents: []string{ + "openshift/conformance/serial", + }, + Qualifiers: []string{`labels.exists(l, l == "Serial")`}, + }) + + ovnTestsExtension.AddSuite(extension.Suite{ + Name: "ovn-kubernetes/conformance/parallel", + Parents: []string{ + "openshift/conformance/parallel", + }, + Qualifiers: []string{`!labels.exists(l, l == "Serial")`}, + }) + + specs, err := ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite(extensiontests.AllTestsIncludingVendored()) + if err != nil { + panic(err) + } + + // Initialization for kube ginkgo test framework needs to run before all tests execute + specs.AddBeforeAll(func() { + if err := initializeTestFramework(os.Getenv("TEST_PROVIDER")); err != nil { + panic(err) + } + }) + + blockingTests := loadBlockingTests() + + specs.Walk(func(spec *extensiontests.ExtensionTestSpec) { + for _, label := range getTestExtensionLabels() { + spec.Labels.Insert(label) + } + + // Exclude Network Segmentation tests on SingleReplica topology (e.g., MicroShift, SNO) + // These tests require at least 2 nodes and will fail on single-node deployments + if spec.Labels.Has("Feature:NetworkSegmentation") { + spec.Exclude(extensiontests.TopologyEquals("SingleReplica")) + } + + if annotations, ok := generated.AppendedAnnotations[spec.Name]; ok { + spec.Name += " " + annotations + } + spec.Name = generatePrependedLabelsStr(spec.Labels) + " " + spec.Name // prepend ginkgo labels to test name + + if !blockingTests[spec.Name] { + spec.Lifecycle = extensiontests.LifecycleInforming + } + }) + + specs = specs.Select(func(spec *extensiontests.ExtensionTestSpec) bool { + return !strings.Contains(spec.Name, "[Disabled:") + }) + + ovnTestsExtension.AddSpecs(specs) + extensionRegistry.Register(ovnTestsExtension) + root := &cobra.Command{ + Long: "OVN-Kubernetes tests extension for OpenShift", + } + root.AddCommand( + cmd.DefaultExtensionCommands(extensionRegistry)..., + ) + if err := func() error { + return root.Execute() + }(); err != nil { + os.Exit(1) + } +} diff --git a/openshift/cmd/ovn-kubernetes-tests-ext/provider.go b/openshift/cmd/ovn-kubernetes-tests-ext/provider.go new file mode 100644 index 0000000000..bf5157c9e3 --- /dev/null +++ b/openshift/cmd/ovn-kubernetes-tests-ext/provider.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + ocphacke2e "github.com/ovn-org/ovn-kubernetes/openshift/test" + ocpdeploymentconfig "github.com/ovn-org/ovn-kubernetes/openshift/test/deploymentconfig" + ocpinfraprovider "github.com/ovn-org/ovn-kubernetes/openshift/test/infraprovider" + "github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig" + "github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/reporters" + "github.com/onsi/ginkgo/v2/types" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + kclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubernetes/test/e2e/framework" +) + +// partially copied from https://github.com/openshift/origin/blob/17371a2c6a91e0426045fdd0ab3455c5b457622a/pkg/test/extensions/binary.go +// and https://github.com/openshift/origin/blob/e0a2fbc82ac1f97dc4fa84a00ed5739c94366926/pkg/clioptions/clusterdiscovery/provider.go +func initializeTestFramework(provider string) error { + if len(provider) == 0 { + provider = "{\"type\":\"skeleton\"}" + } + providerInfo := &ClusterConfiguration{} + if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil { + return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err) + } + if len(providerInfo.ProviderName) == 0 { + return fmt.Errorf("provider must be a JSON object with the 'type' key") + } + config := &ClusterConfiguration{} + if err := json.Unmarshal([]byte(provider), config); err != nil { + return fmt.Errorf("provider must decode into the ClusterConfig object: %v", err) + } + + // update testContext with loaded config + testContext := &framework.TestContext + testContext.Provider = config.ProviderName + testContext.CloudConfig = framework.CloudConfig{ + ProjectID: config.ProjectID, + Region: config.Region, + Zone: config.Zone, + Zones: config.Zones, + NumNodes: config.NumNodes, + MultiMaster: config.MultiMaster, + MultiZone: config.MultiZone, + ConfigFile: config.ConfigFile, + Provider: framework.NullProvider{}, + } + testContext.AllowedNotReadyNodes = 0 + testContext.MinStartupPods = -1 + testContext.MaxNodesToGather = 0 + testContext.KubeConfig = os.Getenv("KUBECONFIG") + gomega.Expect(testContext.KubeConfig).NotTo(gomega.BeEmpty()) + testContext.DeleteNamespace = os.Getenv("DELETE_NAMESPACE") != "false" + testContext.VerifyServiceAccount = false + //TODO: do we really need the file systems? + testContext.KubectlPath = "oc" + if ad := os.Getenv("ARTIFACT_DIR"); len(strings.TrimSpace(ad)) == 0 { + os.Setenv("ARTIFACT_DIR", filepath.Join(os.TempDir(), "artifacts")) + } + // "debian" is used when not set. At least GlusterFS tests need "custom". + // (There is no option for "rhel" or "centos".) + testContext.NodeOSDistro = "custom" + testContext.MasterOSDistro = "custom" + // load and set the host variable for kubectl + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: testContext.KubeConfig}, &clientcmd.ConfigOverrides{}) + cfg, err := clientConfig.ClientConfig() + if err != nil { + return fmt.Errorf("failed to get client config: %v", err) + } + testContext.Host = cfg.Host + testContext.CreateTestingNS = func(ctx context.Context, baseName string, c kclientset.Interface, labels map[string]string) (*corev1.Namespace, error) { + return ocphacke2e.CreateTestingNS(ctx, baseName, c, labels, true) + } + testContext.DumpLogsOnFailure = true + testContext.ReportDir = os.Getenv("TEST_JUNIT_DIR") + ocpInfra, err := ocpinfraprovider.New(cfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ocpInfra).NotTo(gomega.BeNil()) + infraprovider.Set(ocpInfra) + ocpDeployment := ocpdeploymentconfig.New() + gomega.Expect(ocpDeployment).NotTo(gomega.BeNil()) + deploymentconfig.Set(ocpDeployment) + return nil +} + +// WriteJUnitReport generates a JUnit file that is shorter than the one +// normally written by `ginkgo --junit-report`. This is needed because the full +// report can become too large for tools like Spyglass +// (https://github.com/kubernetes/kubernetes/issues/111510). +func writeJUnitReport(report ginkgo.Report, filename string) error { + config := reporters.JunitReportConfig{ + // Remove details for specs where we don't care. + OmitTimelinesForSpecState: types.SpecStatePassed | types.SpecStateSkipped, + + // Don't write . The same text is + // also in the full text for the failure. If we were to write + // both, then tools like kettle and spyglass would concatenate + // the two strings and thus show duplicated information. + OmitFailureMessageAttr: true, + + // All labels are also part of the spec texts in inline [] tags, + // so we don't need to write them separately. + OmitSpecLabels: true, + } + + return reporters.GenerateJUnitReportWithConfig(report, filename, config) +} diff --git a/openshift/cmd/ovn-kubernetes-tests-ext/types.go b/openshift/cmd/ovn-kubernetes-tests-ext/types.go new file mode 100644 index 0000000000..f06ca032de --- /dev/null +++ b/openshift/cmd/ovn-kubernetes-tests-ext/types.go @@ -0,0 +1,69 @@ +package main + +import "k8s.io/apimachinery/pkg/util/sets" + +// copied directly from https://github.com/openshift/origin/blob/64d4f0e016da7540cca0f27ba2e4eedc79e26d1a/pkg/clioptions/clusterdiscovery/cluster.go + +// HypervisorConfig contains configuration for hypervisor-based recovery operations +type HypervisorConfig struct { + HypervisorIP string `json:"hypervisorIP"` + SSHUser string `json:"sshUser"` + PrivateKeyPath string `json:"privateKeyPath"` +} + +type ClusterConfiguration struct { + ProviderName string `json:"type"` + + // These fields (and the "type" tag for ProviderName) chosen to match + // upstream's e2e.CloudConfig. + ProjectID string + Region string + Zone string + NumNodes int + MultiMaster bool + MultiZone bool + Zones []string + ConfigFile string + + // Disconnected is set for test jobs without external internet connectivity + Disconnected bool + + // SingleReplicaTopology is set for disabling disruptive tests or tests + // that require high availability + SingleReplicaTopology bool + + // NetworkPlugin is the "official" plugin name + NetworkPlugin string + // NetworkPluginMode is an optional sub-identifier for the NetworkPlugin. + // (Currently it is only used for OpenShiftSDN.) + NetworkPluginMode string `json:",omitempty"` + + // HasIPv4 and HasIPv6 determine whether IPv4-specific, IPv6-specific, + // and dual-stack-specific tests are run + HasIPv4 bool + HasIPv6 bool + // IPFamily defines default IP stack of the cluster, replaces upstream getDefaultClusterIPFamily + IPFamily string + + // HasSCTP determines whether SCTP connectivity tests can be run in the cluster + HasSCTP bool + + // IsProxied determines whether we are accessing the cluster through an HTTP proxy + IsProxied bool + + // IsIBMROKS determines whether the cluster is Managed IBM Cloud (ROKS) + IsIBMROKS bool + + // IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled + HasNoOptionalCapabilities bool + + // HypervisorConfig contains SSH configuration for hypervisor-based recovery operations + HypervisorConfig *HypervisorConfig + + // APIGroups contains the set of API groups available in the cluster + APIGroups sets.Set[string] `json:"-"` + // EnabledFeatureGates contains the set of enabled feature gates in the cluster + EnabledFeatureGates sets.Set[string] `json:"-"` + // DisabledFeatureGates contains the set of disabled feature gates in the cluster + DisabledFeatureGates sets.Set[string] `json:"-"` +} diff --git a/openshift/go.mod b/openshift/go.mod new file mode 100644 index 0000000000..15a1bcb64e --- /dev/null +++ b/openshift/go.mod @@ -0,0 +1,247 @@ +module github.com/ovn-org/ovn-kubernetes/openshift + +go 1.24.0 + +toolchain go1.24.5 + +require ( + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.1 + github.com/openshift-eng/openshift-tests-extension v0.0.0-20250916161632-d81c09058835 + github.com/openshift/api v0.0.0-20251020135558-286504b695bc + github.com/ovn-org/ovn-kubernetes/go-controller v1.0.0 + github.com/ovn-org/ovn-kubernetes/test/e2e v0.0.0-20250827185716-56d14a3074ba + github.com/spf13/cobra v1.9.1 + k8s.io/api v0.34.1 + k8s.io/apimachinery v0.34.1 + k8s.io/client-go v0.34.1 + k8s.io/component-base v0.34.1 + k8s.io/kubernetes v1.34.1 +) + +require ( + cel.dev/expr v0.24.0 // indirect + github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hnslib v0.1.1 // indirect + github.com/NYTimes/gziphandler v1.1.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect + github.com/aws/aws-sdk-go v1.44.204 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clarketm/json v1.17.1 // indirect + github.com/container-storage-interface/spec v1.9.0 // indirect + github.com/containerd/containerd/api v1.8.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.6 // indirect + github.com/containerd/typeurl/v2 v2.2.2 // indirect + github.com/containernetworking/cni v1.2.3 // indirect + github.com/containernetworking/plugins v1.2.0 // indirect + github.com/coreos/butane v0.18.0 // indirect + github.com/coreos/go-iptables v0.6.0 // indirect + github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/coreos/ignition/v2 v2.15.0 // indirect + github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v26.1.5+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gaissmai/cidrtree v0.1.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.21.1 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/cadvisor v0.52.1 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f // indirect + github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/josharian/native v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/k8snetworkplumbingwg/govdpa v0.1.5-0.20230926073613-07c1031aea47 // indirect + github.com/k8snetworkplumbingwg/ipamclaims v0.5.1-alpha // indirect + github.com/k8snetworkplumbingwg/multi-networkpolicy v1.0.1 // indirect + github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7 // indirect + github.com/k8snetworkplumbingwg/sriovnet v1.2.1-0.20250818105516-24ab680f94f3 // indirect + github.com/karrick/godirwalk v1.17.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/libopenstorage/openstorage v1.0.0 // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 // indirect + github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect + github.com/mdlayher/ndp v1.0.1 // indirect + github.com/mdlayher/packet v1.0.0 // indirect + github.com/mdlayher/socket v0.2.1 // indirect + github.com/metallb/frr-k8s v0.0.21 // indirect + github.com/miekg/dns v1.1.43 // indirect + github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/opencontainers/cgroups v0.0.3 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opencontainers/runc v1.2.5 // indirect + github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/opencontainers/selinux v1.11.1 // indirect + github.com/openshift-kni/k8sreporter v1.0.6 // indirect + github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 // indirect + github.com/openshift/custom-resource-status v1.1.2 // indirect + github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/safchain/ethtool v0.3.1-0.20231027162144-83e5e0097c91 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.10.0 // indirect + github.com/urfave/cli/v2 v2.27.2 // indirect + github.com/vincent-petithory/dataurl v1.0.0 // indirect + github.com/vishvananda/netlink v1.3.1 // indirect + github.com/vishvananda/netns v0.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect + go.etcd.io/etcd/api/v3 v3.6.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect + go.etcd.io/etcd/client/v3 v3.6.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.universe.tf/metallb v0.0.0-00010101000000-000000000000 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.46.0 // indirect + golang.org/x/oauth2 v0.29.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/term v0.36.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.11.0 // indirect + golang.org/x/tools v0.38.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f // indirect + google.golang.org/grpc v1.72.1 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/gcfg.v1 v1.2.3 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.34.1 // indirect + k8s.io/apiserver v0.34.1 // indirect + k8s.io/cloud-provider v0.34.1 // indirect + k8s.io/component-helpers v0.34.1 // indirect + k8s.io/controller-manager v0.34.1 // indirect + k8s.io/cri-api v0.34.1 // indirect + k8s.io/cri-client v0.34.1 // indirect + k8s.io/csi-translation-lib v0.34.1 // indirect + k8s.io/dynamic-resource-allocation v0.34.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kms v0.34.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/kube-scheduler v0.34.1 // indirect + k8s.io/kubectl v0.34.1 // indirect + k8s.io/kubelet v0.34.1 // indirect + k8s.io/mount-utils v0.34.1 // indirect + k8s.io/pod-security-admission v0.34.1 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + kubevirt.io/api v1.4.0 // indirect + kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect + kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 // indirect + sigs.k8s.io/controller-runtime v0.22.1 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/network-policy-api v0.1.5 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) + +replace ( + github.com/coreos/go-iptables => github.com/trozet/go-iptables v0.0.0-20240328221912-077e672b3808 + github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 + github.com/ovn-org/ovn-kubernetes/go-controller => ../go-controller + github.com/ovn-org/ovn-kubernetes/test/e2e => ../test/e2e + go.universe.tf/metallb => github.com/metallb/metallb v0.14.9 + // The dependency must be with downstream kubernetes so that agnhost container image + // index is derived correctly. + k8s.io/api => github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251017123720-96593f323733 + k8s.io/apiextensions-apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251017123720-96593f323733 + k8s.io/apimachinery => github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251017123720-96593f323733 + k8s.io/apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251017123720-96593f323733 + k8s.io/client-go => github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251017123720-96593f323733 + k8s.io/cloud-provider => github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251017123720-96593f323733 + k8s.io/component-base => github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251017123720-96593f323733 + k8s.io/component-helpers => github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251017123720-96593f323733 + k8s.io/controller-manager => github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251017123720-96593f323733 + k8s.io/cri-api => github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251017123720-96593f323733 + k8s.io/cri-client => github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251017123720-96593f323733 + k8s.io/csi-translation-lib => github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251017123720-96593f323733 + k8s.io/dynamic-resource-allocation => github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251017123720-96593f323733 + k8s.io/kms => github.com/openshift/kubernetes/staging/src/k8s.io/kms v0.0.0-20251017123720-96593f323733 + k8s.io/kube-scheduler => github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251017123720-96593f323733 + k8s.io/kubectl => github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251017123720-96593f323733 + k8s.io/kubelet => github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251017123720-96593f323733 + k8s.io/kubernetes => github.com/openshift/kubernetes v1.30.1-0.20251017123720-96593f323733 + k8s.io/mount-utils => github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251017123720-96593f323733 + k8s.io/pod-security-admission => github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251017123720-96593f323733 +) diff --git a/openshift/go.sum b/openshift/go.sum new file mode 100644 index 0000000000..c3ff3ccd0c --- /dev/null +++ b/openshift/go.sum @@ -0,0 +1,921 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hnslib v0.1.1 h1:JsZy681SnvSOUAfCZVAxkX4LgQGp+CZZwPbLV0/pdF8= +github.com/Microsoft/hnslib v0.1.1/go.mod h1:DRQR4IjLae6WHYVhW7uqe44hmFUiNhmaWA+jwMbz5tM= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.44.204 h1:7/tPUXfNOHB390A63t6fJIwmlwVQAkAwcbzKsU2/6OQ= +github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/clarketm/json v1.17.1 h1:U1IxjqJkJ7bRK4L6dyphmoO840P6bdhPdbbLySourqI= +github.com/clarketm/json v1.17.1/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= +github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= +github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= +github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/ttrpc v1.2.6 h1:zG+Kn5EZ6MUYCS1t2Hmt2J4tMVaLSFEJVOraDQwNPC4= +github.com/containerd/ttrpc v1.2.6/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.2 h1:3jN/k2ysKuPCsln5Qv8bzR9cxal8XjkxPogJfSNO31k= +github.com/containerd/typeurl/v2 v2.2.2/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= +github.com/containernetworking/cni v1.2.3 h1:hhOcjNVUQTnzdRJ6alC5XF+wd9mfGIUaj8FuJbEslXM= +github.com/containernetworking/cni v1.2.3/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M= +github.com/containernetworking/plugins v1.2.0 h1:SWgg3dQG1yzUo4d9iD8cwSVh1VqI+bP7mkPDoSfP9VU= +github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+r/s9qIfPdqlLF4TW8c4= +github.com/coreos/butane v0.18.0 h1:WDeUC/dX1MUUVPwiqsQetQZsShNKk+2lrRXlC4ZhnZA= +github.com/coreos/butane v0.18.0/go.mod h1:oLR7xKzJB3o1WIdLtfHlPwrSwvMmeV/zknkwOznRu88= +github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb h1:rmqyI19j3Z/74bIRhuC59RB442rXUazKNueVpfJPxg4= +github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb/go.mod h1:rcFZM3uxVvdyNmsAV2jopgPD1cs5SPWJWU5dOz2LUnw= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/ignition/v2 v2.15.0 h1:v2fQ6QvkcAF+La5PHHpnpBS1eGZo+LYL1wTOPvDKAcs= +github.com/coreos/ignition/v2 v2.15.0/go.mod h1:+7BiKurzCFg3P427Ml0wqnKzIuhLimnil6LhFV2DkJM= +github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 h1:uSmlDgJGbUB0bwQBcZomBTottKwEDF5fF8UjSwKSzWM= +github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687/go.mod h1:Salmysdw7DAVuobBW/LwsKKgpyCPHUhjyJoMJD+ZJiI= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= +github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= +github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gaissmai/cidrtree v0.1.4 h1:/aYnv1LIwjtSDHNr1eNN99WJeh6vLrB+Sgr1tRMhHDc= +github.com/gaissmai/cidrtree v0.1.4/go.mod h1:nrjEeeMZmvoJpLcSvZ3qIVFxw/+9GHKi7wDHHmHKGRI= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= +github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cadvisor v0.52.1 h1:sC8SZ6jio9ds+P2dk51bgbeYeufxo55n0X3tmrpA9as= +github.com/google/cadvisor v0.52.1/go.mod h1:OAhPcx1nOm5YwMh/JhpUOMKyv1YKLRtS9KgzWPndHmA= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f h1:7MmqygqdeJtziBUpm4Z9ThROFZUaVGaePMfcDnluf1E= +github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f h1:5CjVwnuUcp5adK4gmY6i72gpVFVnZDP2h5TmPScB6u4= +github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk= +github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/k8snetworkplumbingwg/govdpa v0.1.5-0.20230926073613-07c1031aea47 h1:iSncnlC+rtlNOIpPa3fbqQMhpTscGJIlkiWaPl1VcS4= +github.com/k8snetworkplumbingwg/govdpa v0.1.5-0.20230926073613-07c1031aea47/go.mod h1:SPaDIyUmwN03Bgn0u/mhoiE4o/+koeKh11VUsdsUX0U= +github.com/k8snetworkplumbingwg/ipamclaims v0.5.1-alpha h1:RPXmPp07hm6l2/a3PSInne+6VnZeSapcFeegpDNklCs= +github.com/k8snetworkplumbingwg/ipamclaims v0.5.1-alpha/go.mod h1:MGaMX1tJ7MlHDee4/xmqp3guQh+eDiuCLAauqD9K11Q= +github.com/k8snetworkplumbingwg/multi-networkpolicy v1.0.1 h1:Egj1hEVYNXWFlKpgzAXxe/2o8VNiVcAJLrKzlinILQo= +github.com/k8snetworkplumbingwg/multi-networkpolicy v1.0.1/go.mod h1:kEJ4WM849yNmXekuSXLRwb+LaZ9usC06O8JgoAIq+f4= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7 h1:z4P744DR+PIpkjwXSEc6TvN3L6LVzmUquFgmNm8wSUc= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7/go.mod h1:CM7HAH5PNuIsqjMN0fGc1ydM74Uj+0VZFhob620nklw= +github.com/k8snetworkplumbingwg/sriovnet v1.2.1-0.20250818105516-24ab680f94f3 h1:uSGOz0UYNPduUVXLdAthKdRjIaaCUxN8j9R30Kx0JxQ= +github.com/k8snetworkplumbingwg/sriovnet v1.2.1-0.20250818105516-24ab680f94f3/go.mod h1:UnAcraX3CxamBrn9H/xCLngKOquy5DyGWiupn05x9Ag= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/libopenstorage/openstorage v1.0.0 h1:GLPam7/0mpdP8ZZtKjbfcXJBTIA/T1O6CBErVEFEyIM= +github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 h1:ql8x//rJsHMjS+qqEag8n3i4azw1QneKh5PieH9UEbY= +github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875/go.mod h1:kfOoFJuHWp76v1RgZCb9/gVUc7XdY877S2uVYbNliGc= +github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 h1:2oDp6OOhLxQ9JBoUuysVz9UZ9uI6oLUbvAZu0x8o+vE= +github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118/go.mod h1:ZFUnHIVchZ9lJoWoEGUg8Q3M4U8aNNWA3CVSUTkW4og= +github.com/mdlayher/ndp v1.0.1 h1:+yAD79/BWyFlvAoeG5ncPS0ItlHP/eVbH7bQ6/+LVA4= +github.com/mdlayher/ndp v1.0.1/go.mod h1:rf3wKaWhAYJEXFKpgF8kQ2AxypxVbfNcZbqoAo6fVzk= +github.com/mdlayher/packet v1.0.0 h1:InhZJbdShQYt6XV2GPj5XHxChzOfhJJOMbvnGAmOfQ8= +github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU= +github.com/mdlayher/socket v0.2.1 h1:F2aaOwb53VsBE+ebRS9bLd7yPOfYUMC8lOODdCBDY6w= +github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= +github.com/metallb/frr-k8s v0.0.21 h1:JLlCeXVlW5BLVdPy2u5sS9UCVlnK9x2vzWbIkxb8Atk= +github.com/metallb/frr-k8s v0.0.21/go.mod h1:VMnCZUVXYy7k0Fsa2L3XKwISFs3Thv0Uord7rSZPQZw= +github.com/metallb/metallb v0.14.9 h1:rjhftr7b0vv56c8pXm7UDo7ad61EwKT6lbGGocrN/VM= +github.com/metallb/metallb v0.14.9/go.mod h1:qUh1zVwYAfp3JLxhZrDH20j55QvYwCkI37QU4gUG3ns= +github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/cgroups v0.0.3 h1:Jc9dWh/0YLGjdy6J/9Ln8NM5BfTA4W2BY0GMozy3aDU= +github.com/opencontainers/cgroups v0.0.3/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runc v1.2.5 h1:8KAkq3Wrem8bApgOHyhRI/8IeLXIfmZ6Qaw6DNSLnA4= +github.com/opencontainers/runc v1.2.5/go.mod h1:dOQeFo29xZKBNeRBI0B19mJtfHv68YgCTh1X+YphA+4= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8= +github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20250916161632-d81c09058835 h1:rkqIIfdYYkasXbF2XKVgh/3f1mhjSQK9By8WtVMgYo8= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20250916161632-d81c09058835/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= +github.com/openshift-kni/k8sreporter v1.0.6 h1:aaxDzZx3s9bo1I3nopR63RGVZxcJgR94j5X87aDihYo= +github.com/openshift-kni/k8sreporter v1.0.6/go.mod h1:tX6LOg0m0oXje7WNLFo8LKHC9Ix8VV0a7vUc6eyeFBQ= +github.com/openshift/api v0.0.0-20251020135558-286504b695bc h1:hUPxgvR9S2Xt6IUeToTVxSr6mYfV0dyedmccLoSrbIo= +github.com/openshift/api v0.0.0-20251020135558-286504b695bc/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= +github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 h1:9JBeIXmnHlpXTQPi7LPmu1jdxznBhAE7bb1K+3D8gxY= +github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235/go.mod h1:L49W6pfrZkfOE5iC1PqEkuLkXG4W0BX4w8b+L2Bv7fM= +github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= +github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= +github.com/openshift/kubernetes v1.30.1-0.20251017123720-96593f323733 h1:Mpab1CmJPLVWGB0CNGoWnup/NScvv55MVPe94c8JgUk= +github.com/openshift/kubernetes v1.30.1-0.20251017123720-96593f323733/go.mod h1:w3+IfrXNp5RosdDXg3LB55yijJqR/FwouvVntYHQf0o= +github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251017123720-96593f323733 h1:42lm41QwjG8JoSicx4FHcuIG2kxHxlUnz6c+ftg2e0E= +github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251017123720-96593f323733/go.mod h1:sRDdfB9W3pU52PnpjJ9RuMVsg/UQ5iLNlVfbRpb250o= +github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251017123720-96593f323733 h1:8NT55In5sAdZVLSDm4jyZ7Q7Gi/DTw/Tns5OQtN4i1w= +github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251017123720-96593f323733/go.mod h1:ZxysqjDkqvJUamd823zDHJXKD4X19Q1HFmkLG63o9eU= +github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251017123720-96593f323733 h1:f/lXWnFFn8f5CKE0obK8PRC4l7fDzmncfvKVxJLBdoU= +github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251017123720-96593f323733/go.mod h1:c6W+CrhzWKfUpUBjoSx/88x7wmaGRznQEcR6jN1H3Tg= +github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251017123720-96593f323733 h1:tUxTpKWhjvFJey3guoLabrkNjNKfrBVl7VFraLon91Q= +github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251017123720-96593f323733/go.mod h1:TRSSqgXggJaDK5vtVtlQ9wEYOk32Pl+9tf0ROf3ljiM= +github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251017123720-96593f323733 h1:62i8XkBwvTM7d9P+1la2JVsuuLMxtJCCd2jR3xkjAj0= +github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251017123720-96593f323733/go.mod h1:pqajivnjOqvKyXx5bPYITDe/uBLBA+Tk6f8E01CGcA4= +github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251017123720-96593f323733 h1:M3wl3m7qduIVzMNYvlXcy+S7dWmD3LZjn/7sbDaxgUM= +github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251017123720-96593f323733/go.mod h1:46jYZR2jZ3bmcRXZPZzHfJLFD7qR44/AcZ72oiAGVsQ= +github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251017123720-96593f323733 h1:fY15tmTbBFYtxIiv3LldWyJHlNWOqUWdWxz523Z3dF4= +github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251017123720-96593f323733/go.mod h1:TYThr4NC8GXH90tsn+yCMH6LiXHj7pGNijDwBN6ZsG0= +github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251017123720-96593f323733 h1:q11kjR6cnzaAh57kaH81Obl5hHFqnVwkD1WOUnvj+Go= +github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251017123720-96593f323733/go.mod h1:/yyEEP5EdBUI2dmEyMKzS9XDXrKQBD1Q3G/UFGyBIy0= +github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251017123720-96593f323733 h1:cK+a41pyUZ7FsJZAiExYXVJc4X8hV4TIgeE/lSRwMWQ= +github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251017123720-96593f323733/go.mod h1:uIPPF88dUOgzUajix3EMCWGA4YChCoOo8ikkPyhwDnI= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251017123720-96593f323733 h1:fSYRAWS1LindhtDYmUjZhoC9lyvHi/H2UF3ammAd4Mc= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251017123720-96593f323733/go.mod h1:SrD2bRkLK0Fra2C8qzzuRWciVrAkVq6qKgQZqY+psvs= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251017123720-96593f323733 h1:EU9R8OFlDHiROJ3MTMXtYM4yrNlhqxo9x7fXECXopAo= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251017123720-96593f323733/go.mod h1:oiryEAfmSayRHtdki0nmpAjQfku0aP4Y+0NIqaqRn3E= +github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251017123720-96593f323733 h1:uFbdQh5m7QGyjpxoiRIgubC3NKlfG/IK7vFTmxgwIEE= +github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251017123720-96593f323733/go.mod h1:jYAZWKz2s5rQTVDh35tpx466iVAoZO+JvuNkt8h2um4= +github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251017123720-96593f323733 h1:VG4UthsljFwvUiDbcpMtw5XOrelJrxU5sVVxBJwLzHY= +github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251017123720-96593f323733/go.mod h1:DdewGEPN49xRm+9KnI5T8nFsDKjSVAfyWtLO7H6Mlsc= +github.com/openshift/kubernetes/staging/src/k8s.io/kms v0.0.0-20251017123720-96593f323733 h1:dPOkP7XiH47lKh+CUecAa+NTMWyD5Jfy4xKTyssZZJk= +github.com/openshift/kubernetes/staging/src/k8s.io/kms v0.0.0-20251017123720-96593f323733/go.mod h1:nBKbnRrEbqRdLb6RWnxuho9oSqgJ4jbxPeIcjx7Ju90= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251017123720-96593f323733 h1:YXrEzhBTClZ195q3eQl6LUtKjnyBGMso6K4HgxLyj1w= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251017123720-96593f323733/go.mod h1:pTSelVB5l12qziWSDxi77oc4P2t5N0dxYhhj4uxjxiM= +github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251017123720-96593f323733 h1:qRJFpBOLD0wpDvUczHdBZgG7pDe07O5QJiGjbzKEM0k= +github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251017123720-96593f323733/go.mod h1:EU/sHfUc/w62dGZ1VmEysozxDAFvARFrIcQmHEmObaY= +github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251017123720-96593f323733 h1:Us43/HbC/zmntiPVmZF38Y37Vnk5SAqUcV1Z89hSUUM= +github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251017123720-96593f323733/go.mod h1:+bTwPbT5dZB8j6eKQHBZRMfNmY6MEryba1wQljr9VWw= +github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251017123720-96593f323733 h1:J8bbx4ZSR4AKl9MYuX9xG66d8Ehjre/v7pxKLKU5y7c= +github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251017123720-96593f323733/go.mod h1:T7oGB72dQtHfSIJWZmeNU4Xo5QvIpzIuJ8X20TWu628= +github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251017123720-96593f323733 h1:2vQPmqKwQU+jpqm7Iv3EU3k8DYYNqZwN/A1AdydMYpc= +github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251017123720-96593f323733/go.mod h1:yuCdx9wLndqpNhmsYZh48wtbgrqc8ql1191ke9zIOfg= +github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 h1:bANtDc8SgetSK4nQehf59x3+H9FqVJCprgjs49/OTg0= +github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5/go.mod h1:OlFFws1AO51uzfc48MsStGE4SFMWlMZD0+f5a/zCtKI= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/safchain/ethtool v0.3.1-0.20231027162144-83e5e0097c91 h1:q815fjV3G+4JvXNo2VwT2m+/msMU0sUkCK68CgHV9Y8= +github.com/safchain/ethtool v0.3.1-0.20231027162144-83e5e0097c91/go.mod h1:qIWCTaK0xQlXNlNlIVoZjKMZFopqfMZcg4JcRqGoYc0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/trozet/go-iptables v0.0.0-20240328221912-077e672b3808 h1:dhannoFtRVRqYF22YA/M0iUD0IQhB9PmT9XRuynUWxg= +github.com/trozet/go-iptables v0.0.0-20240328221912-077e672b3808/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= +github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= +github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/ziutek/telnet v0.0.0-20180329124119-c3b780dc415b/go.mod h1:IZpXDfkJ6tWD3PhBK5YzgQT+xJWh7OsdwiG8hA2MkO4= +go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= +go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= +go.etcd.io/etcd/api/v3 v3.6.4 h1:7F6N7toCKcV72QmoUKa23yYLiiljMrT4xCeBL9BmXdo= +go.etcd.io/etcd/api/v3 v3.6.4/go.mod h1:eFhhvfR8Px1P6SEuLT600v+vrhdDTdcfMzmnxVXXSbk= +go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= +go.etcd.io/etcd/client/pkg/v3 v3.6.4/go.mod h1:sbdzr2cl3HzVmxNw//PH7aLGVtY4QySjQFuaCgcRFAI= +go.etcd.io/etcd/client/v3 v3.6.4 h1:YOMrCfMhRzY8NgtzUsHl8hC2EBSnuqbR3dh84Uryl7A= +go.etcd.io/etcd/client/v3 v3.6.4/go.mod h1:jaNNHCyg2FdALyKWnd7hxZXZxZANb0+KGY+YQaEMISo= +go.etcd.io/etcd/pkg/v3 v3.6.4 h1:fy8bmXIec1Q35/jRZ0KOes8vuFxbvdN0aAFqmEfJZWA= +go.etcd.io/etcd/pkg/v3 v3.6.4/go.mod h1:kKcYWP8gHuBRcteyv6MXWSN0+bVMnfgqiHueIZnKMtE= +go.etcd.io/etcd/server/v3 v3.6.4 h1:LsCA7CzjVt+8WGrdsnh6RhC0XqCsLkBly3ve5rTxMAU= +go.etcd.io/etcd/server/v3 v3.6.4/go.mod h1:aYCL/h43yiONOv0QIR82kH/2xZ7m+IWYjzRmyQfnCAg= +go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= +go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 h1:KemlMZlVwBSEGaO91WKgp41BBFsnWqqj9sKRwmOqC40= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= +go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= +golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= +golang.org/x/telemetry v0.0.0-20250710130107-8d8967aff50b/go.mod h1:4ZwOYna0/zsOKwuR5X/m0QFOJpSZvAxFfkQT+Erd9D4= +golang.org/x/telemetry v0.0.0-20250807160809-1a19826ec488/go.mod h1:fGb/2+tgXXjhjHsTNdVEEMZNWA0quBnfrO+AfoDSAKw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f h1:1FTH6cpXFsENbPR5Bu8NQddPSaUUE6NA2XdZdDSAJK4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/code-generator v0.22.7/go.mod h1:iOZwYADSgFPNGWfqHFfg1V0TNJnl1t0WyZluQp4baqU= +k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v1.4.0 h1:dDLyQLSp9obzsDrv3cyL1olIc/66IWVaGiR3gfPfgT0= +kubevirt.io/api v1.4.0/go.mod h1:qcnumjJeOCo+qdYXf0OjpHGMhad0SAn4i0h6IAP+6Eg= +kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 h1:IWo12+ei3jltSN5jQN1xjgakfvRSF3G3Rr4GXVOOy2I= +kubevirt.io/containerized-data-importer-api v1.57.0-alpha1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 h1:qPrZsv1cwQiFeieFlRqT627fVZ+tyfou/+S5S0H5ua0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.22.1 h1:Ah1T7I+0A7ize291nJZdS1CabF/lB4E++WizgV24Eqg= +sigs.k8s.io/controller-runtime v0.22.1/go.mod h1:FwiwRjkRPbiN+zp2QRp7wlTCzbUXxZ/D4OzuQUDwBHY= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/network-policy-api v0.1.5 h1:xyS7VAaM9EfyB428oFk7WjWaCK6B129i+ILUF4C8l6E= +sigs.k8s.io/network-policy-api v0.1.5/go.mod h1:D7Nkr43VLNd7iYryemnj8qf0N/WjBzTZDxYA+g4u1/Y= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v6 v6.2.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/openshift/hack/build-tests-ext.sh b/openshift/hack/build-tests-ext.sh new file mode 100755 index 0000000000..420681c354 --- /dev/null +++ b/openshift/hack/build-tests-ext.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +HERE=$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")") +ROOT=$(readlink --canonicalize "$HERE/..") +OUTPUT="${ROOT}/bin" +mkdir -vp "$OUTPUT" +if [[ -z "$(command -v go)" ]]; then + cat < 0 { + for _, s := range generator.errors { + fmt.Fprintf(os.Stderr, "appending label failed: %s\n", s) + } + os.Exit(1) + } + ginkgo.GetSuite().WalkTests(generator.appendLabelsToTestName) + if len(generator.missing) > 0 { + var names []string + for name := range generator.missing { + names = append(names, name) + } + sort.Strings(names) + fmt.Fprintf(os.Stderr, "appending label failed:\n%s\n", strings.Join(names, "\n")) + os.Exit(1) + } + + var pairs []string + for testName, labels := range generator.output { + if filter(fmt.Sprintf("%s%s", testName, labels)) { + continue + } + pairs = append(pairs, fmt.Sprintf("%q:\n%q,", testName, labels)) + } + sort.Strings(pairs) + contents := fmt.Sprintf(` +package generated + +import ( + "fmt" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" +) + +var AppendedAnnotations = map[string]string{ +%s +} + +func init() { + ginkgo.GetSuite().SetAnnotateFn(func(name string, node types.TestSpec) { + if newLabels, ok := AppendedAnnotations[name]; ok { + node.AppendText(newLabels) + } else { + panic(fmt.Sprintf("unable to find test %%s", name)) + } + }) +} +`, strings.Join(pairs, "\n\n")) // double space between to ease readability + generatedAnnotationsFileName := os.Args[len(os.Args)-1] + if err := os.WriteFile(generatedAnnotationsFileName, []byte(contents), 0644); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to write generated annotations file at path %q: %v", generatedAnnotationsFileName, err) + os.Exit(1) + } + if _, err := exec.Command("gofmt", "-s", "-w", generatedAnnotationsFileName).Output(); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to format golang file at path %q: %v", generatedAnnotationsFileName, err) + os.Exit(1) + } +} + +// labelAppender is used to select tests and append labels to a ginkgo test name. It can select tests based on labels or regex or +// substring match. Prepended labels to a test name is not supported because ginkgo API does not allow it. +type labelAppender struct { + // all labels we care about applying rules to and defined in openshift/test/pkg/annotate/rules.go derived from LabelToTestNameMatchMaps & LabelToTestNameMatchMaps + labels []string + // substrings match to apply a particular label derived from LabelToTestNameMatchMaps map + stringMatches map[string][]string + // regular expressions match to apply a particular label derived from LabelToTestNameMatchMaps map + matches map[string]*regexp.Regexp + // regular expressions match + // see ExcludedTests in openshift/test/pkg/annotate/rules.go + excludedTestsFilter *regexp.Regexp + // key is label which maps to a list of labels defined in openshift/test/pkg/annotate/rules.go LabelToLabelMaps map + labelToLabel map[string][]string + + // output from the generator and also input for appendLabelsToTestName + output map[string]string + // map of unmatched test names + missing map[string]struct{} + // a list of errors to display + errors []string +} + +func newGenerator(labelToTestNames, labelToLabels map[string][]string) *labelAppender { + allLabelsSet := sets.New[string]() + matches := make(map[string]*regexp.Regexp) + stringMatches := make(map[string][]string) + for _, labels := range labelToLabels { + allLabelsSet.Insert(labels...) + } + + for label, matchTestNames := range labelToTestNames { + sort.Strings(matchTestNames) + allLabelsSet.Insert(label) + var remain []string + for _, matchTestName := range matchTestNames { + // matchTestName maybe regex expression or partial test name or full test name + re := regexp.MustCompile(matchTestName) + if p, ok := re.LiteralPrefix(); ok { + stringMatches[label] = append(stringMatches[label], p) + } else { + remain = append(remain, matchTestName) + } + } + if len(remain) > 0 { + matches[label] = regexp.MustCompile(strings.Join(remain, `|`)) + } + } + + excludedTestsFilter := regexp.MustCompile(strings.Join(ExcludedTests, `|`)) + + return &labelAppender{ + labels: sets.List(allLabelsSet), + stringMatches: stringMatches, + matches: matches, + labelToLabel: labelToLabels, + excludedTestsFilter: excludedTestsFilter, + output: make(map[string]string), + } +} + +func (r *labelAppender) appendLabelsToTestName(name string, node types.TestSpec) { + if newLabels, ok := r.output[name]; ok { + node.AppendText(newLabels) + } else { + r.missing[name] = struct{}{} + } +} + +func (r *labelAppender) generate(name string, node types.TestSpec) { + newLabels := "" + newName := name + + for { + count := 0 + for _, label := range r.labels { + // skip processing this label if it already exists in the test name + if strings.Contains(newName, label) { + continue + } + var isLabelRequired bool + // check if there is a substring match from the test name + for _, segment := range r.stringMatches[label] { + isLabelRequired = strings.Contains(newName, segment) + if isLabelRequired { + break + } + } + // check if there is a match from the test name using a regex + if !isLabelRequired { + if re := r.matches[label]; re != nil { + isLabelRequired = r.matches[label].MatchString(newName) + } + } + // check to see if the label is present - label is extracted from ginkgo labels + if !isLabelRequired { + if potentialLabels, ok := r.labelToLabel[label]; ok { + for _, potentialLabel := range potentialLabels { + isLabelRequired = containsLabel(node.Labels(), potentialLabel) + if isLabelRequired { + break + } + } + } + } + + if isLabelRequired { + count++ + newLabels += label + newName += label + } + } + if count == 0 { + break + } + } + + // Append suite name to test, if it doesn't already have one + if !r.excludedTestsFilter.MatchString(newName) && !strings.Contains(newName, "[Suite:") { + isSerial := strings.Contains(newName, "[Serial]") + isConformance := strings.Contains(newName, "[Conformance]") + switch { + case isSerial && isConformance: + newLabels += "[Suite:openshift/conformance/serial/minimal]" + case isSerial: + newLabels += "[Suite:openshift/conformance/serial]" + case isConformance: + newLabels += "[Suite:openshift/conformance/parallel/minimal]" + default: + newLabels += "[Suite:openshift/conformance/parallel]" + } + } + + if err := checkBalancedBrackets(newName); err != nil { + r.errors = append(r.errors, err.Error()) + } + + r.output[name] = newLabels +} + +// checkBalancedBrackets ensures that square brackets are balanced in generated test +// names. If they are not, it returns an error with the name of the test and a guess +// where the unmatched bracket(s) are. +func checkBalancedBrackets(testName string) error { + stack := make([]int, 0, len(testName)) + for idx, c := range testName { + switch c { + case '[': + stack = append(stack, idx) + case ']': + // case when we start off with a ] + if len(stack) == 0 { + stack = append(stack, idx) + } else { + stack = stack[:len(stack)-1] + } + } + } + + if len(stack) > 0 { + msg := testName + "\n" + outerLoop: + for i := 0; i < len(testName); i++ { + for _, loc := range stack { + if i == loc { + msg += "^" + continue outerLoop + } + } + msg += " " + } + return fmt.Errorf("unbalanced brackets in test name:\n%s", msg) + } + + return nil +} + +// containLabel labels return true if labels are equal. Input args labels slice and/or candidate label may or may not contain brackets. +func containsLabel(labels []string, candidateLabel string) bool { + if len(labels) == 0 { + return false + } + for _, label := range labels { + if stripBrackets(label) == stripBrackets(candidateLabel) { + return true + } + } + return false +} + +func stripBrackets(s string) string { + return strings.TrimFunc(s, func(r rune) bool { + if r == '[' || r == ']' { + return true + } + return false + }) +} diff --git a/openshift/test/annotate/rules.go b/openshift/test/annotate/rules.go new file mode 100644 index 0000000000..c9fba6c3ae --- /dev/null +++ b/openshift/test/annotate/rules.go @@ -0,0 +1,144 @@ +package annotate + +import ( + // ensure all the ginkgo tests are loaded + _ "github.com/ovn-org/ovn-kubernetes/test/e2e" +) + +var ( + // LabelToLabelMaps label -> label (ginkgo label) + // E2E tests are written with the support of ginkgo. ginkgo tests may contain Labels. + LabelToLabelMaps = map[string][]string{ + "[Disabled:Unimplemented]": { + `[Feature:Service]`, + `[Feature:NetworkPolicy]`, + `[Feature:AdminNetworkPolicy]`, + `[Feature:BaselineNetworkPolicy]`, + `[Feature:EgressIP]`, + `[Feature:EgressService]`, + `[Feature:EgressFirewall]`, + `[Feature:EgressQos]`, + `[Feature:ExternalGateway]`, + `[Feature:DisablePacketMTUCheck]`, + `[Feature:VirtualMachineSupport]`, + `[Feature:Interconnect]`, + `[Feature:Multicast]`, + `[Feature:MultiHoming]`, + `[Feature:NetworkConnect]`, + `[Feature:NodeIPMACMigration]`, + `[Feature:OVSCPUPin]`, + `[Feature:Unidle]`, + `[Feature:RouteAdvertisements]`, + }, + } + // if a test name partially or fully contains one of the map value strings, then add the label to the test + // label -> partial or full test name or regex to match a test name + LabelToTestNameMatchMaps = map[string][]string{ + // alpha features that are not gated + "[Disabled:Alpha]": {}, + // tests for features that are not implemented in openshift + "[Disabled:Unimplemented]": { + `Creating a static pod on a node Should successfully create then remove a static pod`, + `Pod to external server PMTUD`, + `Pod to pod TCP with low MTU`, + `blocking ICMP needs frag`, + // UDN test requires egress + `pod2Egress on a user defined primary network`, + `is isolated from the default network`, + // requires host net port collision avoidance + `EndpointSlices mirroring`, + // reference kind nodes + `Should validate connectivity within a namespace of pods on separate nodes`, + // tied to KinD / container runtime + `e2e delete databases`, + `test e2e inter-node connectivity between worker nodes`, + `e2e control plane`, + `test e2e pod connectivity to host addresses`, + `e2e br-int flow monitoring export validation`, + `e2e ingress to host-networked pods traffic validation`, + `e2e ingress traffic validation`, + // pods dont drop privileges + `Should validate the hairpinned traffic is always allowed`, + // refactor to give pod sufficient privs for tcpdump + `should be able to receive multicast IGMP query`, + // refactor to give pod sufficient privs + `UDN Pod should react to k8s.ovn.org/open-default-ports annotations changes`, + // load balancer isn't becoming available from the cloud services. Ensure, we provider the correct provider to the k8 api which spawns the ext LB. + `services on a user defined primary network should be reachable through their cluster IP, node port and load balancer L3 primary UDN, cluster-networked pods, NodePort service`, + `services on a user defined primary network should be reachable through their cluster IP, node port and load balancer L2 primary UDN, cluster-networked pods, NodePort service`, + // test gets interrupted when test timeout expires while testing pod and lack of pod connectivity. + `a user defined primary network created using ClusterUserDefinedNetwork isolates overlapping CIDRs with L3 primary UDN`, + `a user defined primary network created using NetworkAttachmentDefinitions isolates overlapping CIDRs with L3 primary UDN`, + // tests are tied to KinD deployment and select nodes based on KinD deployment. Needs refactoring. + `allow ingress traffic to one pod from a particular namespace`, + // private image in upstream test & need privs for tcpdump + `e2e NetworkQoS validation`, + // ClusterNetworkConnect CR is not in downstream yet + `ClusterNetworkConnect: API validations`, + // unknown rc 7 code + `Network Segmentation: API validations`, + // 'Network allocation failed for at least one node' + `Network Segmentation UserDefinedNetwork CRD Controller should correctly report subsystem error on node subnet allocation`, + // requires implementation of overlay method (provider API) + `Network Segmentation: Localnet using ClusterUserDefinedNetwork CR, pods in different namespaces, should communicate over localnet topology`, + // requires implementation of the SetupUnderlay() method + `Network Segmentation: Localnet should preserve LSPs for IPAM-less localnet pods after ovnkube-node restart`, + // pods dont drop privileges + `should be able to send multicast UDP traffic between nodes`, + // TODO: fix the flakiness with net-seg overlapping CIDRs test in downstream. + "isolates overlapping CIDRs", + // tied to only kind cluster + "Node Shutdown and Startup", + // TODO: Fix flakiness in this test. Pod connectivity checks may need + // to be wrapped in an Eventually block. + "perform east/west traffic between nodes following OVN Kube node pod restart", + }, + // tests that rely on special configuration that we do not yet support + "[Disabled:SpecialConfig]": {}, + // tests that are known broken and need to be fixed upstream or in openshift + // always add an issue here + "[Disabled:Broken]": {}, + // tests that need to be temporarily disabled while the rebase is in progress. + "[Disabled:RebaseInProgress]": {}, + // tests that may work, but we don't support them + "[Disabled:Unsupported]": {}, + // tests too slow to be part of conformance + "[Slow]": {}, + // tests that are known flaky + "[Flaky]": {}, + // tests that must be run without competition + "[Serial]": {}, + // Tests that don't pass on disconnected, either due to requiring + // internet access for GitHub (e.g. many of the s2i builds), or + // because of pullthrough not supporting ICSP (https://bugzilla.redhat.com/show_bug.cgi?id=1918376) + "[Skipped:Disconnected]": {}, + "[Skipped:alibabacloud]": {}, + "[Skipped:aws]": {}, + "[Skipped:azure]": {}, + "[Skipped:baremetal]": {}, + "[Skipped:gce]": {}, + "[Skipped:ibmcloud]": {}, + "[Skipped:kubevirt]": {}, + "[Skipped:nutanix]": {}, + "[Skipped:openstack]": {}, + "[Skipped:ovirt]": {}, + "[Skipped:vsphere]": {}, + // These tests are skipped when openshift-tests needs to use a proxy to reach the + // cluster -- either because the test won't work while proxied, or because the test + // itself is testing a functionality using it's own proxy. + "[Skipped:Proxy]": {}, + "[Skipped:SingleReplicaTopology]": {}, + // Tests which can't be run/don't make sense to run against a cluster with all optional capabilities disabled + "[Skipped:NoOptionalCapabilities]": {}, + "[Skipped:ibmroks]": {}, + } + + ExcludedTests = []string{ + `\[Disabled:`, + `\[Disruptive\]`, + `\[Skipped\]`, + `\[Slow\]`, + `\[Flaky\]`, + `\[Local\]`, + } +) diff --git a/openshift/test/blocking_tests.go b/openshift/test/blocking_tests.go new file mode 100644 index 0000000000..1e45e1164a --- /dev/null +++ b/openshift/test/blocking_tests.go @@ -0,0 +1,63 @@ +package test + +// BlockingTests lists tests that are considered stable and should block CI jobs if they fail. +// Tests NOT in this list or explicitly "Disabled" in annotations will be marked as "informing" +// - they run but failures don't fail the job. +// +// To graduate a test from informing to blocking: +// 1. Add the full test name to this slice (with proper quotes and comma) +// 2. Rebuild: ./hack/build-tests-ext.sh +// 3. Verify: ./bin/ovn-kubernetes-tests-ext list tests | jq -r '.[] | select(.name == "test name here") | .lifecycle' +// +// Used by: openshift/cmd/ovn-kubernetes-tests-ext/main.go +var BlockingTests = []string{ + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L2 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L2 primary UDN with custom network [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L3 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork creates a networkStatus Annotation with UDN interface L2 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork creates a networkStatus Annotation with UDN interface L2 primary UDN with custom network [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork creates a networkStatus Annotation with UDN interface L3 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions can perform east/west traffic between nodes two pods connected over a L2 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions can perform east/west traffic between nodes two pods connected over a L2 primary UDN with custom network [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions can perform east/west traffic between nodes two pods connected over a L3 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions creates a networkStatus Annotation with UDN interface L2 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions creates a networkStatus Annotation with UDN interface L2 primary UDN with custom network [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions creates a networkStatus Annotation with UDN interface L3 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using UserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L2 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using UserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L2 primary UDN with custom network [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using UserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L3 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using UserDefinedNetwork creates a networkStatus Annotation with UDN interface L2 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using UserDefinedNetwork creates a networkStatus Annotation with UDN interface L2 primary UDN with custom network [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network created using UserDefinedNetwork creates a networkStatus Annotation with UDN interface L3 primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation a user defined primary network doesn't cause network name conflict [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation ClusterUserDefinedNetwork CRD Controller pod connected to ClusterUserDefinedNetwork CR & managed NADs cannot be deleted when being used [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation ClusterUserDefinedNetwork CRD Controller should create NAD according to spec in each target namespace and report active namespaces [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation ClusterUserDefinedNetwork CRD Controller should create NAD in new created namespaces that apply to namespace-selector [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation ClusterUserDefinedNetwork CRD Controller when CR is deleted, should delete all managed NAD in each target namespace [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation ClusterUserDefinedNetwork CRD Controller when namespace-selector is mutated should create NAD in namespaces that apply to mutated namespace-selector [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation ClusterUserDefinedNetwork CRD Controller when namespace-selector is mutated should delete managed NAD in namespaces that no longer apply to namespace-selector [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Default network multus annotation ValidatingAdmissionPolicy protection should prevent adding, modifying and removing the default-network annotation on existing pods [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Default network multus annotation when added with static IP and MAC to a pod belonging to primary UDN should create the pod with the specified static IP and MAC address without persistent IPAM enabled [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Default network multus annotation when added with static IP and MAC to a pod belonging to primary UDN should create the pod with the specified static IP and MAC address with persistent IPAM [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Network Policies on a user defined primary network pods within namespace should be isolated when deny policy is present in L2 dualstack primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Network Policies on a user defined primary network pods within namespace should be isolated when deny policy is present in L2 dualstack primary UDN with custom network [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Network Policies on a user defined primary network pods within namespace should be isolated when deny policy is present in L3 dualstack primary UDN [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN duplicate IP validation with primary UDN layer 2 pods should fail when creating second pod with duplicate static IP IPv4 duplicate [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN duplicate IP validation with primary UDN layer 2 pods should fail when creating second pod with duplicate static IP IPv6 duplicate [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN should respect network configuration Layer2 basic configuration [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN should respect network configuration Layer2 with custom subnets [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN should respect network configuration Layer2 with inverted gateway/management IPs [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN unmasked reserved / infrastructure subnets are not allowed Layer2 with unmasked IPv4 infrastructure subnets [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN unmasked reserved / infrastructure subnets are not allowed Layer2 with unmasked IPv4 reserved subnets [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN unmasked reserved / infrastructure subnets are not allowed Layer2 with unmasked IPv6 infrastructure subnets [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: Preconfigured Layer2 UDN unmasked reserved / infrastructure subnets are not allowed Layer2 with unmasked IPv6 reserved subnets [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation: services on a user defined primary network should be reachable through their cluster IP, node port and load balancer L2 primary UDN with custom network, cluster-networked pods, NodePort service [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation UserDefinedNetwork CRD Controller for L2 secondary network pod connected to UserDefinedNetwork cannot be deleted when being used [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation UserDefinedNetwork CRD Controller for L2 secondary network should create NetworkAttachmentDefinition according to spec [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation UserDefinedNetwork CRD Controller for L2 secondary network should delete NetworkAttachmentDefinition when UserDefinedNetwork is deleted [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation UserDefinedNetwork CRD Controller for primary UDN without required namespace label should be able to create pod and it will attach to the cluster default network [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation UserDefinedNetwork CRD Controller for primary UDN without required namespace label should not be able to update the namespace and add the UDN label [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation UserDefinedNetwork CRD Controller for primary UDN without required namespace label should not be able to update the namespace and remove the UDN label [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation when primary network exist, ClusterUserDefinedNetwork status should report not-ready [Suite:openshift/conformance/parallel]", + "[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] Network Segmentation when primary network exist, UserDefinedNetwork status should report not-ready [Suite:openshift/conformance/parallel]", +} diff --git a/openshift/test/deploymentconfig/openshift.go b/openshift/test/deploymentconfig/openshift.go new file mode 100644 index 0000000000..27e074da64 --- /dev/null +++ b/openshift/test/deploymentconfig/openshift.go @@ -0,0 +1,53 @@ +package deploymentconfig + +import ( + "fmt" + "strings" + + "github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig/api" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func IsOpenShift(config *rest.Config) (bool, error) { + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return false, fmt.Errorf("failed to create kubernetes client: %w", err) + } + // Check for OpenShift-specific API groups + groups, err := kubeClient.Discovery().ServerGroups() + if err != nil { + return false, fmt.Errorf("failed to get server groups: %w", err) + } + for _, group := range groups.Groups { + if strings.HasSuffix(group.Name, ".openshift.io") { + return true, nil + } + } + return false, nil +} + +type openshift struct{} + +func New() api.DeploymentConfig { + return openshift{} +} + +func (m openshift) OVNKubernetesNamespace() string { + return "openshift-ovn-kubernetes" +} + +func (m openshift) FRRK8sNamespace() string { + return "openshift-frr-k8s" +} + +func (m openshift) ExternalBridgeName() string { + return "br-ex" +} + +func (m openshift) PrimaryInterfaceName() string { + // support only for baremetald which expects the following interface name + // TODO; dynamically look up primary interface name instead of hardcoding it to baremetald env + return "enp0s3" +} diff --git a/openshift/test/e2e_test.go b/openshift/test/e2e_test.go new file mode 100644 index 0000000000..914aec0929 --- /dev/null +++ b/openshift/test/e2e_test.go @@ -0,0 +1,11 @@ +package test + +import ( + // import OVN-Kubernetes E2Es + _ "github.com/ovn-org/ovn-kubernetes/test/e2e" + + // Ensure that logging flags are part of the command line. + _ "k8s.io/component-base/logs/testinit" +) + +//go:generate go run -mod vendor ../cmd/annotate ./generated/zz_generated.annotations.go diff --git a/openshift/test/generated/zz_generated.annotations.go b/openshift/test/generated/zz_generated.annotations.go new file mode 100644 index 0000000000..4929f618f5 --- /dev/null +++ b/openshift/test/generated/zz_generated.annotations.go @@ -0,0 +1,1815 @@ +package generated + +import ( + "fmt" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" +) + +var AppendedAnnotations = map[string]string{ + "ACL Logging for AdminNetworkPolicy and BaselineAdminNetworkPolicy the ANP ACL logs have the expected log level": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when an invalid value is provided to the allow rule when the allowed destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when an invalid value is provided to the allow rule when the denied destination is poked the logs should have the expected log level": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when both the namespace's ACL logging deny and allow annotation are set to \"\" when the allowed destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when both the namespace's ACL logging deny and allow annotation are set to \"\" when the denied destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when both the namespace's ACL logging deny and allow annotation are set to \"invalid\" when the allowed destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when both the namespace's ACL logging deny and allow annotation are set to \"invalid\" when the denied destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace is brought up with the initial ACL log severity when the allowed destination is poked the logs should have the expected log level": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace is brought up with the initial ACL log severity when the denied destination is poked the logs should have the expected log level": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's ACL logging allow annotation is removed when the allowed destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's ACL logging allow annotation is removed when the denied destination is poked the logs should have the expected log level": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's ACL logging annotation cannot be parsed when the allowed destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's ACL logging annotation cannot be parsed when the denied destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's ACL logging annotation is updated when the allowed destination is poked the logs should have the expected log level": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's ACL logging annotation is updated when the denied destination is poked the logs should have the expected log level": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's entire ACL logging annotation is removed when the allowed destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's entire ACL logging annotation is removed when the denied destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's entire ACL logging annotation is set to {} when the allowed destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for EgressFirewall when the namespace's entire ACL logging annotation is set to {} when the denied destination is poked there should be no trace in the ACL logs": "[Disabled:Unimplemented]", + + "ACL Logging for NetworkPolicy the logs have the expected log level": "[Disabled:Unimplemented]", + + "ACL Logging for NetworkPolicy when the namespace's ACL allow and deny logging annotations are set to invalid values ACL logging is disabled": "[Disabled:Unimplemented]", + + "ACL Logging for NetworkPolicy when the namespace's ACL logging annotation is removed ACL logging is disabled": "[Disabled:Unimplemented]", + + "ACL Logging for NetworkPolicy when the namespace's ACL logging annotation is updated the ACL logs are updated accordingly": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network Can reach KAPI service": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It can be reached by an external server on the same network When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It can be reached by an external server on the same network When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It can reach an external server on the same network When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It can reach an external server on the same network When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It cannot be reached by a cluster node When it is a different node When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It cannot be reached by a cluster node When it is a different node When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It cannot be reached by a cluster node When it is the same node When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It cannot be reached by a cluster node When it is the same node When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It cannot be reached by an external server on a different network When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It cannot be reached by an external server on a different network When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It cannot reach an external server on a different network When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network It cannot reach an external server on a different network When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When other pod runs on the tested network On a different node Backing a ClusterIP service The first pod can reach the ClusterIP service on the same network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When other pod runs on the tested network On a different node Backing a ClusterIP service The first pod can reach the ClusterIP service on the same network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When other pod runs on the tested network On a different node The pods on the tested network can reach each other When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When other pod runs on the tested network On a different node The pods on the tested network can reach each other When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When other pod runs on the tested network On the same node Backing a ClusterIP service The first pod can reach the ClusterIP service on the same network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When other pod runs on the tested network On the same node Backing a ClusterIP service The first pod can reach the ClusterIP service on the same network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When other pod runs on the tested network On the same node The pods on the tested network can reach each other When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When other pod runs on the tested network On the same node The pods on the tested network can reach each other When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 2 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network Can reach KAPI service": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It can be reached by an external server on the same network When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It can be reached by an external server on the same network When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It can reach an external server on the same network When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It can reach an external server on the same network When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It cannot be reached by a cluster node When it is a different node When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It cannot be reached by a cluster node When it is a different node When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It cannot be reached by a cluster node When it is the same node When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It cannot be reached by a cluster node When it is the same node When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It cannot be reached by an external server on a different network When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It cannot be reached by an external server on a different network When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It cannot reach an external server on a different network When the network is IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network It cannot reach an external server on a different network When the network is IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When other pod runs on the tested network On a different node Backing a ClusterIP service The first pod can reach the ClusterIP service on the same network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When other pod runs on the tested network On a different node Backing a ClusterIP service The first pod can reach the ClusterIP service on the same network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When other pod runs on the tested network On a different node The pods on the tested network can reach each other When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When other pod runs on the tested network On a different node The pods on the tested network can reach each other When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When other pod runs on the tested network On the same node Backing a ClusterIP service The first pod can reach the ClusterIP service on the same network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When other pod runs on the tested network On the same node Backing a ClusterIP service The first pod can reach the ClusterIP service on the same network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When other pod runs on the tested network On the same node The pods on the tested network can reach each other When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When other pod runs on the tested network On the same node The pods on the tested network can reach each other When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Default And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 2 UDN non advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 CUDN advertised VRF-Lite And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On a different node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node Backing a ClusterIP service The pod on the tested network cannot reach the service on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node The pod on the other network cannot reach the pod on the tested network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv4": "[Disabled:Unimplemented]", + + "BGP: For a VRF-Lite configured network When the tested network is of type Layer 3 When a pod runs on the tested network When there is other network Of type Layer 3 UDN non advertised And a pod runs on the other network On the same node The pod on the tested network cannot reach the pod on the other network When the networks are IPv6": "[Disabled:Unimplemented]", + + "BGP: Pod to external server when CUDN network is advertised Route Advertisements layer2": "[Disabled:Unimplemented]", + + "BGP: Pod to external server when CUDN network is advertised Route Advertisements layer3": "[Disabled:Unimplemented]", + + "BGP: When default podNetwork is advertised when a client ovnk pod is created can connect to an external server and another cluster node after toggling default network advertisement off and back on": "[Disabled:Unimplemented]", + + "BGP: When default podNetwork is advertised when a client ovnk pod is created tests are run towards the external agnhost echo server": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks UDN pod to a different node should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks UDN pod to local node should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=Cluster] UDN pod to a different node nodeport service in default network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=Cluster] UDN pod to a different node nodeport service in different UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=Cluster] UDN pod to a different node nodeport service in same UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=Cluster] UDN pod to the same node nodeport service in default network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=Cluster] UDN pod to the same node nodeport service in different UDN network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=Cluster] UDN pod to the same node nodeport service in same UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=LOCAL] Default network pod to different node nodeport service in UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=LOCAL] Default network pod to same node nodeport service in UDN network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=LOCAL] UDN pod to a different node nodeport service in default network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=LOCAL] UDN pod to a different node nodeport service in different UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=LOCAL] UDN pod to a different node nodeport service in same UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=LOCAL] UDN pod to the same node nodeport service in default network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=LOCAL] UDN pod to the same node nodeport service in different UDN network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks [ETP=LOCAL] UDN pod to the same node nodeport service in same UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks host to a different node UDN pod should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks host to a local UDN pod should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod in the UDN should be able to access a service in the same network": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod in the UDN should be able to access kapi in default network service": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod in the UDN should be able to access kapi service cluster IP directly": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod in the UDN should not be able to access a default network service": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod in the UDN should not be able to access a service in a different UDN": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod in the default network should not be able to access a UDN service": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod in the default network should not be able to access an advertised UDN pod on a different node": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod in the default network should not be able to access an advertised UDN pod on the same node": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod to pod connectivity on different networks and different nodes": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod to pod connectivity on different networks and same node": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod to pod on the same network and different nodes should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer2 connectivity between networks pod to pod on the same network and same node should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks UDN pod to a different node should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks UDN pod to local node should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=Cluster] UDN pod to a different node nodeport service in default network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=Cluster] UDN pod to a different node nodeport service in different UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=Cluster] UDN pod to a different node nodeport service in same UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=Cluster] UDN pod to the same node nodeport service in default network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=Cluster] UDN pod to the same node nodeport service in different UDN network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=Cluster] UDN pod to the same node nodeport service in same UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=LOCAL] Default network pod to different node nodeport service in UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=LOCAL] Default network pod to same node nodeport service in UDN network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=LOCAL] UDN pod to a different node nodeport service in default network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=LOCAL] UDN pod to a different node nodeport service in different UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=LOCAL] UDN pod to a different node nodeport service in same UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=LOCAL] UDN pod to the same node nodeport service in default network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=LOCAL] UDN pod to the same node nodeport service in different UDN network should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks [ETP=LOCAL] UDN pod to the same node nodeport service in same UDN network should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks host to a different node UDN pod should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks host to a local UDN pod should not work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod in the UDN should be able to access a service in the same network": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod in the UDN should be able to access kapi in default network service": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod in the UDN should be able to access kapi service cluster IP directly": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod in the UDN should not be able to access a default network service": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod in the UDN should not be able to access a service in a different UDN": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod in the default network should not be able to access a UDN service": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod in the default network should not be able to access an advertised UDN pod on a different node": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod in the default network should not be able to access an advertised UDN pod on the same node": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod to pod connectivity on different networks and different nodes": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod to pod connectivity on different networks and same node": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod to pod on the same network and different nodes should work": "[Disabled:Unimplemented]", + + "BGP: isolation between advertised networks Layer3 connectivity between networks pod to pod on the same network and same node should work": "[Disabled:Unimplemented]", + + "Check whether gateway-mtu-support annotation on node is set based on disable-pkt-mtu-check value when DisablePacketMTUCheck is either not set or set to false Verify whether gateway-mtu-support annotation is not set on nodes when DisablePacketMTUCheck is either not set or set to false": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController CNC lifecycle CNC deletion and recreation - tunnel ID is allocated after recreate": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController CNC lifecycle tunnel ID is stable across CNC spec updates": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController full lifecycle workflow comprehensive workflow - create, add, update, remove networks through CNC lifecycle": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC has no matching networks has only tunnel ID annotation": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC is created before networks full matrix created after CNC - annotations are updated with all 8 networks": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC is created before networks multiple networks created after CNC: annotations are updated P-CUDNs (one multi-ns)": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC is created before networks multiple networks created after CNC: annotations are updated P-UDNs": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC is created before networks single network created after CNC: annotations are updated L2 P-CUDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC is created before networks single network created after CNC: annotations are updated L2 P-UDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC is created before networks single network created after CNC: annotations are updated L3 P-CUDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC is created before networks single network created after CNC: annotations are updated L3 P-UDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC selector is updated adding and removing CUDN selector from CNC - count increases then decreases": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC selector is updated adding and removing PUDN selector from CNC - count increases then decreases": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC selector is updated widening then narrowing CUDN selector - count increases then decreases": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when CNC selector is updated widening then narrowing PUDN namespace selector - count increases then decreases": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when multiple CNCs exist deleting one CNC does not affect the other": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when multiple CNCs exist two CNCs matching same network - both track the network (this works but is usually treated as misconfiguration)": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when multiple CNCs exist two CNCs with non-overlapping selectors - each tracks its own networks": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when network or namespace labels are mutated CUDN label mutation - adding then removing label changes CNC count": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when network or namespace labels are mutated namespace label mutation - adding then removing label changes CNC count": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are added to existing CNC adding a network to CNC with existing networks: count increases add L2 P-CUDN to L3 P-CUDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are added to existing CNC adding a network to CNC with existing networks: count increases add L2 P-UDN to L3 P-UDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are added to existing CNC adding a network to CNC with existing networks: count increases add L3 P-CUDN to L2 P-CUDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are added to existing CNC adding a network to CNC with existing networks: count increases add L3 P-UDN to L2 P-UDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are added to existing CNC adding mixed networks (P-UDN + P-CUDN) to existing CNC - all networks appear": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are deleted from CNC deleting mixed networks (P-UDN + P-CUDN) - annotations update correctly": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are deleted from CNC deleting networks from CNC: count decreases to zero delete L2 then L3 P-CUDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are deleted from CNC deleting networks from CNC: count decreases to zero delete L2 then L3 P-UDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are deleted from CNC deleting networks from CNC: count decreases to zero delete L3 then L3 P-CUDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks are deleted from CNC deleting networks from CNC: count decreases to zero delete L3 then L3 P-UDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks exist before CNC creation full matrix (2x each type) - has all 8 networks in subnet annotation": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks exist before CNC creation multiple networks (2xL3 + 2xL2): has all networks in subnet annotation P-CUDNs (one multi-ns)": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks exist before CNC creation multiple networks (2xL3 + 2xL2): has all networks in subnet annotation P-UDNs": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks exist before CNC creation single network: has both subnet and tunnel ID annotations L2 P-CUDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks exist before CNC creation single network: has both subnet and tunnel ID annotations L2 P-UDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks exist before CNC creation single network: has both subnet and tunnel ID annotations L3 P-CUDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect ClusterManagerController when networks exist before CNC creation single network: has both subnet and tunnel ID annotations L3 P-UDN": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect OVN-Kubernetes Controller End-to-end connectivity validation should manage cross-network connectivity through CNC lifecycle": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect OVN-Kubernetes Controller Multiple CNCs with overlapping network selection should maintain non-transitive connectivity when a network is selected by multiple CNCs": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect: API validations api-server should accept valid ClusterNetworkConnect CRs Valid ClusterNetworkConnect configurations": "[Disabled:Unimplemented]", + + "ClusterNetworkConnect: API validations api-server should reject invalid ClusterNetworkConnect CRs Invalid network selector types": "[Disabled:Unimplemented]", + + "Creating a static pod on a node Should successfully create then remove a static pod": "[Disabled:Unimplemented]", + + "EVPN: VTEP API validations api-server should accept valid VTEP CRs Valid VTEP configurations": "[Disabled:Unimplemented]", + + "EVPN: VTEP API validations api-server should reject invalid VTEP CRs Invalid VTEP configurations": "[Disabled:Unimplemented]", + + "EgressService Multiple Networks, external clients sharing ip [LGW] Should validate pods on different networks can reach different clients with same ip without SNAT ipv4 pods": "[Disabled:Unimplemented]", + + "EgressService Multiple Networks, external clients sharing ip [LGW] Should validate pods on different networks can reach different clients with same ip without SNAT ipv6 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate a node with a local ep is selected when ETP=Local ipv4 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate a node with a local ep is selected when ETP=Local ipv6 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate egress service has higher priority than EgressIP when not assigned to the same node ipv4 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate egress service has higher priority than EgressIP when not assigned to the same node ipv6 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate pods' egress is SNATed to the LB's ingress ip with selectors ipv4 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate pods' egress is SNATed to the LB's ingress ip with selectors ipv6 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate pods' egress is SNATed to the LB's ingress ip without selectors ipv4 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate pods' egress is SNATed to the LB's ingress ip without selectors ipv6 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate the egress SVC SNAT functionality against host-networked pods ipv4 pods": "[Disabled:Unimplemented]", + + "EgressService Should validate the egress SVC SNAT functionality against host-networked pods ipv6 pods": "[Disabled:Unimplemented]", + + "EgressService [LGW] Should validate ingress reply traffic uses the Network ipv4 pods": "[Disabled:Unimplemented]", + + "EgressService [LGW] Should validate ingress reply traffic uses the Network ipv6 pods": "[Disabled:Unimplemented]", + + "EgressService [LGW] Should validate pods' egress uses node's IP when setting Network without SNAT ipv4 pods": "[Disabled:Unimplemented]", + + "EgressService [LGW] Should validate pods' egress uses node's IP when setting Network without SNAT ipv6 pods": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still references the same pods with the pod selector IPV4 tcp": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still references the same pods with the pod selector IPV4 udp": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still references the same pods with the pod selector IPV6 tcp": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still references the same pods with the pod selector IPV6 udp": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy IPV4 tcp": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy IPV4 udp": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy IPV6 tcp": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy IPV6 udp": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway annotations and a policy CR and after the annotations are removed ipv4": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod when deleting the annotation and supported by a CR with the same gateway IPs TCP ipv4": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod when deleting the annotation and supported by a CR with the same gateway IPs TCP ipv6": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod when deleting the annotation and supported by a CR with the same gateway IPs UDP ipv4": "[Disabled:Unimplemented]", + + "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod when deleting the annotation and supported by a CR with the same gateway IPs UDP ipv6": "[Disabled:Unimplemented]", + + "External Gateway When validating the Admin Policy Based External Route status Should update the status of a successful and failed CRs": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 tcp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 udp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 tcp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 udp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e non-vxlan external gateway through a dynamic hop Should validate ICMP connectivity to an external gateway's loopback address via a pod with dynamic hop ipv4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e non-vxlan external gateway through a dynamic hop Should validate ICMP connectivity to an external gateway's loopback address via a pod with dynamic hop ipv6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e non-vxlan external gateway through a dynamic hop Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with a dynamic hop TCP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e non-vxlan external gateway through a dynamic hop Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with a dynamic hop TCP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e non-vxlan external gateway through a dynamic hop Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with a dynamic hop UDP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs BFD e2e non-vxlan external gateway through a dynamic hop Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with a dynamic hop UDP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod annotation update": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod deletion timestamp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod not ready": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod annotation update": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod deletion timestamp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod not ready": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod annotation update": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod deletion timestamp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod not ready": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod annotation update": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod deletion timestamp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod not ready": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Static Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Static Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Static Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Static Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 tcp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 udp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 tcp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 udp": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a gateway pod ipv4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a gateway pod ipv6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity even after MAC change (gateway migration) for egress TCP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity even after MAC change (gateway migration) for egress TCP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity even after MAC change (gateway migration) for egress UDP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity even after MAC change (gateway migration) for egress UDP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a gateway pod TCP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a gateway pod TCP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a gateway pod UDP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a gateway pod UDP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV4": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV6": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 tcp": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 udp": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 tcp": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 udp": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled ipv4": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled ipv6": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled TCP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled TCP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled UDP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled UDP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod annotation update": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod deletion timestamp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod not ready": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod annotation update": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod delete": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod deletion timestamp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod not ready": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod annotation update": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod delete": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod deletion timestamp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod not ready": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod annotation update": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod deletion timestamp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod not ready": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV4": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV6": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 tcp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 udp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 tcp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 udp": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway CR ipv4": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway CR ipv6": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled TCP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled TCP ipv6": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled UDP ipv4": "[Disabled:Unimplemented]", + + "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled UDP ipv6": "[Disabled:Unimplemented]", + + "External Gateway e2e ingress gateway traffic validation Should validate ingress connectivity from an external gateway": "[Disabled:Unimplemented]", + + "External Gateway e2e non-vxlan external gateway and update validation Should validate connectivity without vxlan before and after updating the namespace annotation to a new external gateway": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines IP family validation for layer2 primary networks should fail when dual-stack network requests only IPv4": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines IP family validation for layer2 primary networks should fail when dual-stack network requests only IPv6": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines IP family validation for layer2 primary networks should fail when single-stack IPv4 network requests multiple IPv4 IPs": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines IP family validation for layer2 primary networks should fail when single-stack IPv6 network requests multiple IPv6 IPs": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines IP family validation for layer2 primary networks should succeed when dual-stack network requests correct IPs (1 IPv4 + 1 IPv6)": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines duplicate addresses validation should fail when creating second VM with duplicate static IP": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines duplicate addresses validation should fail when creating second VM with duplicate user requested MAC": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines ipv4 subnet exhaustion should fail when subnet is exhausted": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with default pod network when live migration with post-copy succeeds, should keep connectivity": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with default pod network when live migration with pre-copy fails, should keep connectivity": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with default pod network when live migration with pre-copy succeeds, should keep connectivity": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with kubevirt VM using layer2 UDPN should configure IPv4 and IPv6 using DHCP and NDP": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration failed of VirtualMachineInstance with Secondary/Localnet with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration failed of VirtualMachineInstance with interface binding for UDN with Primary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachine with Secondary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachine with Secondary/Localnet with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachine with interface binding for UDN and statics IPs and MAC with Primary/Layer2 with routed ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachine with interface binding for UDN and statics IPs and MAC with Primary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachine with interface binding for UDN with Primary/Layer2 with routed ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachine with interface binding for UDN with Primary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachineInstance with Secondary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachineInstance with Secondary/Localnet with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration of VirtualMachineInstance with interface binding for UDN with Primary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after restart of VirtualMachine with Secondary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after restart of VirtualMachine with Secondary/Localnet with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after restart of VirtualMachine with interface binding for UDN and statics IPs and MAC with Primary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after restart of VirtualMachine with interface binding for UDN with Primary/Layer2 with snat ingress": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks with ipamless localnet topology should maintain tcp connection with minimal downtime after failed live migration": "[Disabled:Unimplemented]", + + "Kubevirt Virtual Machines with user defined networks with ipamless localnet topology should maintain tcp connection with minimal downtime after succeeded live migration": "[Disabled:Unimplemented]", + + "Load Balancer Service Tests with MetalLB Should ensure connectivity works on an external service when mtu changes in intermediate node": "[Disabled:Unimplemented]", + + "Load Balancer Service Tests with MetalLB Should ensure load balancer service works when ETP=local and backend pods are also egressIP served pods": "[Disabled:Unimplemented]", + + "Load Balancer Service Tests with MetalLB Should ensure load balancer service works when ETP=local and session affinity is set": "[Disabled:Unimplemented]", + + "Load Balancer Service Tests with MetalLB Should ensure load balancer service works with 0 node ports when ETP=local": "[Disabled:Unimplemented]", + + "Load Balancer Service Tests with MetalLB Should ensure load balancer service works with 0 node ports when named targetPorts are used and ETP=local": "[Disabled:Unimplemented]", + + "Load Balancer Service Tests with MetalLB Should ensure load balancer service works with pmtud": "[Disabled:Unimplemented]", + + "Multi Homing A pod with multiple attachments to the same OVN-K networks features two different IPs from the same subnet": "[Disabled:Unimplemented]", + + "Multi Homing A pod with multiple attachments to the same secondary NAD features multiple different IPs and connectivity redundancy L2 secondary NAD": "[Disabled:Unimplemented]", + + "Multi Homing A pod with multiple attachments to the same secondary NAD features multiple different IPs and connectivity redundancy L3 secondary NAD": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a client pod in the default network on a different node, when the localnet uses a VLAN and an external router": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a client pod in the default network on a different node, when the localnet uses an IP in the host subnet": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a client pod in the default network on the same node, when the localnet uses a VLAN and an external router": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a client pod in the default network on the same node, when the localnet uses an IP in the host subnet": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a host-networked pod on a different node, when the localnet uses a VLAN and an external router": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a host-networked pod on a different node, when the localnet uses an IP in the host subnet": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a host-networked pod on the same node, when the localnet uses a VLAN and an external router": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a host-networked pod on the same node, when the localnet uses an IP in the host subnet": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can reach a host-network pod on a different node, when the localnet uses a VLAN and an external router": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can reach a host-network pod on the same node, when the localnet uses a VLAN and an external router": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can reach a host-networked pod on a different node, when the localnet uses an IP in the host subnet": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can reach a host-networked pod on the same node, when the localnet uses an IP in the host subnet": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to a localnet - switched - network featuring `excludeCIDR`s": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to a localnet - switched - network with an IPv6 subnet": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to a localnet - switched - network without IPAM": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to a localnet - switched - network": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to an L2 - switched - network featuring `excludeCIDR`s": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to an L2 - switched - network with a dual stack configuration": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to an L2 - switched - network with an IPv6 subnet": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to an L2 - switched - network without IPAM": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to an L2 - switched - network": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to an L3 - routed - network with IPv6 network": "[Disabled:Unimplemented]", + + "Multi Homing A single pod with an OVN-K secondary network is able to get to the Running phase when attaching to an L3 - routed - network": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over a localnet secondary network when the pods are scheduled on different nodes": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over a localnet secondary network with a dual stack configuration when pods are scheduled on different nodes": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over a localnet secondary network with an IPv6 subnet when pods are scheduled on different nodes": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over a localnet secondary network without IPAM when the pods are scheduled on different nodes": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over a localnet secondary network without IPAM when the pods are scheduled on different nodes, with static IPs configured via network selection elements": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L2 - switched - secondary network with `excludeCIDR`s": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L2 - switched - secondary network without IPAM": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L2 secondary network when the pods are scheduled in different nodes": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L2 secondary network with a dual stack configuration": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L2 secondary network with an IPv6 subnet when pods are scheduled in different nodes": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L2 secondary network without IPAM, with static IPs configured via network selection elements": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L3 - routed - secondary network with IPv6 subnet": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L3 - routed - secondary network with a dual stack configuration": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network can communicate over the secondary network can communicate over an L3 - routed - secondary network": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network eventually configures pods that were added to an already existing network before the nad": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay and networkAttachmentDefinition is modified allocates the pod's secondary interface IP in the new range after NetworkAttachmentDefinition reconcile": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay and networkAttachmentDefinition is modified and the service connected to the underlay is reconfigured to connect to the new VLAN-ID can now communicate over a localnet secondary network from pod to the underlay service": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay and networkAttachmentDefinition is modified can no longer communicate over a localnet secondary network from pod to the underlay service": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay and networkAttachmentDefinition is modified sets the new MTU on the pod after NetworkAttachmentDefinition reconcile": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay can communicate over a localnet secondary network from pod to the underlay service": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay correctly sets the MTU on the pod": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay when a policy is provisioned can communicate over a localnet secondary network from pod to gw egress allow all": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay when a policy is provisioned can communicate over a localnet secondary network from pod to gw egress deny all": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay when a policy is provisioned can communicate over a localnet secondary network from pod to gw ingress denyall, egress allow all, ingress policy should have no impact on egress": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay when a policy is provisioned can communicate over a localnet secondary network from pod to gw ingress denyall, ingress policy should have no impact on egress": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a service running on the underlay with multi network policy blocking the traffic can not communicate over a localnet secondary network from pod to the underlay service": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network localnet OVN-K secondary network with a trunked configuration the same bridge mapping can be shared by a separate VLAN by using the physical network name attribute": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that allow all ingress using egress deny-all for a localnet topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that allow all ingress using egress deny-all, ingress allow-all for a localnet topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that allow all ingress using ingress allow-all for a localnet topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using IPBlock for a localnet topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using IPBlock for a pure L2 overlay": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using IPBlock for a routed topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using IPBlock for an IPAMless pure L2 overlay": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using namespace selectors for a localnet topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using namespace selectors for a pure L2 overlay": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using namespace selectors for a routed topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using pod selectors and port range for a pure L2 overlay": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using pod selectors for a localnet topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using pod selectors for a pure L2 overlay": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that configure traffic allow lists using pod selectors for a routed topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that deny traffic using ingress deny-all for a localnet topology": "[Disabled:Unimplemented]", + + "Multi Homing multiple pods connected to the same OVN-K secondary network with multi-network policies that deny traffic using pod selectors and wrong port range for a localnet topology": "[Disabled:Unimplemented]", + + "Multi node zones interconnect Pod interconnectivity": "[Disabled:Unimplemented]", + + "Multicast when multicast enabled for namespace should be able to receive multicast IGMP query": "[Disabled:Unimplemented]", + + "Multicast when multicast enabled for namespace should be able to send multicast UDP traffic between nodes": "[Disabled:Unimplemented]", + + "Network Segmentation ClusterUserDefinedNetwork CRD Controller pod connected to ClusterUserDefinedNetwork CR & managed NADs cannot be deleted when being used": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation ClusterUserDefinedNetwork CRD Controller should create NAD according to spec in each target namespace and report active namespaces": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation ClusterUserDefinedNetwork CRD Controller should create NAD in new created namespaces that apply to namespace-selector": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation ClusterUserDefinedNetwork CRD Controller when CR is deleted, should delete all managed NAD in each target namespace": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation ClusterUserDefinedNetwork CRD Controller when namespace-selector is mutated should create NAD in namespaces that apply to mutated namespace-selector": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation ClusterUserDefinedNetwork CRD Controller when namespace-selector is mutated should delete managed NAD in namespaces that no longer apply to namespace-selector": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using NetworkAttachmentDefinitions does not mirror EndpointSlices in namespaces not using user defined primary networks L2 secondary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using NetworkAttachmentDefinitions does not mirror EndpointSlices in namespaces not using user defined primary networks L3 secondary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using NetworkAttachmentDefinitions mirrors EndpointSlices managed by the default controller for namespaces with user defined primary networks L2 primary UDN, cluster-networked pods": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using NetworkAttachmentDefinitions mirrors EndpointSlices managed by the default controller for namespaces with user defined primary networks L2 primary UDN, host-networked pods": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using NetworkAttachmentDefinitions mirrors EndpointSlices managed by the default controller for namespaces with user defined primary networks L3 primary UDN, cluster-networked pods": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using NetworkAttachmentDefinitions mirrors EndpointSlices managed by the default controller for namespaces with user defined primary networks L3 primary UDN, host-networked pods": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using UserDefinedNetwork does not mirror EndpointSlices in namespaces not using user defined primary networks L2 secondary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using UserDefinedNetwork does not mirror EndpointSlices in namespaces not using user defined primary networks L3 secondary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using UserDefinedNetwork mirrors EndpointSlices managed by the default controller for namespaces with user defined primary networks L2 primary UDN, cluster-networked pods": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using UserDefinedNetwork mirrors EndpointSlices managed by the default controller for namespaces with user defined primary networks L2 primary UDN, host-networked pods": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using UserDefinedNetwork mirrors EndpointSlices managed by the default controller for namespaces with user defined primary networks L3 primary UDN, cluster-networked pods": "[Disabled:Unimplemented]", + + "Network Segmentation EndpointSlices mirroring a user defined primary network created using UserDefinedNetwork mirrors EndpointSlices managed by the default controller for namespaces with user defined primary networks L3 primary UDN, host-networked pods": "[Disabled:Unimplemented]", + + "Network Segmentation Sync perform east/west traffic between nodes following OVN Kube node pod restart L2 with custom network": "[Disabled:Unimplemented]", + + "Network Segmentation Sync perform east/west traffic between nodes following OVN Kube node pod restart L2": "[Disabled:Unimplemented]", + + "Network Segmentation Sync perform east/west traffic between nodes following OVN Kube node pod restart L3": "[Disabled:Unimplemented]", + + "Network Segmentation UDN Pod should react to k8s.ovn.org/open-default-ports annotations changes": "[Disabled:Unimplemented]", + + "Network Segmentation UserDefinedNetwork CRD Controller for L2 secondary network pod connected to UserDefinedNetwork cannot be deleted when being used": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation UserDefinedNetwork CRD Controller for L2 secondary network should create NetworkAttachmentDefinition according to spec": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation UserDefinedNetwork CRD Controller for L2 secondary network should delete NetworkAttachmentDefinition when UserDefinedNetwork is deleted": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation UserDefinedNetwork CRD Controller for primary UDN without required namespace label should be able to create pod and it will attach to the cluster default network": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation UserDefinedNetwork CRD Controller for primary UDN without required namespace label should not be able to update the namespace and add the UDN label": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation UserDefinedNetwork CRD Controller for primary UDN without required namespace label should not be able to update the namespace and remove the UDN label": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation UserDefinedNetwork CRD Controller should correctly report subsystem error on node subnet allocation": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L2 primary UDN with custom network": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L2 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L3 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork creates a networkStatus Annotation with UDN interface L2 primary UDN with custom network": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork creates a networkStatus Annotation with UDN interface L2 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork creates a networkStatus Annotation with UDN interface L3 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork is isolated from the default network with L2 primary UDN with custom network": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork is isolated from the default network with L2 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork is isolated from the default network with L3 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork isolates overlapping CIDRs with L2 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using ClusterUserDefinedNetwork isolates overlapping CIDRs with L3 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions can perform east/west traffic between nodes two pods connected over a L2 primary UDN with custom network": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions can perform east/west traffic between nodes two pods connected over a L2 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions can perform east/west traffic between nodes two pods connected over a L3 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions creates a networkStatus Annotation with UDN interface L2 primary UDN with custom network": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions creates a networkStatus Annotation with UDN interface L2 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions creates a networkStatus Annotation with UDN interface L3 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions is isolated from the default network with L2 primary UDN with custom network": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions is isolated from the default network with L2 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions is isolated from the default network with L3 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions isolates overlapping CIDRs with L2 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using NetworkAttachmentDefinitions isolates overlapping CIDRs with L3 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L2 primary UDN with custom network": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L2 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork can perform east/west traffic between nodes two pods connected over a L3 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork creates a networkStatus Annotation with UDN interface L2 primary UDN with custom network": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork creates a networkStatus Annotation with UDN interface L2 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork creates a networkStatus Annotation with UDN interface L3 primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork is isolated from the default network with L2 primary UDN with custom network": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork is isolated from the default network with L2 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork is isolated from the default network with L3 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork isolates overlapping CIDRs with L2 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network created using UserDefinedNetwork isolates overlapping CIDRs with L3 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network doesn't cause network name conflict": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation a user defined primary network with multicast feature enabled for namespace should be able to receive multicast IGMP query with primary layer3 UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network with multicast feature enabled for namespace should be able to send multicast UDP traffic between nodes with primary layer2 UDN with custom network": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network with multicast feature enabled for namespace should be able to send multicast UDP traffic between nodes with primary layer2 UDN": "[Disabled:Unimplemented]", + + "Network Segmentation a user defined primary network with multicast feature enabled for namespace should be able to send multicast UDP traffic between nodes with primary layer3 UDN": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using ClusterUserDefinedNetwork can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer2 network with custom network": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using ClusterUserDefinedNetwork can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer2 network": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using ClusterUserDefinedNetwork can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer3 network": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using NetworkAttachmentDefinitions can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer2 network with custom network": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using NetworkAttachmentDefinitions can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer2 network": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using NetworkAttachmentDefinitions can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer3 network": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using UserDefinedNetwork can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer2 network with custom network": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using UserDefinedNetwork can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer2 network": "[Disabled:Unimplemented]", + + "Network Segmentation pod2Egress on a user defined primary network created using UserDefinedNetwork can be accessed to from the pods running in the Kubernetes cluster by one pod over a layer3 network": "[Disabled:Unimplemented]", + + "Network Segmentation when primary network exist, ClusterUserDefinedNetwork status should report not-ready": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation when primary network exist, UserDefinedNetwork status should report not-ready": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: API validations api-server should accept valid CRs ClusterUserDefinedNetwork, evpn": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should accept valid CRs ClusterUserDefinedNetwork, layer2": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should accept valid CRs ClusterUserDefinedNetwork, localnet": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should accept valid CRs ClusterUserDefinedNetwork, no-overlay, valid": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should accept valid CRs UserDefinedNetwork, layer2": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, evpn": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, layer2": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, localnet, invalid mtu": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, localnet, invalid physicalNetworkName": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, localnet, invalid role": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, localnet, invalid subnets": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, localnet, invalid vlan": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, mismatch topology and config": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs ClusterUserDefinedNetwork, no-overlay, invalid": "[Disabled:Unimplemented]", + + "Network Segmentation: API validations api-server should reject invalid CRs UserDefinedNetwork, layer2": "[Disabled:Unimplemented]", + + "Network Segmentation: Default network multus annotation ValidatingAdmissionPolicy protection should prevent adding, modifying and removing the default-network annotation on existing pods": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Default network multus annotation when added with static IP and MAC to a pod belonging to primary UDN should create the pod with the specified static IP and MAC address with persistent IPAM": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Default network multus annotation when added with static IP and MAC to a pod belonging to primary UDN should create the pod with the specified static IP and MAC address without persistent IPAM enabled": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Localnet should preserve LSPs for IPAM-less localnet pods after ovnkube-node restart": "[Disabled:Unimplemented]", + + "Network Segmentation: Localnet using ClusterUserDefinedNetwork CR, pods in different namespaces, should communicate over localnet topology": "[Disabled:Unimplemented]", + + "Network Segmentation: Network Policies on a user defined primary network allow ingress traffic to one pod from a particular namespace in L2 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation: Network Policies on a user defined primary network allow ingress traffic to one pod from a particular namespace in L3 primary UDN": "[Disabled:Unimplemented]", + + "Network Segmentation: Network Policies on a user defined primary network pods within namespace should be isolated when deny policy is present in L2 dualstack primary UDN with custom network": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Network Policies on a user defined primary network pods within namespace should be isolated when deny policy is present in L2 dualstack primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Network Policies on a user defined primary network pods within namespace should be isolated when deny policy is present in L3 dualstack primary UDN": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN duplicate IP validation with primary UDN layer 2 pods should fail when creating second pod with duplicate static IP IPv4 duplicate": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN duplicate IP validation with primary UDN layer 2 pods should fail when creating second pod with duplicate static IP IPv6 duplicate": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN should respect network configuration Layer2 basic configuration": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN should respect network configuration Layer2 with custom subnets": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN should respect network configuration Layer2 with inverted gateway/management IPs": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN unmasked reserved / infrastructure subnets are not allowed Layer2 with unmasked IPv4 infrastructure subnets": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN unmasked reserved / infrastructure subnets are not allowed Layer2 with unmasked IPv4 reserved subnets": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN unmasked reserved / infrastructure subnets are not allowed Layer2 with unmasked IPv6 infrastructure subnets": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: Preconfigured Layer2 UDN unmasked reserved / infrastructure subnets are not allowed Layer2 with unmasked IPv6 reserved subnets": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: services on a user defined primary network should be reachable through their cluster IP, node port and load balancer L2 primary UDN with custom network, cluster-networked pods, NodePort service": "[Suite:openshift/conformance/parallel]", + + "Network Segmentation: services on a user defined primary network should be reachable through their cluster IP, node port and load balancer L2 primary UDN, cluster-networked pods, NodePort service": "[Disabled:Unimplemented]", + + "Network Segmentation: services on a user defined primary network should be reachable through their cluster IP, node port and load balancer L3 primary UDN, cluster-networked pods, NodePort service": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv4 address is updated when ETP=Local service with host network backend is configured makes sure that the flows are updated with new IP address (update kubelet first, the IP address later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv4 address is updated when ETP=Local service with host network backend is configured makes sure that the flows are updated with new IP address (update the IP address first, kubelet later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv4 address is updated when EgressIPs are configured makes sure that the EgressIP is still operational (update kubelet first, the IP address later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv4 address is updated when EgressIPs are configured makes sure that the EgressIP is still operational (update the IP address first, kubelet later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv4 address is updated when no EgressIPs are configured makes sure that the cluster is still operational (update kubelet first, the IP address later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv4 address is updated when no EgressIPs are configured makes sure that the cluster is still operational (update the IP address first, kubelet later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv6 address is updated when ETP=Local service with host network backend is configured makes sure that the flows are updated with new IP address (update kubelet first, the IP address later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv6 address is updated when ETP=Local service with host network backend is configured makes sure that the flows are updated with new IP address (update the IP address first, kubelet later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv6 address is updated when EgressIPs are configured makes sure that the EgressIP is still operational (update kubelet first, the IP address later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv6 address is updated when EgressIPs are configured makes sure that the EgressIP is still operational (update the IP address first, kubelet later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv6 address is updated when no EgressIPs are configured makes sure that the cluster is still operational (update kubelet first, the IP address later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when the node IPv6 address is updated when no EgressIPs are configured makes sure that the cluster is still operational (update the IP address first, kubelet later)": "[Disabled:Unimplemented]", + + "Node IP and MAC address migration when when MAC address changes when a nodeport service is configured Ensures flows are updated when MAC address changes": "[Disabled:Unimplemented]", + + "Node Shutdown and Startup should maintain cluster health after node shutdown and startup": "[Disabled:Unimplemented]", + + "OVS CPU affinity pinning can be enabled on specific nodes by creating enable_dynamic_cpu_affinity file": "[Disabled:Unimplemented]", + + "Pod to external server PMTUD when a client ovnk pod targeting an external server is created when tests are run towards the agnhost echo server queries to the hostNetworked server pod on another node shall work for TCP": "[Disabled:Unimplemented]", + + "Pod to external server PMTUD when a client ovnk pod targeting an external server is created when tests are run towards the agnhost echo server queries to the hostNetworked server pod on another node shall work for UDP": "[Disabled:Unimplemented]", + + "Pod to pod TCP with low MTU when a client ovnk pod targeting an ovnk pod server(running on another node) with low mtu when MTU is lowered between the two nodes large queries to the server pod on another node shall work for TCP": "[Disabled:Unimplemented]", + + "Service Hairpin SNAT Should ensure service hairpin traffic is NOT SNATed to hairpin masquerade IP; GR LB": "[Disabled:Unimplemented]", + + "Service Hairpin SNAT Should ensure service hairpin traffic is SNATed to hairpin masquerade IP; Switch LB": "[Disabled:Unimplemented]", + + "Services All service features work when manually listening on a non-default address": "[Disabled:Unimplemented]", + + "Services Allow connection to an external IP using a source port that is equal to a node port": "[Disabled:Unimplemented]", + + "Services Creates a host-network service, and ensures that host-network pods can connect to it": "[Disabled:Unimplemented]", + + "Services Creates a service with session-affinity, and ensures it works after backend deletion": "[Disabled:Unimplemented]", + + "Services does not use host masquerade address as source IP address when communicating externally": "[Disabled:Unimplemented]", + + "Services of type NodePort should be able to preserve UDP traffic when server pod cycles for a NodePort service via a different node": "[Disabled:Unimplemented]", + + "Services of type NodePort should handle IP fragments": "[Disabled:Unimplemented]", + + "Services of type NodePort should listen on each host addresses": "[Disabled:Unimplemented]", + + "Services of type NodePort should work on secondary node interfaces for ETP=local and ETP=cluster when backend pods are also served by EgressIP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:false, namedPort:false, ETP:Cluster is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for TCP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:false, namedPort:false, ETP:Cluster is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for UDP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:false, namedPort:false, ETP:Local is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for TCP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:false, namedPort:false, ETP:Local is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for UDP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:false, namedPort:true, ETP:Cluster is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for TCP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:false, namedPort:true, ETP:Cluster is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for UDP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:false, namedPort:true, ETP:Local is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for TCP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:false, namedPort:true, ETP:Local is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for UDP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:true, namedPort:false, ETP:Cluster is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for TCP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:true, namedPort:false, ETP:Cluster is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for UDP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:true, namedPort:false, ETP:Local is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for TCP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:true, namedPort:false, ETP:Local is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for UDP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:true, namedPort:true, ETP:Cluster is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for TCP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:true, namedPort:true, ETP:Cluster is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for UDP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:true, namedPort:true, ETP:Local is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for TCP": "[Disabled:Unimplemented]", + + "Services when a nodePort service targeting a pod with hostNetwork:true, namedPort:true, ETP:Local is created when tests are run towards the agnhost echo service queries to the nodePort service shall work for UDP": "[Disabled:Unimplemented]", + + "Status manager validation Should validate the egress firewall status when adding a new zone": "[Disabled:Unimplemented]", + + "Status manager validation Should validate the egress firewall status when adding an unknown zone": "[Disabled:Unimplemented]", + + "Unidling Should generate a NeedPods event for traffic destined to idled services": "[Disabled:Unimplemented]", + + "Unidling With annotated service Should connect to an unidled backend at the first attempt": "[Disabled:Unimplemented]", + + "Unidling With annotated service Should generate a NeedPods event for traffic destined to idled services": "[Disabled:Unimplemented]", + + "Unidling With annotated service Should generate a NeedPods event when backends were added and then removed": "[Disabled:Unimplemented]", + + "Unidling With annotated service Should not generate a NeedPods event when has backend": "[Disabled:Unimplemented]", + + "Unidling With annotated service Should not generate a NeedPods event when removing the annotation": "[Disabled:Unimplemented]", + + "Unidling With non annotated service Should generate a NeedPods event when adding the annotation": "[Disabled:Unimplemented]", + + "Unidling With non annotated service Should not generate a NeedPods event for traffic destined to idled services": "[Disabled:Unimplemented]", + + "Unidling With non annotated service Should not generate a NeedPods event when backends were added and then removed": "[Disabled:Unimplemented]", + + "Unidling With non annotated service Should not generate a NeedPods event when has backend": "[Disabled:Unimplemented]", + + "blocking ICMP needs frag when a client VM pod with 1500 MTU targets a host networked pod should be able to send large TCP packet and not get a route cache entry": "[Disabled:Unimplemented]", + + "blocking ICMP needs frag when a client host networked pod with targets a proxy node nodeport service with ovnk networked backend should be able to send large UDP packet and not get a route cache entry": "[Disabled:Unimplemented]", + + "blocking ICMP needs frag when an ovnk pod targets a host networked pod with large UDP should be able to send large UDP packet and not get a route cache entry": "[Disabled:Unimplemented]", + + "e2e EgressQoS validation Should deny resources with bad values": "[Disabled:Unimplemented]", + + "e2e EgressQoS validation Should validate correct DSCP value on EgressQoS resource changes ipv4 pod after resource": "[Disabled:Unimplemented]", + + "e2e EgressQoS validation Should validate correct DSCP value on EgressQoS resource changes ipv4 pod before resource": "[Disabled:Unimplemented]", + + "e2e EgressQoS validation Should validate correct DSCP value on EgressQoS resource changes ipv6 pod after resource": "[Disabled:Unimplemented]", + + "e2e EgressQoS validation Should validate correct DSCP value on EgressQoS resource changes ipv6 pod before resource": "[Disabled:Unimplemented]", + + "e2e EgressQoS validation Should validate correct DSCP value on pod labels changes ipv4 pod": "[Disabled:Unimplemented]", + + "e2e EgressQoS validation Should validate correct DSCP value on pod labels changes ipv6 pod": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Limits egress traffic targeting a pod by protocol and port through a NetworkQoS spec ipv4": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Limits egress traffic targeting a pod by protocol and port through a NetworkQoS spec ipv6": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Limits egress traffic targeting an individual pod by protocol through a NetworkQoS spec ipv4": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Limits egress traffic targeting an individual pod by protocol through a NetworkQoS spec ipv6": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Limits egress traffic to all target pods below the specified rate in NetworkQoS spec ipv4": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Limits egress traffic to all target pods below the specified rate in NetworkQoS spec ipv6": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Should have correct DSCP value for host network traffic when NetworkQoS is applied ipv4": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Should have correct DSCP value for host network traffic when NetworkQoS is applied ipv6": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Should have correct DSCP value for overlay traffic when NetworkQoS is applied ipv4": "[Disabled:Unimplemented]", + + "e2e NetworkQoS validation Should have correct DSCP value for overlay traffic when NetworkQoS is applied ipv6": "[Disabled:Unimplemented]", + + "e2e br-int flow monitoring export validation Should validate flow data of br-int is sent to an external gateway with netflow v5": "[Disabled:Unimplemented]", + + "e2e br-int flow monitoring export validation Should validate flow data of br-int is sent to an external gateway with sflow": "[Disabled:Unimplemented]", + + "e2e control plane should provide Internet connection continuously when all ovnkube-control-plane pods are killed": "[Disabled:Unimplemented]", + + "e2e control plane should provide Internet connection continuously when all pods are killed on node running master instance of ovnkube-control-plane": "[Disabled:Unimplemented]", + + "e2e control plane should provide Internet connection continuously when ovnkube-node pod is killed": "[Disabled:Unimplemented]", + + "e2e control plane should provide Internet connection continuously when pod running master instance of ovnkube-control-plane is killed": "[Disabled:Unimplemented]", + + "e2e control plane should provide connection to external host by DNS name from a pod": "[Disabled:Unimplemented]", + + "e2e control plane test node readiness according to its defaults interface MTU size should get node not ready with a too small MTU": "[Disabled:Unimplemented]", + + "e2e control plane test node readiness according to its defaults interface MTU size should get node ready with a big enough MTU": "[Disabled:Unimplemented]", + + "e2e delete databases Should validate connectivity before and after deleting all the db-pods at once in HA mode": "[Disabled:Unimplemented]", + + "e2e delete databases Should validate connectivity before and after deleting all the db-pods at once in Non-HA mode": "[Disabled:Unimplemented]", + + "e2e delete databases recovering from deleting db files while maintaining connectivity when deleting both db files on ovnkube-db-0": "[Disabled:Unimplemented]", + + "e2e delete databases recovering from deleting db files while maintaining connectivity when deleting both db files on ovnkube-db-1": "[Disabled:Unimplemented]", + + "e2e delete databases recovering from deleting db files while maintaining connectivity when deleting both db files on ovnkube-db-2": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network Should fail if egressip-mark annotation is being added by a regular user": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network Should fail if egressip-mark annotation is present during EgressIP creation": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network Should handle EIP reassignment correctly on namespace and pod label updates, and EIP object updates": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network Should re-assign egress IPs when node readiness / reachability goes down/up": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network Should validate egress IP logic when one pod is managed by more than one egressIP object": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network Should validate the egress IP SNAT functionality for stateful-sets": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network Should validate the egress IP functionality against remote hosts with egress firewall applied": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [OVN network] Should validate the egress IP SNAT functionality against host-networked pods": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding GRCP health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding Legacy health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes with egress-assignable label": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [OVN network] multiple namespaces sharing a role primary network": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [OVN network] multiple namespaces with different primary networks L2 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [OVN network] multiple namespaces with different primary networks L3 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [secondary-host-eip] Multiple EgressIP objects and their Egress IP hosted on the same interface": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv4": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 compressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 uncompressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [secondary-host-eip] Using different methods to disable a node or pod availability for egress": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [secondary-host-eip] should send address advertisements for EgressIP": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network [secondary-host-eip] uses VRF routing table if EIP assigned interface is VRF slave": "[Disabled:Unimplemented]", + + "e2e egress IP validation Cluster Default Network of replies to egress IP packets that require fragmentation [LGW][IPv4]": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary Should fail if egressip-mark annotation is being added by a regular user": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary Should fail if egressip-mark annotation is present during EgressIP creation": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary Should handle EIP reassignment correctly on namespace and pod label updates, and EIP object updates": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary Should re-assign egress IPs when node readiness / reachability goes down/up": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary Should validate egress IP logic when one pod is managed by more than one egressIP object": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary Should validate the egress IP SNAT functionality for stateful-sets": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary Should validate the egress IP functionality against remote hosts with egress firewall applied": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [OVN network] Should validate the egress IP SNAT functionality against host-networked pods": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding GRCP health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding Legacy health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes with egress-assignable label": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [OVN network] multiple namespaces sharing a role primary network": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [OVN network] multiple namespaces with different primary networks L2 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [OVN network] multiple namespaces with different primary networks L3 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [secondary-host-eip] Multiple EgressIP objects and their Egress IP hosted on the same interface": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv4": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 compressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 uncompressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [secondary-host-eip] should send address advertisements for EgressIP": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary [secondary-host-eip] uses VRF routing table if EIP assigned interface is VRF slave": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L2 role primary of replies to egress IP packets that require fragmentation [LGW][IPv4]": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary Should fail if egressip-mark annotation is being added by a regular user": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary Should fail if egressip-mark annotation is present during EgressIP creation": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary Should handle EIP reassignment correctly on namespace and pod label updates, and EIP object updates": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary Should re-assign egress IPs when node readiness / reachability goes down/up": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary Should validate egress IP logic when one pod is managed by more than one egressIP object": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary Should validate the egress IP SNAT functionality for stateful-sets": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary Should validate the egress IP functionality against remote hosts with egress firewall applied": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [OVN network] Should validate the egress IP SNAT functionality against host-networked pods": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding GRCP health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding Legacy health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes with egress-assignable label": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [OVN network] multiple namespaces sharing a role primary network": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [OVN network] multiple namespaces with different primary networks L2 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [OVN network] multiple namespaces with different primary networks L3 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [secondary-host-eip] Multiple EgressIP objects and their Egress IP hosted on the same interface": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv4": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 compressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 uncompressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [secondary-host-eip] should send address advertisements for EgressIP": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary [secondary-host-eip] uses VRF routing table if EIP assigned interface is VRF slave": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv4 L3 role primary of replies to egress IP packets that require fragmentation [LGW][IPv4]": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary Should fail if egressip-mark annotation is being added by a regular user": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary Should fail if egressip-mark annotation is present during EgressIP creation": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary Should handle EIP reassignment correctly on namespace and pod label updates, and EIP object updates": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary Should re-assign egress IPs when node readiness / reachability goes down/up": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary Should validate egress IP logic when one pod is managed by more than one egressIP object": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary Should validate the egress IP SNAT functionality for stateful-sets": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary Should validate the egress IP functionality against remote hosts with egress firewall applied": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [OVN network] Should validate the egress IP SNAT functionality against host-networked pods": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding GRCP health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding Legacy health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes with egress-assignable label": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [OVN network] multiple namespaces sharing a role primary network": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [OVN network] multiple namespaces with different primary networks L2 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [OVN network] multiple namespaces with different primary networks L3 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [secondary-host-eip] Multiple EgressIP objects and their Egress IP hosted on the same interface": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv4": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 compressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 uncompressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [secondary-host-eip] should send address advertisements for EgressIP": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary [secondary-host-eip] uses VRF routing table if EIP assigned interface is VRF slave": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L2 role primary of replies to egress IP packets that require fragmentation [LGW][IPv4]": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary Should fail if egressip-mark annotation is being added by a regular user": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary Should fail if egressip-mark annotation is present during EgressIP creation": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary Should handle EIP reassignment correctly on namespace and pod label updates, and EIP object updates": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary Should re-assign egress IPs when node readiness / reachability goes down/up": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary Should validate egress IP logic when one pod is managed by more than one egressIP object": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary Should validate the egress IP SNAT functionality for stateful-sets": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary Should validate the egress IP functionality against remote hosts with egress firewall applied": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [OVN network] Should validate the egress IP SNAT functionality against host-networked pods": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding GRCP health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes impeding Legacy health check": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [OVN network] Using different methods to disable a node's availability for egress Should validate the egress IP functionality against remote hosts disabling egress nodes with egress-assignable label": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [OVN network] multiple namespaces sharing a role primary network": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [OVN network] multiple namespaces with different primary networks L2 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [OVN network] multiple namespaces with different primary networks L3 Primary UDN": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [secondary-host-eip] Multiple EgressIP objects and their Egress IP hosted on the same interface": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv4": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 compressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress IPv6 uncompressed": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [secondary-host-eip] Using different methods to disable a node or pod availability for egress": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [secondary-host-eip] should send address advertisements for EgressIP": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary [secondary-host-eip] uses VRF routing table if EIP assigned interface is VRF slave": "[Disabled:Unimplemented]", + + "e2e egress IP validation Network Segmentation: IPv6 L3 role primary of replies to egress IP packets that require fragmentation [LGW][IPv4]": "[Disabled:Unimplemented]", + + "e2e egress firewall policy validation with DNS name resolver Should validate that egressfirewall policy functionality for allowed DNS name": "[Disabled:Unimplemented]", + + "e2e egress firewall policy validation with external containers Should validate that egressfirewall supports DNS name in caps": "[Disabled:Unimplemented]", + + "e2e egress firewall policy validation with external containers Should validate the egress firewall allows inbound connections": "[Disabled:Unimplemented]", + + "e2e egress firewall policy validation with external containers Should validate the egress firewall doesn't affect internal connections": "[Disabled:Unimplemented]", + + "e2e egress firewall policy validation with external containers Should validate the egress firewall policy functionality for allowed CIDR and port": "[Disabled:Unimplemented]", + + "e2e egress firewall policy validation with external containers Should validate the egress firewall policy functionality for allowed IP": "[Disabled:Unimplemented]", + + "e2e ingress to host-networked pods traffic validation Validating ingress traffic to Host Networked pods with externalTrafficPolicy=local Should be allowed to node local host-networked endpoints by nodeport services": "[Disabled:Unimplemented]", + + "e2e ingress traffic validation Validating ingress traffic Should be allowed by externalip services": "[Disabled:Unimplemented]", + + "e2e ingress traffic validation Validating ingress traffic Should be allowed by nodeport services after upgrade to DualStack": "[Disabled:Unimplemented]", + + "e2e ingress traffic validation Validating ingress traffic Should be allowed by nodeport services": "[Disabled:Unimplemented]", + + "e2e ingress traffic validation Validating ingress traffic Should be allowed to node local cluster-networked endpoints by nodeport services with externalTrafficPolicy=local": "[Disabled:Unimplemented]", + + "e2e ingress traffic validation Validating ingress traffic to manually added node IPs Should be allowed by externalip services to a new node ip": "[Disabled:Unimplemented]", + + "e2e network policy hairpinning validation Should validate the hairpinned traffic is always allowed": "[Disabled:Unimplemented]", + + "test e2e inter-node connectivity between worker nodes Should validate connectivity within a namespace of pods on separate nodes": "[Disabled:Unimplemented]", + + "test e2e pod connectivity to host addresses Should validate connectivity from a pod to a non-node host address on same node": "[Disabled:Unimplemented]", +} + +func init() { + ginkgo.GetSuite().SetAnnotateFn(func(name string, node types.TestSpec) { + if newLabels, ok := AppendedAnnotations[name]; ok { + node.AppendText(newLabels) + } else { + panic(fmt.Sprintf("unable to find test %s", name)) + } + }) +} diff --git a/openshift/test/infraprovider/openshift.go b/openshift/test/infraprovider/openshift.go new file mode 100644 index 0000000000..ab4bf119a7 --- /dev/null +++ b/openshift/test/infraprovider/openshift.go @@ -0,0 +1,209 @@ +package infraprovider + +import ( + "bytes" + "errors" + "fmt" + "os/exec" + "strings" + "time" + + ovnkconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider/api" + "github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider/portalloc" + + "github.com/onsi/ginkgo/v2" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/kubernetes/test/e2e/framework" +) + +type openshift struct { + externalContainerPortAlloc *portalloc.PortAllocator + hostPortAlloc *portalloc.PortAllocator + kubeClient *kubernetes.Clientset +} + +func (o openshift) ShutdownNode(nodeName string) error { + panic("not implemented") +} + +func (o openshift) StartNode(nodeName string) error { + panic("not implemented") +} + +func (m openshift) GetDefaultTimeoutContext() *framework.TimeoutContext { + timeouts := framework.NewTimeoutContext() + timeouts.PodStart = 10 * time.Minute + return timeouts +} + +func IsProvider(config *rest.Config) (bool, error) { + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return false, fmt.Errorf("failed to create kubernetes client: %w", err) + } + // Check for OpenShift-specific API groups + groups, err := kubeClient.Discovery().ServerGroups() + if err != nil { + return false, fmt.Errorf("failed to get server groups: %w", err) + } + for _, group := range groups.Groups { + if strings.HasSuffix(group.Name, ".openshift.io") { + return true, nil + } + } + return false, nil +} + +func New(config *rest.Config) (api.Provider, error) { + ovnkconfig.Kubernetes.DNSServiceNamespace = "openshift-dns" + ovnkconfig.Kubernetes.DNSServiceName = "dns-default" + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("unable to create kubernetes client: %w", err) + } + return openshift{ + externalContainerPortAlloc: portalloc.New(30000, 32767), + hostPortAlloc: portalloc.New(30000, 32767), + kubeClient: kubeClient, + }, nil +} + +func (o openshift) Name() string { + return "openshift" +} + +func (o openshift) PrimaryNetwork() (api.Network, error) { + panic("not implemented") +} + +func (o openshift) ExternalContainerPrimaryInterfaceName() string { + panic("not implemented") +} + +func (o openshift) GetNetwork(name string) (api.Network, error) { + panic("not implemented") +} + +func (o openshift) GetExternalContainerNetworkInterface(container api.ExternalContainer, network api.Network) (api.NetworkInterface, error) { + panic("not implemented") +} + +func (o openshift) GetK8NodeNetworkInterface(instance string, network api.Network) (api.NetworkInterface, error) { + panic("not implemented") +} + +func (o openshift) GetExternalContainerLogs(container api.ExternalContainer) (string, error) { + panic("not implemented") +} + +func (o openshift) ExecK8NodeCommand(nodeName string, cmd []string) (string, error) { + if len(cmd) == 0 { + panic("ExecK8NodeCommand(): insufficient command arguments") + } + cmd = append([]string{"debug", fmt.Sprintf("node/%s", nodeName), "--to-namespace=default", + "--", "chroot", "/host"}, cmd...) + ocDebugCmd := exec.Command("oc", cmd...) + var stdout, stderr bytes.Buffer + ocDebugCmd.Stdout = &stdout + ocDebugCmd.Stderr = &stderr + + if err := ocDebugCmd.Run(); err != nil { + return "", fmt.Errorf("failed to run command %q on node %s: %v, stdout: %s, stderr: %s", ocDebugCmd.String(), nodeName, err, stdout.String(), stderr.String()) + } + return stdout.String(), nil +} + +func (o openshift) ExecExternalContainerCommand(container api.ExternalContainer, cmd []string) (string, error) { + panic("not implemented") +} + +func (o openshift) GetExternalContainerPort() uint16 { + return o.externalContainerPortAlloc.Allocate() +} + +func (o openshift) GetK8HostPort() uint16 { + return o.hostPortAlloc.Allocate() +} + +func (o openshift) NewTestContext() api.Context { + co := &contextOpenshift{make([]func() error, 0)} + ginkgo.DeferCleanup(co.CleanUp) + return co +} + +type contextOpenshift struct { + cleanUpFns []func() error +} + +func (c *contextOpenshift) GetAllowedExternalContainerPort() int { + panic("not implemented") +} + +func (c *contextOpenshift) CreateExternalContainer(container api.ExternalContainer) (api.ExternalContainer, error) { + panic("not implemented") +} + +func (c *contextOpenshift) DeleteExternalContainer(container api.ExternalContainer) error { + panic("not implemented") +} + +func (c *contextOpenshift) GetExternalContainerLogs(container api.ExternalContainer) (string, error) { + panic("not implemented") +} + +func (c contextOpenshift) CreateNetwork(name string, subnets ...string) (api.Network, error) { + panic("not implemented") +} + +func (c contextOpenshift) DeleteNetwork(network api.Network) error { + panic("not implemented") +} + +func (c *contextOpenshift) GetAttachedNetworks() (api.Networks, error) { + panic("not implemented") +} + +func (c *contextOpenshift) SetupUnderlay(f *framework.Framework, underlay api.Underlay) error { + panic("not implemented") +} + +func (c contextOpenshift) AttachNetwork(network api.Network, instance string) (api.NetworkInterface, error) { + panic("not implemented") +} + +func (c contextOpenshift) DetachNetwork(network api.Network, instance string) error { + panic("not implemented") +} + +func (c *contextOpenshift) AddCleanUpFn(cleanUpFn func() error) { + c.cleanUpFns = append(c.cleanUpFns, cleanUpFn) +} + +func (c *contextOpenshift) CleanUp() error { + ginkgo.By("Cleaning up openshift test context") + var errs []error + // generic cleanup activities + for i := len(c.cleanUpFns) - 1; i >= 0; i-- { + if err := c.cleanUpFns[i](); err != nil { + errs = append(errs, err) + } + } + c.cleanUpFns = nil + return condenseErrors(errs) +} + +func condenseErrors(errs []error) error { + switch len(errs) { + case 0: + return nil + case 1: + return errs[0] + } + err := errs[0] + for _, e := range errs[1:] { + err = errors.Join(err, e) + } + return err +} diff --git a/openshift/test/namespace.go b/openshift/test/namespace.go new file mode 100644 index 0000000000..b9f043ec42 --- /dev/null +++ b/openshift/test/namespace.go @@ -0,0 +1,130 @@ +package test + +import ( + "context" + "fmt" + "runtime/debug" + "strings" + + "github.com/onsi/ginkgo/v2" + projectv1 "github.com/openshift/api/project/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + kclientset "k8s.io/client-go/kubernetes" + rbacv1client "k8s.io/client-go/kubernetes/typed/rbac/v1" + "k8s.io/client-go/util/retry" + "k8s.io/kubernetes/test/e2e/framework" +) + +// CreateTestingNS ensures that kubernetes e2e tests have their service accounts in the privileged and anyuid SCCs +func CreateTestingNS(ctx context.Context, baseName string, c kclientset.Interface, labels map[string]string, isKubeNamespace bool) (*corev1.Namespace, error) { + if !strings.HasPrefix(baseName, "e2e-") { + baseName = "e2e-" + baseName + } + + if labels == nil { + labels = map[string]string{} + } + // turn off the OpenShift label syncer so that it does not attempt to sync + // the PodSecurity admission labels + labels["security.openshift.io/scc.podSecurityLabelSync"] = "false" + + if isKubeNamespace { + labels["security.openshift.io/disable-securitycontextconstraints"] = "true" + } + + ns, err := framework.CreateTestingNS(ctx, baseName, c, labels) + if err != nil { + return ns, err + } + + if !isKubeNamespace { + return ns, err + } + + // Add anyuid and privileged permissions for upstream tests + clientConfig, err := framework.LoadConfig() + if err != nil { + return ns, err + } + + rbacClient, err := rbacv1client.NewForConfig(clientConfig) + if err != nil { + return ns, err + } + framework.Logf("About to run a Kube e2e test, ensuring namespace/%s is privileged", ns.Name) + // add the "privileged" scc to ensure pods that explicitly + // request extra capabilities are not rejected + addRoleToE2EServiceAccounts(ctx, rbacClient, []corev1.Namespace{*ns}, "system:openshift:scc:privileged") + // add the "anyuid" scc to ensure pods that don't specify a + // uid don't get forced into a range (mimics upstream + // behavior) + addRoleToE2EServiceAccounts(ctx, rbacClient, []corev1.Namespace{*ns}, "system:openshift:scc:anyuid") + // add the "hostmount-anyuid" scc to ensure pods using hostPath + // can execute tests + addRoleToE2EServiceAccounts(ctx, rbacClient, []corev1.Namespace{*ns}, "system:openshift:scc:hostmount-anyuid") + + // The intra-pod test requires that the service account have + // permission to retrieve service endpoints. + addRoleToE2EServiceAccounts(ctx, rbacClient, []corev1.Namespace{*ns}, "view") + + // in practice too many kube tests ignore scheduling constraints + allowAllNodeScheduling(ctx, c, ns.Name) + + return ns, err +} + +var longRetry = wait.Backoff{Steps: 100} + +func fatalErr(msg interface{}) { + // the path that leads to this being called isn't always clear... + fmt.Fprintln(ginkgo.GinkgoWriter, string(debug.Stack())) + framework.Failf("%v", msg) +} + +func addRoleToE2EServiceAccounts(ctx context.Context, rbacClient rbacv1client.RbacV1Interface, namespaces []corev1.Namespace, roleName string) { + err := retry.RetryOnConflict(longRetry, func() error { + for _, ns := range namespaces { + if ns.Status.Phase != corev1.NamespaceTerminating { + _, err := rbacClient.RoleBindings(ns.Name).Create(ctx, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "default-" + roleName, Namespace: ns.Name}, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: roleName, + }, + Subjects: []rbacv1.Subject{ + {Name: "default", Namespace: ns.Name, Kind: rbacv1.ServiceAccountKind}, + }, + }, metav1.CreateOptions{}) + if err != nil { + framework.Logf("Warning: Failed to add role to e2e service account: %v", err) + } + } + } + return nil + }) + if err != nil { + fatalErr(err) + } +} + +// allowAllNodeScheduling sets the annotation on namespace that allows all nodes to be scheduled onto. +func allowAllNodeScheduling(ctx context.Context, c kclientset.Interface, namespace string) { + err := retry.RetryOnConflict(longRetry, func() error { + ns, err := c.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err != nil { + return err + } + if ns.Annotations == nil { + ns.Annotations = make(map[string]string) + } + ns.Annotations[projectv1.ProjectNodeSelector] = "" + _, err = c.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}) + return err + }) + if err != nil { + fatalErr(err) + } +} diff --git a/test/Makefile b/test/Makefile index 146bddff59..0124c750a0 100644 --- a/test/Makefile +++ b/test/Makefile @@ -12,7 +12,7 @@ DUALSTACK_CONVERSION?=false COREDUMP_DIR?=/tmp/kind/logs/coredumps # Processes to skip when checking for coredumps (pipe-separated for grep) # https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5782 -SKIPPED_COREDUMPS?=zebra|bgpd|mgmtd +SKIPPED_COREDUMPS?=zebra|bgpd|mgmtd|bfdd # Check for coredumps and fail if any are found (excluding skipped processes) # Usage: $(call check-coredumps) diff --git a/test/e2e/cluster_network_connect.go b/test/e2e/cluster_network_connect.go index ee588986ae..5634bb3509 100644 --- a/test/e2e/cluster_network_connect.go +++ b/test/e2e/cluster_network_connect.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net" "strings" "time" @@ -16,14 +17,36 @@ import ( "k8s.io/apimachinery/pkg/util/rand" clientset "k8s.io/client-go/kubernetes" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" "github.com/ovn-org/ovn-kubernetes/test/e2e/feature" ) +// ============================================================================ +// CNC Test Constants +// ============================================================================ const ( // Annotation keys used by the CNC controller ovnNetworkConnectSubnetAnnotation = "k8s.ovn.org/network-connect-subnet" ovnConnectRouterTunnelKeyAnnotation = "k8s.ovn.org/connect-router-tunnel-key" + + // CNC connect subnet configuration + cncConnectSubnetIPv4CIDR = "192.168.0.0/16" + cncConnectSubnetIPv4Prefix = 24 + // IPv6 networkPrefix must satisfy: 32 - ipv4Prefix == 128 - ipv6Prefix + // With ipv4Prefix=24: 32-24=8, so ipv6Prefix must be 128-8=120 + cncConnectSubnetIPv6CIDR = "fd00:10::/112" + cncConnectSubnetIPv6Prefix = 120 + + // Layer3 UDN CIDRs with hostSubnet (IPv4: /24, IPv6: /64) + layer3UserDefinedNetworkIPv4CIDR = "172.31.0.0/16" + layer3UserDefinedNetworkIPv4HostSubnet = 24 + layer3UserDefinedNetworkIPv6CIDR = "2014:100:200::0/60" + layer3UserDefinedNetworkIPv6HostSubnet = 64 + + // Layer2 UDN CIDRs + layer2UserDefinedNetworkIPv4CIDR = "10.200.0.0/16" + layer2UserDefinedNetworkIPv6CIDR = "2015:100:200::0/60" ) // cncAnnotationSubnet represents the subnet annotation structure @@ -32,113 +55,145 @@ type cncAnnotationSubnet struct { IPv6 string `json:"ipv6,omitempty"` } -var _ = Describe("ClusterNetworkConnect ClusterManagerController", feature.NetworkConnect, func() { - f := wrappedTestFramework("cnc-controller") - // disable automatic namespace creation, we need to add the required UDN label - f.SkipNamespaceCreation = true +// ============================================================================ +// CNC Test Global Utilities +// ============================================================================ - var ( - cs clientset.Interface - ) +// generateCNCName generates a random CNC name +func generateCNCName() string { + return fmt.Sprintf("test-cnc-%s", rand.String(5)) +} - const ( - cncConnectSubnetIPv4CIDR = "192.168.0.0/16" - cncConnectSubnetIPv4Prefix = 24 - // IPv6 networkPrefix must satisfy: 32 - ipv4Prefix == 128 - ipv6Prefix - // With ipv4Prefix=24: 32-24=8, so ipv6Prefix must be 128-8=120 - cncConnectSubnetIPv6CIDR = "fd00:10::/112" - cncConnectSubnetIPv6Prefix = 120 - // Layer3 UDN CIDRs with hostSubnet (IPv4: /24, IPv6: /64) - layer3UserDefinedNetworkIPv4CIDR = "172.31.0.0/16" - layer3UserDefinedNetworkIPv4HostSubnet = 24 - layer3UserDefinedNetworkIPv6CIDR = "2014:100:200::0/60" - layer3UserDefinedNetworkIPv6HostSubnet = 64 - // Layer2 UDN CIDRs - layer2UserDefinedNetworkIPv4CIDR = "10.200.0.0/16" - layer2UserDefinedNetworkIPv6CIDR = "2015:100:200::0/60" - ) +// generateConnectSubnets generates connectSubnets YAML based on cluster IP family support +func generateConnectSubnets(cs clientset.Interface) string { + return generateConnectSubnetsWithCIDRs(cs, cncConnectSubnetIPv4CIDR, cncConnectSubnetIPv4Prefix, + cncConnectSubnetIPv6CIDR, cncConnectSubnetIPv6Prefix) +} - BeforeEach(func() { - cs = f.ClientSet - }) +// generateConnectSubnetsWithCIDRs generates connectSubnets YAML with custom CIDRs +func generateConnectSubnetsWithCIDRs(cs clientset.Interface, v4CIDR string, v4Prefix int, v6CIDR string, v6Prefix int) string { + var subnets []string + if isIPv4Supported(cs) { + subnets = append(subnets, fmt.Sprintf(` - cidr: "%s" + networkPrefix: %d`, v4CIDR, v4Prefix)) + } + if isIPv6Supported(cs) { + subnets = append(subnets, fmt.Sprintf(` - cidr: "%s" + networkPrefix: %d`, v6CIDR, v6Prefix)) + } + return strings.Join(subnets, "\n") +} + +// generateNetworkSubnets generates subnets YAML with custom CIDRs +// Pass empty strings to use defaults +func generateNetworkSubnets(cs clientset.Interface, topology, v4Subnet, v6Subnet string) string { + // Use custom subnets if provided, otherwise fall back to defaults + l3v4CIDR := layer3UserDefinedNetworkIPv4CIDR + l3v6CIDR := layer3UserDefinedNetworkIPv6CIDR + l2v4CIDR := layer2UserDefinedNetworkIPv4CIDR + l2v6CIDR := layer2UserDefinedNetworkIPv6CIDR + + if v4Subnet != "" { + l3v4CIDR = v4Subnet + l2v4CIDR = v4Subnet + } + if v6Subnet != "" { + l3v6CIDR = v6Subnet + l2v6CIDR = v6Subnet + } - // Helper to generate connectSubnets YAML based on cluster IP family support - generateConnectSubnets := func() string { + if topology == "Layer3" { var subnets []string if isIPv4Supported(cs) { - subnets = append(subnets, fmt.Sprintf(` - cidr: "%s" - networkPrefix: %d`, cncConnectSubnetIPv4CIDR, cncConnectSubnetIPv4Prefix)) + subnets = append(subnets, fmt.Sprintf(`{cidr: "%s", hostSubnet: %d}`, l3v4CIDR, layer3UserDefinedNetworkIPv4HostSubnet)) } if isIPv6Supported(cs) { - subnets = append(subnets, fmt.Sprintf(` - cidr: "%s" - networkPrefix: %d`, cncConnectSubnetIPv6CIDR, cncConnectSubnetIPv6Prefix)) + subnets = append(subnets, fmt.Sprintf(`{cidr: "%s", hostSubnet: %d}`, l3v6CIDR, layer3UserDefinedNetworkIPv6HostSubnet)) } - return strings.Join(subnets, "\n") + return fmt.Sprintf("[%s]", strings.Join(subnets, ",")) } - - // Helper to create a namespace with UDN label - createUDNNamespace := func(baseName string, labels map[string]string) *corev1.Namespace { - if labels == nil { - labels = map[string]string{} - } - labels[RequiredUDNNamespaceLabel] = "" - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: baseName + "-" + rand.String(5), - Labels: labels, - }, - } - createdNs, err := cs.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - return createdNs + // Layer2 format + var quotedCidrs []string + if isIPv4Supported(cs) { + quotedCidrs = append(quotedCidrs, fmt.Sprintf(`"%s"`, l2v4CIDR)) } + if isIPv6Supported(cs) { + quotedCidrs = append(quotedCidrs, fmt.Sprintf(`"%s"`, l2v6CIDR)) + } + return fmt.Sprintf("[%s]", strings.Join(quotedCidrs, ",")) +} - // Helper to generate a random CNC name - generateCNCName := func() string { - return fmt.Sprintf("test-cnc-%s", rand.String(5)) +// createUDNNamespaceWithName creates a namespace with UDN label and optional additional labels +func createUDNNamespaceWithName(cs clientset.Interface, name string, labels map[string]string) *corev1.Namespace { + if labels == nil { + labels = map[string]string{} } + labels[RequiredUDNNamespaceLabel] = "" + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + } + createdNs, err := cs.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + return createdNs +} - // Helper to create or update a CNC with CUDN and/or PUDN selectors - // Pass nil for a selector type you don't want to use, but at least one must be non-nil - // Uses kubectl apply, so can be called to update an existing CNC - createOrUpdateCNC := func(cncName string, cudnLabelSelector, pudnLabelSelector map[string]string) { - // CNC requires at least one selector (MinItems=1 on NetworkSelectors type) - Expect(cudnLabelSelector != nil || pudnLabelSelector != nil).To(BeTrue(), - "createOrUpdateCNC requires at least one selector (cudnLabelSelector or pudnLabelSelector)") - - var networkSelectors []string - - if cudnLabelSelector != nil { - cudnLabelSelectorStr := "" - for k, v := range cudnLabelSelector { - if cudnLabelSelectorStr != "" { - cudnLabelSelectorStr += "\n " - } - cudnLabelSelectorStr += fmt.Sprintf("%s: \"%s\"", k, v) +// createUDNNamespace creates a namespace with UDN label and a random suffix +func createUDNNamespace(cs clientset.Interface, baseName string, labels map[string]string) *corev1.Namespace { + return createUDNNamespaceWithName(cs, baseName+"-"+rand.String(5), labels) +} + +// deleteNamespace deletes a namespace +func deleteNamespace(cs clientset.Interface, nsName string) { + _ = cs.CoreV1().Namespaces().Delete(context.Background(), nsName, metav1.DeleteOptions{}) +} + +// createOrUpdateCNC creates or updates a CNC with CUDN and/or PUDN selectors using default connect subnets +// Uses kubectl apply, so can be called to update an existing CNC +func createOrUpdateCNC(cs clientset.Interface, cncName string, cudnLabelSelector, udnLabelSelector map[string]string) { + createOrUpdateCNCWithSubnets(cncName, cudnLabelSelector, udnLabelSelector, generateConnectSubnets(cs)) +} + +// createOrUpdateCNCWithSubnets creates or updates a CNC with custom connect subnets +func createOrUpdateCNCWithSubnets(cncName string, cudnLabelSelector, udnLabelSelector map[string]string, connectSubnets string) { + Expect(cudnLabelSelector != nil || udnLabelSelector != nil).To(BeTrue(), + "createOrUpdateCNCWithSubnets requires at least one selector (cudnLabelSelector or udnLabelSelector)") + + var networkSelectors []string + + if cudnLabelSelector != nil { + cudnLabelSelectorStr := "" + for k, v := range cudnLabelSelector { + if cudnLabelSelectorStr != "" { + cudnLabelSelectorStr += "\n " } - networkSelectors = append(networkSelectors, fmt.Sprintf(` - networkSelectionType: "ClusterUserDefinedNetworks" + cudnLabelSelectorStr += fmt.Sprintf("%s: \"%s\"", k, v) + } + networkSelectors = append(networkSelectors, fmt.Sprintf(` - networkSelectionType: "ClusterUserDefinedNetworks" clusterUserDefinedNetworkSelector: networkSelector: matchLabels: %s`, cudnLabelSelectorStr)) - } + } - if pudnLabelSelector != nil { - pudnLabelSelectorStr := "" - for k, v := range pudnLabelSelector { - if pudnLabelSelectorStr != "" { - pudnLabelSelectorStr += "\n " - } - pudnLabelSelectorStr += fmt.Sprintf("%s: \"%s\"", k, v) + if udnLabelSelector != nil { + udnLabelSelectorStr := "" + for k, v := range udnLabelSelector { + if udnLabelSelectorStr != "" { + udnLabelSelectorStr += "\n " } - networkSelectors = append(networkSelectors, fmt.Sprintf(` - networkSelectionType: "PrimaryUserDefinedNetworks" + udnLabelSelectorStr += fmt.Sprintf("%s: \"%s\"", k, v) + } + networkSelectors = append(networkSelectors, fmt.Sprintf(` - networkSelectionType: "PrimaryUserDefinedNetworks" primaryUserDefinedNetworkSelector: namespaceSelector: matchLabels: - %s`, pudnLabelSelectorStr)) - } + %s`, udnLabelSelectorStr)) + } - manifest := fmt.Sprintf(` + manifest := fmt.Sprintf(` apiVersion: k8s.ovn.org/v1 kind: ClusterNetworkConnect metadata: @@ -149,47 +204,34 @@ spec: connectSubnets: %s connectivity: ["PodNetwork"] -`, cncName, strings.Join(networkSelectors, "\n"), generateConnectSubnets()) - _, err := e2ekubectl.RunKubectlInput("", manifest, "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred()) - } +`, cncName, strings.Join(networkSelectors, "\n"), connectSubnets) + _, err := e2ekubectl.RunKubectlInput("", manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred()) +} - // Helper to generate subnets YAML based on topology and cluster IP family support - // Layer3 uses [{cidr: "...", hostSubnet: N}] format, Layer2 uses ["..."] format - generateNetworkSubnets := func(topology string) string { - if topology == "Layer3" { - var subnets []string - if isIPv4Supported(cs) { - subnets = append(subnets, fmt.Sprintf(`{cidr: "%s", hostSubnet: %d}`, layer3UserDefinedNetworkIPv4CIDR, layer3UserDefinedNetworkIPv4HostSubnet)) - } - if isIPv6Supported(cs) { - subnets = append(subnets, fmt.Sprintf(`{cidr: "%s", hostSubnet: %d}`, layer3UserDefinedNetworkIPv6CIDR, layer3UserDefinedNetworkIPv6HostSubnet)) - } - return fmt.Sprintf("[%s]", strings.Join(subnets, ",")) - } - // Layer2 format - var quotedCidrs []string - if isIPv4Supported(cs) { - quotedCidrs = append(quotedCidrs, fmt.Sprintf(`"%s"`, layer2UserDefinedNetworkIPv4CIDR)) - } - if isIPv6Supported(cs) { - quotedCidrs = append(quotedCidrs, fmt.Sprintf(`"%s"`, layer2UserDefinedNetworkIPv6CIDR)) - } - return fmt.Sprintf("[%s]", strings.Join(quotedCidrs, ",")) - } +// deleteCNC deletes a CNC +func deleteCNC(cncName string) { + _, _ = e2ekubectl.RunKubectl("", "delete", "clusternetworkconnect", cncName, "--ignore-not-found") +} - // Helper to create a primary CUDN with specified topology - createPrimaryCUDN := func(cudnName, topology string, labels map[string]string, targetNamespaces ...string) { - targetNs := strings.Join(targetNamespaces, ",") - labelAnnotations := "" - for k, v := range labels { - if labelAnnotations != "" { - labelAnnotations += "\n " - } - labelAnnotations += fmt.Sprintf("%s: \"%s\"", k, v) +// createPrimaryCUDN creates a primary CUDN with specified topology +func createPrimaryCUDN(cs clientset.Interface, cudnName, topology string, labels map[string]string, targetNamespaces ...string) { + createPrimaryCUDNWithSubnets(cs, cudnName, topology, labels, "", "", targetNamespaces...) +} + +// createPrimaryCUDNWithSubnets creates a primary CUDN with specified topology and custom subnets. +// Pass empty strings for v4Subnet/v6Subnet to use defaults. +func createPrimaryCUDNWithSubnets(cs clientset.Interface, cudnName, topology string, labels map[string]string, v4Subnet, v6Subnet string, targetNamespaces ...string) { + targetNs := strings.Join(targetNamespaces, ",") + labelAnnotations := "" + for k, v := range labels { + if labelAnnotations != "" { + labelAnnotations += "\n " } - topologyLower := strings.ToLower(topology) - manifest := fmt.Sprintf(` + labelAnnotations += fmt.Sprintf("%s: \"%s\"", k, v) + } + topologyLower := strings.ToLower(topology) + manifest := fmt.Sprintf(` apiVersion: k8s.ovn.org/v1 kind: ClusterUserDefinedNetwork metadata: @@ -207,23 +249,26 @@ spec: %s: role: Primary subnets: %s -`, cudnName, labelAnnotations, targetNs, topology, topologyLower, generateNetworkSubnets(topology)) - _, err := e2ekubectl.RunKubectlInput("", manifest, "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred()) - } +`, cudnName, labelAnnotations, targetNs, topology, topologyLower, generateNetworkSubnets(cs, topology, v4Subnet, v6Subnet)) + _, err := e2ekubectl.RunKubectlInput("", manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred()) +} - // Convenience wrappers for Layer3/Layer2 CUDN creation - createLayer3PrimaryCUDN := func(cudnName string, labels map[string]string, targetNamespaces ...string) { - createPrimaryCUDN(cudnName, "Layer3", labels, targetNamespaces...) - } - createLayer2PrimaryCUDN := func(cudnName string, labels map[string]string, targetNamespaces ...string) { - createPrimaryCUDN(cudnName, "Layer2", labels, targetNamespaces...) - } +// deleteCUDN deletes a CUDN +func deleteCUDN(cudnName string) { + _, _ = e2ekubectl.RunKubectl("", "delete", "clusteruserdefinednetwork", cudnName, "--wait", "--timeout=60s", "--ignore-not-found") +} - // Helper to create a primary UDN with specified topology - createPrimaryUDN := func(namespace, udnName, topology string) { - topologyLower := strings.ToLower(topology) - manifest := fmt.Sprintf(` +// createPrimaryUDN creates a primary UDN with specified topology +func createPrimaryUDN(cs clientset.Interface, namespace, udnName, topology string) { + createPrimaryUDNWithSubnets(cs, namespace, udnName, topology, "", "") +} + +// createPrimaryUDNWithSubnets creates a primary UDN with specified topology and custom subnets. +// Pass empty strings for v4Subnet/v6Subnet to use defaults. +func createPrimaryUDNWithSubnets(cs clientset.Interface, namespace, udnName, topology, v4Subnet, v6Subnet string) { + topologyLower := strings.ToLower(topology) + manifest := fmt.Sprintf(` apiVersion: k8s.ovn.org/v1 kind: UserDefinedNetwork metadata: @@ -233,164 +278,199 @@ spec: %s: role: Primary subnets: %s -`, udnName, topology, topologyLower, generateNetworkSubnets(topology)) - _, err := e2ekubectl.RunKubectlInput(namespace, manifest, "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred()) - } +`, udnName, topology, topologyLower, generateNetworkSubnets(cs, topology, v4Subnet, v6Subnet)) + _, err := e2ekubectl.RunKubectlInput(namespace, manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred()) +} - // Convenience wrappers for Layer3/Layer2 UDN creation - createLayer3PrimaryUDN := func(namespace, udnName string) { - createPrimaryUDN(namespace, udnName, "Layer3") - } - createLayer2PrimaryUDN := func(namespace, udnName string) { - createPrimaryUDN(namespace, udnName, "Layer2") - } +// deleteUDN deletes a UDN +func deleteUDN(namespace, udnName string) { + _, _ = e2ekubectl.RunKubectl(namespace, "delete", "userdefinednetwork", udnName, "--wait", "--timeout=60s", "--ignore-not-found") +} - // Helper to delete a CNC - deleteCNC := func(cncName string) { - _, _ = e2ekubectl.RunKubectl("", "delete", "clusternetworkconnect", cncName, "--ignore-not-found") - } +// createLayer3PrimaryCUDN creates a Layer3 primary CUDN (convenience function) +func createLayer3PrimaryCUDN(cs clientset.Interface, cudnName string, labels map[string]string, targetNamespaces ...string) { + createPrimaryCUDN(cs, cudnName, "Layer3", labels, targetNamespaces...) +} - // Helper to delete a CUDN - deleteCUDN := func(cudnName string) { - _, _ = e2ekubectl.RunKubectl("", "delete", "clusteruserdefinednetwork", cudnName, "--wait", "--timeout=60s", "--ignore-not-found") - } +// createLayer3PrimaryCUDNWithSubnets creates a Layer3 primary CUDN with custom subnets +func createLayer3PrimaryCUDNWithSubnets(cs clientset.Interface, cudnName string, labels map[string]string, v4Subnet, v6Subnet string, targetNamespaces ...string) { + createPrimaryCUDNWithSubnets(cs, cudnName, "Layer3", labels, v4Subnet, v6Subnet, targetNamespaces...) +} - // Helper to delete a UDN - deleteUDN := func(namespace, udnName string) { - _, _ = e2ekubectl.RunKubectl(namespace, "delete", "userdefinednetwork", udnName, "--wait", "--timeout=60s", "--ignore-not-found") - } +// createLayer2PrimaryCUDN creates a Layer2 primary CUDN (convenience function) +func createLayer2PrimaryCUDN(cs clientset.Interface, cudnName string, labels map[string]string, targetNamespaces ...string) { + createPrimaryCUDN(cs, cudnName, "Layer2", labels, targetNamespaces...) +} - // Helper to get CNC annotations - getCNCAnnotations := func(cncName string) (map[string]string, error) { - annotationsJSON, err := e2ekubectl.RunKubectl("", "get", "clusternetworkconnect", cncName, "-o", "jsonpath={.metadata.annotations}") - if err != nil { - return nil, err - } - if annotationsJSON == "" { - return map[string]string{}, nil - } - var annotations map[string]string - if err := json.Unmarshal([]byte(annotationsJSON), &annotations); err != nil { - return nil, err - } - return annotations, nil - } +// createLayer2PrimaryCUDNWithSubnets creates a Layer2 primary CUDN with custom subnets +func createLayer2PrimaryCUDNWithSubnets(cs clientset.Interface, cudnName string, labels map[string]string, v4Subnet, v6Subnet string, targetNamespaces ...string) { + createPrimaryCUDNWithSubnets(cs, cudnName, "Layer2", labels, v4Subnet, v6Subnet, targetNamespaces...) +} - // Helper to verify CNC has only tunnel ID annotation - verifyCNCHasOnlyTunnelIDAnnotation := func(cncName string) { - Eventually(func(g Gomega) { - annotations, err := getCNCAnnotations(cncName) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(annotations).To(HaveKey(ovnConnectRouterTunnelKeyAnnotation), "CNC should have tunnel ID annotation") - if subnetAnnotation, exists := annotations[ovnNetworkConnectSubnetAnnotation]; exists { - g.Expect(subnetAnnotation).To(Equal("{}"), "subnet annotation should be empty when no networks match") - } - }, 30*time.Second, 1*time.Second).Should(Succeed()) - } +// createLayer3PrimaryUDN creates a Layer3 primary UDN (convenience function) +func createLayer3PrimaryUDN(cs clientset.Interface, namespace, udnName string) { + createPrimaryUDN(cs, namespace, udnName, "Layer3") +} - // Helper to verify CNC has both annotations - verifyCNCHasBothAnnotations := func(cncName string) { - Eventually(func(g Gomega) { - annotations, err := getCNCAnnotations(cncName) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(annotations).To(HaveKey(ovnConnectRouterTunnelKeyAnnotation), "CNC should have tunnel ID annotation") - g.Expect(annotations).To(HaveKey(ovnNetworkConnectSubnetAnnotation), "CNC should have subnet annotation") - subnetAnnotation := annotations[ovnNetworkConnectSubnetAnnotation] - g.Expect(subnetAnnotation).NotTo(Equal("{}"), "subnet annotation should not be empty when networks match") - var subnets map[string]cncAnnotationSubnet - err = json.Unmarshal([]byte(subnetAnnotation), &subnets) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(subnets)).To(BeNumerically(">", 0), "should have at least one network subnet") - }, 60*time.Second, 2*time.Second).Should(Succeed()) - } +// createLayer3PrimaryUDNWithSubnets creates a Layer3 primary UDN with custom subnets +func createLayer3PrimaryUDNWithSubnets(cs clientset.Interface, namespace, udnName, v4Subnet, v6Subnet string) { + createPrimaryUDNWithSubnets(cs, namespace, udnName, "Layer3", v4Subnet, v6Subnet) +} + +// createLayer2PrimaryUDN creates a Layer2 primary UDN (convenience function) +func createLayer2PrimaryUDN(cs clientset.Interface, namespace, udnName string) { + createPrimaryUDN(cs, namespace, udnName, "Layer2") +} + +// createLayer2PrimaryUDNWithSubnets creates a Layer2 primary UDN with custom subnets +func createLayer2PrimaryUDNWithSubnets(cs clientset.Interface, namespace, udnName, v4Subnet, v6Subnet string) { + createPrimaryUDNWithSubnets(cs, namespace, udnName, "Layer2", v4Subnet, v6Subnet) +} - // Helper to verify CNC subnet annotation count - verifyCNCSubnetAnnotationNetworkCount := func(cncName string, expectedCount int) { - Eventually(func(g Gomega) { - annotations, err := getCNCAnnotations(cncName) - g.Expect(err).NotTo(HaveOccurred()) - subnetAnnotation := annotations[ovnNetworkConnectSubnetAnnotation] - var subnets map[string]cncAnnotationSubnet - err = json.Unmarshal([]byte(subnetAnnotation), &subnets) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(subnets)).To(Equal(expectedCount), fmt.Sprintf("should have %d network subnets", expectedCount)) - }, 60*time.Second, 2*time.Second).Should(Succeed()) +// getCNCAnnotations gets CNC annotations +func getCNCAnnotations(cncName string) (map[string]string, error) { + annotationsJSON, err := e2ekubectl.RunKubectl("", "get", "clusternetworkconnect", cncName, "-o", "jsonpath={.metadata.annotations}") + if err != nil { + return nil, err + } + if annotationsJSON == "" { + return map[string]string{}, nil } + var annotations map[string]string + if err := json.Unmarshal([]byte(annotationsJSON), &annotations); err != nil { + return nil, err + } + return annotations, nil +} - // Helper to verify subnet annotation content: key format, topology counts, and CIDR format - // expectedTopologies is a list of expected topologies (e.g., ["Layer3", "Layer2", "Layer3"]) - verifyCNCSubnetAnnotationContent := func(cncName string, expectedTopologies []string) { - Eventually(func(g Gomega) { - annotations, err := getCNCAnnotations(cncName) - g.Expect(err).NotTo(HaveOccurred()) - subnetAnnotation := annotations[ovnNetworkConnectSubnetAnnotation] - var subnets map[string]cncAnnotationSubnet - err = json.Unmarshal([]byte(subnetAnnotation), &subnets) - g.Expect(err).NotTo(HaveOccurred()) - - // Count topologies found - topologyCounts := map[string]int{"layer2": 0, "layer3": 0} - for networkKey, subnet := range subnets { - // Key format should be _ e.g., "layer3_1", "layer2_2" - g.Expect(networkKey).To(MatchRegexp(`^(layer2|layer3)_\d+$`), - fmt.Sprintf("network key %s should match format _", networkKey)) - - if strings.HasPrefix(networkKey, "layer2_") { - topologyCounts["layer2"]++ - } else if strings.HasPrefix(networkKey, "layer3_") { - topologyCounts["layer3"]++ - } +// getCNCTunnelID gets CNC tunnel ID from annotations +func getCNCTunnelID(cncName string) string { + annotations, err := getCNCAnnotations(cncName) + Expect(err).NotTo(HaveOccurred()) + return annotations[ovnConnectRouterTunnelKeyAnnotation] +} - // Verify at least one of IPv4 or IPv6 is present - hasIPv4 := subnet.IPv4 != "" - hasIPv6 := subnet.IPv6 != "" - g.Expect(hasIPv4 || hasIPv6).To(BeTrue(), - fmt.Sprintf("network %s should have at least one subnet", networkKey)) - - isLayer2 := strings.HasPrefix(networkKey, "layer2_") - - // Verify IPv4 format if present (should be CIDR within connectSubnets range) - if hasIPv4 { - g.Expect(subnet.IPv4).To(MatchRegexp(`^192\.168\.\d+\.\d+/\d+$`), - fmt.Sprintf("network %s IPv4 subnet should be in connectSubnets range", networkKey)) - // Layer2 networks use point-to-point /31 subnets - if isLayer2 { - g.Expect(subnet.IPv4).To(HaveSuffix("/31"), - fmt.Sprintf("Layer2 network %s IPv4 should have /31 mask", networkKey)) - } - } +// verifyCNCHasOnlyTunnelIDAnnotation verifies CNC has only tunnel ID annotation +func verifyCNCHasOnlyTunnelIDAnnotation(cncName string) { + Eventually(func(g Gomega) { + annotations, err := getCNCAnnotations(cncName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(annotations).To(HaveKey(ovnConnectRouterTunnelKeyAnnotation), "CNC should have tunnel ID annotation") + if subnetAnnotation, exists := annotations[ovnNetworkConnectSubnetAnnotation]; exists { + g.Expect(subnetAnnotation).To(Equal("{}"), "subnet annotation should be empty when no networks match") + } + }, 30*time.Second, 1*time.Second).Should(Succeed()) +} - // Verify IPv6 format if present (should be CIDR within connectSubnets range) - if hasIPv6 { - g.Expect(subnet.IPv6).To(MatchRegexp(`^fd00:10::[0-9a-f:]*/\d+$`), - fmt.Sprintf("network %s IPv6 subnet should be in connectSubnets range", networkKey)) - // Layer2 networks use point-to-point /127 subnets - if isLayer2 { - g.Expect(subnet.IPv6).To(HaveSuffix("/127"), - fmt.Sprintf("Layer2 network %s IPv6 should have /127 mask", networkKey)) - } +// verifyCNCHasBothAnnotations verifies CNC has both tunnel ID and subnet annotations +func verifyCNCHasBothAnnotations(cncName string) { + Eventually(func(g Gomega) { + annotations, err := getCNCAnnotations(cncName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(annotations).To(HaveKey(ovnConnectRouterTunnelKeyAnnotation), "CNC should have tunnel ID annotation") + g.Expect(annotations).To(HaveKey(ovnNetworkConnectSubnetAnnotation), "CNC should have subnet annotation") + subnetAnnotation := annotations[ovnNetworkConnectSubnetAnnotation] + g.Expect(subnetAnnotation).NotTo(Equal("{}"), "subnet annotation should not be empty when networks match") + var subnets map[string]cncAnnotationSubnet + err = json.Unmarshal([]byte(subnetAnnotation), &subnets) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(subnets)).To(BeNumerically(">", 0), "should have at least one network subnet") + }, 60*time.Second, 2*time.Second).Should(Succeed()) +} + +// verifyCNCSubnetAnnotationNetworkCount verifies CNC subnet annotation has expected network count +func verifyCNCSubnetAnnotationNetworkCount(cncName string, expectedCount int) { + Eventually(func(g Gomega) { + annotations, err := getCNCAnnotations(cncName) + g.Expect(err).NotTo(HaveOccurred()) + subnetAnnotation := annotations[ovnNetworkConnectSubnetAnnotation] + var subnets map[string]cncAnnotationSubnet + err = json.Unmarshal([]byte(subnetAnnotation), &subnets) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(subnets)).To(Equal(expectedCount), fmt.Sprintf("should have %d network subnets", expectedCount)) + }, 60*time.Second, 2*time.Second).Should(Succeed()) +} + +// verifyCNCSubnetAnnotationContent verifies subnet annotation content: key format, topology counts, and CIDR format +// expectedTopologies is a list of expected topologies (e.g., ["Layer3", "Layer2", "Layer3"]) +func verifyCNCSubnetAnnotationContent(cncName string, expectedTopologies []string) { + Eventually(func(g Gomega) { + annotations, err := getCNCAnnotations(cncName) + g.Expect(err).NotTo(HaveOccurred()) + subnetAnnotation := annotations[ovnNetworkConnectSubnetAnnotation] + var subnets map[string]cncAnnotationSubnet + err = json.Unmarshal([]byte(subnetAnnotation), &subnets) + g.Expect(err).NotTo(HaveOccurred()) + + // Count topologies found + topologyCounts := map[string]int{"layer2": 0, "layer3": 0} + for networkKey, subnet := range subnets { + // Key format should be _ e.g., "layer3_1", "layer2_2" + g.Expect(networkKey).To(MatchRegexp(`^(layer2|layer3)_\d+$`), + fmt.Sprintf("network key %s should match format _", networkKey)) + + if strings.HasPrefix(networkKey, "layer2_") { + topologyCounts["layer2"]++ + } else if strings.HasPrefix(networkKey, "layer3_") { + topologyCounts["layer3"]++ + } + + // Verify at least one of IPv4 or IPv6 is present + hasIPv4 := subnet.IPv4 != "" + hasIPv6 := subnet.IPv6 != "" + g.Expect(hasIPv4 || hasIPv6).To(BeTrue(), + fmt.Sprintf("network %s should have at least one subnet", networkKey)) + + isLayer2 := strings.HasPrefix(networkKey, "layer2_") + + // Verify IPv4 format if present (should be CIDR within connectSubnets range) + if hasIPv4 { + g.Expect(subnet.IPv4).To(MatchRegexp(`^192\.16[89]\.\d+\.\d+/\d+$`), + fmt.Sprintf("network %s IPv4 subnet should be in connectSubnets range (192.168.x.x or 192.169.x.x)", networkKey)) + // Layer2 networks use point-to-point /31 subnets + if isLayer2 { + g.Expect(subnet.IPv4).To(HaveSuffix("/31"), + fmt.Sprintf("Layer2 network %s IPv4 should have /31 mask", networkKey)) } } - // Verify expected topology counts match - expectedCounts := map[string]int{"layer2": 0, "layer3": 0} - for _, topo := range expectedTopologies { - expectedCounts[strings.ToLower(topo)]++ + // Verify IPv6 format if present (should be CIDR within connectSubnets range) + if hasIPv6 { + g.Expect(subnet.IPv6).To(MatchRegexp(`^fd00:1[01]::[0-9a-f:]*/\d+$`), + fmt.Sprintf("network %s IPv6 subnet should be in connectSubnets range (fd00:10:: or fd00:11::)", networkKey)) + // Layer2 networks use point-to-point /127 subnets + if isLayer2 { + g.Expect(subnet.IPv6).To(HaveSuffix("/127"), + fmt.Sprintf("Layer2 network %s IPv6 should have /127 mask", networkKey)) + } } - g.Expect(topologyCounts["layer2"]).To(Equal(expectedCounts["layer2"]), - fmt.Sprintf("expected %d Layer2 networks, got %d", expectedCounts["layer2"], topologyCounts["layer2"])) - g.Expect(topologyCounts["layer3"]).To(Equal(expectedCounts["layer3"]), - fmt.Sprintf("expected %d Layer3 networks, got %d", expectedCounts["layer3"], topologyCounts["layer3"])) - }, 60*time.Second, 2*time.Second).Should(Succeed()) - } + } - // Helper to get CNC tunnel ID - getCNCTunnelID := func(cncName string) string { - annotations, err := getCNCAnnotations(cncName) - Expect(err).NotTo(HaveOccurred()) - return annotations[ovnConnectRouterTunnelKeyAnnotation] - } + // Verify expected topology counts match + expectedCounts := map[string]int{"layer2": 0, "layer3": 0} + for _, topo := range expectedTopologies { + expectedCounts[strings.ToLower(topo)]++ + } + g.Expect(topologyCounts["layer2"]).To(Equal(expectedCounts["layer2"]), + fmt.Sprintf("expected %d Layer2 networks, got %d", expectedCounts["layer2"], topologyCounts["layer2"])) + g.Expect(topologyCounts["layer3"]).To(Equal(expectedCounts["layer3"]), + fmt.Sprintf("expected %d Layer3 networks, got %d", expectedCounts["layer3"], topologyCounts["layer3"])) + }, 60*time.Second, 2*time.Second).Should(Succeed()) +} + +var _ = Describe("ClusterNetworkConnect ClusterManagerController", feature.NetworkConnect, func() { + f := wrappedTestFramework("cnc-controller") + // disable automatic namespace creation, we need to add the required UDN label + f.SkipNamespaceCreation = true + + var ( + cs clientset.Interface + ) + + BeforeEach(func() { + cs = f.ClientSet + }) // =========================================== // Group 1: No Matching Networks (1 test) @@ -403,7 +483,7 @@ spec: }) By("creating a CNC with selector that matches no networks") - createOrUpdateCNC(cncName, map[string]string{"nonexistent": "label"}, nil) + createOrUpdateCNC(cs, cncName, map[string]string{"nonexistent": "label"}, nil) By("verifying CNC has only tunnel ID annotation") verifyCNCHasOnlyTunnelIDAnnotation(cncName) @@ -426,7 +506,7 @@ spec: testLabel := map[string]string{fmt.Sprintf("test-%s-%s", strings.ToLower(kind), strings.ToLower(topology)): "true"} if kind == "UDN" { - ns := createUDNNamespace(fmt.Sprintf("test-%s-%s", strings.ToLower(kind), strings.ToLower(topology)), testLabel) + ns := createUDNNamespace(cs, fmt.Sprintf("test-%s-%s", strings.ToLower(kind), strings.ToLower(topology)), testLabel) DeferCleanup(func() { deleteCNC(cncName) deleteUDN(ns.Name, networkName) @@ -434,15 +514,15 @@ spec: }) By(fmt.Sprintf("creating a %s primary UDN", topology)) - createPrimaryUDN(ns.Name, networkName, topology) + createPrimaryUDN(cs, ns.Name, networkName, topology) By("waiting for UDN to be ready") Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, ns.Name, networkName), 30*time.Second, time.Second).Should(Succeed()) By("creating a CNC with PUDN selector") - createOrUpdateCNC(cncName, nil, testLabel) + createOrUpdateCNC(cs, cncName, nil, testLabel) } else { - ns := createUDNNamespace(fmt.Sprintf("test-%s-%s", strings.ToLower(kind), strings.ToLower(topology)), nil) + ns := createUDNNamespace(cs, fmt.Sprintf("test-%s-%s", strings.ToLower(kind), strings.ToLower(topology)), nil) DeferCleanup(func() { deleteCNC(cncName) deleteCUDN(networkName) @@ -450,13 +530,13 @@ spec: }) By(fmt.Sprintf("creating a %s primary CUDN", topology)) - createPrimaryCUDN(networkName, topology, testLabel, ns.Name) + createPrimaryCUDN(cs, networkName, topology, testLabel, ns.Name) By("waiting for CUDN to be ready") Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, networkName), 30*time.Second, time.Second).Should(Succeed()) By("creating a CNC with CUDN selector") - createOrUpdateCNC(cncName, testLabel, nil) + createOrUpdateCNC(cs, cncName, testLabel, nil) } By("verifying CNC has both subnet and tunnel ID annotations") @@ -482,7 +562,7 @@ spec: if kind == "UDN" { // Create 4 namespaces with the same label for PUDN selector for i := 1; i <= 4; i++ { - namespaces = append(namespaces, createUDNNamespace(fmt.Sprintf("test-udn-%d", i), testLabel)) + namespaces = append(namespaces, createUDNNamespace(cs, fmt.Sprintf("test-udn-%d", i), testLabel)) networkNames = append(networkNames, fmt.Sprintf("udn%d", i)) } @@ -495,13 +575,13 @@ spec: }) By("creating 2 Layer3 and 2 Layer2 primary UDNs") - createLayer3PrimaryUDN(namespaces[0].Name, networkNames[0]) + createLayer3PrimaryUDN(cs, namespaces[0].Name, networkNames[0]) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryUDN(namespaces[1].Name, networkNames[1]) + createLayer3PrimaryUDN(cs, namespaces[1].Name, networkNames[1]) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer2PrimaryUDN(namespaces[2].Name, networkNames[2]) + createLayer2PrimaryUDN(cs, namespaces[2].Name, networkNames[2]) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryUDN(namespaces[3].Name, networkNames[3]) + createLayer2PrimaryUDN(cs, namespaces[3].Name, networkNames[3]) expectedTopologies = append(expectedTopologies, "Layer2") By("waiting for all UDNs to be ready") @@ -510,11 +590,11 @@ spec: } By("creating a CNC with PUDN selector") - createOrUpdateCNC(cncName, nil, testLabel) + createOrUpdateCNC(cs, cncName, nil, testLabel) } else { // CUDN case - one CUDN targets multiple namespaces for i := 1; i <= 5; i++ { // 5 namespaces for multi-ns CUDN test - namespaces = append(namespaces, createUDNNamespace(fmt.Sprintf("test-cudn-ns%d", i), nil)) + namespaces = append(namespaces, createUDNNamespace(cs, fmt.Sprintf("test-cudn-ns%d", i), nil)) networkNames = append(networkNames, fmt.Sprintf("cudn-%d-%s", i, rand.String(5))) } @@ -529,13 +609,13 @@ spec: }) By("creating 2 Layer3 and 2 Layer2 primary CUDNs (one L3 targets multiple namespaces)") - createLayer3PrimaryCUDN(networkNames[0], testLabel, namespaces[0].Name) + createPrimaryCUDN(cs, networkNames[0], "Layer3", testLabel, namespaces[0].Name) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryCUDN(networkNames[1], testLabel, namespaces[1].Name, namespaces[4].Name) // multi-ns + createPrimaryCUDN(cs, networkNames[1], "Layer3", testLabel, namespaces[1].Name, namespaces[4].Name) // multi-ns expectedTopologies = append(expectedTopologies, "Layer3") - createLayer2PrimaryCUDN(networkNames[2], testLabel, namespaces[2].Name) + createPrimaryCUDN(cs, networkNames[2], "Layer2", testLabel, namespaces[2].Name) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryCUDN(networkNames[3], testLabel, namespaces[3].Name) + createPrimaryCUDN(cs, networkNames[3], "Layer2", testLabel, namespaces[3].Name) expectedTopologies = append(expectedTopologies, "Layer2") By("waiting for all CUDNs to be ready") @@ -544,7 +624,7 @@ spec: } By("creating a CNC with CUDN selector") - createOrUpdateCNC(cncName, testLabel, nil) + createOrUpdateCNC(cs, cncName, testLabel, nil) } By("verifying CNC has 4 networks in subnet annotation") @@ -559,7 +639,7 @@ spec: It("full matrix (2x each type) - has all 8 networks in subnet annotation", func() { cncName := generateCNCName() cudnLabel := map[string]string{"test-full-matrix": "true"} - pudnLabel := map[string]string{"test-full-matrix": "true"} + udnLabel := map[string]string{"test-full-matrix": "true"} var cudnNames []string var udnNames []string @@ -571,8 +651,8 @@ spec: for i := 1; i <= 4; i++ { cudnNames = append(cudnNames, fmt.Sprintf("fm-cudn-%d-%s", i, rand.String(5))) udnNames = append(udnNames, fmt.Sprintf("udn%d", i)) - cudnNamespaces = append(cudnNamespaces, createUDNNamespace(fmt.Sprintf("fm-cudn-ns%d", i), nil)) - udnNamespaces = append(udnNamespaces, createUDNNamespace(fmt.Sprintf("fm-udn-ns%d", i), pudnLabel)) + cudnNamespaces = append(cudnNamespaces, createUDNNamespace(cs, fmt.Sprintf("fm-cudn-ns%d", i), nil)) + udnNamespaces = append(udnNamespaces, createUDNNamespace(cs, fmt.Sprintf("fm-udn-ns%d", i), udnLabel)) } DeferCleanup(func() { @@ -592,23 +672,23 @@ spec: }) By("creating 4 CUDNs (2xL3 + 2xL2)") - createLayer3PrimaryCUDN(cudnNames[0], cudnLabel, cudnNamespaces[0].Name) + createLayer3PrimaryCUDN(cs, cudnNames[0], cudnLabel, cudnNamespaces[0].Name) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryCUDN(cudnNames[1], cudnLabel, cudnNamespaces[1].Name) + createLayer3PrimaryCUDN(cs, cudnNames[1], cudnLabel, cudnNamespaces[1].Name) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer2PrimaryCUDN(cudnNames[2], cudnLabel, cudnNamespaces[2].Name) + createLayer2PrimaryCUDN(cs, cudnNames[2], cudnLabel, cudnNamespaces[2].Name) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryCUDN(cudnNames[3], cudnLabel, cudnNamespaces[3].Name) + createLayer2PrimaryCUDN(cs, cudnNames[3], cudnLabel, cudnNamespaces[3].Name) expectedTopologies = append(expectedTopologies, "Layer2") By("creating 4 UDNs (2xL3 + 2xL2)") - createLayer3PrimaryUDN(udnNamespaces[0].Name, udnNames[0]) + createLayer3PrimaryUDN(cs, udnNamespaces[0].Name, udnNames[0]) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryUDN(udnNamespaces[1].Name, udnNames[1]) + createLayer3PrimaryUDN(cs, udnNamespaces[1].Name, udnNames[1]) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer2PrimaryUDN(udnNamespaces[2].Name, udnNames[2]) + createLayer2PrimaryUDN(cs, udnNamespaces[2].Name, udnNames[2]) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryUDN(udnNamespaces[3].Name, udnNames[3]) + createLayer2PrimaryUDN(cs, udnNamespaces[3].Name, udnNames[3]) expectedTopologies = append(expectedTopologies, "Layer2") By("waiting for all networks to be ready") @@ -620,7 +700,7 @@ spec: } By("creating a CNC with both CUDN and PUDN selectors") - createOrUpdateCNC(cncName, cudnLabel, pudnLabel) + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) By("verifying CNC has all 8 networks in subnet annotation") verifyCNCHasBothAnnotations(cncName) @@ -642,7 +722,7 @@ spec: var expectedTopologies []string if kind == "UDN" { - ns := createUDNNamespace(fmt.Sprintf("test-dyn-%s-%s", strings.ToLower(kind), strings.ToLower(topology)), testLabel) + ns := createUDNNamespace(cs, fmt.Sprintf("test-dyn-%s-%s", strings.ToLower(kind), strings.ToLower(topology)), testLabel) DeferCleanup(func() { deleteCNC(cncName) deleteUDN(ns.Name, networkName) @@ -650,19 +730,19 @@ spec: }) By("creating a CNC with PUDN selector (no matching networks yet)") - createOrUpdateCNC(cncName, nil, testLabel) + createOrUpdateCNC(cs, cncName, nil, testLabel) By("verifying CNC has only tunnel ID annotation initially") verifyCNCHasOnlyTunnelIDAnnotation(cncName) By(fmt.Sprintf("creating a %s primary UDN", topology)) - createPrimaryUDN(ns.Name, networkName, topology) + createPrimaryUDN(cs, ns.Name, networkName, topology) expectedTopologies = append(expectedTopologies, topology) By("waiting for UDN to be ready") Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, ns.Name, networkName), 30*time.Second, time.Second).Should(Succeed()) } else { - ns := createUDNNamespace(fmt.Sprintf("test-dyn-%s-%s", strings.ToLower(kind), strings.ToLower(topology)), nil) + ns := createUDNNamespace(cs, fmt.Sprintf("test-dyn-%s-%s", strings.ToLower(kind), strings.ToLower(topology)), nil) DeferCleanup(func() { deleteCNC(cncName) deleteCUDN(networkName) @@ -670,13 +750,13 @@ spec: }) By("creating a CNC with CUDN selector (no matching networks yet)") - createOrUpdateCNC(cncName, testLabel, nil) + createOrUpdateCNC(cs, cncName, testLabel, nil) By("verifying CNC has only tunnel ID annotation initially") verifyCNCHasOnlyTunnelIDAnnotation(cncName) By(fmt.Sprintf("creating a %s primary CUDN", topology)) - createPrimaryCUDN(networkName, topology, testLabel, ns.Name) + createPrimaryCUDN(cs, networkName, topology, testLabel, ns.Name) expectedTopologies = append(expectedTopologies, topology) By("waiting for CUDN to be ready") @@ -706,7 +786,7 @@ spec: if kind == "UDN" { // Create namespaces first (with label for PUDN selector) for i := 1; i <= 4; i++ { - namespaces = append(namespaces, createUDNNamespace(fmt.Sprintf("test-dyn-udn-%d", i), testLabel)) + namespaces = append(namespaces, createUDNNamespace(cs, fmt.Sprintf("test-dyn-udn-%d", i), testLabel)) networkNames = append(networkNames, fmt.Sprintf("udn%d", i)) } @@ -719,19 +799,19 @@ spec: }) By("creating a CNC with PUDN selector (no matching networks yet)") - createOrUpdateCNC(cncName, nil, testLabel) + createOrUpdateCNC(cs, cncName, nil, testLabel) By("verifying CNC has only tunnel ID annotation initially") verifyCNCHasOnlyTunnelIDAnnotation(cncName) By("creating 2 Layer3 and 2 Layer2 primary UDNs") - createLayer3PrimaryUDN(namespaces[0].Name, networkNames[0]) + createLayer3PrimaryUDN(cs, namespaces[0].Name, networkNames[0]) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryUDN(namespaces[1].Name, networkNames[1]) + createLayer3PrimaryUDN(cs, namespaces[1].Name, networkNames[1]) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer2PrimaryUDN(namespaces[2].Name, networkNames[2]) + createLayer2PrimaryUDN(cs, namespaces[2].Name, networkNames[2]) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryUDN(namespaces[3].Name, networkNames[3]) + createLayer2PrimaryUDN(cs, namespaces[3].Name, networkNames[3]) expectedTopologies = append(expectedTopologies, "Layer2") By("waiting for all UDNs to be ready") @@ -741,7 +821,7 @@ spec: } else { // CUDN case for i := 1; i <= 5; i++ { - namespaces = append(namespaces, createUDNNamespace(fmt.Sprintf("test-dyn-cudn-ns%d", i), nil)) + namespaces = append(namespaces, createUDNNamespace(cs, fmt.Sprintf("test-dyn-cudn-ns%d", i), nil)) networkNames = append(networkNames, fmt.Sprintf("dyn-cudn-%d-%s", i, rand.String(5))) } @@ -756,19 +836,19 @@ spec: }) By("creating a CNC with CUDN selector (no matching networks yet)") - createOrUpdateCNC(cncName, testLabel, nil) + createOrUpdateCNC(cs, cncName, testLabel, nil) By("verifying CNC has only tunnel ID annotation initially") verifyCNCHasOnlyTunnelIDAnnotation(cncName) By("creating 2 Layer3 and 2 Layer2 primary CUDNs (one L3 targets multiple namespaces)") - createLayer3PrimaryCUDN(networkNames[0], testLabel, namespaces[0].Name) + createPrimaryCUDN(cs, networkNames[0], "Layer3", testLabel, namespaces[0].Name) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryCUDN(networkNames[1], testLabel, namespaces[1].Name, namespaces[4].Name) + createPrimaryCUDN(cs, networkNames[1], "Layer3", testLabel, namespaces[1].Name, namespaces[4].Name) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer2PrimaryCUDN(networkNames[2], testLabel, namespaces[2].Name) + createPrimaryCUDN(cs, networkNames[2], "Layer2", testLabel, namespaces[2].Name) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryCUDN(networkNames[3], testLabel, namespaces[3].Name) + createPrimaryCUDN(cs, networkNames[3], "Layer2", testLabel, namespaces[3].Name) expectedTopologies = append(expectedTopologies, "Layer2") By("waiting for all CUDNs to be ready") @@ -789,7 +869,7 @@ spec: It("full matrix created after CNC - annotations are updated with all 8 networks", func() { cncName := generateCNCName() cudnLabel := map[string]string{"test-dyn-full-matrix": "true"} - pudnLabel := map[string]string{"test-dyn-full-matrix": "true"} + udnLabel := map[string]string{"test-dyn-full-matrix": "true"} var cudnNames []string var udnNames []string @@ -801,8 +881,8 @@ spec: for i := 1; i <= 4; i++ { cudnNames = append(cudnNames, fmt.Sprintf("dyn-fm-cudn-%d-%s", i, rand.String(5))) udnNames = append(udnNames, fmt.Sprintf("udn%d", i)) - cudnNamespaces = append(cudnNamespaces, createUDNNamespace(fmt.Sprintf("dyn-fm-cudn-ns%d", i), nil)) - udnNamespaces = append(udnNamespaces, createUDNNamespace(fmt.Sprintf("dyn-fm-udn-ns%d", i), pudnLabel)) + cudnNamespaces = append(cudnNamespaces, createUDNNamespace(cs, fmt.Sprintf("dyn-fm-cudn-ns%d", i), nil)) + udnNamespaces = append(udnNamespaces, createUDNNamespace(cs, fmt.Sprintf("dyn-fm-udn-ns%d", i), udnLabel)) } DeferCleanup(func() { @@ -822,29 +902,29 @@ spec: }) By("creating a CNC with both CUDN and PUDN selectors (no matching networks yet)") - createOrUpdateCNC(cncName, cudnLabel, pudnLabel) + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) By("verifying CNC has only tunnel ID annotation initially") verifyCNCHasOnlyTunnelIDAnnotation(cncName) By("creating 4 CUDNs (2xL3 + 2xL2)") - createLayer3PrimaryCUDN(cudnNames[0], cudnLabel, cudnNamespaces[0].Name) + createLayer3PrimaryCUDN(cs, cudnNames[0], cudnLabel, cudnNamespaces[0].Name) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryCUDN(cudnNames[1], cudnLabel, cudnNamespaces[1].Name) + createLayer3PrimaryCUDN(cs, cudnNames[1], cudnLabel, cudnNamespaces[1].Name) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer2PrimaryCUDN(cudnNames[2], cudnLabel, cudnNamespaces[2].Name) + createLayer2PrimaryCUDN(cs, cudnNames[2], cudnLabel, cudnNamespaces[2].Name) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryCUDN(cudnNames[3], cudnLabel, cudnNamespaces[3].Name) + createLayer2PrimaryCUDN(cs, cudnNames[3], cudnLabel, cudnNamespaces[3].Name) expectedTopologies = append(expectedTopologies, "Layer2") By("creating 4 UDNs (2xL3 + 2xL2)") - createLayer3PrimaryUDN(udnNamespaces[0].Name, udnNames[0]) + createLayer3PrimaryUDN(cs, udnNamespaces[0].Name, udnNames[0]) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryUDN(udnNamespaces[1].Name, udnNames[1]) + createLayer3PrimaryUDN(cs, udnNamespaces[1].Name, udnNames[1]) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer2PrimaryUDN(udnNamespaces[2].Name, udnNames[2]) + createLayer2PrimaryUDN(cs, udnNamespaces[2].Name, udnNames[2]) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryUDN(udnNamespaces[3].Name, udnNames[3]) + createLayer2PrimaryUDN(cs, udnNamespaces[3].Name, udnNames[3]) expectedTopologies = append(expectedTopologies, "Layer2") By("waiting for all networks to be ready") @@ -878,7 +958,7 @@ spec: if kind == "UDN" { // Create 2 namespaces - one for initial, one for added for i := 1; i <= 2; i++ { - namespaces = append(namespaces, createUDNNamespace(fmt.Sprintf("test-add-udn-%d", i), testLabel)) + namespaces = append(namespaces, createUDNNamespace(cs, fmt.Sprintf("test-add-udn-%d", i), testLabel)) networkNames = append(networkNames, fmt.Sprintf("udn%d", i)) } @@ -891,12 +971,12 @@ spec: }) By(fmt.Sprintf("creating initial %s primary UDN", initialTopology)) - createPrimaryUDN(namespaces[0].Name, networkNames[0], initialTopology) + createPrimaryUDN(cs, namespaces[0].Name, networkNames[0], initialTopology) expectedTopologies = append(expectedTopologies, initialTopology) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, namespaces[0].Name, networkNames[0]), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with PUDN selector") - createOrUpdateCNC(cncName, nil, testLabel) + createOrUpdateCNC(cs, cncName, nil, testLabel) By("verifying CNC has 1 network in subnet annotation") verifyCNCHasBothAnnotations(cncName) @@ -904,13 +984,13 @@ spec: verifyCNCSubnetAnnotationContent(cncName, expectedTopologies) By(fmt.Sprintf("adding a %s primary UDN", addedTopology)) - createPrimaryUDN(namespaces[1].Name, networkNames[1], addedTopology) + createPrimaryUDN(cs, namespaces[1].Name, networkNames[1], addedTopology) expectedTopologies = append(expectedTopologies, addedTopology) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, namespaces[1].Name, networkNames[1]), 30*time.Second, time.Second).Should(Succeed()) } else { // CUDN case for i := 1; i <= 2; i++ { - namespaces = append(namespaces, createUDNNamespace(fmt.Sprintf("test-add-cudn-ns%d", i), nil)) + namespaces = append(namespaces, createUDNNamespace(cs, fmt.Sprintf("test-add-cudn-ns%d", i), nil)) networkNames = append(networkNames, fmt.Sprintf("add-cudn-%d-%s", i, rand.String(5))) } @@ -925,12 +1005,12 @@ spec: }) By(fmt.Sprintf("creating initial %s primary CUDN", initialTopology)) - createPrimaryCUDN(networkNames[0], initialTopology, testLabel, namespaces[0].Name) + createPrimaryCUDN(cs, networkNames[0], initialTopology, testLabel, namespaces[0].Name) expectedTopologies = append(expectedTopologies, initialTopology) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, networkNames[0]), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with CUDN selector") - createOrUpdateCNC(cncName, testLabel, nil) + createOrUpdateCNC(cs, cncName, testLabel, nil) By("verifying CNC has 1 network in subnet annotation") verifyCNCHasBothAnnotations(cncName) @@ -938,7 +1018,7 @@ spec: verifyCNCSubnetAnnotationContent(cncName, expectedTopologies) By(fmt.Sprintf("adding a %s primary CUDN", addedTopology)) - createPrimaryCUDN(networkNames[1], addedTopology, testLabel, namespaces[1].Name) + createPrimaryCUDN(cs, networkNames[1], addedTopology, testLabel, namespaces[1].Name) expectedTopologies = append(expectedTopologies, addedTopology) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, networkNames[1]), 30*time.Second, time.Second).Should(Succeed()) } @@ -956,20 +1036,20 @@ spec: It("adding mixed networks (P-UDN + P-CUDN) to existing CNC - all networks appear", func() { cncName := generateCNCName() cudnLabel := map[string]string{"test-add-mixed": "true"} - pudnLabel := map[string]string{"test-add-mixed": "true"} + udnLabel := map[string]string{"test-add-mixed": "true"} var expectedTopologies []string // Initial: 1 L3 CUDN + 1 L3 UDN initialCudnName := fmt.Sprintf("add-mixed-cudn-init-%s", rand.String(5)) initialUdnName := "udn-init" - cudnNs := createUDNNamespace("test-add-mixed-cudn", nil) - udnNs := createUDNNamespace("test-add-mixed-udn", pudnLabel) + cudnNs := createUDNNamespace(cs, "test-add-mixed-cudn", nil) + udnNs := createUDNNamespace(cs, "test-add-mixed-udn", udnLabel) // Added: 1 L2 CUDN + 1 L2 UDN addedCudnName := fmt.Sprintf("add-mixed-cudn-add-%s", rand.String(5)) addedUdnName := "udn-add" - addedCudnNs := createUDNNamespace("test-add-mixed-cudn2", nil) - addedUdnNs := createUDNNamespace("test-add-mixed-udn2", pudnLabel) + addedCudnNs := createUDNNamespace(cs, "test-add-mixed-cudn2", nil) + addedUdnNs := createUDNNamespace(cs, "test-add-mixed-udn2", udnLabel) DeferCleanup(func() { deleteCNC(cncName) @@ -983,16 +1063,16 @@ spec: }) By("creating initial L3 CUDN and L3 UDN") - createLayer3PrimaryCUDN(initialCudnName, cudnLabel, cudnNs.Name) + createLayer3PrimaryCUDN(cs, initialCudnName, cudnLabel, cudnNs.Name) expectedTopologies = append(expectedTopologies, "Layer3") - createLayer3PrimaryUDN(udnNs.Name, initialUdnName) + createLayer3PrimaryUDN(cs, udnNs.Name, initialUdnName) expectedTopologies = append(expectedTopologies, "Layer3") Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, initialCudnName), 30*time.Second, time.Second).Should(Succeed()) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, udnNs.Name, initialUdnName), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with both selectors") - createOrUpdateCNC(cncName, cudnLabel, pudnLabel) + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) By("verifying CNC has 2 networks initially") verifyCNCHasBothAnnotations(cncName) @@ -1000,9 +1080,9 @@ spec: verifyCNCSubnetAnnotationContent(cncName, expectedTopologies) By("adding L2 CUDN and L2 UDN") - createLayer2PrimaryCUDN(addedCudnName, cudnLabel, addedCudnNs.Name) + createLayer2PrimaryCUDN(cs, addedCudnName, cudnLabel, addedCudnNs.Name) expectedTopologies = append(expectedTopologies, "Layer2") - createLayer2PrimaryUDN(addedUdnNs.Name, addedUdnName) + createLayer2PrimaryUDN(cs, addedUdnNs.Name, addedUdnName) expectedTopologies = append(expectedTopologies, "Layer2") Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, addedCudnName), 30*time.Second, time.Second).Should(Succeed()) @@ -1029,7 +1109,7 @@ spec: if kind == "UDN" { // Create 2 namespaces with 2 networks for i := 1; i <= 2; i++ { - namespaces = append(namespaces, createUDNNamespace(fmt.Sprintf("test-del-udn-%d", i), testLabel)) + namespaces = append(namespaces, createUDNNamespace(cs, fmt.Sprintf("test-del-udn-%d", i), testLabel)) networkNames = append(networkNames, fmt.Sprintf("udn%d", i)) } @@ -1042,14 +1122,14 @@ spec: }) By("creating 2 primary UDNs (L3 + topology)") - createLayer3PrimaryUDN(namespaces[0].Name, networkNames[0]) - createPrimaryUDN(namespaces[1].Name, networkNames[1], topology) + createLayer3PrimaryUDN(cs, namespaces[0].Name, networkNames[0]) + createPrimaryUDN(cs, namespaces[1].Name, networkNames[1], topology) for i, ns := range namespaces { Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, ns.Name, networkNames[i]), 30*time.Second, time.Second).Should(Succeed()) } By("creating CNC with PUDN selector") - createOrUpdateCNC(cncName, nil, testLabel) + createOrUpdateCNC(cs, cncName, nil, testLabel) By("verifying CNC has 2 networks initially") verifyCNCHasBothAnnotations(cncName) @@ -1068,7 +1148,7 @@ spec: } else { // CUDN case for i := 1; i <= 2; i++ { - namespaces = append(namespaces, createUDNNamespace(fmt.Sprintf("test-del-cudn-ns%d", i), nil)) + namespaces = append(namespaces, createUDNNamespace(cs, fmt.Sprintf("test-del-cudn-ns%d", i), nil)) networkNames = append(networkNames, fmt.Sprintf("del-cudn-%d-%s", i, rand.String(5))) } @@ -1081,14 +1161,14 @@ spec: }) By("creating 2 primary CUDNs (L3 + topology)") - createLayer3PrimaryCUDN(networkNames[0], testLabel, namespaces[0].Name) - createPrimaryCUDN(networkNames[1], topology, testLabel, namespaces[1].Name) + createLayer3PrimaryCUDN(cs, networkNames[0], testLabel, namespaces[0].Name) + createPrimaryCUDN(cs, networkNames[1], topology, testLabel, namespaces[1].Name) for i := 0; i < 2; i++ { Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, networkNames[i]), 30*time.Second, time.Second).Should(Succeed()) } By("creating CNC with CUDN selector") - createOrUpdateCNC(cncName, testLabel, nil) + createOrUpdateCNC(cs, cncName, testLabel, nil) By("verifying CNC has 2 networks initially") verifyCNCHasBothAnnotations(cncName) @@ -1118,13 +1198,13 @@ spec: It("deleting mixed networks (P-UDN + P-CUDN) - annotations update correctly", func() { cncName := generateCNCName() cudnLabel := map[string]string{"test-del-mixed": "true"} - pudnLabel := map[string]string{"test-del-mixed": "true"} + udnLabel := map[string]string{"test-del-mixed": "true"} // Create 2 CUDNs + 2 UDNs - cudnNs1 := createUDNNamespace("test-del-mixed-cudn1", nil) - cudnNs2 := createUDNNamespace("test-del-mixed-cudn2", nil) - udnNs1 := createUDNNamespace("test-del-mixed-udn1", pudnLabel) - udnNs2 := createUDNNamespace("test-del-mixed-udn2", pudnLabel) + cudnNs1 := createUDNNamespace(cs, "test-del-mixed-cudn1", nil) + cudnNs2 := createUDNNamespace(cs, "test-del-mixed-cudn2", nil) + udnNs1 := createUDNNamespace(cs, "test-del-mixed-udn1", udnLabel) + udnNs2 := createUDNNamespace(cs, "test-del-mixed-udn2", udnLabel) cudnName1 := fmt.Sprintf("del-mixed-cudn1-%s", rand.String(5)) cudnName2 := fmt.Sprintf("del-mixed-cudn2-%s", rand.String(5)) @@ -1142,10 +1222,10 @@ spec: }) By("creating 2 CUDNs (L3 + L2) and 2 UDNs (L3 + L2)") - createLayer3PrimaryCUDN(cudnName1, cudnLabel, cudnNs1.Name) - createLayer2PrimaryCUDN(cudnName2, cudnLabel, cudnNs2.Name) - createLayer3PrimaryUDN(udnNs1.Name, udnName1) - createLayer2PrimaryUDN(udnNs2.Name, udnName2) + createLayer3PrimaryCUDN(cs, cudnName1, cudnLabel, cudnNs1.Name) + createLayer2PrimaryCUDN(cs, cudnName2, cudnLabel, cudnNs2.Name) + createLayer3PrimaryUDN(cs, udnNs1.Name, udnName1) + createLayer2PrimaryUDN(cs, udnNs2.Name, udnName2) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName1), 30*time.Second, time.Second).Should(Succeed()) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName2), 30*time.Second, time.Second).Should(Succeed()) @@ -1153,7 +1233,7 @@ spec: Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, udnNs2.Name, udnName2), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with both selectors") - createOrUpdateCNC(cncName, cudnLabel, pudnLabel) + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) By("verifying CNC has 4 networks initially") verifyCNCHasBothAnnotations(cncName) @@ -1187,8 +1267,8 @@ spec: commonLabel := map[string]string{"test-cudn-sel": "true"} specificLabel := map[string]string{"test-cudn-sel": "true", "specific": "true"} - ns1 := createUDNNamespace("test-cudn-sel-ns1", nil) - ns2 := createUDNNamespace("test-cudn-sel-ns2", nil) + ns1 := createUDNNamespace(cs, "test-cudn-sel-ns1", nil) + ns2 := createUDNNamespace(cs, "test-cudn-sel-ns2", nil) cudnName1 := fmt.Sprintf("cudn-sel1-%s", rand.String(5)) cudnName2 := fmt.Sprintf("cudn-sel2-%s", rand.String(5)) @@ -1201,13 +1281,13 @@ spec: }) By("creating 2 CUDNs - both with common label, second also has specific label") - createLayer3PrimaryCUDN(cudnName1, commonLabel, ns1.Name) - createLayer2PrimaryCUDN(cudnName2, specificLabel, ns2.Name) + createLayer3PrimaryCUDN(cs, cudnName1, commonLabel, ns1.Name) + createLayer2PrimaryCUDN(cs, cudnName2, specificLabel, ns2.Name) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName1), 30*time.Second, time.Second).Should(Succeed()) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName2), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with specific selector (matches only second CUDN)") - createOrUpdateCNC(cncName, specificLabel, nil) + createOrUpdateCNC(cs, cncName, specificLabel, nil) By("verifying CNC has 1 network initially") verifyCNCHasBothAnnotations(cncName) @@ -1215,14 +1295,14 @@ spec: verifyCNCSubnetAnnotationContent(cncName, []string{"Layer2"}) By("widening CNC selector to common label - count increases") - createOrUpdateCNC(cncName, commonLabel, nil) + createOrUpdateCNC(cs, cncName, commonLabel, nil) By("verifying CNC now has 2 networks") verifyCNCSubnetAnnotationNetworkCount(cncName, 2) verifyCNCSubnetAnnotationContent(cncName, []string{"Layer3", "Layer2"}) By("narrowing CNC selector back to specific - count decreases") - createOrUpdateCNC(cncName, specificLabel, nil) + createOrUpdateCNC(cs, cncName, specificLabel, nil) By("verifying CNC now has 1 network") verifyCNCSubnetAnnotationNetworkCount(cncName, 1) @@ -1234,8 +1314,8 @@ spec: commonLabel := map[string]string{"test-pudn-sel": "true"} specificLabel := map[string]string{"test-pudn-sel": "true", "specific": "true"} - ns1 := createUDNNamespace("test-pudn-sel-ns1", commonLabel) - ns2 := createUDNNamespace("test-pudn-sel-ns2", specificLabel) + ns1 := createUDNNamespace(cs, "test-pudn-sel-ns1", commonLabel) + ns2 := createUDNNamespace(cs, "test-pudn-sel-ns2", specificLabel) udnName1 := "udn1" udnName2 := "udn2" @@ -1248,13 +1328,13 @@ spec: }) By("creating 2 UDNs in namespaces - both with common label, second also has specific") - createLayer3PrimaryUDN(ns1.Name, udnName1) - createLayer2PrimaryUDN(ns2.Name, udnName2) + createLayer3PrimaryUDN(cs, ns1.Name, udnName1) + createLayer2PrimaryUDN(cs, ns2.Name, udnName2) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, ns1.Name, udnName1), 30*time.Second, time.Second).Should(Succeed()) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, ns2.Name, udnName2), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with specific selector (matches only second namespace)") - createOrUpdateCNC(cncName, nil, specificLabel) + createOrUpdateCNC(cs, cncName, nil, specificLabel) By("verifying CNC has 1 network initially") verifyCNCHasBothAnnotations(cncName) @@ -1262,14 +1342,14 @@ spec: verifyCNCSubnetAnnotationContent(cncName, []string{"Layer2"}) By("widening CNC selector to common label - count increases") - createOrUpdateCNC(cncName, nil, commonLabel) + createOrUpdateCNC(cs, cncName, nil, commonLabel) By("verifying CNC now has 2 networks") verifyCNCSubnetAnnotationNetworkCount(cncName, 2) verifyCNCSubnetAnnotationContent(cncName, []string{"Layer3", "Layer2"}) By("narrowing CNC selector back to specific - count decreases") - createOrUpdateCNC(cncName, nil, specificLabel) + createOrUpdateCNC(cs, cncName, nil, specificLabel) By("verifying CNC now has 1 network") verifyCNCSubnetAnnotationNetworkCount(cncName, 1) @@ -1279,10 +1359,10 @@ spec: It("adding and removing PUDN selector from CNC - count increases then decreases", func() { cncName := generateCNCName() cudnLabel := map[string]string{"test-toggle-pudn-sel": "true"} - pudnLabel := map[string]string{"test-toggle-pudn-sel": "true"} + udnLabel := map[string]string{"test-toggle-pudn-sel": "true"} - cudnNs := createUDNNamespace("test-toggle-pudn-sel-cudn", nil) - udnNs := createUDNNamespace("test-toggle-pudn-sel-udn", pudnLabel) + cudnNs := createUDNNamespace(cs, "test-toggle-pudn-sel-cudn", nil) + udnNs := createUDNNamespace(cs, "test-toggle-pudn-sel-udn", udnLabel) cudnName := fmt.Sprintf("toggle-pudn-sel-cudn-%s", rand.String(5)) udnName := "udn1" @@ -1295,13 +1375,13 @@ spec: }) By("creating L3 CUDN and L2 UDN") - createLayer3PrimaryCUDN(cudnName, cudnLabel, cudnNs.Name) - createLayer2PrimaryUDN(udnNs.Name, udnName) + createLayer3PrimaryCUDN(cs, cudnName, cudnLabel, cudnNs.Name) + createLayer2PrimaryUDN(cs, udnNs.Name, udnName) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName), 30*time.Second, time.Second).Should(Succeed()) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, udnNs.Name, udnName), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with only CUDN selector") - createOrUpdateCNC(cncName, cudnLabel, nil) + createOrUpdateCNC(cs, cncName, cudnLabel, nil) By("verifying CNC has 1 network initially (CUDN only)") verifyCNCHasBothAnnotations(cncName) @@ -1309,7 +1389,7 @@ spec: verifyCNCSubnetAnnotationContent(cncName, []string{"Layer3"}) By("adding PUDN selector to CNC - count increases") - createOrUpdateCNC(cncName, cudnLabel, pudnLabel) + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) By("verifying CNC now has 2 networks (CUDN + PUDN)") verifyCNCHasBothAnnotations(cncName) @@ -1317,7 +1397,7 @@ spec: verifyCNCSubnetAnnotationContent(cncName, []string{"Layer3", "Layer2"}) By("removing PUDN selector from CNC - count decreases") - createOrUpdateCNC(cncName, cudnLabel, nil) + createOrUpdateCNC(cs, cncName, cudnLabel, nil) By("verifying CNC now has 1 network (CUDN only)") verifyCNCHasBothAnnotations(cncName) @@ -1328,10 +1408,10 @@ spec: It("adding and removing CUDN selector from CNC - count increases then decreases", func() { cncName := generateCNCName() cudnLabel := map[string]string{"test-toggle-cudn-sel": "true"} - pudnLabel := map[string]string{"test-toggle-cudn-sel": "true"} + udnLabel := map[string]string{"test-toggle-cudn-sel": "true"} - cudnNs := createUDNNamespace("test-toggle-cudn-sel-cudn", nil) - udnNs := createUDNNamespace("test-toggle-cudn-sel-udn", pudnLabel) + cudnNs := createUDNNamespace(cs, "test-toggle-cudn-sel-cudn", nil) + udnNs := createUDNNamespace(cs, "test-toggle-cudn-sel-udn", udnLabel) cudnName := fmt.Sprintf("toggle-cudn-sel-cudn-%s", rand.String(5)) udnName := "udn1" @@ -1344,13 +1424,13 @@ spec: }) By("creating L3 CUDN and L2 UDN") - createLayer3PrimaryCUDN(cudnName, cudnLabel, cudnNs.Name) - createLayer2PrimaryUDN(udnNs.Name, udnName) + createLayer3PrimaryCUDN(cs, cudnName, cudnLabel, cudnNs.Name) + createLayer2PrimaryUDN(cs, udnNs.Name, udnName) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName), 30*time.Second, time.Second).Should(Succeed()) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, udnNs.Name, udnName), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with only PUDN selector") - createOrUpdateCNC(cncName, nil, pudnLabel) + createOrUpdateCNC(cs, cncName, nil, udnLabel) By("verifying CNC has 1 network initially (PUDN only)") verifyCNCHasBothAnnotations(cncName) @@ -1358,7 +1438,7 @@ spec: verifyCNCSubnetAnnotationContent(cncName, []string{"Layer2"}) By("adding CUDN selector to CNC - count increases") - createOrUpdateCNC(cncName, cudnLabel, pudnLabel) + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) By("verifying CNC now has 2 networks (CUDN + PUDN)") verifyCNCHasBothAnnotations(cncName) @@ -1366,7 +1446,7 @@ spec: verifyCNCSubnetAnnotationContent(cncName, []string{"Layer3", "Layer2"}) By("removing CUDN selector from CNC - count decreases") - createOrUpdateCNC(cncName, nil, pudnLabel) + createOrUpdateCNC(cs, cncName, nil, udnLabel) By("verifying CNC now has 1 network (PUDN only)") verifyCNCHasBothAnnotations(cncName) @@ -1374,7 +1454,7 @@ spec: verifyCNCSubnetAnnotationContent(cncName, []string{"Layer2"}) By("changing PUDN selector to non-matching label - count decreases to 0") - createOrUpdateCNC(cncName, nil, map[string]string{"nonexistent": "label"}) + createOrUpdateCNC(cs, cncName, nil, map[string]string{"nonexistent": "label"}) By("verifying CNC has no networks remaining") verifyCNCHasOnlyTunnelIDAnnotation(cncName) // No networks match, so subnet annotation is empty @@ -1390,8 +1470,8 @@ spec: cncName := generateCNCName() cncLabel := map[string]string{"test-cudn-label": "true"} - ns1 := createUDNNamespace("test-cudn-label-ns1", nil) - ns2 := createUDNNamespace("test-cudn-label-ns2", nil) + ns1 := createUDNNamespace(cs, "test-cudn-label-ns1", nil) + ns2 := createUDNNamespace(cs, "test-cudn-label-ns2", nil) cudnName1 := fmt.Sprintf("cudn-label1-%s", rand.String(5)) cudnName2 := fmt.Sprintf("cudn-label2-%s", rand.String(5)) @@ -1404,13 +1484,13 @@ spec: }) By("creating 2 CUDNs - first with matching label, second without") - createLayer3PrimaryCUDN(cudnName1, cncLabel, ns1.Name) - createLayer2PrimaryCUDN(cudnName2, map[string]string{"other": "label"}, ns2.Name) + createLayer3PrimaryCUDN(cs, cudnName1, cncLabel, ns1.Name) + createLayer2PrimaryCUDN(cs, cudnName2, map[string]string{"other": "label"}, ns2.Name) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName1), 30*time.Second, time.Second).Should(Succeed()) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName2), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with CUDN selector") - createOrUpdateCNC(cncName, cncLabel, nil) + createOrUpdateCNC(cs, cncName, cncLabel, nil) By("verifying CNC has 1 network initially") verifyCNCHasBothAnnotations(cncName) @@ -1446,8 +1526,8 @@ spec: cncName := generateCNCName() cncLabel := map[string]string{"test-ns-label": "true"} - ns1 := createUDNNamespace("test-ns-label-ns1", cncLabel) - ns2 := createUDNNamespace("test-ns-label-ns2", nil) // no matching label initially + ns1 := createUDNNamespace(cs, "test-ns-label-ns1", cncLabel) + ns2 := createUDNNamespace(cs, "test-ns-label-ns2", nil) // no matching label initially udnName1 := "udn1" udnName2 := "udn2" @@ -1460,13 +1540,13 @@ spec: }) By("creating 2 UDNs - first in namespace with matching label, second without") - createLayer3PrimaryUDN(ns1.Name, udnName1) - createLayer2PrimaryUDN(ns2.Name, udnName2) + createLayer3PrimaryUDN(cs, ns1.Name, udnName1) + createLayer2PrimaryUDN(cs, ns2.Name, udnName2) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, ns1.Name, udnName1), 30*time.Second, time.Second).Should(Succeed()) Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, ns2.Name, udnName2), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC with PUDN namespace selector") - createOrUpdateCNC(cncName, nil, cncLabel) + createOrUpdateCNC(cs, cncName, nil, cncLabel) By("verifying CNC has 1 network initially") verifyCNCHasBothAnnotations(cncName) @@ -1512,14 +1592,22 @@ spec: // Group 8: Multiple CNCs - multiple CNCs in cluster (3 tests) // =========================================== Context("when multiple CNCs exist", func() { + // Second CNC connect subnet configuration (must be different from first CNC) + const ( + cnc2ConnectSubnetIPv4CIDR = "192.169.0.0/16" + cnc2ConnectSubnetIPv4Prefix = 24 + cnc2ConnectSubnetIPv6CIDR = "fd00:11::/112" + cnc2ConnectSubnetIPv6Prefix = 120 + ) + It("two CNCs with non-overlapping selectors - each tracks its own networks", func() { cncName1 := generateCNCName() cncName2 := generateCNCName() label1 := map[string]string{"test-multi-cnc-1": "true"} label2 := map[string]string{"test-multi-cnc-2": "true"} - ns1 := createUDNNamespace("test-multi-cnc-ns1", nil) - ns2 := createUDNNamespace("test-multi-cnc-ns2", nil) + ns1 := createUDNNamespace(cs, "test-multi-cnc-ns1", nil) + ns2 := createUDNNamespace(cs, "test-multi-cnc-ns2", nil) cudnName1 := fmt.Sprintf("multi-cnc-cudn1-%s", rand.String(5)) cudnName2 := fmt.Sprintf("multi-cnc-cudn2-%s", rand.String(5)) @@ -1533,16 +1621,16 @@ spec: }) By("creating 2 CUDNs with different labels") - createLayer3PrimaryCUDN(cudnName1, label1, ns1.Name) - createLayer2PrimaryCUDN(cudnName2, label2, ns2.Name) + createLayer3PrimaryCUDN(cs, cudnName1, label1, ns1.Name) + createLayer2PrimaryCUDN(cs, cudnName2, label2, ns2.Name) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName1), 30*time.Second, time.Second).Should(Succeed()) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName2), 30*time.Second, time.Second).Should(Succeed()) - By("creating first CNC matching first CUDN") - createOrUpdateCNC(cncName1, label1, nil) + By("creating first CNC matching first CUDN (with first connect subnet)") + createOrUpdateCNCWithSubnets(cncName1, label1, nil, generateConnectSubnets(cs)) - By("creating second CNC matching second CUDN") - createOrUpdateCNC(cncName2, label2, nil) + By("creating second CNC matching second CUDN (with different connect subnet)") + createOrUpdateCNCWithSubnets(cncName2, label2, nil, generateConnectSubnetsWithCIDRs(cs, cnc2ConnectSubnetIPv4CIDR, cnc2ConnectSubnetIPv4Prefix, cnc2ConnectSubnetIPv6CIDR, cnc2ConnectSubnetIPv6Prefix)) By("verifying first CNC has only first network") verifyCNCHasBothAnnotations(cncName1) @@ -1568,7 +1656,7 @@ spec: cncName2 := generateCNCName() sharedLabel := map[string]string{"test-shared-cudn": "true"} - ns := createUDNNamespace("test-shared-cudn-ns", nil) + ns := createUDNNamespace(cs, "test-shared-cudn-ns", nil) cudnName := fmt.Sprintf("shared-cudn-%s", rand.String(5)) DeferCleanup(func() { @@ -1579,14 +1667,14 @@ spec: }) By("creating a CUDN with shared label") - createLayer3PrimaryCUDN(cudnName, sharedLabel, ns.Name) + createLayer3PrimaryCUDN(cs, cudnName, sharedLabel, ns.Name) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName), 30*time.Second, time.Second).Should(Succeed()) - By("creating first CNC matching the CUDN") - createOrUpdateCNC(cncName1, sharedLabel, nil) + By("creating first CNC matching the CUDN (with first connect subnet)") + createOrUpdateCNCWithSubnets(cncName1, sharedLabel, nil, generateConnectSubnets(cs)) - By("creating second CNC also matching the CUDN") - createOrUpdateCNC(cncName2, sharedLabel, nil) + By("creating second CNC also matching the CUDN (with different connect subnet)") + createOrUpdateCNCWithSubnets(cncName2, sharedLabel, nil, generateConnectSubnetsWithCIDRs(cs, cnc2ConnectSubnetIPv4CIDR, cnc2ConnectSubnetIPv4Prefix, cnc2ConnectSubnetIPv6CIDR, cnc2ConnectSubnetIPv6Prefix)) By("verifying both CNCs have the network in their annotations") verifyCNCHasBothAnnotations(cncName1) @@ -1612,8 +1700,8 @@ spec: label1 := map[string]string{"test-cnc-delete-1": "true"} label2 := map[string]string{"test-cnc-delete-2": "true"} - ns1 := createUDNNamespace("test-cnc-delete-ns1", nil) - ns2 := createUDNNamespace("test-cnc-delete-ns2", nil) + ns1 := createUDNNamespace(cs, "test-cnc-delete-ns1", nil) + ns2 := createUDNNamespace(cs, "test-cnc-delete-ns2", nil) cudnName1 := fmt.Sprintf("cnc-delete-cudn1-%s", rand.String(5)) cudnName2 := fmt.Sprintf("cnc-delete-cudn2-%s", rand.String(5)) @@ -1626,14 +1714,14 @@ spec: }) By("creating 2 CUDNs with different labels") - createLayer3PrimaryCUDN(cudnName1, label1, ns1.Name) - createLayer2PrimaryCUDN(cudnName2, label2, ns2.Name) + createLayer3PrimaryCUDN(cs, cudnName1, label1, ns1.Name) + createLayer2PrimaryCUDN(cs, cudnName2, label2, ns2.Name) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName1), 30*time.Second, time.Second).Should(Succeed()) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName2), 30*time.Second, time.Second).Should(Succeed()) - By("creating two CNCs with different selectors") - createOrUpdateCNC(cncName1, label1, nil) - createOrUpdateCNC(cncName2, label2, nil) + By("creating two CNCs with different selectors and different connect subnets") + createOrUpdateCNCWithSubnets(cncName1, label1, nil, generateConnectSubnets(cs)) + createOrUpdateCNCWithSubnets(cncName2, label2, nil, generateConnectSubnetsWithCIDRs(cs, cnc2ConnectSubnetIPv4CIDR, cnc2ConnectSubnetIPv4Prefix, cnc2ConnectSubnetIPv6CIDR, cnc2ConnectSubnetIPv6Prefix)) By("verifying both CNCs have their networks") verifyCNCHasBothAnnotations(cncName1) @@ -1659,7 +1747,7 @@ spec: cncName := generateCNCName() cncLabel := map[string]string{"test-cnc-lifecycle": "true"} - ns := createUDNNamespace("test-cnc-lifecycle-ns", nil) + ns := createUDNNamespace(cs, "test-cnc-lifecycle-ns", nil) cudnName := fmt.Sprintf("cnc-lifecycle-cudn-%s", rand.String(5)) DeferCleanup(func() { @@ -1669,11 +1757,11 @@ spec: }) By("creating a CUDN") - createLayer3PrimaryCUDN(cudnName, cncLabel, ns.Name) + createLayer3PrimaryCUDN(cs, cudnName, cncLabel, ns.Name) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC") - createOrUpdateCNC(cncName, cncLabel, nil) + createOrUpdateCNC(cs, cncName, cncLabel, nil) By("verifying CNC has network and tunnel ID") verifyCNCHasBothAnnotations(cncName) @@ -1691,7 +1779,7 @@ spec: }, 30*time.Second, time.Second).Should(BeTrue()) By("recreating CNC with same name") - createOrUpdateCNC(cncName, cncLabel, nil) + createOrUpdateCNC(cs, cncName, cncLabel, nil) By("verifying CNC has network again") verifyCNCHasBothAnnotations(cncName) @@ -1708,8 +1796,8 @@ spec: label1 := map[string]string{"test-tunnel-stable-1": "true"} label2 := map[string]string{"test-tunnel-stable-2": "true"} - ns1 := createUDNNamespace("test-tunnel-stable-ns1", nil) - ns2 := createUDNNamespace("test-tunnel-stable-ns2", nil) + ns1 := createUDNNamespace(cs, "test-tunnel-stable-ns1", nil) + ns2 := createUDNNamespace(cs, "test-tunnel-stable-ns2", nil) cudnName1 := fmt.Sprintf("tunnel-stable-cudn1-%s", rand.String(5)) cudnName2 := fmt.Sprintf("tunnel-stable-cudn2-%s", rand.String(5)) @@ -1722,13 +1810,13 @@ spec: }) By("creating 2 CUDNs with different labels") - createLayer3PrimaryCUDN(cudnName1, label1, ns1.Name) - createLayer2PrimaryCUDN(cudnName2, label2, ns2.Name) + createLayer3PrimaryCUDN(cs, cudnName1, label1, ns1.Name) + createLayer2PrimaryCUDN(cs, cudnName2, label2, ns2.Name) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName1), 30*time.Second, time.Second).Should(Succeed()) Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName2), 30*time.Second, time.Second).Should(Succeed()) By("creating CNC matching first CUDN") - createOrUpdateCNC(cncName, label1, nil) + createOrUpdateCNC(cs, cncName, label1, nil) By("verifying CNC has network and recording tunnel ID") verifyCNCHasBothAnnotations(cncName) @@ -1736,7 +1824,7 @@ spec: originalTunnelID := getCNCTunnelID(cncName) By("updating CNC to match second CUDN instead") - createOrUpdateCNC(cncName, label2, nil) + createOrUpdateCNC(cs, cncName, label2, nil) By("verifying CNC now has second network") verifyCNCSubnetAnnotationNetworkCount(cncName, 1) @@ -1751,7 +1839,7 @@ spec: // Add label1 to second CUDN so we can match both _, err := e2ekubectl.RunKubectl("", "label", "clusteruserdefinednetwork", cudnName2, "test-tunnel-stable-1=true") Expect(err).NotTo(HaveOccurred()) - createOrUpdateCNC(cncName, label1, nil) + createOrUpdateCNC(cs, cncName, label1, nil) By("verifying CNC now has both networks") verifyCNCSubnetAnnotationNetworkCount(cncName, 2) @@ -1771,14 +1859,14 @@ spec: It("comprehensive workflow - create, add, update, remove networks through CNC lifecycle", func() { cncName := generateCNCName() cudnLabel := map[string]string{"test-lifecycle": "true"} - pudnLabel := map[string]string{"test-lifecycle": "true"} + udnLabel := map[string]string{"test-lifecycle": "true"} var expectedTopologies []string // Create namespaces - cudnNs1 := createUDNNamespace("lifecycle-cudn-ns1", nil) - cudnNs2 := createUDNNamespace("lifecycle-cudn-ns2", nil) - udnNs1 := createUDNNamespace("lifecycle-udn-ns1", pudnLabel) - udnNs2 := createUDNNamespace("lifecycle-udn-ns2", pudnLabel) + cudnNs1 := createUDNNamespace(cs, "lifecycle-cudn-ns1", nil) + cudnNs2 := createUDNNamespace(cs, "lifecycle-cudn-ns2", nil) + udnNs1 := createUDNNamespace(cs, "lifecycle-udn-ns1", udnLabel) + udnNs2 := createUDNNamespace(cs, "lifecycle-udn-ns2", udnLabel) cudnName1 := fmt.Sprintf("lifecycle-cudn1-%s", rand.String(5)) cudnName2 := fmt.Sprintf("lifecycle-cudn2-%s", rand.String(5)) @@ -1798,13 +1886,13 @@ spec: // Phase 1: Create CNC with no matching networks By("Phase 1: Creating CNC with no matching networks yet") - createOrUpdateCNC(cncName, cudnLabel, pudnLabel) + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) verifyCNCHasOnlyTunnelIDAnnotation(cncName) originalTunnelID := getCNCTunnelID(cncName) // Phase 2: Create first L3 CUDN - count goes to 1 By("Phase 2: Creating first L3 CUDN") - createLayer3PrimaryCUDN(cudnName1, cudnLabel, cudnNs1.Name) + createLayer3PrimaryCUDN(cs, cudnName1, cudnLabel, cudnNs1.Name) expectedTopologies = append(expectedTopologies, "Layer3") Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName1), 30*time.Second, time.Second).Should(Succeed()) verifyCNCSubnetAnnotationNetworkCount(cncName, 1) @@ -1812,7 +1900,7 @@ spec: // Phase 3: Create first L2 UDN - count goes to 2 By("Phase 3: Creating first L2 UDN") - createLayer2PrimaryUDN(udnNs1.Name, udnName1) + createLayer2PrimaryUDN(cs, udnNs1.Name, udnName1) expectedTopologies = append(expectedTopologies, "Layer2") Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, udnNs1.Name, udnName1), 30*time.Second, time.Second).Should(Succeed()) verifyCNCSubnetAnnotationNetworkCount(cncName, 2) @@ -1820,7 +1908,7 @@ spec: // Phase 4: Create second L2 CUDN - count goes to 3 By("Phase 4: Creating second L2 CUDN") - createLayer2PrimaryCUDN(cudnName2, cudnLabel, cudnNs2.Name) + createLayer2PrimaryCUDN(cs, cudnName2, cudnLabel, cudnNs2.Name) expectedTopologies = append(expectedTopologies, "Layer2") Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, cudnName2), 30*time.Second, time.Second).Should(Succeed()) verifyCNCSubnetAnnotationNetworkCount(cncName, 3) @@ -1828,7 +1916,7 @@ spec: // Phase 5: Create second L3 UDN - count goes to 4 By("Phase 5: Creating second L3 UDN") - createLayer3PrimaryUDN(udnNs2.Name, udnName2) + createLayer3PrimaryUDN(cs, udnNs2.Name, udnName2) expectedTopologies = append(expectedTopologies, "Layer3") Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, udnNs2.Name, udnName2), 30*time.Second, time.Second).Should(Succeed()) verifyCNCSubnetAnnotationNetworkCount(cncName, 4) @@ -1840,7 +1928,7 @@ spec: // Phase 6: Remove PUDN selector - count goes to 2 (only CUDNs remain) By("Phase 6: Removing PUDN selector from CNC") - createOrUpdateCNC(cncName, cudnLabel, nil) + createOrUpdateCNC(cs, cncName, cudnLabel, nil) verifyCNCSubnetAnnotationNetworkCount(cncName, 2) verifyCNCSubnetAnnotationContent(cncName, []string{"Layer3", "Layer2"}) // cudnName1 is L3, cudnName2 is L2 @@ -1856,7 +1944,7 @@ spec: // Phase 8: Add PUDN selector back - count goes to 3 (1 CUDN + 2 UDNs) By("Phase 8: Adding PUDN selector back to CNC") - createOrUpdateCNC(cncName, cudnLabel, pudnLabel) + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) verifyCNCSubnetAnnotationNetworkCount(cncName, 3) verifyCNCSubnetAnnotationContent(cncName, []string{"Layer2", "Layer2", "Layer3"}) // cudnName2(L2), udn1(L2), udn2(L3) @@ -1894,3 +1982,830 @@ spec: }) }) }) + +// ============================================================================ +// OVN Database Side Testing - End-to-End Connectivity Validation +// ============================================================================ +var _ = Describe("ClusterNetworkConnect OVN-Kubernetes Controller", feature.NetworkConnect, func() { + f := wrappedTestFramework("cnc-ovndb") + // disable automatic namespace creation, we need to add the required UDN label + f.SkipNamespaceCreation = true + + var ( + cs clientset.Interface + ) + + BeforeEach(func() { + cs = f.ClientSet + }) + + // httpServerPodConfig returns a podConfiguration for an HTTP server pod + httpServerPodConfig := func(podName, namespace string) podConfiguration { + cfg := *podConfig(podName, withCommand(func() []string { + return httpServerContainerCmd(8080) + })) + cfg.namespace = namespace + return cfg + } + + // getPrimaryNetworkPodIPs gets the pod IPs for supported IP families on a given primary network + // Returns a slice of IP strings based on cluster's supported IP families + // Uses Eventually to wait for OVN annotations to be populated + getPrimaryNetworkPodIPs := func(namespace, podName, networkName string) []string { + var ips []string + Eventually(func() error { + ips = make([]string, 0, 2) + for _, family := range getSupportedIPFamiliesSlice(cs) { + ip, err := getPodAnnotationIPsForPrimaryNetworkByIPFamily(cs, namespace, podName, networkName, family) + if err != nil { + return err + } + if ip != "" { + ips = append(ips, ip) + } + } + if len(ips) == 0 { + return fmt.Errorf("pod %s/%s has no IPs on network %s yet", namespace, podName, networkName) + } + return nil + }, 2*time.Minute, 5*time.Second).Should(Succeed(), + fmt.Sprintf("waiting for pod %s/%s to have IPs on network %s", namespace, podName, networkName)) + return ips + } + + // checkConnectivity checks connectivity from one pod to another + // Returns true if connection succeeds/fails as expected + checkConnectivity := func(fromNamespace, fromPodName, toIP string, expectSuccess bool) bool { + // net.JoinHostPort properly handles IPv6 addresses by adding brackets + url := fmt.Sprintf("http://%s/hostname", net.JoinHostPort(toIP, "8080")) + stdout, err := e2ekubectl.RunKubectl(fromNamespace, "exec", fromPodName, "--", + "curl", "--connect-timeout", "1", "-s", "-o", "/dev/null", "-w", "%{http_code}", url) + if expectSuccess { + return err == nil && stdout == "200" + } + return err != nil || stdout != "200" + } + + // verifyCrossNetworkConnectivity verifies connectivity from a set of pods to another set + // Supports dual-stack by testing all IPs for each pod + // Uses Eventually for expected success + // For expected failure: first waits for connectivity to fail (Eventually), then verifies it stays failed (Consistently) + verifyCrossNetworkConnectivity := func(fromPods map[string]*corev1.Pod, toPodIPs map[string][]string, expectSuccess bool) { + for fromName, fromPod := range fromPods { + for toName, toIPs := range toPodIPs { + for _, toIP := range toIPs { + msg := fmt.Sprintf("cross-network connectivity from %s to %s (%s) expectSuccess=%v", fromName, toName, toIP, expectSuccess) + if expectSuccess { + Eventually(func() bool { + return checkConnectivity(fromPod.Namespace, fromPod.Name, toIP, true) + }, 10*time.Second, 1*time.Second).Should(BeTrue(), msg) + } else { + // First wait for connectivity to fail (OVN flows take time to update) + Eventually(func() bool { + return checkConnectivity(fromPod.Namespace, fromPod.Name, toIP, false) + }, 5*time.Second, 1*time.Second).Should(BeTrue(), msg+" (waiting for failure)") + // Then verify it stays failed consistently + Consistently(func() bool { + return checkConnectivity(fromPod.Namespace, fromPod.Name, toIP, false) + }, 10*time.Second, 1*time.Second).Should(BeTrue(), msg+" (consistent failure)") + } + } + } + } + } + + // verifyFullMeshConnectivity verifies connectivity from source pods to all target pod IPs + // Uses Eventually for expected success, Eventually+Consistently for expected failure + verifyFullMeshConnectivity := func(srcPods map[string]*corev1.Pod, dstPodIPs map[string][]string, expectSuccess bool) { + for fromName, fromPod := range srcPods { + for toName, toIPs := range dstPodIPs { + if fromName == toName { + continue // Skip self-connectivity + } + for _, toIP := range toIPs { + msg := fmt.Sprintf("connectivity from %s to %s (%s) expectSuccess=%v", fromName, toName, toIP, expectSuccess) + if expectSuccess { + Eventually(func() bool { + return checkConnectivity(fromPod.Namespace, fromPod.Name, toIP, true) + }, 10*time.Second, 1*time.Second).Should(BeTrue(), msg) + } else { + // First wait for connectivity to fail (OVN flows take time to update) + Eventually(func() bool { + return checkConnectivity(fromPod.Namespace, fromPod.Name, toIP, false) + }, 5*time.Second, 1*time.Second).Should(BeTrue(), msg+" (waiting for failure)") + // Then verify it stays failed consistently + Consistently(func() bool { + return checkConnectivity(fromPod.Namespace, fromPod.Name, toIP, false) + }, 10*time.Second, 1*time.Second).Should(BeTrue(), msg+" (consistent failure)") + } + } + } + } + } + + /* + This test validates end-to-end connectivity through CNC (ClusterNetworkConnect). + + Test Scenario: + - Create 2 CUDNs: "black" (L3) and "white" (L2), each serving 2 namespaces with pods on different nodes + - Create 2 UDNs: "blue" (L3) and "green" (L2), each in its own namespace with 2 pods on different nodes + - Initially verify pods cannot communicate across different networks + - Create CNC selecting all 4 networks, verify cross-network connectivity (same-node and cross-node) + - Test all network deselection/reselection methods: + * UDN via namespace label change (blue - L3) + * UDN via CNC selector update (green - L2) + * CUDN via network label change (black - L3) + * CUDN via CNC selector update (black+white) + - Verify proper isolation when networks are disconnected + - Test CNC deletion and re-creation + + Steps: + 1. Create 2 CUDNs: black (L3) and white (L2), each with 2 namespaces and pods on different nodes + 2. Create 2 UDNs: blue (L3) and green (L2), each with 2 pods on different nodes + 3. Verify initial isolation - pods cannot talk across networks + 4. Create CNC selecting all 4 networks + 5. Verify CNC annotations are set correctly (subnet allocation) + 6. Verify pods can communicate across all networks (same-node and cross-node) + 7. Deselect blue UDN (L3) via namespace label removal + 8. Verify blue pods isolated, other networks still connected + 9. Deselect green UDN (L2) via CNC selector update (remove PUDN selector) + 10. Verify green pods isolated, CUDNs still connected + 11. Re-select green UDN via CNC selector update (blue still deselected - no label) + 12. Re-select blue UDN via namespace label restoration + 13. Deselect black CUDN (L3) via CUDN label removal + 14. Verify black pods isolated, other networks still connected + 15. Re-select black CUDN via CUDN label restoration + 16. Verify black pods can communicate again + 17. Deselect both CUDNs (black+white) via CNC selector update (remove CUDN selector) + 18. Verify CUDN pods isolated, UDNs still connected + 19. Re-select CUDNs via CNC selector update + 20. Verify CUDN pods can communicate again + 21. Delete CNC + 22. Verify all cross-network connectivity disabled + 23. Re-create CNC + 24. Verify all cross-network connectivity restored + */ + Context("End-to-end connectivity validation", func() { + const nodeHostnameKey = "kubernetes.io/hostname" + + It("should manage cross-network connectivity through CNC lifecycle", func() { + // Test identifiers + testID := rand.String(5) + cncName := generateCNCName() + + // Get 2 schedulable nodes for cross-node testing + nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), cs, 2) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodes.Items)).To(BeNumerically(">=", 2), "test requires at least 2 schedulable nodes") + node1Name, node2Name := nodes.Items[0].Name, nodes.Items[1].Name + + // Network names + blackCUDN := fmt.Sprintf("black-cudn-%s", testID) + whiteCUDN := fmt.Sprintf("white-cudn-%s", testID) + blueUDN := "blue-udn" + greenUDN := "green-udn" + + // Namespace names (fixed for predictability in CUDN selectors) + blackNs0 := fmt.Sprintf("black-ns-0-%s", testID) + blackNs1 := fmt.Sprintf("black-ns-1-%s", testID) + whiteNs0 := fmt.Sprintf("white-ns-0-%s", testID) + whiteNs1 := fmt.Sprintf("white-ns-1-%s", testID) + blueNs := fmt.Sprintf("blue-ns-%s", testID) + greenNs := fmt.Sprintf("green-ns-%s", testID) + + // Labels for CNC selection + cudnLabel := map[string]string{"cnc-test": testID, "type": "cudn"} + udnLabel := map[string]string{"cnc-test": testID, "type": "pudn"} + + // Store pods and their IPs for connectivity testing (supports dual-stack) + pods := make(map[string]*corev1.Pod) + podIPs := make(map[string][]string) + + // Cleanup + DeferCleanup(func() { + By("Cleanup: Deleting all test resources") + deleteCNC(cncName) + + // Delete pods first + for _, pod := range pods { + _ = cs.CoreV1().Pods(pod.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) + } + + // Delete UDNs + deleteUDN(blueNs, blueUDN) + deleteUDN(greenNs, greenUDN) + + // Delete CUDNs + deleteCUDN(blackCUDN) + deleteCUDN(whiteCUDN) + + // Delete namespaces + deleteNamespace(cs, blackNs0) + deleteNamespace(cs, blackNs1) + deleteNamespace(cs, whiteNs0) + deleteNamespace(cs, whiteNs1) + deleteNamespace(cs, blueNs) + deleteNamespace(cs, greenNs) + }) + + // ===================================================================== + // Step 1: Create 2 CUDNs (black, white) each with 2 namespaces and pods + // ===================================================================== + By("1. Creating namespaces for black and white CUDNs") + createUDNNamespaceWithName(cs, blackNs0, nil) + createUDNNamespaceWithName(cs, blackNs1, nil) + createUDNNamespaceWithName(cs, whiteNs0, nil) + createUDNNamespaceWithName(cs, whiteNs1, nil) + + By("1. Creating black CUDN targeting black-ns-0 and black-ns-1") + createLayer3PrimaryCUDNWithSubnets(cs, blackCUDN, cudnLabel, "10.128.0.0/16", "2014:100:200::0/60", blackNs0, blackNs1) + + By("1. Waiting for black CUDN to be ready") + Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, blackCUDN), 60*time.Second, time.Second).Should(Succeed()) + + By("1. Creating white CUDN targeting white-ns-0 and white-ns-1") + createLayer2PrimaryCUDNWithSubnets(cs, whiteCUDN, cudnLabel, "10.129.0.0/16", "2014:100:300::0/60", whiteNs0, whiteNs1) + + By("1. Waiting for white CUDN to be ready") + Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, whiteCUDN), 60*time.Second, time.Second).Should(Succeed()) + + By("1. Creating pods in black CUDN namespaces (on different nodes for cross-node testing)") + blackPodConfig0 := httpServerPodConfig("black-pod-0", blackNs0) + blackPodConfig0.nodeSelector = map[string]string{nodeHostnameKey: node1Name} + blackPodConfig1 := httpServerPodConfig("black-pod-1", blackNs1) + blackPodConfig1.nodeSelector = map[string]string{nodeHostnameKey: node2Name} + pods["black-pod-0"] = runUDNPod(cs, blackNs0, blackPodConfig0, nil) + pods["black-pod-1"] = runUDNPod(cs, blackNs1, blackPodConfig1, nil) + podIPs["black-pod-0"] = getPrimaryNetworkPodIPs(blackNs0, "black-pod-0", blackCUDN) + podIPs["black-pod-1"] = getPrimaryNetworkPodIPs(blackNs1, "black-pod-1", blackCUDN) + + By("1. Creating pods in white CUDN namespaces (on different nodes for cross-node testing)") + whitePodConfig0 := httpServerPodConfig("white-pod-0", whiteNs0) + whitePodConfig0.nodeSelector = map[string]string{nodeHostnameKey: node1Name} + whitePodConfig1 := httpServerPodConfig("white-pod-1", whiteNs1) + whitePodConfig1.nodeSelector = map[string]string{nodeHostnameKey: node2Name} + pods["white-pod-0"] = runUDNPod(cs, whiteNs0, whitePodConfig0, nil) + pods["white-pod-1"] = runUDNPod(cs, whiteNs1, whitePodConfig1, nil) + podIPs["white-pod-0"] = getPrimaryNetworkPodIPs(whiteNs0, "white-pod-0", whiteCUDN) + podIPs["white-pod-1"] = getPrimaryNetworkPodIPs(whiteNs1, "white-pod-1", whiteCUDN) + + // ===================================================================== + // Step 2: Create 2 UDNs (blue, green) each with 1 namespace and 2 pods on different nodes + // ===================================================================== + By("2. Creating namespaces for blue and green UDNs with PUDN labels") + createUDNNamespaceWithName(cs, blueNs, udnLabel) + createUDNNamespaceWithName(cs, greenNs, udnLabel) + + By("2. Creating blue UDN (L3)") + createLayer3PrimaryUDNWithSubnets(cs, blueNs, blueUDN, "10.130.0.0/16", "2014:100:400::0/60") + + By("2. Waiting for blue UDN to be ready") + Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, blueNs, blueUDN), 60*time.Second, time.Second).Should(Succeed()) + + By("2. Creating green UDN (L2)") + createLayer2PrimaryUDNWithSubnets(cs, greenNs, greenUDN, "10.131.0.0/16", "2014:100:500::0/60") + + By("2. Waiting for green UDN to be ready") + Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, greenNs, greenUDN), 60*time.Second, time.Second).Should(Succeed()) + + By("2. Creating pods in blue UDN namespace (on different nodes for same and cross-node testing)") + bluePodConfig0 := httpServerPodConfig("blue-pod-0", blueNs) + bluePodConfig0.nodeSelector = map[string]string{nodeHostnameKey: node1Name} + bluePodConfig1 := httpServerPodConfig("blue-pod-1", blueNs) + bluePodConfig1.nodeSelector = map[string]string{nodeHostnameKey: node2Name} + pods["blue-pod-0"] = runUDNPod(cs, blueNs, bluePodConfig0, nil) + pods["blue-pod-1"] = runUDNPod(cs, blueNs, bluePodConfig1, nil) + podIPs["blue-pod-0"] = getPrimaryNetworkPodIPs(blueNs, "blue-pod-0", blueUDN) + podIPs["blue-pod-1"] = getPrimaryNetworkPodIPs(blueNs, "blue-pod-1", blueUDN) + + By("2. Creating pods in green UDN namespace (on different nodes for same and cross-node testing)") + greenPodConfig0 := httpServerPodConfig("green-pod-0", greenNs) + greenPodConfig0.nodeSelector = map[string]string{nodeHostnameKey: node1Name} + greenPodConfig1 := httpServerPodConfig("green-pod-1", greenNs) + greenPodConfig1.nodeSelector = map[string]string{nodeHostnameKey: node2Name} + pods["green-pod-0"] = runUDNPod(cs, greenNs, greenPodConfig0, nil) + pods["green-pod-1"] = runUDNPod(cs, greenNs, greenPodConfig1, nil) + podIPs["green-pod-0"] = getPrimaryNetworkPodIPs(greenNs, "green-pod-0", greenUDN) + podIPs["green-pod-1"] = getPrimaryNetworkPodIPs(greenNs, "green-pod-1", greenUDN) + + // ===================================================================== + // Step 3: Verify initial isolation - pods cannot talk across networks + // ===================================================================== + By("3. Verifying initial isolation - black pods cannot reach white pods") + blackPods := map[string]*corev1.Pod{"black-pod-0": pods["black-pod-0"], "black-pod-1": pods["black-pod-1"]} + whitePodIPs := map[string][]string{"white-pod-0": podIPs["white-pod-0"], "white-pod-1": podIPs["white-pod-1"]} + verifyCrossNetworkConnectivity(blackPods, whitePodIPs, false) + + By("3. Verifying initial isolation - white pods cannot reach blue pods") + whitePods := map[string]*corev1.Pod{"white-pod-0": pods["white-pod-0"], "white-pod-1": pods["white-pod-1"]} + bluePodIPs := map[string][]string{"blue-pod-0": podIPs["blue-pod-0"], "blue-pod-1": podIPs["blue-pod-1"]} + verifyCrossNetworkConnectivity(whitePods, bluePodIPs, false) + + By("3. Verifying initial isolation - blue pods cannot reach green pods") + bluePods := map[string]*corev1.Pod{"blue-pod-0": pods["blue-pod-0"], "blue-pod-1": pods["blue-pod-1"]} + greenPodIPs := map[string][]string{"green-pod-0": podIPs["green-pod-0"], "green-pod-1": podIPs["green-pod-1"]} + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, false) + + // ===================================================================== + // Step 4: Create CNC selecting all 4 networks + // ===================================================================== + By("4. Creating CNC selecting all 4 networks (2 CUDNs + 2 UDNs)") + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) + + // ===================================================================== + // Step 5: Verify CM annotations are set correctly + // ===================================================================== + By("5. Verifying CNC has both tunnel ID and subnet annotations") + verifyCNCHasBothAnnotations(cncName) + + By("5. Verifying CNC subnet annotation has 4 networks") + verifyCNCSubnetAnnotationNetworkCount(cncName, 4) + + // ===================================================================== + // Step 6: Verify pods can now communicate across all networks + // ===================================================================== + By("6. Verifying pods can communicate across all connected networks") + // Use one pod per network as source (targets include both nodes for same/cross-node coverage) + srcPods := map[string]*corev1.Pod{ + "black-pod-0": pods["black-pod-0"], + "white-pod-0": pods["white-pod-0"], + "blue-pod-0": pods["blue-pod-0"], + "green-pod-0": pods["green-pod-0"], + } + verifyFullMeshConnectivity(srcPods, podIPs, true) + + // ===================================================================== + // Step 7: Deselect blue UDN by changing namespace label + // ===================================================================== + By("7. Removing PUDN label from blue namespace to deselect blue UDN") + _, err = cs.CoreV1().Namespaces().Patch(context.Background(), blueNs, + types.MergePatchType, + []byte(`{"metadata":{"labels":{"type":null}}}`), + metav1.PatchOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("7. Verifying CNC subnet annotation now has 3 networks") + verifyCNCSubnetAnnotationNetworkCount(cncName, 3) + + // ===================================================================== + // Step 8: Verify blue pods cannot talk to other network pods + // ===================================================================== + By("8. Verifying blue pods cannot reach other network pods") + verifyCrossNetworkConnectivity(bluePods, whitePodIPs, false) + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, false) + + By("8. Verifying other networks can still communicate with each other") + verifyCrossNetworkConnectivity(blackPods, whitePodIPs, true) + verifyCrossNetworkConnectivity(whitePods, greenPodIPs, true) + + // ===================================================================== + // Step 9: Deselect green UDN by updating CNC to remove PUDN selector + // ===================================================================== + By("9. Updating CNC to remove PUDN selector (deselects green UDN)") + createOrUpdateCNC(cs, cncName, cudnLabel, nil) + + By("9. Verifying CNC subnet annotation now has 2 networks (only CUDNs)") + verifyCNCSubnetAnnotationNetworkCount(cncName, 2) + + // ===================================================================== + // Step 10: Verify green pods cannot talk to other network pods + // ===================================================================== + greenPods := map[string]*corev1.Pod{"green-pod-0": pods["green-pod-0"], "green-pod-1": pods["green-pod-1"]} + By("10. Verifying green pods cannot reach other network pods") + verifyCrossNetworkConnectivity(greenPods, whitePodIPs, false) + verifyCrossNetworkConnectivity(greenPods, bluePodIPs, false) + + By("10. Verifying black and white CUDNs can still communicate") + verifyCrossNetworkConnectivity(blackPods, whitePodIPs, true) + verifyCrossNetworkConnectivity(whitePods, map[string][]string{"black-pod-0": podIPs["black-pod-0"]}, true) + + // ===================================================================== + // Step 11: Re-select green UDN via CNC selector update + // (blue still deselected - doesn't have namespace label from step 7) + // ===================================================================== + By("11. Updating CNC to add PUDN selector back (re-selects green UDN)") + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) + + By("11. Verifying CNC subnet annotation has 3 networks (green re-selected, blue still deselected)") + verifyCNCSubnetAnnotationNetworkCount(cncName, 3) + + By("11. Verifying green pods can communicate with black and white CUDNs pods") + blackPodIPs := map[string][]string{"black-pod-0": podIPs["black-pod-0"], "black-pod-1": podIPs["black-pod-1"]} + verifyCrossNetworkConnectivity(greenPods, whitePodIPs, true) + verifyCrossNetworkConnectivity(greenPods, blackPodIPs, true) + + // ===================================================================== + // Step 12: Re-select blue UDN via namespace label + // ===================================================================== + By("12. Adding PUDN label back to blue namespace (re-selects blue UDN)") + _, err = cs.CoreV1().Namespaces().Patch(context.Background(), blueNs, + types.MergePatchType, + []byte(`{"metadata":{"labels":{"type":"pudn"}}}`), + metav1.PatchOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("12. Verifying CNC subnet annotation has 4 networks again") + verifyCNCSubnetAnnotationNetworkCount(cncName, 4) + + By("12. Verifying blue pods can communicate with other network pods") + verifyCrossNetworkConnectivity(bluePods, whitePodIPs, true) + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, true) + + // ===================================================================== + // Step 13: Deselect black CUDN via CUDN label change + // ===================================================================== + By("13. Removing CNC-matching label from black CUDN (deselects black CUDN)") + _, err = e2ekubectl.RunKubectl("", "label", "clusteruserdefinednetwork", blackCUDN, "type-") + Expect(err).NotTo(HaveOccurred()) + + By("13. Verifying CNC subnet annotation now has 3 networks") + verifyCNCSubnetAnnotationNetworkCount(cncName, 3) + + // ===================================================================== + // Step 14: Verify black pods cannot talk with other networks + // ===================================================================== + By("14. Verifying black pods cannot reach other network pods") + verifyCrossNetworkConnectivity(blackPods, whitePodIPs, false) + verifyCrossNetworkConnectivity(blackPods, bluePodIPs, false) + verifyCrossNetworkConnectivity(blackPods, greenPodIPs, false) + + By("14. Verifying white, blue, and green networks can still communicate") + verifyCrossNetworkConnectivity(whitePods, bluePodIPs, true) + verifyCrossNetworkConnectivity(whitePods, greenPodIPs, true) + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, true) + + // ===================================================================== + // Step 15: Re-select black CUDN via CUDN label + // ===================================================================== + By("15. Adding CNC-matching label back to black CUDN (re-selects black CUDN)") + _, err = e2ekubectl.RunKubectl("", "label", "clusteruserdefinednetwork", blackCUDN, "type=cudn") + Expect(err).NotTo(HaveOccurred()) + + By("15. Verifying CNC subnet annotation has 4 networks again") + verifyCNCSubnetAnnotationNetworkCount(cncName, 4) + + // ===================================================================== + // Step 16: Verify black pods can communicate again + // ===================================================================== + By("16. Verifying black pods can reach other network pods again") + verifyCrossNetworkConnectivity(blackPods, whitePodIPs, true) + verifyCrossNetworkConnectivity(blackPods, bluePodIPs, true) + verifyCrossNetworkConnectivity(blackPods, greenPodIPs, true) + + // ===================================================================== + // Step 17: Deselect both CUDNs (black+white) via CNC selector update + // ===================================================================== + By("17. Updating CNC to remove CUDN selector (deselects black and white CUDNs)") + createOrUpdateCNC(cs, cncName, nil, udnLabel) + + By("17. Verifying CNC subnet annotation now has 2 networks (only UDNs)") + verifyCNCSubnetAnnotationNetworkCount(cncName, 2) + + // ===================================================================== + // Step 18: Verify both CUDN pods cannot talk with other networks + // ===================================================================== + By("18. Verifying black and white CUDN pods cannot reach UDN pods") + verifyCrossNetworkConnectivity(blackPods, bluePodIPs, false) + verifyCrossNetworkConnectivity(blackPods, greenPodIPs, false) + verifyCrossNetworkConnectivity(whitePods, bluePodIPs, false) + verifyCrossNetworkConnectivity(whitePods, greenPodIPs, false) + + By("18. Verifying blue and green UDNs can still communicate") + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, true) + + // ===================================================================== + // Step 19: Re-select CUDNs via CNC selector update + // ===================================================================== + By("19. Updating CNC to add CUDN selector back (re-selects black and white CUDNs)") + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) + + By("19. Verifying CNC subnet annotation has 4 networks again") + verifyCNCSubnetAnnotationNetworkCount(cncName, 4) + + // ===================================================================== + // Step 20: Verify CUDN pods can communicate again + // ===================================================================== + By("20. Verifying black and white CUDN pods can reach other network pods again") + verifyCrossNetworkConnectivity(blackPods, whitePodIPs, true) + verifyCrossNetworkConnectivity(blackPods, bluePodIPs, true) + verifyCrossNetworkConnectivity(whitePods, greenPodIPs, true) + + // ===================================================================== + // Step 21: Delete CNC + // ===================================================================== + By("21. Deleting CNC") + deleteCNC(cncName) + + By("21. Waiting for CNC deletion to complete") + Eventually(func() bool { + _, err := getCNCAnnotations(cncName) + return err != nil + }, 60*time.Second, 2*time.Second).Should(BeTrue()) + + // ===================================================================== + // Step 22: Verify all pods cannot talk to pods in other networks + // ===================================================================== + By("22. Verifying all cross-network connectivity is disabled") + verifyCrossNetworkConnectivity(blackPods, whitePodIPs, false) + verifyCrossNetworkConnectivity(blackPods, bluePodIPs, false) + verifyCrossNetworkConnectivity(blackPods, greenPodIPs, false) + verifyCrossNetworkConnectivity(whitePods, bluePodIPs, false) + verifyCrossNetworkConnectivity(whitePods, greenPodIPs, false) + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, false) + + // ===================================================================== + // Step 23: Re-create the CNC + // ===================================================================== + By("23. Re-creating CNC selecting all 4 networks") + createOrUpdateCNC(cs, cncName, cudnLabel, udnLabel) + + By("23. Verifying CNC has both tunnel ID and subnet annotations") + verifyCNCHasBothAnnotations(cncName) + + By("23. Verifying CNC subnet annotation has 4 networks") + verifyCNCSubnetAnnotationNetworkCount(cncName, 4) + + // ===================================================================== + // Step 24: Verify pods can communicate across all networks again + // ===================================================================== + By("24. Verifying pods can communicate across all connected networks again") + verifyFullMeshConnectivity(srcPods, podIPs, true) + + By("Test completed successfully - CNC lifecycle validated") + }) + }) + + /* + Multiple CNCs with overlapping network selection: + + This test validates behavior when multiple CNCs exist in the cluster with + overlapping network selections. It creates: + - CNC-1: selects blue (L2 UDN) and red (L3 CUDN) + - CNC-2: selects blue (L2 UDN) and green (L3 UDN) + + Network topology: + - Blue UDN (L2): shared between both CNCs + - Red CUDN (L3): only in CNC-1 + - Green UDN (L3): only in CNC-2 + + Expected connectivity: + - blue <-> red (via CNC-1) + - blue <-> green (via CNC-2) + - red <-/-> green (no direct connection - non-transitive) + + This validates that: + 1. CNCs can be created before networks exist + 2. Networks get dynamically added to existing CNCs when they match selectors + 3. A network can be part of multiple CNCs simultaneously + 4. Connectivity is non-transitive (no indirect routes through shared networks) + 5. Each CNC maintains independent routing domains + + Steps: + 1. Create CNC-1 selecting blue and red (no networks exist yet) + 2. Create CNC-2 selecting blue and green (no networks exist yet) + 3. Create blue UDN (L2) with pods - gets added to both CNCs + 4. Create red CUDN (L3) with pods - gets added to CNC-1 + 5. Create green UDN (L3) with pods - gets added to CNC-2 + 6. Verify blue <-> red connectivity via CNC-1 + 7. Verify blue <-> green connectivity via CNC-2 + 8. Verify red <-/-> green (non-transitive - no connectivity) + 9. Delete CNC-1, verify blue <-> green still works, blue <-/-> red + 10. Delete CNC-2, verify all networks isolated + */ + Context("Multiple CNCs with overlapping network selection", func() { + const nodeHostnameKey = "kubernetes.io/hostname" + + // Second CNC connect subnet configuration (must be different from first CNC) + const ( + cnc2ConnectSubnetIPv4CIDR = "192.169.0.0/16" + cnc2ConnectSubnetIPv4Prefix = 24 + cnc2ConnectSubnetIPv6CIDR = "fd00:11::/112" + cnc2ConnectSubnetIPv6Prefix = 120 + ) + + It("should maintain non-transitive connectivity when a network is selected by multiple CNCs", func() { + // Test identifiers + testID := rand.String(5) + cnc1Name := fmt.Sprintf("color-1-%s", testID) + cnc2Name := fmt.Sprintf("color-2-%s", testID) + + // Get 2 schedulable nodes for cross-node testing + nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), cs, 2) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodes.Items)).To(BeNumerically(">=", 2), "test requires at least 2 schedulable nodes") + node1Name, node2Name := nodes.Items[0].Name, nodes.Items[1].Name + + // Network names + blueUDN := "blue-udn" // L2 UDN - shared between both CNCs + redCUDN := fmt.Sprintf("red-cudn-%s", testID) // L3 CUDN - CNC-1 only + greenUDN := "green-udn" // L3 UDN - CNC-2 only + + // Namespace names + blueNs := fmt.Sprintf("blue-ns-%s", testID) + redNs := fmt.Sprintf("red-ns-%s", testID) + greenNs := fmt.Sprintf("green-ns-%s", testID) + + // Labels for CNC selection + // Blue needs labels for both CNC-1 (via blueLabel) and CNC-2 (via cnc2Label) + blueLabel := map[string]string{"network-color": "blue", "test-id": testID, "cnc2-member": "true"} + redLabel := map[string]string{"network-color": "red", "test-id": testID} + // Green needs label for CNC-2 + greenLabel := map[string]string{"network-color": "green", "test-id": testID, "cnc2-member": "true"} + // CNC-2 selector - matches both blue and green namespaces + cnc2Label := map[string]string{"cnc2-member": "true"} + + // Store pods and their IPs + pods := make(map[string]*corev1.Pod) + podIPs := make(map[string][]string) + + // Cleanup + DeferCleanup(func() { + By("Cleanup: Deleting all test resources") + deleteCNC(cnc1Name) + deleteCNC(cnc2Name) + + for _, pod := range pods { + _ = cs.CoreV1().Pods(pod.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) + } + + deleteUDN(blueNs, blueUDN) + deleteUDN(greenNs, greenUDN) + deleteCUDN(redCUDN) + + deleteNamespace(cs, blueNs) + deleteNamespace(cs, redNs) + deleteNamespace(cs, greenNs) + }) + + // ===================================================================== + // Step 1: Create CNC-1 selecting blue and red (no networks exist yet) + // Each CNC must have different connect subnets + // ===================================================================== + By("1. Creating CNC-1 (color-1) selecting blue UDN and red CUDN with first connect subnet") + createOrUpdateCNCWithSubnets(cnc1Name, redLabel, blueLabel, generateConnectSubnets(cs)) + + By("1. Verifying CNC-1 has only tunnel ID annotation (no networks yet)") + verifyCNCHasOnlyTunnelIDAnnotation(cnc1Name) + + // ===================================================================== + // Step 2: Create CNC-2 selecting blue and green (no networks exist yet) + // ===================================================================== + By("2. Creating CNC-2 (color-2) selecting blue UDN and green UDN with second connect subnet") + createOrUpdateCNCWithSubnets(cnc2Name, nil, cnc2Label, generateConnectSubnetsWithCIDRs(cs, cnc2ConnectSubnetIPv4CIDR, cnc2ConnectSubnetIPv4Prefix, cnc2ConnectSubnetIPv6CIDR, cnc2ConnectSubnetIPv6Prefix)) + + By("2. Verifying CNC-2 has only tunnel ID annotation (no networks yet)") + verifyCNCHasOnlyTunnelIDAnnotation(cnc2Name) + + // ===================================================================== + // Step 3: Create blue UDN (L2) with pods - gets added to both CNCs + // ===================================================================== + By("3. Creating blue namespace and L2 UDN") + createUDNNamespaceWithName(cs, blueNs, blueLabel) + createLayer2PrimaryUDNWithSubnets(cs, blueNs, blueUDN, "10.128.0.0/16", "2014:100:200::0/60") + + By("3. Waiting for blue UDN to be ready") + Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, blueNs, blueUDN), 60*time.Second, time.Second).Should(Succeed()) + + By("3. Verifying CNC-1 now has 1 network (blue)") + verifyCNCSubnetAnnotationNetworkCount(cnc1Name, 1) + + By("3. Verifying CNC-2 now has 1 network (blue)") + verifyCNCSubnetAnnotationNetworkCount(cnc2Name, 1) + + By("3. Creating pods in blue UDN namespace") + bluePodConfig0 := httpServerPodConfig("blue-pod-0", blueNs) + bluePodConfig0.nodeSelector = map[string]string{nodeHostnameKey: node1Name} + bluePodConfig1 := httpServerPodConfig("blue-pod-1", blueNs) + bluePodConfig1.nodeSelector = map[string]string{nodeHostnameKey: node2Name} + pods["blue-pod-0"] = runUDNPod(cs, blueNs, bluePodConfig0, nil) + pods["blue-pod-1"] = runUDNPod(cs, blueNs, bluePodConfig1, nil) + podIPs["blue-pod-0"] = getPrimaryNetworkPodIPs(blueNs, "blue-pod-0", blueUDN) + podIPs["blue-pod-1"] = getPrimaryNetworkPodIPs(blueNs, "blue-pod-1", blueUDN) + + // ===================================================================== + // Step 4: Create red CUDN (L3) with pods - gets added to CNC-1 + // ===================================================================== + By("4. Creating red namespace and L3 CUDN") + createUDNNamespaceWithName(cs, redNs, nil) + createLayer3PrimaryCUDNWithSubnets(cs, redCUDN, redLabel, "10.129.0.0/16", "2014:100:300::0/60", redNs) + + By("4. Waiting for red CUDN to be ready") + Eventually(clusterUserDefinedNetworkReadyFunc(f.DynamicClient, redCUDN), 60*time.Second, time.Second).Should(Succeed()) + + By("4. Verifying CNC-1 now has 2 networks (blue + red)") + verifyCNCSubnetAnnotationNetworkCount(cnc1Name, 2) + + By("4. Verifying CNC-2 still has 1 network (blue only)") + verifyCNCSubnetAnnotationNetworkCount(cnc2Name, 1) + + By("4. Creating pods in red CUDN namespace") + redPodConfig0 := httpServerPodConfig("red-pod-0", redNs) + redPodConfig0.nodeSelector = map[string]string{nodeHostnameKey: node1Name} + redPodConfig1 := httpServerPodConfig("red-pod-1", redNs) + redPodConfig1.nodeSelector = map[string]string{nodeHostnameKey: node2Name} + pods["red-pod-0"] = runUDNPod(cs, redNs, redPodConfig0, nil) + pods["red-pod-1"] = runUDNPod(cs, redNs, redPodConfig1, nil) + podIPs["red-pod-0"] = getPrimaryNetworkPodIPs(redNs, "red-pod-0", redCUDN) + podIPs["red-pod-1"] = getPrimaryNetworkPodIPs(redNs, "red-pod-1", redCUDN) + + // ===================================================================== + // Step 5: Create green UDN (L3) with pods - gets added to CNC-2 + // ===================================================================== + By("5. Creating green namespace and L3 UDN") + createUDNNamespaceWithName(cs, greenNs, greenLabel) + createLayer3PrimaryUDNWithSubnets(cs, greenNs, greenUDN, "10.130.0.0/16", "2014:100:400::0/60") + + By("5. Waiting for green UDN to be ready") + Eventually(userDefinedNetworkReadyFunc(f.DynamicClient, greenNs, greenUDN), 60*time.Second, time.Second).Should(Succeed()) + + By("5. Verifying CNC-1 still has 2 networks (blue + red)") + verifyCNCSubnetAnnotationNetworkCount(cnc1Name, 2) + + By("5. Verifying CNC-2 now has 2 networks (blue + green)") + verifyCNCSubnetAnnotationNetworkCount(cnc2Name, 2) + + By("5. Creating pods in green UDN namespace") + greenPodConfig0 := httpServerPodConfig("green-pod-0", greenNs) + greenPodConfig0.nodeSelector = map[string]string{nodeHostnameKey: node1Name} + greenPodConfig1 := httpServerPodConfig("green-pod-1", greenNs) + greenPodConfig1.nodeSelector = map[string]string{nodeHostnameKey: node2Name} + pods["green-pod-0"] = runUDNPod(cs, greenNs, greenPodConfig0, nil) + pods["green-pod-1"] = runUDNPod(cs, greenNs, greenPodConfig1, nil) + podIPs["green-pod-0"] = getPrimaryNetworkPodIPs(greenNs, "green-pod-0", greenUDN) + podIPs["green-pod-1"] = getPrimaryNetworkPodIPs(greenNs, "green-pod-1", greenUDN) + + // Define pod groups for connectivity testing + bluePods := map[string]*corev1.Pod{"blue-pod-0": pods["blue-pod-0"], "blue-pod-1": pods["blue-pod-1"]} + redPods := map[string]*corev1.Pod{"red-pod-0": pods["red-pod-0"], "red-pod-1": pods["red-pod-1"]} + greenPods := map[string]*corev1.Pod{"green-pod-0": pods["green-pod-0"], "green-pod-1": pods["green-pod-1"]} + + bluePodIPs := map[string][]string{"blue-pod-0": podIPs["blue-pod-0"], "blue-pod-1": podIPs["blue-pod-1"]} + redPodIPs := map[string][]string{"red-pod-0": podIPs["red-pod-0"], "red-pod-1": podIPs["red-pod-1"]} + greenPodIPs := map[string][]string{"green-pod-0": podIPs["green-pod-0"], "green-pod-1": podIPs["green-pod-1"]} + + // ===================================================================== + // Step 6: Verify blue <-> red connectivity via CNC-1 + // ===================================================================== + By("6. Verifying blue <-> red connectivity via CNC-1") + verifyCrossNetworkConnectivity(bluePods, redPodIPs, true) + verifyCrossNetworkConnectivity(redPods, bluePodIPs, true) + + // ===================================================================== + // Step 7: Verify blue <-> green connectivity via CNC-2 + // ===================================================================== + By("7. Verifying blue <-> green connectivity via CNC-2") + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, true) + verifyCrossNetworkConnectivity(greenPods, bluePodIPs, true) + + // ===================================================================== + // Step 8: Verify red <-/-> green (non-transitive) + // ===================================================================== + By("8. Verifying red <-/-> green (non-transitive - no direct connectivity)") + verifyCrossNetworkConnectivity(redPods, greenPodIPs, false) + verifyCrossNetworkConnectivity(greenPods, redPodIPs, false) + + By("8. Verifying blue still connected to both red and green") + verifyCrossNetworkConnectivity(bluePods, redPodIPs, true) + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, true) + + // ===================================================================== + // Step 9: Delete CNC-1, verify blue <-> green still works + // ===================================================================== + By("9. Deleting CNC-1") + deleteCNC(cnc1Name) + + By("9. Waiting for CNC-1 deletion to complete") + Eventually(func() bool { + _, err := getCNCAnnotations(cnc1Name) + return err != nil + }, 60*time.Second, 2*time.Second).Should(BeTrue()) + + By("9. Verifying blue <-> green still connected via CNC-2") + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, true) + verifyCrossNetworkConnectivity(greenPods, bluePodIPs, true) + + By("9. Verifying blue <-/-> red (CNC-1 deleted)") + verifyCrossNetworkConnectivity(bluePods, redPodIPs, false) + verifyCrossNetworkConnectivity(redPods, bluePodIPs, false) + + // ===================================================================== + // Step 10: Delete CNC-2, verify all networks isolated + // ===================================================================== + By("10. Deleting CNC-2") + deleteCNC(cnc2Name) + + By("10. Waiting for CNC-2 deletion to complete") + Eventually(func() bool { + _, err := getCNCAnnotations(cnc2Name) + return err != nil + }, 60*time.Second, 2*time.Second).Should(BeTrue()) + + By("10. Verifying all networks are now isolated") + verifyCrossNetworkConnectivity(bluePods, redPodIPs, false) + verifyCrossNetworkConnectivity(bluePods, greenPodIPs, false) + verifyCrossNetworkConnectivity(redPods, greenPodIPs, false) + + By("Test completed successfully - Multiple CNCs with non-transitive connectivity validated") + }) + }) +}) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 87323cdb2b..be0313b305 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -1066,9 +1066,11 @@ var _ = ginkgo.Describe("test e2e pod connectivity to host addresses", func() { framework.Failf("Test requires >= 1 Ready nodes, but there are only %v nodes", len(nodes.Items)) } workerNodeName = nodes.Items[0].Name - // Add another IP address to the worker + // Add another IP address to the worker with preferred_lft 0 to mark it as deprecated. + // This prevents the IP from being selected as the node's primary gateway IP while still + // allowing the test to verify pod-to-host connectivity to non-node IPs. _, err = infraprovider.Get().ExecK8NodeCommand(workerNodeName, []string{"ip", "a", "add", - fmt.Sprintf("%s/%s", targetIP, singleIPMask), "dev", deploymentconfig.Get().ExternalBridgeName()}) + fmt.Sprintf("%s/%s", targetIP, singleIPMask), "dev", deploymentconfig.Get().ExternalBridgeName(), "preferred_lft", "0"}) framework.ExpectNoError(err, "failed to add IP to %s", workerNodeName) }) @@ -1946,6 +1948,20 @@ var _ = ginkgo.Describe("e2e br-int flow monitoring export validation", func() { return fmt.Sprintf(collectorContainerTemplate, port) } + getCollectorArgs := func(protocol flowMonitoringProtocol, port uint16) []string { + args := []string{"-kafka=false"} + switch protocol { + case sflow: + // Disable other collectors to avoid non-deterministic startup ordering in logs. + args = append(args, "-nf=false", "-nfl=false", "-sflow=true", fmt.Sprintf("-sflow.port=%d", port)) + case netflow_v5: + args = append(args, "-nf=false", "-sflow=false", "-nfl=true", fmt.Sprintf("-nfl.port=%d", port)) + case ipfix: + args = append(args, "-nfl=false", "-sflow=false", "-nf=true", fmt.Sprintf("-nf.port=%d", port)) + } + return args + } + keywordInLogs := map[flowMonitoringProtocol]string{ netflow_v5: "NETFLOW_V5", ipfix: "IPFIX", sflow: "SFLOW_5"} @@ -1966,7 +1982,7 @@ var _ = ginkgo.Describe("e2e br-int flow monitoring export validation", func() { primaryProviderNetwork, err := infraprovider.Get().PrimaryNetwork() framework.ExpectNoError(err, "failed to get primary network") collectorExternalContainer := infraapi.ExternalContainer{Name: getContainerName(collectorPort), Image: "cloudflare/goflow", - Network: primaryProviderNetwork, CmdArgs: []string{"-kafka=false"}, ExtPort: collectorPort} + Network: primaryProviderNetwork, CmdArgs: getCollectorArgs(protocol, collectorPort), ExtPort: collectorPort} collectorExternalContainer, err = providerCtx.CreateExternalContainer(collectorExternalContainer) if err != nil { framework.Failf("failed to start flow collector container %s: %v", getContainerName(collectorPort), err) @@ -1984,6 +2000,58 @@ var _ = ginkgo.Describe("e2e br-int flow monitoring export validation", func() { setEnv := map[string]string{ovnEnvVar: addressAndPort} setUnsetTemplateContainerEnv(f.ClientSet, ovnKubeNamespace, "daemonset/ovnkube-node", getNodeContainerName(), setEnv) + ovnKubeNodePods, err := f.ClientSet.CoreV1().Pods(ovnKubeNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app=ovnkube-node", + }) + if err != nil { + framework.Failf("could not get ovnkube-node pods: %v", err) + } + + if protocol == sflow { + ginkgo.By("Waiting for ovnkube-node to configure br-int sflow and setting sampling/polling for better signal") + for _, ovnKubeNodePod := range ovnKubeNodePods.Items { + var sFlowUUID string + err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { + getSFlowExecOptions := e2epod.ExecOptions{ + Command: []string{"ovs-vsctl", "--if-exists", "get", "bridge", "br-int", "sflow"}, + Namespace: ovnKubeNamespace, + PodName: ovnKubeNodePod.Name, + ContainerName: getNodeContainerName(), + CaptureStdout: true, + CaptureStderr: true, + } + rawUUID, stderr, execErr := e2epod.ExecWithOptions(f, getSFlowExecOptions) + if execErr != nil { + framework.Logf("waiting for sflow row on %s: query failed: %v, stderr: %s", + ovnKubeNodePod.Name, execErr, stderr) + return false, nil + } + rawUUID = strings.TrimSpace(strings.Trim(rawUUID, "\"")) + if rawUUID == "" || rawUUID == "[]" { + framework.Logf("waiting for sflow row on %s: br-int has no sflow row yet", ovnKubeNodePod.Name) + return false, nil + } + sFlowUUID = rawUUID + return true, nil + }) + framework.ExpectNoError(err, "timed out waiting for br-int sflow row on %s", ovnKubeNodePod.Name) + + setSFlowExecOptions := e2epod.ExecOptions{ + Command: []string{"ovs-vsctl", "--if-exists", "set", "sflow", sFlowUUID, "sampling=1", "polling=1"}, + Namespace: ovnKubeNamespace, + PodName: ovnKubeNodePod.Name, + ContainerName: getNodeContainerName(), + CaptureStdout: true, + CaptureStderr: true, + } + _, setStderr, setErr := e2epod.ExecWithOptions(f, setSFlowExecOptions) + if setErr != nil { + framework.Logf("skipping sflow sampling tuning on %s: failed to set sampling/polling for row %s: %v, stderr: %s", + ovnKubeNodePod.Name, sFlowUUID, setErr, setStderr) + } + } + } + ginkgo.By(fmt.Sprintf("Checking that the collector container received %s data", protocolStr)) keyword := keywordInLogs[protocol] collectorContainerLogsTest := func() wait.ConditionFunc { @@ -1995,14 +2063,14 @@ var _ = ginkgo.Describe("e2e br-int flow monitoring export validation", func() { } collectorContainerLogs = strings.TrimSuffix(collectorContainerLogs, "\n") logLines := strings.Split(collectorContainerLogs, "\n") - lastLine := logLines[len(logLines)-1] // check that flow monitoring traffic has been logged - if strings.Contains(lastLine, keyword) { - framework.Logf("Successfully found string %s in last log line of"+ - " the collector: %s", keyword, lastLine) - return true, nil + for _, line := range logLines { + if strings.Contains(line, keyword) { + framework.Logf("Successfully found string %s in collector logs line: %s", keyword, line) + return true, nil + } } - framework.Logf("%s not found in last log line: %s", keyword, lastLine) + framework.Logf("%s not found in collector logs", keyword) return false, nil } } @@ -2014,7 +2082,7 @@ var _ = ginkgo.Describe("e2e br-int flow monitoring export validation", func() { ginkgo.By(fmt.Sprintf("Unsetting %s variable in ovnkube-node daemonset", ovnEnvVar)) setUnsetTemplateContainerEnv(f.ClientSet, ovnKubeNamespace, "daemonset/ovnkube-node", getNodeContainerName(), nil, ovnEnvVar) - ovnKubeNodePods, err := f.ClientSet.CoreV1().Pods(ovnKubeNamespace).List(context.TODO(), metav1.ListOptions{ + ovnKubeNodePods, err = f.ClientSet.CoreV1().Pods(ovnKubeNamespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: "app=ovnkube-node", }) if err != nil { @@ -2032,9 +2100,9 @@ var _ = ginkgo.Describe("e2e br-int flow monitoring export validation", func() { CaptureStderr: true, } - targets, stderr, _ := e2epod.ExecWithOptions(f, execOptions) + targets, stderr, execErr := e2epod.ExecWithOptions(f, execOptions) framework.Logf("execOptions are %v", execOptions) - if err != nil { + if execErr != nil { framework.Failf("could not lookup ovs %s targets: %v", protocolStr, stderr) } gomega.Expect(targets).To(gomega.BeEmpty()) diff --git a/test/e2e/egressip.go b/test/e2e/egressip.go index 403a9384af..6e7d75f147 100644 --- a/test/e2e/egressip.go +++ b/test/e2e/egressip.go @@ -36,6 +36,7 @@ import ( e2enode "k8s.io/kubernetes/test/e2e/framework/node" "k8s.io/kubernetes/test/e2e/framework/pod" e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" utilnet "k8s.io/utils/net" ) @@ -225,6 +226,40 @@ func isSupportedAgnhostForEIP(externalContainer infraapi.ExternalContainer) bool return true } +// Create EgressIP Manifest +func createEIPManifest(name string, podLabel, namespaceLabel map[string]string, egressIPs ...string) string { + var ipsYAML string + for _, ip := range egressIPs { + ipsYAML += fmt.Sprintf("\n - %s", ip) + } + + var podLabelYaml string + for k, v := range podLabel { + podLabelYaml = fmt.Sprintf("%s: %s", k, v) + } + + var namespaceLabelYaml string + for k, v := range namespaceLabel { + namespaceLabelYaml = fmt.Sprintf("%s: %s", k, v) + } + egressIPConfig := fmt.Sprintf(`apiVersion: k8s.ovn.org/v1 +kind: EgressIP +metadata: + name: %s +spec: + egressIPs:%s + podSelector: + matchLabels: + %s + namespaceSelector: + matchLabels: + %s +`, name, ipsYAML, podLabelYaml, namespaceLabelYaml) + + return egressIPConfig + +} + // targetHostNetworkContainerAndTest targets the internal host network test container from // our test pods, collects its logs and verifies that the logs have traces // of the `verifyIPs` provided. We need to target the test @@ -3181,6 +3216,357 @@ spec: err = wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(primaryTargetExternalContainer, pod2OtherNetworkNamespace, pod2Name, true, []string{egressIP1.String()})) framework.ExpectNoError(err, "Step 11. Check connectivity from other pod and verify that the srcIP is the expected egressIP and verify that the srcIP is the expected nodeIP, failed: %v", err) }) + /* + This test does the following: + 0. Add the "k8s.ovn.org/egress-assignable" label to one node + 1. Create an EgressIP object1 with one egress IP1 defined + 2. Create an EgressIP object2 with one egress IP2 defined + 3. Check that status of both EgressIP objects is of length one + 4. Create one pod matching the EgressIP object1 + 5. Update namespace labels match EgressIP object1, + 6. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object1 + 7. Verify source IP is NOT the node IP + 8. Update namespace labels match EgressIP object2 + 9. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object2 + 10. Verify source IP is NOT the node IP + 11. Update both EgressIP objects that namespace having same labels, and different pod selectors labels + 12. Check that status of both EgressIP objects is of length one + 13. Check connectivity from that one to an external \"node\" and verify that the IP is the node IP. + 14. Update pod labels match EgressIP object1 + 15. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object1 + 16. Verify source IP is NOT the node IP + 17. Update pod labels match EgressIP object2 + 15. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object2 + 16. Verify source IP is NOT the node IP + 17. Update EgressIP object1 to match the current pod label, EgressIP object2 not match pod label + 18. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object1 + 19. Verify source IP is NOT the node IP + 20. Update EgressIP object2 to match the current pod label, EgressIP object1 not match pod label + 21. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object2 + 22. Verify source IP is NOT the node IP + */ + ginkgo.It("Should handle EIP reassignment correctly on namespace and pod label updates, and EIP object updates", func() { + if isUserDefinedNetwork(netConfigParams) { + ginkgo.Skip("Unsupported for UDNs") + } + + ginkgo.By("0. Add the \"k8s.ovn.org/egress-assignable\" label to one node") + e2enode.AddOrUpdateLabelOnNode(f.ClientSet, egress1Node.name, "k8s.ovn.org/egress-assignable", "dummy") + + ginkgo.By("1. Create an EgressIP object with one egress IP1 defined") + var egressIP1 net.IP + var err error + var retryTimeout2 = 2 * retryInterval + if utilnet.IsIPv6String(egress1Node.nodeIP) { + egressIP1, err = ipalloc.NewPrimaryIPv6() + } else { + egressIP1, err = ipalloc.NewPrimaryIPv4() + } + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "must allocate new EgressIP") + podNamespace := f.Namespace + egressLabels := map[string]string{ + "wants": "egress", + } + egressIPConfig := createEIPManifest(egressIPName, egressLabels, egressLabels, egressIP1.String()) + if err := os.WriteFile(egressIPYaml, []byte(egressIPConfig), 0644); err != nil { + framework.Failf("Unable to write CRD config to disk: %v", err) + } + defer func() { + if err := os.Remove(egressIPYaml); err != nil { + framework.Logf("Unable to remove the CRD config from disk: %v", err) + } + }() + framework.Logf("Create the EgressIP configuration") + e2ekubectl.RunKubectlOrDie("default", "create", "-f", egressIPYaml) + + ginkgo.By("2. Create second EgressIP object with one egress IP2 defined") + var egressIP2 net.IP + if utilnet.IsIPv6String(egress1Node.nodeIP) { + egressIP2, err = ipalloc.NewPrimaryIPv6() + } else { + egressIP2, err = ipalloc.NewPrimaryIPv4() + } + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "must allocate new EgressIP") + egressLabels2 := map[string]string{ + "wants": "egress2", + } + egressIPConfig2 := createEIPManifest(egressIPName2, egressLabels, egressLabels2, egressIP2.String()) + if err := os.WriteFile(egressIPYaml, []byte(egressIPConfig2), 0644); err != nil { + framework.Failf("Unable to write CRD config to disk: %v", err) + } + defer func() { + if err := os.Remove(egressIPYaml); err != nil { + framework.Logf("Unable to remove the CRD config from disk: %v", err) + } + }() + framework.Logf("Create the EgressIP configuration") + e2ekubectl.RunKubectlOrDie("default", "create", "-f", egressIPYaml) + + ginkgo.By("3. Check that status of both EgressIP objects is of length one") + verifySpecificEgressIPStatusLengthEquals(egressIPName, 1, nil) + verifySpecificEgressIPStatusLengthEquals(egressIPName2, 1, nil) + + ginkgo.By("4. Create one pod matching the EgressIP") + _, err = createGenericPodWithLabel(f, pod1Name, pod1Node.name, f.Namespace.Name, getAgnHostHTTPPortBindFullCMD(clusterNetworkHTTPPort), egressLabels) + framework.ExpectNoError(err, "failed to create pod %s/%s", f.Namespace.Name, pod1Name) + framework.Logf("Created pod %s on node %s", pod1Name, pod1Node.name) + + // Run namespace label updates multiple times to ensure EIP reassignment works well + for i := 1; i <= 5; i++ { + ginkgo.By(fmt.Sprintf("5.%d. Update namespace labels match egressIP %s selectors (iteration %d)", i, egressIPName, i)) + podNamespace = getNamespace(f, podNamespace.Name) + updateNamespaceLabels(f, podNamespace, egressLabels) + ginkgo.By(fmt.Sprintf("6.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d)", i, egressIPName, i)) + err := wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{egressIP1.String()}).WithContext()) + framework.ExpectNoError(err, "6.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d): %v", i, egressIPName, i, err) + ginkgo.By(fmt.Sprintf("7.%d. Verify source IP is NOT the node IP (iteration %d)", i, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout2, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{pod1Node.nodeIP}).WithContext()) + gomega.Expect(err).To(gomega.HaveOccurred(), "Node IP should NOT be used as source IP - connection should succeed but node IP should not be found") + + ginkgo.By(fmt.Sprintf("8.%d. Update namespace labels to match egressIP %s selectors (iteration %d)", i, egressIPName2, i)) + podNamespace = getNamespace(f, podNamespace.Name) + updateNamespaceLabels(f, podNamespace, egressLabels2) + ginkgo.By(fmt.Sprintf("9.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d)", i, egressIPName2, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{egressIP2.String()}).WithContext()) + framework.ExpectNoError(err, "9.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d): %v", i, egressIPName2, i, err) + ginkgo.By(fmt.Sprintf("10.%d. Verify source IP is NOT the node IP (iteration %d)", i, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout2, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{pod1Node.nodeIP}).WithContext()) + gomega.Expect(err).To(gomega.HaveOccurred(), "Node IP should NOT be used as source IP - connection should succeed but node IP should not be found") + } + + ginkgo.By("11. Update both egressIP objects such that they have same namespace selector but different pod selector") + egressLabelsJSON, err := json.Marshal(egressLabels) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressLabels2JSON, err := json.Marshal(egressLabels2) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + specString := fmt.Sprintf("{\"spec\":{\"podSelector\":{\"matchLabels\":%s},\"namespaceSelector\": {\"matchLabels\":%s}}}", + string(egressLabelsJSON), string(egressLabelsJSON)) + e2ekubectl.RunKubectlOrDie("default", "patch", "EgressIP/"+egressIPName, "-p", specString, "--type=merge") + specString = fmt.Sprintf("{\"spec\":{\"podSelector\":{\"matchLabels\":%s},\"namespaceSelector\": {\"matchLabels\":%s}}}", + string(egressLabels2JSON), string(egressLabelsJSON)) + e2ekubectl.RunKubectlOrDie("default", "patch", "EgressIP/"+egressIPName2, "-p", specString, "--type=merge") + + ginkgo.By("12. Check that status of both EgressIP objects is of length one") + verifySpecificEgressIPStatusLengthEquals(egressIPName, 1, nil) + verifySpecificEgressIPStatusLengthEquals(egressIPName2, 1, nil) + ginkgo.By("13. Check connectivity from that one to an external \"node\" and verify that the IP is the node IP.") + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{pod1Node.nodeIP}).WithContext()) + framework.ExpectNoError(err, "Step 13. Check connectivity from that one to an external \"node\" and verify that the IP is the node IP, failed, err: %v", err) + ginkgo.By("Update namespace label to match the change in step 11") + podNamespace = getNamespace(f, podNamespace.Name) + updateNamespaceLabels(f, podNamespace, egressLabels) + ginkgo.By(fmt.Sprintf("14. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s ", egressIPName)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{egressIP1.String()}).WithContext()) + framework.ExpectNoError(err, "14. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s ", egressIPName) + + // Run pod label updates multiple times to ensure EIP reassignment works well + for i := 1; i <= 5; i++ { + ginkgo.By(fmt.Sprintf("15.%d. Update pod labels match egressIP %s selectors (iteration %d)", i, egressIPName, i)) + pod1 := getPod(f, pod1Name) + pod1.Labels = egressLabels + updatePod(f, pod1) + ginkgo.By(fmt.Sprintf("16.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d)", i, egressIPName, i)) + err := wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{egressIP1.String()}).WithContext()) + framework.ExpectNoError(err, "16.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d): %v", i, egressIPName, i, err) + ginkgo.By(fmt.Sprintf("17.%d. Verify source IP is NOT the node IP (iteration %d)", i, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout2, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{pod1Node.nodeIP}).WithContext()) + gomega.Expect(err).To(gomega.HaveOccurred(), "Node IP should NOT be used as source IP - connection should succeed but node IP should not be found") + + ginkgo.By(fmt.Sprintf("18.%d. Update pod labels to match egressIP object2 %s selectors (iteration %d)", i, egressIPName2, i)) + pod1 = getPod(f, pod1Name) + pod1.Labels = egressLabels2 + updatePod(f, pod1) + ginkgo.By(fmt.Sprintf("19.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object2 %s (iteration %d)", i, egressIPName2, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{egressIP2.String()}).WithContext()) + framework.ExpectNoError(err, "19.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object2 %s (iteration %d): %v", i, egressIPName2, i, err) + ginkgo.By(fmt.Sprintf("20.%d. Verify source IP is NOT the node IP (iteration %d)", i, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout2, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{pod1Node.nodeIP}).WithContext()) + gomega.Expect(err).To(gomega.HaveOccurred(), "Node IP should NOT be used as source IP - connection should succeed but node IP should not be found") + } + + // Run EIP object updates multiple times to ensure EIP reassignment works well + for i := 1; i <= 5; i++ { + ginkgo.By(fmt.Sprintf("21.%d. Update EgressIP %s selectors to match pod labels and EgressIP %s not matching pod labels,(iteration %d)", i, egressIPName, egressIPName2, i)) + specString = fmt.Sprintf("{\"spec\":{\"podSelector\":{\"matchLabels\":%s},\"namespaceSelector\": {\"matchLabels\":%s}}}", + string(egressLabels2JSON), string(egressLabelsJSON)) + e2ekubectl.RunKubectlOrDie("default", "patch", "EgressIP/"+egressIPName, "-p", specString, "--type=merge") + specString = fmt.Sprintf("{\"spec\":{\"podSelector\":{\"matchLabels\":%s},\"namespaceSelector\": {\"matchLabels\":%s}}}", + string(egressLabelsJSON), string(egressLabelsJSON)) + e2ekubectl.RunKubectlOrDie("default", "patch", "EgressIP/"+egressIPName2, "-p", specString, "--type=merge") + ginkgo.By(fmt.Sprintf("22.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from object1 %s (iteration %d)", i, egressIPName, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{egressIP1.String()}).WithContext()) + framework.ExpectNoError(err, "22.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d): %v", i, egressIPName, i, err) + ginkgo.By(fmt.Sprintf("23.%d. Verify source IP is NOT the node IP (iteration %d)", i, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout2, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{pod1Node.nodeIP}).WithContext()) + gomega.Expect(err).To(gomega.HaveOccurred(), "Node IP should NOT be used as source IP - connection should succeed but node IP should not be found") + + ginkgo.By(fmt.Sprintf("24.%d. Update EgressIP %s selectors to match pod labels and EgressIP %s not matching pod labels,(iteration %d)", i, egressIPName2, egressIPName, i)) + specString = fmt.Sprintf("{\"spec\":{\"podSelector\":{\"matchLabels\":%s},\"namespaceSelector\": {\"matchLabels\":%s}}}", + string(egressLabelsJSON), string(egressLabelsJSON)) + e2ekubectl.RunKubectlOrDie("default", "patch", "EgressIP/"+egressIPName, "-p", specString, "--type=merge") + specString = fmt.Sprintf("{\"spec\":{\"podSelector\":{\"matchLabels\":%s},\"namespaceSelector\": {\"matchLabels\":%s}}}", + string(egressLabels2JSON), string(egressLabelsJSON)) + e2ekubectl.RunKubectlOrDie("default", "patch", "EgressIP/"+egressIPName2, "-p", specString, "--type=merge") + ginkgo.By(fmt.Sprintf("25.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d)", i, egressIPName2, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{egressIP2.String()}).WithContext()) + framework.ExpectNoError(err, "25.%d. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP from %s (iteration %d): %v", i, egressIPName2, i, err) + ginkgo.By(fmt.Sprintf("26.%d. Verify source IP is NOT the node IP (iteration %d)", i, i)) + err = wait.PollUntilContextTimeout(context.TODO(), retryInterval, retryTimeout2, + true, targetExternalContainerAndTest(primaryTargetExternalContainer, + podNamespace.Name, pod1Name, true, []string{pod1Node.nodeIP}).WithContext()) + gomega.Expect(err).To(gomega.HaveOccurred(), "Node IP should NOT be used as source IP - connection should succeed but node IP should not be found") + } + }) + + ginkgo.It("Should fail if egressip-mark annotation is present during EgressIP creation", func() { + // This check can be removed when https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5879 is addressed + if isHelmEnabled() { + e2eskipper.Skipf("Skipping this test for HELM environment as we dont create required Validatingadmissionpolicy in a HELM environment") + } + + ginkgo.By("1. Create an EgressIP object with one egress IP defined") + var egressIP1 net.IP + var err error + if utilnet.IsIPv6String(egress1Node.nodeIP) { + egressIP1, err = ipalloc.NewPrimaryIPv6() + } else { + egressIP1, err = ipalloc.NewPrimaryIPv4() + } + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "must allocate new Node IP") + + var egressIPConfig = `apiVersion: k8s.ovn.org/v1 +kind: EgressIP +metadata: + name: ` + egressIPName + ` + annotations: + ` + util.EgressIPMarkAnnotation + `: "50000" +spec: + egressIPs: + - ` + egressIP1.String() + ` + namespaceSelector: + matchLabels: + name: ` + f.Namespace.Name + ` +` + if err := os.WriteFile(egressIPYaml, []byte(egressIPConfig), 0644); err != nil { + framework.Failf("Unable to write CRD config to disk: %v", err) + } + defer func() { + if err := os.Remove(egressIPYaml); err != nil { + framework.Logf("Unable to remove the CRD config from disk: %v", err) + } + }() + + ginkgo.By("2. Create an EgressIP with k8s.ovn.org/egressip-mark annotation defined") + _, err = e2ekubectl.RunKubectl("default", "create", "-f", egressIPYaml) + gomega.Expect(err).To(gomega.HaveOccurred(), "Should fail if k8s.ovn.org/egressip-mark annotation is present during creation") + gomega.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("EgressIP resources cannot be created with the \"k8s.ovn.org/egressip-mark\" annotation. This annotation is managed by the system."))) + }) + + ginkgo.It("Should fail if egressip-mark annotation is being added by a regular user", func() { + // This check can be removed when https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5879 is addressed + if isHelmEnabled() { + e2eskipper.Skipf("Skipping this test for HELM environment as we dont create required Validatingadmissionpolicy in a HELM environment") + } + + ginkgo.By("1. Add the \"k8s.ovn.org/egress-assignable\" label to egress1Node node") + egressNodeAvailabilityHandler := egressNodeAvailabilityHandlerViaLabel{f} + egressNodeAvailabilityHandler.Enable(egress1Node.name) + defer egressNodeAvailabilityHandler.Restore(egress1Node.name) + + podNamespace := f.Namespace + labels := map[string]string{ + "name": f.Namespace.Name, + } + updateNamespaceLabels(f, podNamespace, labels) + + ginkgo.By("2. Create an EgressIP object with one egress IP defined") + var egressIP1 net.IP + var err error + if utilnet.IsIPv6String(egress1Node.nodeIP) { + egressIP1, err = ipalloc.NewPrimaryIPv6() + } else { + egressIP1, err = ipalloc.NewPrimaryIPv4() + } + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "must allocate new Node IP") + + var egressIPConfig = `apiVersion: k8s.ovn.org/v1 +kind: EgressIP +metadata: + name: ` + egressIPName + ` +spec: + egressIPs: + - ` + egressIP1.String() + ` + namespaceSelector: + matchLabels: + name: ` + f.Namespace.Name + ` +` + if err := os.WriteFile(egressIPYaml, []byte(egressIPConfig), 0644); err != nil { + framework.Failf("Unable to write CRD config to disk: %v", err) + } + defer func() { + if err := os.Remove(egressIPYaml); err != nil { + framework.Logf("Unable to remove the CRD config from disk: %v", err) + } + }() + + framework.Logf("Create the EgressIP configuration") + e2ekubectl.RunKubectlOrDie("default", "create", "-f", egressIPYaml) + + ginkgo.By("3. Check that the status is of length one and that it is assigned to egress1Node") + statuses := verifyEgressIPStatusLengthEquals(1, nil) + if statuses[0].Node != egress1Node.name { + framework.Failf("Step 3. Check that the status is of length one and that it is assigned to egress1Node, failed") + } + + ginkgo.By("4. Try updating k8s.ovn.org/egressip-mark annotation") + // Get the current annotation value to ensure we try to overwrite with a different value + annotationsJSON, err := e2ekubectl.RunKubectl("", "get", "egressip", egressIPName, "-o", "jsonpath={.metadata.annotations}") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get annotations") + var annotations map[string]string + err = json.Unmarshal([]byte(annotationsJSON), &annotations) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to unmarshal annotations JSON") + currentValue := annotations[util.EgressIPMarkAnnotation] + + newValue := 50000 + if currentValue == "50000" { + newValue = 50001 + } + + _, err = e2ekubectl.RunKubectl("", "annotate", "--overwrite", "egressip", egressIPName, fmt.Sprintf("%s=%d", util.EgressIPMarkAnnotation, newValue)) + gomega.Expect(err).To(gomega.HaveOccurred(), "Should fail if k8s.ovn.org/egressip-mark is being updated") + gomega.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("The \"k8s.ovn.org/egressip-mark\" annotation cannot be modified or removed once set. This annotation is managed by the system."))) + + ginkgo.By("5. Try removing k8s.ovn.org/egressip-mark annotation") + _, err = e2ekubectl.RunKubectl("", "annotate", "--overwrite", "egressip", egressIPName, fmt.Sprintf("%s-", util.EgressIPMarkAnnotation)) + gomega.Expect(err).To(gomega.HaveOccurred(), "Should fail if k8s.ovn.org/egressip-mark is being removed") + gomega.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("The \"k8s.ovn.org/egressip-mark\" annotation cannot be modified or removed once set. This annotation is managed by the system."))) + }) ginkgo.DescribeTable("[OVN network] multiple namespaces with different primary networks", func(otherNetworkAttachParms networkAttachmentConfigParams) { if !isNetworkSegmentationEnabled() { diff --git a/test/e2e/feature/features.go b/test/e2e/feature/features.go index 0a995c1b37..c322d594ae 100644 --- a/test/e2e/feature/features.go +++ b/test/e2e/feature/features.go @@ -15,6 +15,7 @@ var ( EgressService = New("EgressService") EgressFirewall = New("EgressFirewall") EgressQos = New("EgressQos") + EVPN = New("EVPN") ExternalGateway = New("ExternalGateway") DisablePacketMTUCheck = New("DisablePacketMTUCheck") VirtualMachineSupport = New("VirtualMachineSupport") diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 971f2fbe3a..506173a5d4 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -51,9 +51,9 @@ require ( github.com/gaissmai/cidrtree v0.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonpointer v0.21.1 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/cel-go v0.26.0 // indirect @@ -70,13 +70,13 @@ require ( github.com/k8snetworkplumbingwg/govdpa v0.1.5-0.20230926073613-07c1031aea47 // indirect github.com/k8snetworkplumbingwg/sriovnet v1.2.1-0.20250818105516-24ab680f94f3 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 // indirect github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect github.com/mdlayher/ndp v1.0.1 // indirect github.com/mdlayher/packet v1.0.0 // indirect github.com/mdlayher/socket v0.2.1 // indirect - github.com/metallb/frr-k8s v0.0.16 // indirect + github.com/metallb/frr-k8s v0.0.21 // indirect github.com/miekg/dns v1.1.43 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/spdystream v0.5.0 // indirect @@ -92,9 +92,9 @@ require ( github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.22.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/safchain/ethtool v0.3.1-0.20231027162144-83e5e0097c91 // indirect github.com/spf13/afero v1.14.0 // indirect @@ -123,11 +123,11 @@ require ( golang.org/x/crypto v0.43.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.46.0 // indirect - golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/oauth2 v0.29.0 // indirect golang.org/x/sys v0.37.0 // indirect golang.org/x/term v0.36.0 // indirect golang.org/x/text v0.30.0 // indirect - golang.org/x/time v0.9.0 // indirect + golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 3f6fd31e56..9236cb7764 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -141,8 +141,8 @@ github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= +github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= @@ -151,8 +151,8 @@ github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDsl github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -281,8 +281,9 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 h1:ql8x//rJsHMjS+qqEag8n3i4azw1QneKh5PieH9UEbY= github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875/go.mod h1:kfOoFJuHWp76v1RgZCb9/gVUc7XdY877S2uVYbNliGc= github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 h1:2oDp6OOhLxQ9JBoUuysVz9UZ9uI6oLUbvAZu0x8o+vE= @@ -293,8 +294,8 @@ github.com/mdlayher/packet v1.0.0 h1:InhZJbdShQYt6XV2GPj5XHxChzOfhJJOMbvnGAmOfQ8 github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU= github.com/mdlayher/socket v0.2.1 h1:F2aaOwb53VsBE+ebRS9bLd7yPOfYUMC8lOODdCBDY6w= github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= -github.com/metallb/frr-k8s v0.0.16 h1:tlNIDW5wtBlKE7wUgXBs9GklyTCTFtdOGz8fvEl40vU= -github.com/metallb/frr-k8s v0.0.16/go.mod h1:TjrGoAf+v00hYGlI8jUdyDxY5udMAOs2GWwrvLWnA4E= +github.com/metallb/frr-k8s v0.0.21 h1:JLlCeXVlW5BLVdPy2u5sS9UCVlnK9x2vzWbIkxb8Atk= +github.com/metallb/frr-k8s v0.0.21/go.mod h1:VMnCZUVXYy7k0Fsa2L3XKwISFs3Thv0Uord7rSZPQZw= github.com/metallb/metallb v0.14.9 h1:rjhftr7b0vv56c8pXm7UDo7ad61EwKT6lbGGocrN/VM= github.com/metallb/metallb v0.14.9/go.mod h1:qUh1zVwYAfp3JLxhZrDH20j55QvYwCkI37QU4gUG3ns= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= @@ -364,12 +365,12 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= @@ -548,8 +549,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= +golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -632,8 +633,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/test/e2e/infraprovider/providers/kind/kind.go b/test/e2e/infraprovider/providers/kind/kind.go index 58a25d9774..532a45fe5d 100644 --- a/test/e2e/infraprovider/providers/kind/kind.go +++ b/test/e2e/infraprovider/providers/kind/kind.go @@ -721,51 +721,45 @@ func getNetworkInterface(containerName, networkName string) (api.NetworkInterfac return valueStr, nil } - getIPFamilyFlagForIPRoute2 := func(ipStr string) (string, error) { + getIPFamilyForIPRoute2 := func(ipStr string) (string, error) { ip := net.ParseIP(ipStr) if ip == nil { return "", fmt.Errorf("invalid IP address: %s", ipStr) } if utilnet.IsIPv6(ip) { - return "-6", nil + return "inet6", nil } - return "-4", nil + return "inet", nil } getInterfaceNameUsingIP := func(ip string) (string, error) { - ipFlag, err := getIPFamilyFlagForIPRoute2(ip) + ipFamily, err := getIPFamilyForIPRoute2(ip) if err != nil { - return "", fmt.Errorf("failed to get IP family flag for %s: %w", ip, err) + return "", fmt.Errorf("failed to get IP family for %s: %w", ip, err) } - allInfAddrBytes, err := exec.Command(containerengine.Get().String(), "exec", "-i", containerName, "ip", "-br", ipFlag, "a", "sh").CombinedOutput() + cmdArgs := []string{"exec", "-i", containerName, "ip", "-o", "-f", ipFamily, "addr", "show"} + allInfAddrBytes, err := exec.Command(containerengine.Get().String(), cmdArgs...).CombinedOutput() if err != nil { - return "", fmt.Errorf("failed to find interface with IP %s on container %s with command 'ip -br a sh': err %v, out: %s", ip, containerName, - err, allInfAddrBytes) + return "", fmt.Errorf("failed to find interface with IP %s on container %s with command %q: err %v, out: %s", ip, containerName, + strings.Join(cmdArgs[3:], " "), err, allInfAddrBytes) } - var ipLine string + var infName string for _, line := range strings.Split(string(allInfAddrBytes), "\n") { if strings.Contains(line, ip) { - ipLine = line + fields := strings.Fields(line) + if len(fields) < 2 { + return "", fmt.Errorf("failed to parse 'ip addr' output line %q", line) + } + infName = strings.TrimSuffix(fields[1], ":") + if strings.Contains(infName, "@") { + infName = strings.SplitN(infName, "@", 2)[0] + } break } } - if ipLine == "" { + if infName == "" { return "", fmt.Errorf("failed to find IP %q within 'ip a' command on container %q:\n\n%q", ip, containerName, string(allInfAddrBytes)) } - ipLineSplit := strings.Split(ipLine, " ") - if len(ipLine) == 0 { - return "", fmt.Errorf("failed to find interface name from 'ip a' output line %q", ipLine) - } - infNames := ipLineSplit[0] - splitChar := " " - if strings.Contains(infNames, "@") { - splitChar = "@" - } - infNamesSplit := strings.Split(infNames, splitChar) - if len(infNamesSplit) == 0 { - return "", fmt.Errorf("failed to extract inf name + veth name from %q splitting by %q", infNames, splitChar) - } - infName := infNamesSplit[0] // validate its an interface name on the Node with iproute2 out, err := exec.Command(containerengine.Get().String(), "exec", "-i", containerName, "ip", "link", "show", infName).CombinedOutput() if err != nil { @@ -805,7 +799,7 @@ func getNetworkInterface(containerName, networkName string) (api.NetworkInterfac if ni.IPv6 != "" { ni.InfName, err = getInterfaceNameUsingIP(ni.IPv6) if err != nil { - framework.Logf("failed to get network interface name using IPv4 address %s: %v", ni.IPv6, err) + framework.Logf("failed to get network interface name using IPv6 address %s: %v", ni.IPv6, err) } } ni.IPv6Prefix, err = getContainerNetwork(inspectNetworkIPv6PrefixKeyStr) diff --git a/test/e2e/kubevirt.go b/test/e2e/kubevirt.go index 2ebdcd0d0d..2f05e722d6 100644 --- a/test/e2e/kubevirt.go +++ b/test/e2e/kubevirt.go @@ -339,7 +339,7 @@ var _ = Describe("Kubevirt Virtual Machines", feature.VirtualMachineSupport, fun } // Fail fast Expect(iperfLog).NotTo(ContainSubstring("iperf3: error"), stage+": "+iperfLogFile) - //Remove last carriage return to propertly split by new line. + // Remove last carriage return to properly split by new line. iperfLog = strings.TrimSuffix(iperfLog, "\n") iperfLogLines := strings.Split(iperfLog, "\n") if len(iperfLogLines) == 0 { @@ -744,7 +744,6 @@ var _ = Describe("Kubevirt Virtual Machines", feature.VirtualMachineSupport, fun } return familyFn(*iface), nil } - } addressesFromStatus = func(vmi *kubevirtv1.VirtualMachineInstance) func() ([]string, error) { @@ -1186,7 +1185,6 @@ passwd: } liveMigrateAndCheck(vm.Name, td.mode, endpoints, "after live migration to node owning the subnet") } - } checkPodHasIPAtStatus = func(g Gomega, pod *corev1.Pod) { @@ -1248,7 +1246,7 @@ fi if err != nil { return nil, err } - for _ = range idx { + for range idx { ip = iputils.NextIP(ip) } ipNet.IP = ip @@ -1277,7 +1275,6 @@ fi } pod.Spec.Containers[0].Image = images.IPerf3() pod.Spec.Containers[0].Args = []string{iperfServerScript + "\n sleep infinity"} - }) if err != nil { return nil, err @@ -1353,7 +1350,7 @@ fi removeImagesInNodes = func(imageURL string) error { nodesList, err := fr.ClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) - for nodeIdx, _ := range nodesList.Items { + for nodeIdx := range nodesList.Items { err = removeImagesInNode(nodesList.Items[nodeIdx].Name, imageURL) if err != nil { return err @@ -1417,7 +1414,6 @@ fi }) Context("with default pod network", Ordered, func() { - BeforeEach(func() { ns, err := fr.CreateNamespace(context.TODO(), fr.BaseName, map[string]string{ "e2e-framework": fr.BaseName, @@ -1468,7 +1464,6 @@ fi } prepareHTTPServerPods(map[string]string{}, checkPodHasIPAtStatus) - }) AfterEach(func() { @@ -1488,9 +1483,7 @@ fi if td.mode == kubevirtv1.MigrationPostCopy && os.Getenv("KUBEVIRT_SKIP_MIGRATE_POST_COPY") == "true" { Skip("Post copy live migration explicitly skipped") } - var ( - err error - ) + var err error Expect(err).NotTo(HaveOccurred()) @@ -1973,7 +1966,8 @@ ip route add %[3]s via %[4]s By("Check egress src ip is not node IP on 'routed' ingress mode") for _, vmAddress := range expectedAddreses { output, err := infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{ - "bash", "-c", fmt.Sprintf("grep 'connected to %s' /tmp/test_*", vmAddress)}) + "bash", "-c", fmt.Sprintf("grep 'connected to %s' /tmp/test_*", vmAddress), + }) Expect(err).NotTo(HaveOccurred(), step+": "+output) } checkNorthSouthEgressIperfTraffic(vmi, externalContainerIPs, iperf3DefaultPort, step) @@ -2213,7 +2207,6 @@ ip route add %[3]s via %[4]s By("Reconfigure primary UDN interface to use dhcp/nd for ipv4 and ipv6") _, err = virtLauncherCommand(kubevirt.GenerateAddressDiscoveryConfigurationCommand("ovn-udn1")) Expect(err).NotTo(HaveOccurred()) - }) It("should configure IPv4 and IPv6 using DHCP and NDP", func() { dnsService, err := fr.ClientSet.CoreV1().Services(config.Kubernetes.DNSServiceNamespace). @@ -2266,7 +2259,6 @@ ip route add %[3]s via %[4]s Expect(primaryUDNValueForConnection("IP6.ROUTE")).To(ContainElement(ContainSubstring(fmt.Sprintf("dst = %s", cidrIPv6)))) Expect(primaryUDNValueForDevice("GENERAL.MTU")).To(ConsistOf("1300")) } - }) }) Context("with user defined networks with ipamless localnet topology", Ordered, func() { @@ -2289,6 +2281,8 @@ ip route add %[3]s via %[4]s vmiMAC = "0A:58:0A:80:00:64" staticIPsNetworkData = func(ips []string) (string, error) { type Ethernet struct { + DHCP4 *bool `json:"dhcp4,omitempty"` + DHCP6 *bool `json:"dhcp6,omitempty"` Addresses []string `json:"addresses,omitempty"` } networkData, err := yaml.Marshal(&struct { @@ -2298,6 +2292,8 @@ ip route add %[3]s via %[4]s Version: 2, Ethernets: map[string]Ethernet{ "eth0": { + DHCP4: ptr.To(false), + DHCP6: ptr.To(false), Addresses: ips, }, }, @@ -2328,7 +2324,8 @@ chpasswd: { expire: False } iperfServerTestPods, err = createIperfServerPods(selectedNodes, cudn.Name, cudn.Spec.Network.Localnet.Role, filterCIDRs(fr.ClientSet, ipv4CIDR, ipv6CIDR)) Expect(err).NotTo(HaveOccurred()) - networkData, err := staticIPsNetworkData(filterCIDRs(fr.ClientSet, vmiIPv4, vmiIPv6)) + filteredCIDRs := filterCIDRs(fr.ClientSet, vmiIPv4, vmiIPv6) + networkData, err := staticIPsNetworkData(filteredCIDRs) Expect(err).NotTo(HaveOccurred()) vm := fedoraWithTestToolingVM(nil /*labels*/, nil /*annotations*/, nil /*nodeSelector*/, kubevirtv1.NetworkSource{ @@ -2384,16 +2381,31 @@ chpasswd: { expire: False } Expect(err).ToNot(HaveOccurred(), output) step = by(vmi.Name, fmt.Sprintf("Force kill qemu at node %q where VM is running on", vmi.Status.NodeName)) - Expect(kubevirt.ForceKillVirtLauncherAtNode(infraprovider.Get(), vmi.Status.NodeName, vmi.Namespace, vmi.Name)).To(Succeed()) + Expect(kubevirt.ForceKillVirtLauncherAtNode(infraprovider.Get(), vmi.Status.NodeName, vmi.Namespace, vmi.Name)).To(Succeed(), step) step = by(vmi.Name, "Waiting for failed restarted VMI to reach ready state") waitVirtualMachineInstanceFailed(vmi) waitVirtualMachineInstanceReadiness(vmi) - Expect(crClient.Get(context.TODO(), crclient.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) + Expect(crClient.Get(context.TODO(), crclient.ObjectKeyFromObject(vmi), vmi)).To(Succeed(), step) step = by(vmi.Name, "Login to virtual machine after virtual machine instance force killed") Expect(virtClient.LoginToFedora(vmi, "fedora", "fedora")).To(Succeed(), step) + step = by(vmi.Name, "Wait for cloud init to finish after vm restart") + output, err = virtClient.RunCommand(vmi, "cloud-init status --wait", time.Minute) + Expect(err).NotTo(HaveOccurred(), step+": "+output) + + step = by(vmi.Name, "Verify static IP is configured after vm restart") + filteredIPs := []string{} + for _, filteredCIDR := range filteredCIDRs { + filteredIPs = append(filteredIPs, strings.Split(filteredCIDR, "/")[0]) + } + Eventually(kubevirt.RetrieveAllGlobalAddressesFromGuest). + WithArguments(virtClient, vmi). + WithTimeout(5*time.Second). + WithPolling(time.Second). + Should(ConsistOf(filteredIPs), step) + step = by(vmi.Name, "Restart iperf traffic after forcing a vm failure") Expect(startEastWestIperfTraffic(vmi, testPodsIPs, step)).To(Succeed(), step) checkEastWestIperfTraffic(vmi, testPodsIPs, step) diff --git a/test/e2e/kubevirt/ip.go b/test/e2e/kubevirt/ip.go index 3e11bd9b92..f3bbf2037f 100644 --- a/test/e2e/kubevirt/ip.go +++ b/test/e2e/kubevirt/ip.go @@ -3,6 +3,7 @@ package kubevirt import ( "encoding/json" "fmt" + "net" "time" v1 "kubevirt.io/api/core/v1" @@ -32,8 +33,11 @@ func RetrieveAllGlobalAddressesFromGuest(cli *Client, vmi *v1.VirtualMachineInst continue } for _, address := range iface.Addresses { - // Skip non DHCPv6 address - if address.Family == "inet6" && address.PrefixLen != 128 { + ip := net.ParseIP(address.Local) + if ip == nil { + return nil, fmt.Errorf("invalid ip address %q", address.Local) + } + if ip.IsLinkLocalUnicast() { continue } addresses = append(addresses, address.Local) diff --git a/test/e2e/multihoming.go b/test/e2e/multihoming.go index 16a5181570..e48b10fdab 100644 --- a/test/e2e/multihoming.go +++ b/test/e2e/multihoming.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "strings" "time" @@ -16,6 +17,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2enode "k8s.io/kubernetes/test/e2e/framework/node" mnpapi "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/apis/k8s.cni.cncf.io/v1beta1" @@ -2382,6 +2384,329 @@ ip a add %[4]s/24 dev %[2]s Expect(inRange(secondaryFlatL2NetworkCIDR, netStatus[1].IPs[0])) }) }) + + Context("A pod with multiple attachments to the same secondary NAD", func() { + const ( + testNadName = "test-multi-secondary-nad" + testPodName = "test-pod-multi-secondary-nad" + ) + + var pod *v1.Pod + + DescribeTable("features multiple different IPs and connectivity redundancy", + func(netConfigParams networkAttachmentConfigParams) { + netConfig := newNetworkAttachmentConfig(netConfigParams) + netConfig.namespace = f.Namespace.Name + netConfig.name = testNadName + + By("creating the secondary network attachment definition") + _, err := nadClient.NetworkAttachmentDefinitions(netConfig.namespace).Create( + context.Background(), + generateNAD(netConfig, f.ClientSet), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for controller to sync the NAD") + time.Sleep(5 * time.Second) + + By("creating a pod with multiple attachments to the same secondary NAD") + // Specify the same NAD name multiple times to test GetIndexedNADKey functionality + // This will create indexed NAD keys like "ns/nad", "ns/nad/1" + podConfig := podConfiguration{ + attachments: []nadapi.NetworkSelectionElement{ + {Name: testNadName, Namespace: netConfig.namespace}, // First attachment - will be indexed as "ns/nad" + {Name: testNadName, Namespace: netConfig.namespace}, // Second attachment - will be indexed as "ns/nad/1" + }, + name: testPodName, + namespace: f.Namespace.Name, + isPrivileged: true, // Required for ip link set commands to manipulate network interfaces + } + pod, err = cs.CoreV1().Pods(podConfig.namespace).Create( + context.Background(), + generatePodSpec(podConfig), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + By("asserting the pod gets to the `Running` phase") + Eventually(func() v1.PodPhase { + updatedPod, err := cs.CoreV1().Pods(podConfig.namespace).Get(context.Background(), pod.GetName(), metav1.GetOptions{}) + if err != nil { + return v1.PodFailed + } + pod = updatedPod + return updatedPod.Status.Phase + }, 2*time.Minute, 6*time.Second).Should(Equal(v1.PodRunning)) + + By("verifying the pod has two network status entries for the same secondary NAD") + netStatus, err := podNetworkStatus(pod, func(status nadapi.NetworkStatus) bool { + return !status.Default && strings.Contains(status.Name, testNadName) + }) + Expect(err).NotTo(HaveOccurred()) + Expect(netStatus).To(HaveLen(2), "Pod should have two network status entries for the secondary NAD attachments") + + By("verifying both interfaces have IPs") + for i := 0; i < 2; i++ { + Expect(netStatus[i].IPs).NotTo(BeEmpty(), fmt.Sprintf("Interface %d should have at least one IP", i)) + } + + By("verifying both IPs are different") + ips := make([]string, 2) + for i := 0; i < 2; i++ { + ips[i] = netStatus[i].IPs[0] + } + Expect(ips[0]).NotTo(Equal(ips[1]), "First and second interface IPs should be different") + + By("verifying all IPs are from the configured secondary NAD subnet") + subnet, err := getNetCIDRSubnet(netConfig.cidr) + Expect(err).NotTo(HaveOccurred()) + for i, ip := range ips { + Expect(inRange(subnet, ip)).To(Succeed(), fmt.Sprintf("IP[%d] %s should be in subnet %s", i, ip, subnet)) + By(fmt.Sprintf("Verified IP[%d] %s is from subnet %s", i, ip, subnet)) + } + + By("verifying both interfaces have unique interface names") + interfaceNames := make([]string, 2) + for i := 0; i < 2; i++ { + Expect(netStatus[i].Interface).NotTo(BeEmpty(), fmt.Sprintf("Interface %d should have a name", i)) + interfaceNames[i] = netStatus[i].Interface + } + Expect(interfaceNames[0]).NotTo(Equal(interfaceNames[1]), "First and second interface names should be different") + + By(fmt.Sprintf("Successfully validated two interfaces with IPs: %s, %s", ips[0], ips[1])) + + // For L3 secondary NADs, verify ECMP routes are added in the pod + if netConfigParams.topology == "layer3" { + By("verifying ECMP routes are added for L3 secondary NAD with multiple attachments") + + // Calculate the gateway IP from the pod's IP - gateway is the .1 address of the subnet + // For example, if pod IP is 172.31.0.3/24, gateway is 172.31.0.1 + podIP := net.ParseIP(ips[0]) + Expect(podIP).NotTo(BeNil(), "Pod IP should be valid") + podIPv4 := podIP.To4() + Expect(podIPv4).NotTo(BeNil(), "Pod IP should be IPv4") + expectedGateway := net.IPv4(podIPv4[0], podIPv4[1], podIPv4[2], 1).String() + + // Get routes for the secondary network CIDR + routeOutput, err := e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "ip", + "route", + "show", + secondaryNetworkCIDR, + ) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Should be able to get routes from pod for dest %s", secondaryNetworkCIDR)) + + // ECMP routes should contain "nexthop" entries for both interfaces + // Example format: + // 172.31.0.0/16 + // nexthop via 172.31.0.1 dev net1 weight 1 + // nexthop via 172.31.0.1 dev net2 weight 1 + By(fmt.Sprintf("ECMP routes to %s", routeOutput)) + routes := strings.Split(routeOutput, "\n") + // output should be at lease 3 + Expect(len(routes)).To(BeNumerically(">", 2)) + Expect(routes[1]).To(ContainSubstring(fmt.Sprintf("nexthop via %s dev %s weight 1", expectedGateway, interfaceNames[0])), fmt.Sprintf("ECMP routes should include interface %s", interfaceNames[0])) + Expect(routes[2]).To(ContainSubstring(fmt.Sprintf("nexthop via %s dev %s weight 1", expectedGateway, interfaceNames[1])), fmt.Sprintf("ECMP routes should include interface %s", interfaceNames[1])) + + By(fmt.Sprintf("Successfully verified ECMP routes to %s via gateway %s with interfaces %s and %s", + secondaryNetworkCIDR, expectedGateway, interfaceNames[0], interfaceNames[1])) + } + + By("creating a second pod with a single attachment to the same secondary NAD") + serverPodName := "test-pod-multi-secondary-nad-server" + serverPodConfig := podConfiguration{ + attachments: []nadapi.NetworkSelectionElement{ + {Name: testNadName, Namespace: netConfig.namespace}, + }, + name: serverPodName, + namespace: f.Namespace.Name, + containerCmd: httpServerContainerCmd(8080), + } + serverPod, err := cs.CoreV1().Pods(serverPodConfig.namespace).Create( + context.Background(), + generatePodSpec(serverPodConfig), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + By("asserting the server pod gets to the `Running` phase") + Eventually(func() v1.PodPhase { + updatedPod, err := cs.CoreV1().Pods(serverPodConfig.namespace).Get(context.Background(), serverPod.GetName(), metav1.GetOptions{}) + if err != nil { + return v1.PodFailed + } + serverPod = updatedPod + return updatedPod.Status.Phase + }, 2*time.Minute, 6*time.Second).Should(Equal(v1.PodRunning)) + + By("getting the server pod's IP on the secondary NAD") + serverNetStatus, err := podNetworkStatus(serverPod, func(status nadapi.NetworkStatus) bool { + return !status.Default && strings.Contains(status.Name, testNadName) + }) + Expect(err).NotTo(HaveOccurred()) + Expect(serverNetStatus).To(HaveLen(1), "Server pod should have one network status entry for the secondary NAD") + Expect(serverNetStatus[0].IPs).NotTo(BeEmpty(), "Server pod should have at least one IP") + serverIP := serverNetStatus[0].IPs[0] + By(fmt.Sprintf("Server pod IP on secondary NAD: %s", serverIP)) + + By("verifying connectivity from client pod to server pod with all interfaces up") + Eventually(func() error { + _, err := e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "curl", + "--connect-timeout", + "2", + fmt.Sprintf("http://%s:8080/hostname", serverIP), + ) + return err + }, 30*time.Second, 2*time.Second).Should(Succeed(), "Should be able to reach server pod from client pod") + + // Test connectivity explicitly using the first interface + interfaceName := interfaceNames[0] + By(fmt.Sprintf("verifying connectivity explicitly through interface %d (%s)", 0, interfaceName)) + + Eventually(func() error { + _, err := e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "curl", + "--connect-timeout", + "2", + "--interface", + interfaceName, + fmt.Sprintf("http://%s:8080/hostname", serverIP), + ) + return err + }, 30*time.Second, 2*time.Second).Should(Succeed(), fmt.Sprintf("Should be able to reach server through interface %s", interfaceName)) + + By(fmt.Sprintf("Successfully verified connectivity through interface %d (%s)", 0, interfaceName)) + + // Test redundancy: verify that when the first interface is down, we can still reach the server + interfaceToDisable := interfaceNames[0] + workingInterface := interfaceNames[1] + + By(fmt.Sprintf("bringing down interface 0 (%s) and verifying connectivity through interface 1 (%s)", interfaceToDisable, workingInterface)) + + _, err = e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "ip", + "link", + "set", + interfaceToDisable, + "down", + ) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Should be able to bring down interface %s", interfaceToDisable)) + + Eventually(func() error { + _, err := e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "curl", + "--connect-timeout", + "2", + "--interface", + workingInterface, + fmt.Sprintf("http://%s:8080/hostname", serverIP), + ) + return err + }, 30*time.Second, 2*time.Second).Should(Succeed(), fmt.Sprintf("Should be able to reach server through working interface %s when %s is down", workingInterface, interfaceToDisable)) + + _, err = e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "ip", + "link", + "set", + interfaceToDisable, + "up", + ) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Should be able to bring interface %s back up", interfaceToDisable)) + + // Test the reverse + interfaceToDisable = interfaceNames[1] + workingInterface = interfaceNames[0] + + By(fmt.Sprintf("bringing down interface 1 (%s) and verifying connectivity through interface 0 (%s)", interfaceToDisable, workingInterface)) + + _, err = e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "ip", + "link", + "set", + interfaceToDisable, + "down", + ) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Should be able to bring down interface %s", interfaceToDisable)) + + Eventually(func() error { + _, err := e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "curl", + "--connect-timeout", + "2", + "--interface", + workingInterface, + fmt.Sprintf("http://%s:8080/hostname", serverIP), + ) + return err + }, 30*time.Second, 2*time.Second).Should(Succeed(), fmt.Sprintf("Should be able to reach server through working interface %s when %s is down", workingInterface, interfaceToDisable)) + + _, err = e2ekubectl.RunKubectl( + podConfig.namespace, + "exec", + pod.Name, + "--", + "ip", + "link", + "set", + interfaceToDisable, + "up", + ) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Should be able to bring interface %s back up", interfaceToDisable)) + + By("Successfully verified that both interfaces can reach the same secondary NAD and provide redundancy") + }, + Entry("L2 secondary NAD", + networkAttachmentConfigParams{ + name: testNadName, + topology: "layer2", + cidr: secondaryFlatL2NetworkCIDR, + role: "secondary", + }, + ), + Entry("L3 secondary NAD", + networkAttachmentConfigParams{ + name: testNadName, + topology: "layer3", + cidr: netCIDR(secondaryNetworkCIDR, netPrefixLengthPerNode), + role: "secondary", + }, + ), + ) + }) }) func kickstartPod(cs clientset.Interface, configuration podConfiguration) *v1.Pod { diff --git a/test/e2e/network_segmentation_api_validations.go b/test/e2e/network_segmentation_api_validations.go index e10151d5d6..776285548a 100644 --- a/test/e2e/network_segmentation_api_validations.go +++ b/test/e2e/network_segmentation_api_validations.go @@ -31,6 +31,7 @@ var _ = Describe("Network Segmentation: API validations", feature.NetworkSegment Entry("ClusterUserDefinedNetwork, localnet, invalid mtu", testscenariocudn.LocalnetInvalidMTU), Entry("ClusterUserDefinedNetwork, localnet, invalid vlan", testscenariocudn.LocalnetInvalidVLAN), Entry("ClusterUserDefinedNetwork, layer2", testscenariocudn.Layer2CUDNInvalid), + Entry("ClusterUserDefinedNetwork, evpn", feature.RouteAdvertisements, feature.EVPN, testscenariocudn.EVPNCUDNInvalid), Entry("UserDefinedNetwork, layer2", testscenariocudn.Layer2UDNInvalid), Entry("ClusterUserDefinedNetwork, no-overlay, invalid", testscenariocudn.NoOverlayInvalid), ) @@ -48,6 +49,7 @@ var _ = Describe("Network Segmentation: API validations", feature.NetworkSegment }, Entry("ClusterUserDefinedNetwork, localnet", testscenariocudn.LocalnetValid), Entry("ClusterUserDefinedNetwork, layer2", testscenariocudn.Layer2CUDNValid), + Entry("ClusterUserDefinedNetwork, evpn", feature.RouteAdvertisements, feature.EVPN, testscenariocudn.EVPNCUDNValid), Entry("UserDefinedNetwork, layer2", testscenariocudn.Layer2UDNValid), Entry("ClusterUserDefinedNetwork, no-overlay, valid", testscenariocudn.NoOverlayValid), ) diff --git a/test/e2e/network_segmentation_services.go b/test/e2e/network_segmentation_services.go index 124f206fc5..e728048b4c 100644 --- a/test/e2e/network_segmentation_services.go +++ b/test/e2e/network_segmentation_services.go @@ -99,6 +99,8 @@ var _ = Describe("Network Segmentation: services", feature.NetworkSegmentation, namespace := f.Namespace.Name jig := e2eservice.NewTestJig(cs, namespace, "udn-service") + dynamicUDNEnabled := isDynamicUDNEnabled() + if netConfigParams.topology == "layer2" && !isInterconnectEnabled() { const upstreamIssue = "https://github.com/ovn-org/ovn-kubernetes/issues/4703" e2eskipper.Skipf( @@ -174,8 +176,10 @@ ips=$(ip -o addr show dev $iface| grep global |awk '{print $4}' | cut -d/ -f1 | By("Connect to the UDN service nodePort on all 3 nodes from the UDN client pod") checkConnectionToLoadBalancers(f, udnClientPod, udnService, udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[0], "endpoint node", udnServerPod.Name) - checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[1], "other node", udnServerPod.Name) - checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + if !dynamicUDNEnabled { + checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[1], "other node", udnServerPod.Name) + checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + } By(fmt.Sprintf("Creating a UDN client pod on a different node (%s)", clientNode)) udnClientPod2 := e2epod.NewAgnhostPod(namespace, "udn-client2", nil, nil, nil) udnClientPod2.Spec.NodeName = clientNode @@ -186,14 +190,20 @@ ips=$(ip -o addr show dev $iface| grep global |awk '{print $4}' | cut -d/ -f1 | checkConnectionToLoadBalancers(f, udnClientPod2, udnService, udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[1], "local node", udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[0], "server node", udnServerPod.Name) - checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + if !dynamicUDNEnabled { + checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + } By("Connect to the UDN service from the UDN client external container") externalContainer := infraapi.ExternalContainer{Name: "frr"} - checkConnectionToLoadBalancersFromExternalContainer(f, externalContainer, udnService, udnServerPod.Name) + if !dynamicUDNEnabled { + checkConnectionToLoadBalancersFromExternalContainer(f, externalContainer, udnService, udnServerPod.Name) + } checkConnectionToNodePortFromExternalContainer(externalContainer, udnService, &nodes.Items[0], "server node", udnServerPod.Name) - checkConnectionToNodePortFromExternalContainer(externalContainer, udnService, &nodes.Items[1], "other node", udnServerPod.Name) - checkConnectionToNodePortFromExternalContainer(externalContainer, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + if !dynamicUDNEnabled { + checkConnectionToNodePortFromExternalContainer(externalContainer, udnService, &nodes.Items[1], "other node", udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(externalContainer, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + } // Default network -> UDN // Check that it cannot connect @@ -216,7 +226,9 @@ ips=$(ip -o addr show dev $iface| grep global |awk '{print $4}' | cut -d/ -f1 | checkNoConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[1], "local node") // TODO change to checkConnectionToNodePort when we have full UDN support in ovnkube-node checkConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[0], "server node", udnServerPod.Name) - checkConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + if !dynamicUDNEnabled { + checkConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + } // UDN -> Default network // Create a backend pod and service in the default network and verify that the client pod in the UDN diff --git a/test/e2e/route_advertisements.go b/test/e2e/route_advertisements.go index 9fcc0d26f5..c9335c2a95 100644 --- a/test/e2e/route_advertisements.go +++ b/test/e2e/route_advertisements.go @@ -882,6 +882,7 @@ var _ = ginkgo.Describe("BGP: Pod to external server when CUDN network is advert var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks", feature.RouteAdvertisements, func(cudnATemplate, cudnBTemplate *udnv1.ClusterUserDefinedNetwork) { + const curlConnectionResetCode = "56" const curlConnectionTimeoutCode = "28" f := wrappedTestFramework("bgp-network-isolation") @@ -893,7 +894,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" // podNetB is in cudnB hosted on nodes[1], podNetDefault is in the default network hosted on nodes[1] - done in BeforeEach var podNetB, podNetDefault *corev1.Pod - var svcNetA, svcNetB, svcNetDefault *corev1.Service + var svcNodePortNetA, svcNodePortNetB, svcNodePortNetDefault, svcNodePortETPLocalDefault, svcNodePortETPLocalNetA *corev1.Service var cudnA, cudnB *udnv1.ClusterUserDefinedNetwork var ra *rav1.RouteAdvertisements var hostNetworkPort int @@ -1002,7 +1003,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" familyPolicy := corev1.IPFamilyPolicyPreferDualStack svc.Spec.IPFamilyPolicy = &familyPolicy svc.Spec.Type = corev1.ServiceTypeNodePort - svcNetA, err = f.ClientSet.CoreV1().Services(pod.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}) + svcNodePortNetA, err = f.ClientSet.CoreV1().Services(pod.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) pod.Name = fmt.Sprintf("pod-1-%s-net-%s", nodes.Items[1].Name, cudnB.Name) @@ -1014,7 +1015,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" svc.Name = fmt.Sprintf("service-%s", cudnB.Name) svc.Namespace = pod.Namespace svc.Spec.Selector = pod.Labels - svcNetB, err = f.ClientSet.CoreV1().Services(pod.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}) + svcNodePortNetB, err = f.ClientSet.CoreV1().Services(pod.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) pod.Name = fmt.Sprintf("pod-1-%s-net-default", nodes.Items[1].Name) @@ -1022,11 +1023,25 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" pod.Labels = map[string]string{"network": "default"} podNetDefault = e2epod.PodClientNS(f, "default").CreateSync(context.TODO(), pod) - svc.Name = fmt.Sprintf("service-default") + svc.Name = "service-default" svc.Namespace = "default" svc.Spec.Selector = pod.Labels svc.Spec.Type = corev1.ServiceTypeNodePort - svcNetDefault, err = f.ClientSet.CoreV1().Services(pod.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}) + svcNodePortNetDefault, err = f.ClientSet.CoreV1().Services(pod.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // create one nodePort service with externalTrafficPolicy=Local in default namespace + svc.Name = "nodeport-default-etp-local" + svc.Spec.Type = corev1.ServiceTypeNodePort + svc.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal + svcNodePortETPLocalDefault, err = f.ClientSet.CoreV1().Services(svc.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // create one nodePort service with externalTrafficPolicy=Local in udnNamespaceA + svc.Name = fmt.Sprintf("nodeport-etp-local-%s", cudnA.Name) + svc.Namespace = udnNamespaceA.Name + svc.Spec.Selector = map[string]string{"network": cudnA.Name} + svcNodePortETPLocalNetA, err = f.ClientSet.CoreV1().Services(svc.Namespace).Create(context.Background(), svc, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Expose networks") @@ -1099,7 +1114,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" gomega.Eventually(func() bool { _, err := udnClient.K8sV1().ClusterUserDefinedNetworks().Get(context.TODO(), cudnB.Name, metav1.GetOptions{}) return apierrors.IsNotFound(err) - }, time.Second*30).Should(gomega.BeTrue()) + }, time.Second*60).Should(gomega.BeTrue()) cudnB = nil } if cudnA != nil { @@ -1108,7 +1123,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" gomega.Eventually(func() bool { _, err := udnClient.K8sV1().ClusterUserDefinedNetworks().Get(context.TODO(), cudnA.Name, metav1.GetOptions{}) return apierrors.IsNotFound(err) - }, time.Second*30).Should(gomega.BeTrue()) + }, time.Second*60).Should(gomega.BeTrue()) cudnA = nil } @@ -1117,10 +1132,16 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" gomega.Expect(err).NotTo(gomega.HaveOccurred()) podNetDefault = nil } - if svcNetDefault != nil { - err = f.ClientSet.CoreV1().Services(svcNetDefault.Namespace).Delete(context.Background(), svcNetDefault.Name, metav1.DeleteOptions{}) + + if svcNodePortNetDefault != nil { + err = f.ClientSet.CoreV1().Services(svcNodePortNetDefault.Namespace).Delete(context.Background(), svcNodePortNetDefault.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + svcNodePortNetDefault = nil + } + if svcNodePortETPLocalDefault != nil { + err = f.ClientSet.CoreV1().Services(svcNodePortETPLocalDefault.Namespace).Delete(context.Background(), svcNodePortETPLocalDefault.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - svcNetDefault = nil + svcNodePortETPLocalDefault = nil } raClient, err := raclientset.NewForConfig(f.ClientConfig()) @@ -1293,12 +1314,12 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" }), ginkgo.Entry("pod in the default network should not be able to access a UDN service", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { - return podNetDefault.Name, podNetDefault.Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNetA.Spec.ClusterIPs), "8080") + "/clientip", + return podNetDefault.Name, podNetDefault.Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNodePortNetA.Spec.ClusterIPs), "8080") + "/clientip", curlConnectionTimeoutCode, true }), ginkgo.Entry("pod in the UDN should be able to access a service in the same network", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { - return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNetA.Spec.ClusterIPs), "8080") + "/clientip", "", false + return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNodePortNetA.Spec.ClusterIPs), "8080") + "/clientip", "", false }), ginkgo.Entry("pod in the UDN should not be able to access a default network service", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { @@ -1311,7 +1332,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" // this causes curl timeout with code 7 host unreachable instead of code 28 out = "" } - return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNetDefault.Spec.ClusterIPs), "8080") + "/clientip", out, err + return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNodePortNetDefault.Spec.ClusterIPs), "8080") + "/clientip", out, err }), ginkgo.Entry("pod in the UDN should be able to access kapi in default network service", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { @@ -1337,7 +1358,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" }), ginkgo.Entry("pod in the UDN should not be able to access a service in a different UDN", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { - return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNetB.Spec.ClusterIPs), "8080") + "/clientip", + return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNodePortNetB.Spec.ClusterIPs), "8080") + "/clientip", curlConnectionTimeoutCode, true }), ginkgo.Entry("host to a local UDN pod should not work", @@ -1413,7 +1434,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" // pod -> node traffic should use the node's IP as the source for advertised UDNs. return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(hostNetworkPort)) + "/clientip", clientNodeIP, false }), - ginkgo.Entry("UDN pod to the same node nodeport service in default network should not work", + ginkgo.Entry("[ETP=Cluster] UDN pod to the same node nodeport service in default network should not work", // FIXME: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5410 func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] @@ -1425,11 +1446,11 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" if ipFamily == utilnet.IPv6 { nodeIP = nodeIPv6 } - nodePort := svcNetDefault.Spec.Ports[0].NodePort + nodePort := svcNodePortNetDefault.Spec.Ports[0].NodePort return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", curlConnectionTimeoutCode, true }), - ginkgo.Entry("UDN pod to a different node nodeport service in default network should work", + ginkgo.Entry("[ETP=Cluster] UDN pod to a different node nodeport service in default network should work", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // podsNetA[0] is on nodes[0]. We need a different node. podNetDefault is on nodes[1]. @@ -1441,11 +1462,11 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" if ipFamily == utilnet.IPv6 { nodeIP = nodeIPv6 } - nodePort := svcNetDefault.Spec.Ports[0].NodePort + nodePort := svcNodePortNetDefault.Spec.Ports[0].NodePort return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", "", false }), - ginkgo.Entry("UDN pod to the same node nodeport service in same UDN network should work", + ginkgo.Entry("[ETP=Cluster] UDN pod to the same node nodeport service in same UDN network should work", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // The service is backed by pods in podsNetA. @@ -1458,13 +1479,13 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" if ipFamily == utilnet.IPv6 { nodeIP = nodeIPv6 } - nodePort := svcNetA.Spec.Ports[0].NodePort + nodePort := svcNodePortNetA.Spec.Ports[0].NodePort // The service can be backed by any of the pods in podsNetA, so we can't reliably check the output hostname. // Just check that the connection is successful. return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", "", false }), - ginkgo.Entry("UDN pod to a different node nodeport service in same UDN network should work", + ginkgo.Entry("[ETP=Cluster] UDN pod to a different node nodeport service in same UDN network should work", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // The service is backed by pods in podsNetA. @@ -1477,12 +1498,12 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" if ipFamily == utilnet.IPv6 { nodeIP = nodeIPv6 } - nodePort := svcNetA.Spec.Ports[0].NodePort + nodePort := svcNodePortNetA.Spec.Ports[0].NodePort // sourceIP will be joinSubnetIP for nodeports, so only using hostname endpoint return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", "", false }), - ginkgo.Entry("UDN pod to the same node nodeport service in different UDN network should not work", + ginkgo.Entry("[ETP=Cluster] UDN pod to the same node nodeport service in different UDN network should not work", // FIXME: This test should work: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5419 // This traffic flow is expected to work eventually but doesn't work today on Layer3 (v4 and v6) and Layer2 (v4 and v6) networks. // Reason it doesn't work today is because UDN networks don't have MAC bindings for masqueradeIPs of other networks. @@ -1502,11 +1523,11 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" if ipFamily == utilnet.IPv6 { nodeIP = nodeIPv6 } - nodePort := svcNetB.Spec.Ports[0].NodePort + nodePort := svcNodePortNetB.Spec.Ports[0].NodePort // sourceIP will be joinSubnetIP for nodeports, so only using hostname endpoint return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", curlConnectionTimeoutCode, true }), - ginkgo.Entry("UDN pod to a different node nodeport service in different UDN network should work", + ginkgo.Entry("[ETP=Cluster] UDN pod to a different node nodeport service in different UDN network should work", func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // The service is backed by podNetB. @@ -1519,11 +1540,171 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" if ipFamily == utilnet.IPv6 { nodeIP = nodeIPv6 } - nodePort := svcNetB.Spec.Ports[0].NodePort + nodePort := svcNodePortNetB.Spec.Ports[0].NodePort // sourceIP will be joinSubnetIP for nodeports, so only using hostname endpoint return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", "", false }), + ginkgo.Entry("[ETP=LOCAL] UDN pod to the same node nodeport service in same UDN network should work", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + clientPod := podsNetA[0] + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), clientPod.Spec.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } + nodePortA := svcNodePortETPLocalNetA.Spec.Ports[0].NodePort + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePortA)) + "/hostname", "", false + }), + + ginkgo.Entry("[ETP=LOCAL] UDN pod to a different node nodeport service in same UDN network should work", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + clientPod := podsNetA[0] + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), podsNetA[2].Spec.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } + nodePortA := svcNodePortETPLocalNetA.Spec.Ports[0].NodePort + out := "" + errBool := false + // FIXME https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5531#issuecomment-3749407414 + // There is a new option on ovn 25.03 and further called "ct-commit-all" that can be set for each LR. + // This should avoid the mentioned issue. + if IsGatewayModeLocal(f.ClientSet) { + // FIXME: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5846 + // its supposed to fail with 56 error code which is fine + // but due to this fwmark bug it ends up failing wtih 28 error code that's not expected. + out = curlConnectionTimeoutCode + errBool = true + if ipFamily == utilnet.IPv4 || (ipFamily == utilnet.IPv6 && !isIPv4Supported(f.ClientSet)) { + out = curlConnectionResetCode + } + } + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePortA)) + "/hostname", out, errBool + }), + + ginkgo.Entry("[ETP=LOCAL] UDN pod to the same node nodeport service in different UDN network should not work", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + // FIXME: This test should work: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5419 + clientPod := podNetB + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), clientPod.Spec.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } + nodePortA := svcNodePortETPLocalNetA.Spec.Ports[0].NodePort + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePortA)) + "/hostname", curlConnectionTimeoutCode, true + }), + ginkgo.Entry("[ETP=LOCAL] UDN pod to a different node nodeport service in different UDN network should work", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + clientPod := podNetB + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), podsNetA[0].Spec.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } + nodePortA := svcNodePortETPLocalNetA.Spec.Ports[0].NodePort + out := "" + errBool := false + + // FIXME https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5531#issuecomment-3749407414 + // There is a new option on ovn 25.03 and further called "ct-commit-all" that can be set for each LR. + // This should avoid the mentioned issue. + if IsGatewayModeLocal(f.ClientSet) { + // FIXME: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5846 + // its supposed to fail with 56 error code which is fine + // but due to this fwmark bug it ends up failing wtih 28 error code that's not expected. + out = curlConnectionTimeoutCode + errBool = true + if ipFamily == utilnet.IPv4 || (ipFamily == utilnet.IPv6 && !isIPv4Supported(f.ClientSet)) { + out = curlConnectionResetCode + } + } + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePortA)) + "/hostname", out, errBool + }), + ginkgo.Entry("[ETP=LOCAL] UDN pod to the same node nodeport service in default network should not work", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + // FIXME: This test should work: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5419 + clientPod := podNetB + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), clientPod.Spec.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } + nodePortB := svcNodePortETPLocalDefault.Spec.Ports[0].NodePort + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePortB)) + "/hostname", curlConnectionTimeoutCode, true + }), + ginkgo.Entry("[ETP=LOCAL] UDN pod to a different node nodeport service in default network should work", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + // podsNetA[0] is on nodes[0]. We need a different node. podNetDefault is on nodes[1]. + // So we hit nodeport on nodes[1]. + clientPod := podsNetA[0] + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), podNetDefault.Spec.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } + nodePortB := svcNodePortETPLocalDefault.Spec.Ports[0].NodePort + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePortB)) + "/hostname", "", false + }), + ginkgo.Entry("[ETP=LOCAL] Default network pod to same node nodeport service in UDN network should not work", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + // FIXME: This test should work: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5419 + clientPod := podNetDefault + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), clientPod.Spec.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } + nodePortA := svcNodePortETPLocalNetA.Spec.Ports[0].NodePort + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePortA)) + "/hostname", curlConnectionTimeoutCode, true + }), + ginkgo.Entry("[ETP=LOCAL] Default network pod to different node nodeport service in UDN network should work", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + // podNetDefault is on nodes[1]. We need a different node. podsNetA[0] is on nodes[0]. + // So we hit nodeport on nodes[0]. + clientPod := podNetDefault + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), podsNetA[0].Spec.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } + nodePortA := svcNodePortETPLocalNetA.Spec.Ports[0].NodePort + out := "" + errBool := false + + // FIXME https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5531#issuecomment-3749407414 + // There is a new option on ovn 25.03 and further called "ct-commit-all" that can be set for each LR. + // This should avoid the mentioned issue. + if IsGatewayModeLocal(f.ClientSet) { + // FIXME: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5846 + // its supposed to fail with 56 error code which is fine + // but due to this fwmark bug it ends up failing wtih 28 error code that's not expected. + out = curlConnectionTimeoutCode + errBool = true + if ipFamily == utilnet.IPv4 || (ipFamily == utilnet.IPv6 && !isIPv4Supported(f.ClientSet)) { + out = curlConnectionResetCode + } + } + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePortA)) + "/hostname", out, errBool + }), ) }, @@ -2276,7 +2457,7 @@ type templateInputFRR struct { var ratestdata embed.FS var tmplDir = filepath.Join("testdata", "routeadvertisements") -const frrImage = "quay.io/frrouting/frr:9.1.3" +const frrImage = "quay.io/frrouting/frr:10.4.1" // generateFRRConfiguration to establish a BGP session towards the provided // neighbors in the network's VRF configured to advertised the provided diff --git a/test/e2e/testscenario/cudn/invalid-scenarios-evpn.go b/test/e2e/testscenario/cudn/invalid-scenarios-evpn.go new file mode 100644 index 0000000000..5dffc7b4b1 --- /dev/null +++ b/test/e2e/testscenario/cudn/invalid-scenarios-evpn.go @@ -0,0 +1,475 @@ +package cudn + +import "github.com/ovn-org/ovn-kubernetes/test/e2e/testscenario" + +var EVPNCUDNInvalid = []testscenario.ValidateCRScenario{ + { + Description: "EVPN transport requires evpn configuration field", + ExpectedErr: `spec.evpn field is required when transport is 'EVPN'`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-no-config +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN +`, + }, + { + Description: "evpn configuration field is forbidden when transport is not 'EVPN'", + ExpectedErr: `spec.evpn field is forbidden when transport is not 'EVPN'`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-config-no-transport +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 +`, + }, + { + Description: "EVPN is not supported for Secondary networks", + ExpectedErr: `transport 'EVPN' is only supported for Layer2 or Layer3 primary networks`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-secondary +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Secondary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 +`, + }, + { + Description: "EVPN is not supported for Localnet topology", + ExpectedErr: `transport 'EVPN' is only supported for Layer2 or Layer3 primary networks`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-localnet +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Localnet + localnet: + role: Secondary + physicalNetworkName: physnet1 + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 +`, + }, + { + Description: "Layer2 EVPN requires macVRF", + ExpectedErr: `spec.evpn.macVRF field is required for Layer2 topology when transport is 'EVPN'`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l2-evpn-no-macvrf +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + ipVRF: + vni: 100 +`, + }, + { + Description: "Layer3 EVPN requires ipVRF", + ExpectedErr: `spec.evpn.ipVRF field is required for Layer3 topology when transport is 'EVPN'`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l3-evpn-no-ipvrf +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer3 + layer3: + role: Primary + subnets: + - cidr: "10.20.100.0/16" + hostSubnet: 24 + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 +`, + }, + { + Description: "Layer3 EVPN forbids macVRF", + ExpectedErr: `spec.evpn.macVRF field is forbidden for Layer3 topology when transport is 'EVPN'`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l3-evpn-with-macvrf +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer3 + layer3: + role: Primary + subnets: + - cidr: "10.20.100.0/16" + hostSubnet: 24 + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + ipVRF: + vni: 101 +`, + }, + { + Description: "evpn requires at least macVRF or ipVRF", + ExpectedErr: `at least one of macVRF or ipVRF must be specified`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-no-vrf +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep +`, + }, + { + Description: "VNI must be at least 1", + ExpectedErr: `spec.network.evpn.macVRF.vni: Invalid value`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-vni-zero +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 0 +`, + }, + { + Description: "VNI must not exceed 16777215", + ExpectedErr: `spec.network.evpn.macVRF.vni: Invalid value`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-vni-too-large +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 16777216 +`, + }, + { + Description: "routeTarget must be in valid format", + ExpectedErr: `RT must contain exactly one colon`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-format +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "invalid" +`, + }, + { + Description: "routeTarget 4-byte AS requires 2-byte local admin (6-byte constraint)", + ExpectedErr: `RT with 4-byte ASN global administrator must have format GHJK:MN where GHJK <= 4294967295 and MN <= 65535`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-4byte-as +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "4200000000:70000" +`, + }, + { + Description: "routeTarget IPv4 format requires valid IPv4 address", + ExpectedErr: `RT global administrator must be either '*', an IPv4 address, or a number`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-ipv4 +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "999.999.999.999:100" +`, + }, + { + Description: "routeTarget IPv4 format requires 2-byte local admin", + ExpectedErr: `RT with IPv4 global administrator must have format A.B.C.D:MN where MN <= 65535`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-ipv4-local +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "192.168.1.1:70000" +`, + }, + { + Description: "routeTarget format must have exactly one colon", + ExpectedErr: `RT must contain exactly one colon`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-multiple-colons +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65000:100:200" +`, + }, + { + Description: "routeTarget format must include colon separator", + ExpectedErr: `RT must contain exactly one colon`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-no-colon +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65000" +`, + }, + { + Description: "VTEP name is required in evpn", + ExpectedErr: `spec.network.evpn.vtep`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-no-vtep +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + macVRF: + vni: 100 +`, + }, + { + Description: "VTEP name cannot be empty", + ExpectedErr: `spec.network.evpn.vtep`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-empty-vtep +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: "" + macVRF: + vni: 100 +`, + }, + { + Description: "routeTarget local administrator must be a number", + ExpectedErr: `RT local administrator must be a number`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-local-nan +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65000:abc" +`, + }, + { + Description: "routeTarget wildcard format local admin must not exceed 4294967295", + ExpectedErr: `RT with wildcard global administrator must have format *:OPQR where OPQR <= 4294967295`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-wildcard-overflow +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "*:5000000000" +`, + }, + { + Description: "routeTarget exceeds maximum length of 21 characters", + ExpectedErr: `Too long: may not be longer than 21`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-invalid-rt-too-long +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "255.255.255.255:655350" +`, + }, +} diff --git a/test/e2e/testscenario/cudn/invalid-scenarios-no-overlay.go b/test/e2e/testscenario/cudn/invalid-scenarios-no-overlay.go index 3f52313a8d..7938d2c55d 100644 --- a/test/e2e/testscenario/cudn/invalid-scenarios-no-overlay.go +++ b/test/e2e/testscenario/cudn/invalid-scenarios-no-overlay.go @@ -20,7 +20,7 @@ spec: subnets: - 10.10.0.0/16 transport: NoOverlay - noOverlayOptions: + noOverlay: outboundSNAT: Enabled routing: Managed `, @@ -43,7 +43,7 @@ spec: - cidr: 10.10.0.0/16 hostSubnet: 24 transport: NoOverlay - noOverlayOptions: + noOverlay: outboundSNAT: Enabled routing: Managed `, @@ -66,14 +66,14 @@ spec: subnets: - 10.10.0.0/16 transport: NoOverlay - noOverlayOptions: + noOverlay: outboundSNAT: Enabled routing: Managed `, }, { - Description: "noOverlayOptions is required when transport is NoOverlay", - ExpectedErr: `noOverlayOptions is required when transport is 'NoOverlay'`, + Description: "noOverlay is required when transport is NoOverlay", + ExpectedErr: `spec.noOverlay is required when type transport is 'NoOverlay'`, Manifest: ` apiVersion: k8s.ovn.org/v1 kind: ClusterUserDefinedNetwork @@ -92,8 +92,8 @@ spec: `, }, { - Description: "noOverlayOptions is forbidden when transport is Geneve", - ExpectedErr: `noOverlayOptions is forbidden when transport is not 'NoOverlay'`, + Description: "noOverlay is forbidden when transport is Geneve", + ExpectedErr: `spec.noOverlay is forbidden when transport type is not 'NoOverlay'`, Manifest: ` apiVersion: k8s.ovn.org/v1 kind: ClusterUserDefinedNetwork @@ -109,14 +109,14 @@ spec: - cidr: 10.10.0.0/16 hostSubnet: 24 transport: Geneve - noOverlayOptions: + noOverlay: outboundSNAT: Enabled routing: Managed `, }, { - Description: "noOverlayOptions is forbidden when transport is not set (defaults to Geneve)", - ExpectedErr: `noOverlayOptions is forbidden when transport is not 'NoOverlay'`, + Description: "noOverlay is forbidden when transport is not set (defaults to Geneve)", + ExpectedErr: `spec.noOverlay is forbidden when transport type is not 'NoOverlay'`, Manifest: ` apiVersion: k8s.ovn.org/v1 kind: ClusterUserDefinedNetwork @@ -131,7 +131,7 @@ spec: subnets: - cidr: 10.10.0.0/16 hostSubnet: 24 - noOverlayOptions: + noOverlay: outboundSNAT: Enabled routing: Managed `, diff --git a/test/e2e/testscenario/cudn/valid-scenarios-evpn.go b/test/e2e/testscenario/cudn/valid-scenarios-evpn.go new file mode 100644 index 0000000000..105a454844 --- /dev/null +++ b/test/e2e/testscenario/cudn/valid-scenarios-evpn.go @@ -0,0 +1,425 @@ +package cudn + +import "github.com/ovn-org/ovn-kubernetes/test/e2e/testscenario" + +var EVPNCUDNValid = []testscenario.ValidateCRScenario{ + { + Description: "valid Layer2 Primary EVPN with macVRF", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l2-evpn-primary +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 +`, + }, + { + Description: "valid Layer2 Primary EVPN with macVRF and routeTarget", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l2-evpn-with-rt +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65000:100" +`, + }, + { + Description: "valid Layer2 Primary EVPN with both macVRF and ipVRF", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l2-evpn-mac-ip-vrf +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + ipVRF: + vni: 101 +`, + }, + { + Description: "valid Layer3 Primary EVPN with ipVRF", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l3-evpn-primary +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer3 + layer3: + role: Primary + subnets: + - cidr: "10.20.100.0/16" + hostSubnet: 24 + transport: EVPN + evpn: + vtep: evpn-vtep + ipVRF: + vni: 200 +`, + }, + { + Description: "valid Layer3 Primary EVPN with ipVRF and routeTarget", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l3-evpn-with-rt +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer3 + layer3: + role: Primary + subnets: + - cidr: "10.20.100.0/16" + hostSubnet: 24 + transport: EVPN + evpn: + vtep: evpn-vtep + ipVRF: + vni: 200 + routeTarget: "65000:200" +`, + }, + { + Description: "valid Layer2 Primary EVPN dual-stack", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: l2-evpn-dual-stack +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24", "2001:db8::/64"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 300 +`, + }, + { + Description: "valid EVPN with max VNI value", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-max-vni +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 16777215 +`, + }, + { + Description: "valid EVPN with 4-byte ASN in routeTarget (2-byte local admin)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-4byte-asn +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "4200000000:100" +`, + }, + { + Description: "valid EVPN with 2-byte ASN and 4-byte local admin", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-2byte-asn-4byte-local +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65000:16777215" +`, + }, + { + Description: "valid EVPN with IPv4 address format routeTarget", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-ipv4-rt +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "192.168.1.1:100" +`, + }, + { + Description: "valid EVPN with wildcard AS routeTarget", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-wildcard-rt +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "*:100" +`, + }, + { + Description: "valid EVPN with wildcard AS and large local admin", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-wildcard-large-local +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "*:16777215" +`, + }, + { + Description: "valid EVPN with AS=0 (FRR allows)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-as-zero +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "0:100" +`, + }, + { + Description: "valid EVPN with local admin=0 (FRR allows)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-local-zero +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65000:0" +`, + }, + { + Description: "valid EVPN with wildcard and local admin=0 (FRR allows)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-wildcard-zero +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "*:0" +`, + }, + { + Description: "valid EVPN with max 2-byte AS boundary (65535)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-2byte-as-max +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65535:100" +`, + }, + { + Description: "valid EVPN with min 4-byte AS boundary (65536)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-4byte-as-min +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65536:100" +`, + }, + { + Description: "valid EVPN with max local admin for 2-byte AS (4294967295)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-2byte-as-max-local +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "65000:4294967295" +`, + }, + { + Description: "valid EVPN with max length routeTarget (21 chars: 255.255.255.255:65535)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: evpn-max-length-rt +spec: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: red}} + network: + topology: Layer2 + layer2: + role: Primary + subnets: ["10.20.100.0/24"] + transport: EVPN + evpn: + vtep: evpn-vtep + macVRF: + vni: 100 + routeTarget: "255.255.255.255:65535" +`, + }, +} diff --git a/test/e2e/testscenario/cudn/valid-scenarios-no-overlay.go b/test/e2e/testscenario/cudn/valid-scenarios-no-overlay.go index 260fa68fae..94dababda8 100644 --- a/test/e2e/testscenario/cudn/valid-scenarios-no-overlay.go +++ b/test/e2e/testscenario/cudn/valid-scenarios-no-overlay.go @@ -21,7 +21,7 @@ spec: - cidr: 10.10.0.0/16 hostSubnet: 24 transport: NoOverlay - noOverlayOptions: + noOverlay: outboundSNAT: Enabled routing: Managed `, @@ -44,7 +44,7 @@ spec: - cidr: 10.20.0.0/16 hostSubnet: 24 transport: NoOverlay - noOverlayOptions: + noOverlay: outboundSNAT: Disabled routing: Unmanaged `, @@ -67,7 +67,7 @@ spec: - cidr: 10.30.0.0/16 hostSubnet: 24 transport: NoOverlay - noOverlayOptions: + noOverlay: outboundSNAT: Disabled routing: Managed `, @@ -90,7 +90,7 @@ spec: - cidr: 10.40.0.0/16 hostSubnet: 24 transport: NoOverlay - noOverlayOptions: + noOverlay: outboundSNAT: Enabled routing: Unmanaged `, @@ -115,7 +115,7 @@ spec: - cidr: fd00:10:50::/48 hostSubnet: 64 transport: NoOverlay - noOverlayOptions: + noOverlay: outboundSNAT: Enabled routing: Managed `, diff --git a/test/e2e/testscenario/vtep/invalid-scenarios.go b/test/e2e/testscenario/vtep/invalid-scenarios.go new file mode 100644 index 0000000000..6cf6ce4f97 --- /dev/null +++ b/test/e2e/testscenario/vtep/invalid-scenarios.go @@ -0,0 +1,101 @@ +package vtep + +import "github.com/ovn-org/ovn-kubernetes/test/e2e/testscenario" + +var Invalid = []testscenario.ValidateCRScenario{ + { + Description: "CIDR must be a valid network address (not a host IP)", + ExpectedErr: `CIDR must be a valid network address`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-invalid-cidr-not-network +spec: + cidrs: + - "10.20.100.1/24" +`, + }, + { + Description: "CIDR cannot be empty", + ExpectedErr: `spec.cidrs: Invalid value`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-empty-cidrs +spec: + cidrs: [] +`, + }, + { + Description: "CIDRs cannot have more than 2 items", + ExpectedErr: `spec.cidrs: Too many`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-too-many-cidrs +spec: + cidrs: + - "10.20.100.0/24" + - "10.30.100.0/24" + - "10.40.100.0/24" +`, + }, + { + Description: "Dual-stack CIDRs must be from different IP families", + ExpectedErr: `When 2 CIDRs are set, they must be from different IP families`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-same-family-cidrs +spec: + cidrs: + - "10.20.100.0/24" + - "10.30.100.0/24" +`, + }, + { + Description: "Dual-stack CIDRs must be from different IP families (IPv6)", + ExpectedErr: `When 2 CIDRs are set, they must be from different IP families`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-same-family-cidrs-ipv6 +spec: + cidrs: + - "fd00:10:20::/64" + - "fd00:10:30::/64" +`, + }, + { + Description: "CIDR must be in valid format", + ExpectedErr: `CIDR must be a valid network address`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-invalid-cidr-format +spec: + cidrs: + - "invalid-cidr" +`, + }, + { + Description: "Mode must be a valid enum value", + ExpectedErr: `spec.mode: Unsupported value`, + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-invalid-mode +spec: + cidrs: + - "10.20.100.0/24" + mode: "InvalidMode" +`, + }, +} diff --git a/test/e2e/testscenario/vtep/valid-scenarios.go b/test/e2e/testscenario/vtep/valid-scenarios.go new file mode 100644 index 0000000000..15bf42f996 --- /dev/null +++ b/test/e2e/testscenario/vtep/valid-scenarios.go @@ -0,0 +1,82 @@ +package vtep + +import "github.com/ovn-org/ovn-kubernetes/test/e2e/testscenario" + +var Valid = []testscenario.ValidateCRScenario{ + { + Description: "Valid VTEP with single IPv4 CIDR and default mode", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-ipv4-default +spec: + cidrs: + - "100.64.0.0/24" +`, + }, + { + Description: "Valid VTEP with single IPv4 CIDR and Managed mode", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-ipv4-managed +spec: + cidrs: + - "100.65.0.0/24" + mode: Managed +`, + }, + { + Description: "Valid VTEP with single IPv4 CIDR and Unmanaged mode", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-ipv4-unmanaged +spec: + cidrs: + - "100.66.0.0/24" + mode: Unmanaged +`, + }, + { + Description: "Valid VTEP with single IPv6 CIDR", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-ipv6 +spec: + cidrs: + - "fd00:100:64::/64" +`, + }, + { + Description: "Valid VTEP with dual-stack CIDRs", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-dualstack +spec: + cidrs: + - "100.67.0.0/24" + - "fd00:100:67::/64" +`, + }, + { + Description: "Valid VTEP with dual-stack CIDRs (IPv6 first)", + Manifest: ` +apiVersion: k8s.ovn.org/v1 +kind: VTEP +metadata: + name: vtep-dualstack-ipv6-first +spec: + cidrs: + - "fd00:100:68::/64" + - "100.68.0.0/24" +`, + }, +} diff --git a/test/e2e/util.go b/test/e2e/util.go index d23208bc31..5a9715d4a8 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -657,11 +657,14 @@ func waitClusterHealthy(f *framework.Framework, numControlPlanePods int, control }) } -// waitForRollout waits for the daemon set in a given namespace to be +// updateAndWaitForRollout waits for the resource in a given namespace to be // successfully rolled out following an update. // +// The updateFunc parameter is a callback that performs the update operation +// (e.g., applying a new configuration). +// // If allowedNotReadyNodes is -1, this method returns immediately without waiting. -func waitForRollout(c kubernetes.Interface, ns string, resource string, allowedNotReadyNodes int32, timeout time.Duration) error { +func updateAndWaitForRollout(c kubernetes.Interface, ns string, resource string, allowedNotReadyNodes int32, timeout time.Duration, updateFunc func()) error { if allowedNotReadyNodes == -1 { return nil } @@ -673,8 +676,25 @@ func waitForRollout(c kubernetes.Interface, ns string, resource string, allowedN resourceType := resourceAtoms[0] resourceName := resourceAtoms[1] + var oldGeneration int64 + switch resourceType { + case "daemonset", "daemonsets", "ds": + ds, err := c.AppsV1().DaemonSets(ns).Get(context.TODO(), resourceName, metav1.GetOptions{}) + if err != nil { + return err + } + oldGeneration = ds.Generation + case "deployment", "deployments", "deploy": + dp, err := c.AppsV1().Deployments(ns).Get(context.TODO(), resourceName, metav1.GetOptions{}) + if err != nil { + return err + } + oldGeneration = dp.Generation + } + updateFunc() + start := time.Now() - framework.Logf("Waiting up to %v for daemonset %s in namespace %s to update", + framework.Logf("Waiting up to %v for %s in namespace %s to update", timeout, resource, ns) return wait.Poll(framework.Poll, timeout, func() (bool, error) { @@ -710,6 +730,10 @@ func waitForRollout(c kubernetes.Interface, ns string, resource string, allowedN } if generation <= observedGeneration { + if generation <= oldGeneration { + framework.Logf("Waiting for %s generation to increase (currently %d)...", resource, generation) + return false, nil + } if updated < desired { framework.Logf("Waiting for %s rollout to finish: %d out of %d new pods have been updated (%d seconds elapsed)", resource, updated, desired, int(time.Since(start).Seconds())) @@ -1143,7 +1167,7 @@ func wrappedTestFramework(basename string) *framework.Framework { coredumpDir := "/tmp/kind/logs/coredumps" dbLocation := "/var/lib/openvswitch" // https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5782 - skippedCoredumps := []string{"zebra", "bgpd", "mgmtd"} + skippedCoredumps := []string{"zebra", "bgpd", "mgmtd", "bfdd"} // Check for coredumps on host var coredumpFiles []string @@ -1219,6 +1243,7 @@ func wrappedTestFramework(basename string) *framework.Framework { func newPrivelegedTestFramework(basename string) *framework.Framework { f := framework.NewDefaultFramework(basename) f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + f.NamespacePodSecurityWarnLevel = admissionapi.LevelPrivileged f.DumpAllNamespaceInfo = func(ctx context.Context, f *framework.Framework, namespace string) { debug.DumpAllNamespaceInfo(context.TODO(), f.ClientSet, namespace) } @@ -1271,17 +1296,30 @@ func setUnsetTemplateContainerEnv(c kubernetes.Interface, namespace, resource, c args := []string{"set", "env", resource, "-c", container} env := make([]string, 0, len(set)+len(unset)) for k, v := range set { - env = append(env, fmt.Sprintf("%s=%s", k, v)) + currentValue := getTemplateContainerEnv(namespace, resource, container, k) + if currentValue != v { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } } for _, k := range unset { - env = append(env, fmt.Sprintf("%s-", k)) + currentValue := getTemplateContainerEnv(namespace, resource, container, k) + if currentValue != "" { + env = append(env, fmt.Sprintf("%s-", k)) + } + } + + if len(env) == 0 { + framework.Logf("No environment changes needed for %s container %s in namespace %s, skipping update", resource, container, namespace) + return } + framework.Logf("Setting environment in %s container %s of namespace %s to %v", resource, container, namespace, env) - e2ekubectl.RunKubectlOrDie(namespace, append(args, env...)...) // Make sure the change has rolled out // TODO (Change this to use the exported upstream function) - err := waitForRollout(c, namespace, resource, 0, rolloutTimeout) + err := updateAndWaitForRollout(c, namespace, resource, 0, rolloutTimeout, func() { + e2ekubectl.RunKubectlOrDie(namespace, append(args, env...)...) + }) framework.ExpectNoError(err) } @@ -1392,6 +1430,11 @@ func isInterconnectEnabled() bool { return present && val == "true" } +func isDynamicUDNEnabled() bool { + val, present := os.LookupEnv("DYNAMIC_UDN_ALLOCATION") + return present && val == "true" +} + func isNetworkSegmentationEnabled() bool { val, present := os.LookupEnv("ENABLE_NETWORK_SEGMENTATION") return present && val == "true" @@ -1402,6 +1445,11 @@ func isLocalGWModeEnabled() bool { return present && val == "local" } +func isHelmEnabled() bool { + val, present := os.LookupEnv("USE_HELM") + return present && val == "true" +} + func isPreConfiguredUdnAddressesEnabled() bool { ovnKubeNamespace := deploymentconfig.Get().OVNKubernetesNamespace() val := getTemplateContainerEnv(ovnKubeNamespace, "daemonset/ovnkube-node", getNodeContainerName(), "OVN_PRE_CONF_UDN_ADDR_ENABLE") diff --git a/test/e2e/vtep_api_validations.go b/test/e2e/vtep_api_validations.go new file mode 100644 index 0000000000..80f338731e --- /dev/null +++ b/test/e2e/vtep_api_validations.go @@ -0,0 +1,52 @@ +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + + "github.com/ovn-org/ovn-kubernetes/test/e2e/feature" + "github.com/ovn-org/ovn-kubernetes/test/e2e/testscenario" + testscenariovtep "github.com/ovn-org/ovn-kubernetes/test/e2e/testscenario/vtep" +) + +var _ = Describe("EVPN: VTEP API validations", feature.RouteAdvertisements, feature.EVPN, func() { + DescribeTable("api-server should reject invalid VTEP CRs", + func(scenarios []testscenario.ValidateCRScenario) { + DeferCleanup(func() { + cleanupVTEPCRsTest(scenarios) + }) + for _, s := range scenarios { + By(s.Description) + _, stderr, err := runKubectlInputWithFullOutput("", s.Manifest, "create", "-f", "-") + Expect(err).To(HaveOccurred(), "should fail to create invalid VTEP CR") + Expect(stderr).To(ContainSubstring(s.ExpectedErr)) + } + }, + Entry("Invalid VTEP configurations", testscenariovtep.Invalid), + ) + + DescribeTable("api-server should accept valid VTEP CRs", + func(scenarios []testscenario.ValidateCRScenario) { + DeferCleanup(func() { + cleanupVTEPCRsTest(scenarios) + }) + for _, s := range scenarios { + By(s.Description) + _, err := e2ekubectl.RunKubectlInput("", s.Manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "should create valid VTEP CR successfully") + } + }, + Entry("Valid VTEP configurations", testscenariovtep.Valid), + ) +}) + +func cleanupVTEPCRsTest(scenarios []testscenario.ValidateCRScenario) { + for _, s := range scenarios { + e2ekubectl.RunKubectlInput("", s.Manifest, "delete", "-f", "-") + } + _, stderr, err := e2ekubectl.RunKubectlWithFullOutput("", "get", "vteps") + Expect(err).NotTo(HaveOccurred()) + Expect(stderr).To(Equal("No resources found\n")) +} diff --git a/test/scripts/e2e-cp.sh b/test/scripts/e2e-cp.sh index a786f22f0f..e9cee42c41 100755 --- a/test/scripts/e2e-cp.sh +++ b/test/scripts/e2e-cp.sh @@ -161,9 +161,8 @@ if [[ "${WHAT}" = "$SERIAL_LABEL" ]]; then shift # don't "focus" on Serial since we filter by label fi -BGP_TESTS="BGP" if [ "$ENABLE_ROUTE_ADVERTISEMENTS" != true ]; then - skip $BGP_TESTS + skip_label "Feature:RouteAdvertisements" else if [ "$ADVERTISE_DEFAULT_NETWORK" = true ]; then # Filter out extended RouteAdvertisements tests to keep job run time down @@ -188,6 +187,11 @@ else # https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5569 skip "Multi Homing" fi + if [ "$PLATFORM_IPV4_SUPPORT" == true ] && [ "$PLATFORM_IPV6_SUPPORT" == false ]; then + # Skip IPv6/dual-stack multihoming secondary network tests in IPv4-only clusters. + skip "Multi Homing.*L3 - routed - secondary network with IPv6 subnet" + skip "Multi Homing.*L3 - routed - secondary network with a dual stack configuration" + fi # these tests require metallb but the configuration we do for it is not compatible with the configuration we do to advertise the default network # TODO: consolidate configuration skip "Load Balancer Service Tests with MetalLB" @@ -201,6 +205,7 @@ else skip "e2e egress IP validation Cluster Default Network Should validate egress IP logic when one pod is managed by more than one egressIP object" skip "e2e egress IP validation Cluster Default Network Should re-assign egress IPs when node readiness / reachability goes down/up" skip "Pod to external server PMTUD when a client ovnk pod targeting an external server is created when tests are run towards the agnhost echo server queries to the hostNetworked server pod on another node shall work for UDP" + skip "e2e egress IP validation Cluster Default Network Should handle EIP reassignment correctly on namespace and pod label updates, and EIP object updates" # https://issues.redhat.com/browse/OCPBUGS-55028 skip "e2e egress IP validation Cluster Default Network \[secondary-host-eip\]" @@ -225,6 +230,20 @@ if [ "${PARALLEL:-false}" = "true" ]; then skip_label "$SERIAL_LABEL" fi +if [ "$ENABLE_NO_OVERLAY" == true ]; then + # No-overlay mode uses underlying network infrastructure directly. + # Overlay-dependent features are not supported. + skip_label "Feature:Multicast" + skip_label "Feature:EgressIP" + skip_label "Feature:EgressService" + # This test validates MTU reduction behavior specific to overlay mode (1500->1400). + # In no-overlay mode, pods use the full underlying network MTU without reduction. + skip "blocking ICMP needs frag" + # This test validates MTU reduction due to Geneve encapsulation overhead (1400->1342). + # In no-overlay mode, there is no encapsulation and thus no MTU overhead. + skip "Pod to pod TCP with low MTU" +fi + # setting these is required to make RuntimeClass tests work ... :/ export KUBE_CONTAINER_RUNTIME=remote export KUBE_CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock diff --git a/test/scripts/e2e-kind.sh b/test/scripts/e2e-kind.sh index 0d086e62b8..42b3ddc3ca 100755 --- a/test/scripts/e2e-kind.sh +++ b/test/scripts/e2e-kind.sh @@ -125,9 +125,6 @@ should function for service endpoints using hostNetwork # Skips when default pod network is advertised through BGP RA_SKIPPED_TESTS=" -# Pod to ETP local nodeport on a different node is broken -# https://github.com/ovn-org/ovn-kubernetes/issues/4804 -\[sig-network\] Services should fallback to local terminating endpoints when there are no ready endpoints with externalTrafficPolicy=Local " # Github CI doesn´t offer IPv6 connectivity, so always skip IPv6 only tests.