Rarp

.gitignore

@@ -7,6 +7,7 @@ dist/images/kube-ovn-cmd
 dist/images/kube-ovn-daemon
 dist/images/kube-ovn-pinger
 dist/images/kube-ovn-webhook
+dist/images/vpcnatgateway/send-rarp-request
 dist/windows/kube-ovn.exe
 dist/windows/kube-ovn-daemon.exe
 ovnnb_db.*.backup

Makefile

@@ -103,6 +103,7 @@ build-go:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -v ./cmd
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon -v ./cmd/daemon
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-pinger -v ./cmd/pinger
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/vpcnatgateway/send-rarp-request -v ./cmd/send-rarp-request
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/test-server -v ./test/server
 
 .PHONY: build-go-windows
@@ -117,6 +118,7 @@ build-go-arm:
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -v ./cmd
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon -v ./cmd/daemon
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-pinger -v ./cmd/pinger
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/vpcnatgateway/send-rarp-request -v ./cmd/send-rarp-request
 
 .PHONY: build-kube-ovn
 build-kube-ovn: build-debug build-go
@@ -170,7 +172,7 @@ image-kube-ovn-dpdk: build-go
 	docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk --build-arg VERSION=$(RELEASE_TAG) --build-arg BASE_TAG=$(RELEASE_TAG)-dpdk -o type=docker -f dist/images/Dockerfile dist/images/
 
 .PHONY: image-vpc-nat-gateway
-image-vpc-nat-gateway:
+image-vpc-nat-gateway: build-go
 	docker buildx build --platform linux/amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
 
 .PHOONY: image-test
@@ -181,7 +183,7 @@ image-test: build-go
 release: lint image-kube-ovn image-vpc-nat-gateway
 
 .PHONY: release-arm
-release-arm: release-arm-debug image-kube-ovn-arm64
+release-arm: build-go-arm release-arm-debug image-kube-ovn-arm64
 	docker buildx build --platform linux/arm64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
 
 .PHONY: release-arm-debug

cmd/send-rarp-request/main.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"os"
+
+	"github.com/mdlayher/arp"
+	"github.com/mdlayher/ethernet"
+	"github.com/mdlayher/packet"
+)
+
+const rarpProtocol = 0x8035
+const rarpRequestOp = 3
+
+func exit(msg string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, msg, args...)
+	os.Exit(1)
+}
+
+func exitOnError(err error, msg string, args ...interface{}) {
+	if err != nil {
+		exit(msg, append(args, err)...)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 && len(os.Args) != 3 {
+		exit("Usage: %s <interface> [ip address]\n", os.Args[0])
+	}
+
+	addr := netip.IPv4Unspecified()
+	if len(os.Args) == 3 {
+		var err error
+		addr, err = netip.ParseAddr(os.Args[2])
+		exitOnError(err, "failed to parse ip address %q: %v", os.Args[2])
+		if !addr.Is4() {
+			exit("Invalid IPv4 addresses %q\n", os.Args[2])
+		}
+	}
+
+	iface, err := net.InterfaceByName(os.Args[1])
+	exitOnError(err, "failed to get interface %q: %v", os.Args[1])
+
+	p, err := packet.Listen(iface, packet.Raw, rarpProtocol, nil)
+	exitOnError(err, "failed to listen on interface %s: %v", os.Args[1])
+	defer p.Close()
+
+	arp, err := arp.NewPacket(rarpRequestOp, iface.HardwareAddr, addr, ethernet.Broadcast, addr)
+	exitOnError(err, "failed to create rarp request packet: %v")
+
+	pb, err := arp.MarshalBinary()
+	exitOnError(err, "failed to convert rarp request packet to binary: %v")
+
+	f := &ethernet.Frame{
+		Destination: ethernet.Broadcast,
+		Source:      arp.SenderHardwareAddr,
+		EtherType:   rarpProtocol,
+		Payload:     pb,
+	}
+
+	fb, err := f.MarshalBinary()
+	exitOnError(err, "failed to convert ethernet frame to binary: %v")
+
+	_, err = p.WriteTo(fb, &packet.Addr{HardwareAddr: arp.SenderHardwareAddr})
+	exitOnError(err, "failed to send rarp request packet: %v")
+}

dist/images/Dockerfile.base

@@ -55,9 +55,7 @@ RUN cd /usr/src/ && git clone -b branch-24.03 --depth=1 https://github.com/ovn-o
     # fix lr-lb dnat with multiple distributed gateway ports
     curl -s https://github.com/kubeovn/ovn/commit/3eb25ce85393e97814c6a2b70101afdc66c6626c.patch | git apply && \
     # northd: skip arp/nd request for lrp addresses from localnet ports
-    curl -s https://github.com/kubeovn/ovn/commit/1aa952f2f1b348739d2585618a6f2afff2ed4501.patch | git apply && \
-    # ovn-controller: make activation strategy work for single chassis
-    curl -s https://github.com/kubeovn/ovn/commit/487cd1c6c0cc9def7cedaaaf3dd4bc99c23974c4.patch | git apply
+    curl -s https://github.com/kubeovn/ovn/commit/1aa952f2f1b348739d2585618a6f2afff2ed4501.patch | git apply
 
 RUN apt install -y build-essential fakeroot \
     autoconf automake bzip2 debhelper-compat dh-exec dh-python dh-sequence-python3 dh-sequence-sphinxdoc \

dist/images/vpcnatgateway/Dockerfile

@@ -14,3 +14,4 @@ RUN set -ex \
 WORKDIR /kube-ovn
 COPY nat-gateway.sh /kube-ovn/
 COPY lb-svc.sh /kube-ovn/
+COPY send-rarp-request /bin/

dist/images/vpcnatgateway/nat-gateway.sh

@@ -41,6 +41,10 @@ function init() {
     iptables -t nat -A SNAT_FILTER -j SHARED_SNAT
 }
 
+function activate() {
+    echo "activating interface eth0 by sending a RARP request"
+    exec_cmd "send-rarp-request eth0"
+}
 
 function get_iptables_version() {
   exec_cmd "iptables --version"
@@ -406,6 +410,9 @@ case $opt in
         echo "init $rules"
         init $rules
         ;;
+    activate)
+        activate
+        ;;
     subnet-route-add)
         echo "subnet-route-add $rules"
         add_vpc_internal_route $rules
@@ -475,7 +482,7 @@ case $opt in
         qos_del $rules
         ;;
     *)
-        echo "Usage: $0 [init|subnet-route-add|subnet-route-del|eip-add|eip-del|floating-ip-add|floating-ip-del|dnat-add|dnat-del|snat-add|snat-del] ..."
+        echo "Usage: $0 [init|activate|subnet-route-add|subnet-route-del|eip-add|eip-del|floating-ip-add|floating-ip-del|dnat-add|dnat-del|snat-add|snat-del] ..."
         exit 1
         ;;
 esac

go.mod

@@ -23,6 +23,8 @@ require (
 	github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589
 	github.com/kubeovn/ovsdb v0.0.0-20240410091831-5dd26006c475
 	github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875
+	github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118
+	github.com/mdlayher/packet v1.1.2
 	github.com/moby/sys/mountinfo v0.7.2
 	github.com/onsi/ginkgo/v2 v2.20.0
 	github.com/onsi/gomega v1.34.1
@@ -150,8 +152,6 @@ require (
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect
-	github.com/mdlayher/packet v1.1.2 // indirect
 	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect

pkg/controller/controller.go

@@ -1154,6 +1154,9 @@ func (c *Controller) startWorkers(ctx context.Context) {
 	go wait.Until(func() {
 		c.resyncVpcNatGwConfig()
 	}, time.Second, ctx.Done())
+	go wait.Until(func() {
+		c.activateVpcNatGateway()
+	}, time.Second, ctx.Done())
 
 	go wait.Until(func() {
 		c.resyncVpcNatConfig()

pkg/controller/vpc_nat_gateway.go

@@ -37,6 +37,7 @@
 
 const (
 	natGwInit             = "init"
+	natGwActivate         = "activate"
 	natGwEipAdd           = "eip-add"
 	natGwEipDel           = "eip-del"
 	natGwDnatAdd          = "dnat-add"
@@ -94,6 +95,96 @@
 	klog.Info("finish establishing vpc-nat-gateway")
 }
 
+func (c *Controller) activateVpcNatGateway() {
+	gateways, err := c.vpcNatGatewayLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list vpc nat gateway: %v", err)
+		return
+	}
+
+	eips, err := c.iptablesEipsLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list eips: %v", err)
+		return
+	}
+
+	snats, err := c.iptablesSnatRulesLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list snats: %v", err)
+		return
+	}
+
+LOOP:
+	for _, gw := range gateways {
+		cachedPod, err := c.getNatGwPod(gw.Name)
+		if err != nil {
+			klog.Infof("failed to get nat gw %s pod: %v", gw.Name, err)
+			continue
+		}
+
+		pod := cachedPod.DeepCopy()
+		if cachedPod.Status.Phase != corev1.PodRunning {
+			continue
+		}
+		if pod.Annotations[util.VpcNatGatewayInitAnnotation] != "true" ||
+			pod.Annotations[util.VpcNatGatewayActivatedAnnotation] == "true" {
+			continue
+		}
+
+		startTime := pod.Status.ContainerStatuses[0].State.Running.StartedAt.Time
+		fnCheckRedo := func(redo string) bool {
+			if redo == "" {
+				return true
+			}
+			t, err := time.ParseInLocation("2006-01-02T15:04:05", redo, time.Local)
+			if err != nil {
+				klog.Errorf("failed to parse redo time %s: %v", redo, err)
+				return false
+			}
+			return t.Before(startTime)
+		}
+
+		// TODO: wait routes to be configured
+		// TODO: performace optimization
+		for _, eip := range eips {
+			if eip.Spec.NatGwDp != gw.Name {
+				continue
+			}
+			if !eip.Status.Ready || !fnCheckRedo(eip.Status.Redo) {
+				continue LOOP
+			}
+
+			for _, snat := range snats {
+				if snat.Spec.EIP != eip.Name {
+					continue
+				}
+				if !snat.Status.Ready || !fnCheckRedo(snat.Status.Redo) {
+					continue LOOP
+				}
+			}
+		}
+
+		klog.Infof("activating vpc nat gw %s", gw.Name)
+		if err = c.execNatGwRules(pod, natGwActivate, nil); err != nil {
+			klog.Errorf("failed to activate vpc nat gateway %s: %v", gw.Name, err)
+			continue
+		}
+
+		pod.Annotations[util.VpcNatGatewayActivatedAnnotation] = "true"
+		patch, err := util.GenerateStrategicMergePatchPayload(cachedPod, pod)
+		if err != nil {
+			klog.Error(err)
+			continue
+		}
+		if _, err = c.config.KubeClient.CoreV1().Pods(pod.Namespace).Patch(context.Background(), pod.Name,
+			types.StrategicMergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {
+			err := fmt.Errorf("patch pod %s/%s failed %v", pod.Name, pod.Namespace, err)
+			klog.Error(err)
+			continue
+		}
+	}
+}
+
 func (c *Controller) enqueueAddVpcNatGw(obj interface{}) {
 	var key string
 	var err error
@@ -323,6 +414,7 @@
 			return err
 		}
 	}
+
 	// if update qos success, will update nat gw status
 	if gw.Spec.QoSPolicy != gw.Status.QoSPolicy {
 		if err = c.patchNatGwQoSStatus(key, gw.Spec.QoSPolicy); err != nil {