Skip to content

OCPBUGS-70130: [release-4.21] Remove IC zone migration HACK code #2988

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aswinsuryan wants to merge 1 commit into from
Closed

OCPBUGS-70130: [release-4.21] Remove IC zone migration HACK code #2988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 3 additions & 186 deletions go-controller/pkg/node/default_node_network_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,60 +702,6 @@ func getOVNSBZone() (string, error) {
return dbZone, nil
}

/** HACK BEGIN **/
// TODO(tssurya): Remove this HACK a few months from now.
// checkOVNSBNodeLRSR returns true if the logical router static route for the
// the given nodeSubnet is present in the SBDB
func checkOVNSBNodeLRSR(nodeSubnet *net.IPNet) bool {
var matchv4, matchv6 string
v6 := true
v4 := true
if config.IPv6Mode && utilnet.IsIPv6CIDR(nodeSubnet) {
matchv6 = fmt.Sprintf("match=\"reg7 == 0 && ip6.dst == %s\"", nodeSubnet)
stdout, stderr, err := util.RunOVNSbctl("--bare", "--columns", "_uuid", "find", "logical_flow", matchv6)
klog.Infof("Upgrade Hack: checkOVNSBNodeLRSR for node - %s : match %s : stdout - %s : stderr - %s : err %v",
nodeSubnet, matchv6, stdout, stderr, err)
v6 = (err == nil && stderr == "" && stdout != "")
}
if config.IPv4Mode && !utilnet.IsIPv6CIDR(nodeSubnet) {
matchv4 = fmt.Sprintf("match=\"reg7 == 0 && ip4.dst == %s\"", nodeSubnet)
stdout, stderr, err := util.RunOVNSbctl("--bare", "--columns", "_uuid", "find", "logical_flow", matchv4)
klog.Infof("Upgrade Hack: checkOVNSBNodeLRSR for node - %s : match %s : stdout - %s : stderr - %s : err %v",
nodeSubnet, matchv4, stdout, stderr, err)
v4 = (err == nil && stderr == "" && stdout != "")
}
return v6 && v4
}

func fetchLBNames() string {
stdout, stderr, err := util.RunOVNSbctl("--bare", "--columns", "name", "find", "Load_Balancer")
if err != nil || stderr != "" {
klog.Errorf("Upgrade hack: fetchLBNames could not fetch services %v/%v", err, stderr)
return stdout // will be empty and we will retry
}
klog.Infof("Upgrade Hack: fetchLBNames: stdout - %s : stderr - %s : err %v", stdout, stderr, err)
return stdout
}

// lbExists returns true if the OVN load balancer for the corresponding namespace/name
// was created
func lbExists(lbNames, namespace, name string) bool {
stitchedServiceName := "Service_" + namespace + "/" + name
match := strings.Contains(lbNames, stitchedServiceName)
klog.Infof("Upgrade Hack: lbExists for service - %s/%s/%s : match - %v",
namespace, name, stitchedServiceName, match)
return match
}

func portExists(namespace, name string) bool {
lspName := fmt.Sprintf("logical_port=%s", util.GetLogicalPortName(namespace, name))
stdout, stderr, err := util.RunOVNSbctl("--bare", "--columns", "_uuid", "find", "Port_Binding", lspName)
klog.Infof("Upgrade Hack: portExists for pod - %s/%s : stdout - %s : stderr - %s", namespace, name, stdout, stderr)
return err == nil && stderr == "" && stdout != ""
}

/** HACK END **/

// Init executes the first steps to start the DefaultNodeNetworkController.
// It is split from Start() and executed before UserDefinedNodeNetworkController (UDNNC)
// to allow UDNNC to reference the openflow manager created in Init.
Expand Down Expand Up @@ -826,12 +772,9 @@ func (nc *DefaultNodeNetworkController) Init(ctx context.Context) error {
return fmt.Errorf("timed out waiting for the node zone %s to match the OVN Southbound db zone, err: %v, err1: %v", config.Default.Zone, err, err1)
}

// if its nonIC OR IC=true and if its phase1 OR if its IC to IC upgrades
if !config.OVNKubernetesFeature.EnableInterconnect || sbZone == types.OvnDefaultZone || util.HasNodeMigratedZone(node) { // if its nonIC or if its phase1
for _, auth := range []config.OvnAuthConfig{config.OvnNorth, config.OvnSouth} {
if err := auth.SetDBAuth(); err != nil {
return err
}
for _, auth := range []config.OvnAuthConfig{config.OvnNorth, config.OvnSouth} {
if err := auth.SetDBAuth(); err != nil {
return err
}
}

Expand Down Expand Up @@ -965,7 +908,6 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error {
klog.Infof("Starting the default node network controller")

var err error
var node *corev1.Node

if nc.mgmtPortController == nil {
return fmt.Errorf("default node network controller hasn't been pre-started")
Expand All @@ -977,12 +919,6 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error {
if err := level.Set("5"); err != nil {
klog.Errorf("Setting klog \"loglevel\" to 5 failed, err: %v", err)
}

if node, err = nc.watchFactory.GetNode(nc.name); err != nil {
return fmt.Errorf("error retrieving node %s: %v", nc.name, err)
}

nodeAnnotator := kube.NewNodeAnnotator(nc.Kube, node.Name)
waiter := newStartupWaiter()

// Complete gateway initialization
Expand Down Expand Up @@ -1010,125 +946,6 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error {
}
}

/** HACK BEGIN **/
// TODO(tssurya): Remove this HACK a few months from now. This has been added only to
// minimize disruption for upgrades when moving to interconnect=true.
// We want the legacy ovnkube-master to wait for remote ovnkube-node to
// signal it using "k8s.ovn.org/remote-zone-migrated" annotation before
// considering a node as remote when we upgrade from "global" (1 zone IC)
// zone to multi-zone. This is so that network disruption for the existing workloads
// is negligible and until the point where ovnkube-node flips the switch to connect
// to the new SBDB, it would continue talking to the legacy RAFT ovnkube-sbdb to ensure
// OVN/OVS flows are intact.
// STEP1: ovnkube-node start's up in remote zone and sets the "k8s.ovn.org/zone-name" above.
// STEP2: We delay the flip of connection for ovnkube-node(ovn-controller) to the new remote SBDB
// until the new remote ovnkube-controller has finished programming all the K8s core objects
// like routes, services and pods. Until then the ovnkube-node will talk to legacy SBDB.
// STEP3: Once we get the signal that the new SBDB is ready, we set the "k8s.ovn.org/remote-zone-migrated" annotation
// STEP4: We call setDBAuth to now point to new SBDB
// STEP5: Legacy ovnkube-master sees "k8s.ovn.org/remote-zone-migrated" annotation on this node and now knows that
// this node has remote-zone-migrated successfully and tears down old setup and creates new IC resource
// plumbing (takes 80ms based on what we saw in CI runs so we might still have that small window of disruption).
// NOTE: ovnkube-node in DPU host mode doesn't go through upgrades for OVN-IC and has no SBDB to connect to. Thus this part shall be skipped.
var syncNodes, syncServices, syncPods bool
if config.OvnKubeNode.Mode != types.NodeModeDPUHost && config.OVNKubernetesFeature.EnableInterconnect && nc.sbZone != types.OvnDefaultZone && !util.HasNodeMigratedZone(node) {
klog.Info("Upgrade Hack: Interconnect is enabled")
var err1 error
start := time.Now()
err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 300*time.Second, true, func(_ context.Context) (bool, error) {
// we loop through all the nodes in the cluster and ensure ovnkube-controller has finished creating the LRSR required for pod2pod overlay communication
if !syncNodes {
nodes, err := nc.watchFactory.GetNodes()
if err != nil {
err1 = fmt.Errorf("upgrade hack: error retrieving node %s: %v", nc.name, err)
return false, nil
}
for _, node := range nodes {
node := *node
if nc.name != node.Name && util.GetNodeZone(&node) != config.Default.Zone && !util.NoHostSubnet(&node) {
nodeSubnets, err := util.ParseNodeHostSubnetAnnotation(&node, types.DefaultNetworkName)
if err != nil {
if util.IsAnnotationNotSetError(err) {
klog.Infof("Skipping node %q. k8s.ovn.org/node-subnets annotation was not found", node.Name)
continue
}
err1 = fmt.Errorf("unable to fetch node-subnet annotation for node %s: err, %v", node.Name, err)
return false, nil
}
for _, nodeSubnet := range nodeSubnets {
klog.Infof("Upgrade Hack: node %s, subnet %s", node.Name, nodeSubnet)
if !checkOVNSBNodeLRSR(nodeSubnet) {
err1 = fmt.Errorf("upgrade hack: unable to find LRSR for node %s", node.Name)
return false, nil
}
}
}
}
klog.Infof("Upgrade Hack: Syncing nodes took %v", time.Since(start))
syncNodes = true
}
// we loop through all existing services in the cluster and ensure ovnkube-controller has finished creating LoadBalancers required for services to work
if !syncServices {
services, err := nc.watchFactory.GetServices()
if err != nil {
err1 = fmt.Errorf("upgrade hack: error retrieving the services %v", err)
return false, nil
}
lbNames := fetchLBNames()
for _, s := range services {
// don't process headless service
if !util.ServiceTypeHasClusterIP(s) || !util.IsClusterIPSet(s) {
continue
}
if !lbExists(lbNames, s.Namespace, s.Name) {
return false, nil
}
}
klog.Infof("Upgrade Hack: Syncing services took %v", time.Since(start))
syncServices = true
}
if !syncPods {
pods, err := nc.watchFactory.GetAllPods()
if err != nil {
err1 = fmt.Errorf("upgrade hack: error retrieving the services %v", err)
return false, nil
}
for _, p := range pods {
if !util.PodScheduled(p) || util.PodCompleted(p) || util.PodWantsHostNetwork(p) {
continue
}
if p.Spec.NodeName != nc.name {
// remote pod
continue
}
if !portExists(p.Namespace, p.Name) {
return false, nil
}
}
klog.Infof("Upgrade Hack: Syncing pods took %v", time.Since(start))
syncPods = true
}
return true, nil
})
if err != nil {
return fmt.Errorf("upgrade hack: failed while waiting for the remote ovnkube-controller to be ready: %v, %v", err, err1)
}
if err := util.SetNodeZoneMigrated(nodeAnnotator, nc.sbZone); err != nil {
return fmt.Errorf("upgrade hack: failed to set node zone annotation for node %s: %w", nc.name, err)
}
if err := nodeAnnotator.Run(); err != nil {
return fmt.Errorf("upgrade hack: failed to set node %s annotations: %w", nc.name, err)
}
klog.Infof("ovnkube-node %s finished annotating node with remote-zone-migrated; took: %v", nc.name, time.Since(start))
for _, auth := range []config.OvnAuthConfig{config.OvnNorth, config.OvnSouth} {
if err := auth.SetDBAuth(); err != nil {
return fmt.Errorf("upgrade hack: Unable to set the authentication towards OVN local dbs")
}
}
klog.Infof("Upgrade hack: ovnkube-node %s finished setting DB Auth; took: %v", nc.name, time.Since(start))
}
/** HACK END **/

// Wait for management port and gateway resources to be created by the master
klog.Infof("Waiting for gateway and management port readiness...")
start := time.Now()
Expand Down
14 changes: 0 additions & 14 deletions go-controller/pkg/ovn/base_network_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -984,20 +984,6 @@ func (bnc *BaseNetworkController) GetLocalZoneNodes() ([]*corev1.Node, error) {

// isLocalZoneNode returns true if the node is part of the local zone.
func (bnc *BaseNetworkController) isLocalZoneNode(node *corev1.Node) bool {
/** HACK BEGIN **/
// TODO(tssurya): Remove this HACK a few months from now. This has been added only to
// minimize disruption for upgrades when moving to interconnect=true.
// We want the legacy ovnkube-master to wait for remote ovnkube-node to
// signal it using "k8s.ovn.org/remote-zone-migrated" annotation before
// considering a node as remote when we upgrade from "global" (1 zone IC)
// zone to multi-zone. This is so that network disruption for the existing workloads
// is negligible and until the point where ovnkube-node flips the switch to connect
// to the new SBDB, it would continue talking to the legacy RAFT ovnkube-sbdb to ensure
// OVN/OVS flows are intact.
if bnc.zone == types.OvnDefaultZone {
return !util.HasNodeMigratedZone(node)
}
/** HACK END **/
return util.GetNodeZone(node) == bnc.zone
}

Expand Down
27 changes: 1 addition & 26 deletions go-controller/pkg/ovn/controller/services/node_tracker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,6 @@ type nodeInfo struct {

// The node's zone
zone string
/** HACK BEGIN **/
// has the node migrated to remote?
migrated bool
/** HACK END **/

// The list of node's management IPs
mgmtIPs []net.IP
Expand Down Expand Up @@ -127,7 +123,6 @@ func (nt *nodeTracker) Start(nodeInformer coreinformers.NodeInformer) (cache.Res
oldObj.Name != newObj.Name ||
util.NodeHostCIDRsAnnotationChanged(oldObj, newObj) ||
util.NodeZoneAnnotationChanged(oldObj, newObj) ||
util.NodeMigratedZoneAnnotationChanged(oldObj, newObj) ||
util.NoHostSubnet(oldObj) != util.NoHostSubnet(newObj) {
nt.updateNode(newObj)
}
Expand All @@ -154,7 +149,7 @@ func (nt *nodeTracker) Start(nodeInformer coreinformers.NodeInformer) (cache.Res

// updateNodeInfo updates the node info cache, and syncs all services
// if it changed.
func (nt *nodeTracker) updateNodeInfo(nodeName, switchName, routerName, chassisID string, l3gatewayAddresses, hostAddresses []net.IP, podSubnets []*net.IPNet, mgmtIPs []net.IP, zone string, nodePortDisabled, migrated bool) {
func (nt *nodeTracker) updateNodeInfo(nodeName, switchName, routerName, chassisID string, l3gatewayAddresses, hostAddresses []net.IP, podSubnets []*net.IPNet, mgmtIPs []net.IP, zone string, nodePortDisabled bool) {
ni := nodeInfo{
name: nodeName,
l3gatewayAddresses: l3gatewayAddresses,
Expand All @@ -166,7 +161,6 @@ func (nt *nodeTracker) updateNodeInfo(nodeName, switchName, routerName, chassisI
chassisID: chassisID,
nodePortDisabled: nodePortDisabled,
zone: zone,
migrated: migrated,
}
for i := range podSubnets {
ni.podSubnets = append(ni.podSubnets, *podSubnets[i]) // de-pointer
Expand Down Expand Up @@ -275,7 +269,6 @@ func (nt *nodeTracker) updateNode(node *corev1.Node) {
mgmtIPs,
util.GetNodeZone(node),
!nodePortEnabled,
util.HasNodeMigratedZone(node),
)
}

Expand All @@ -285,24 +278,6 @@ func (nt *nodeTracker) updateNode(node *corev1.Node) {
func (nt *nodeTracker) getZoneNodes() []nodeInfo {
out := make([]nodeInfo, 0, len(nt.nodes))
for _, node := range nt.nodes {
/** HACK BEGIN **/
// TODO(tssurya): Remove this HACK a few months from now. This has been added only to
// minimize disruption for upgrades when moving to interconnect=true.
// We want the legacy ovnkube-master to wait for remote ovnkube-node to
// signal it using "k8s.ovn.org/remote-zone-migrated" annotation before
// considering a node as remote when we upgrade from "global" (1 zone IC)
// zone to multi-zone. This is so that network disruption for the existing workloads
// is negligible and until the point where ovnkube-node flips the switch to connect
// to the new SBDB, it would continue talking to the legacy RAFT ovnkube-sbdb to ensure
// OVN/OVS flows are intact. Legacy ovnkube-master must not delete the service load
// balancers for this node till it has finished migration
if nt.zone == types.OvnDefaultZone {
if !node.migrated {
out = append(out, node)
}
continue
}
/** HACK END **/
if node.zone == nt.zone {
out = append(out, node)
}
Expand Down
14 changes: 0 additions & 14 deletions go-controller/pkg/ovn/egressip.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1178,20 +1178,6 @@ func (e *EgressIPController) isPodScheduledinLocalZone(pod *corev1.Pod) bool {

// isLocalZoneNode returns true if the node is part of the local zone.
func (e *EgressIPController) isLocalZoneNode(node *corev1.Node) bool {
/** HACK BEGIN **/
// TODO(tssurya): Remove this HACK a few months from now. This has been added only to
// minimize disruption for upgrades when moving to interconnect=true.
// We want the legacy ovnkube-master to wait for remote ovnkube-node to
// signal it using "k8s.ovn.org/remote-zone-migrated" annotation before
// considering a node as remote when we upgrade from "global" (1 zone IC)
// zone to multi-zone. This is so that network disruption for the existing workloads
// is negligible and until the point where ovnkube-node flips the switch to connect
// to the new SBDB, it would continue talking to the legacy RAFT ovnkube-sbdb to ensure
// OVN/OVS flows are intact.
if e.zone == types.OvnDefaultZone {
return !util.HasNodeMigratedZone(node)
}
/** HACK END **/
return util.GetNodeZone(node) == e.zone
}

Expand Down
Loading