Skip to content

Commit

Permalink
[SecondaryNetwork] Attach OVS uplink after removing flow-restore-wait…
Browse files Browse the repository at this point in the history
… flag

Physcial network interface is attached on the OVS bridge with SecondaryNetwork
feature enabled. Antrea uses a global configuration flow-restore-wait='true'
to ensure that OVS OpenFlow entries can start working after the dependencies are
ready. A connectivity issue exists if a setup uses the Node NIC as secondary
network interface and connects the NIC to OVS bridge before removing the
flow-restore-wait option.

This change ensures agent attaches the physical network interface to the secondary
OVS bridge after the global flow-restore-wait option is removed.

Signed-off-by: Wenying Dong <[email protected]>
  • Loading branch information
wenyingd committed Jul 10, 2024
1 parent 75699cf commit ad15933
Show file tree
Hide file tree
Showing 7 changed files with 165 additions and 135 deletions.
22 changes: 16 additions & 6 deletions cmd/antrea-agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -731,14 +731,15 @@ func run(o *Options) error {
go ipamController.Run(stopCh)
}

var secondaryNetworkController *secondarynetwork.Controller
if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
defer secondarynetwork.RestoreHostInterfaceConfiguration(&o.config.SecondaryNetwork)
if err := secondarynetwork.Initialize(
secondaryNetworkController, err = secondarynetwork.NewController(
o.config.ClientConnection, o.config.KubeAPIServerOverride,
k8sClient, localPodInformer.Get(), nodeConfig.Name,
podUpdateChannel, stopCh,
&o.config.SecondaryNetwork, ovsdbConnection); err != nil {
return fmt.Errorf("failed to initialize secondary network: %v", err)
k8sClient, localPodInformer.Get(),
podUpdateChannel,
&o.config.SecondaryNetwork, ovsdbConnection)
if err != nil {
return fmt.Errorf("failed to create secondary network controller: %w", err)
}
}

Expand Down Expand Up @@ -864,6 +865,15 @@ func run(o *Options) error {
return fmt.Errorf("failed to connect uplink to OVS bridge: %w", err)
}
}
// secondaryNetworkController Initialize must be run after FlowRestoreComplete for the case that Node
// IPs are moved to the secondary OVS bridge
if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
defer secondaryNetworkController.Restore()
if err = secondaryNetworkController.Initialize(); err != nil {
return fmt.Errorf("failed to initialize secondary network: %v", err)
}
go secondaryNetworkController.Run(stopCh)
}

// statsCollector collects stats and reports to the antrea-controller periodically. For now it's only used for
// NetworkPolicy stats and Multicast stats.
Expand Down
109 changes: 109 additions & 0 deletions pkg/agent/secondarynetwork/init.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright 2024 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package secondarynetwork

import (
"fmt"
"k8s.io/klog/v2"

"github.com/TomCodeLV/OVSDB-golang-lib/pkg/ovsdb"
netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
componentbaseconfig "k8s.io/component-base/config"

"antrea.io/antrea/pkg/agent/secondarynetwork/podwatch"
agentconfig "antrea.io/antrea/pkg/config/agent"
"antrea.io/antrea/pkg/ovs/ovsconfig"
"antrea.io/antrea/pkg/util/channel"
"antrea.io/antrea/pkg/util/k8s"
)

var (
newOVSBridgeFn = ovsconfig.NewOVSBridge
)

type Controller struct {
ovsBridgeClient ovsconfig.OVSBridgeClient
secNetConfig *agentconfig.SecondaryNetworkConfig
podController *podwatch.PodController
}

func NewController(
clientConnectionConfig componentbaseconfig.ClientConnectionConfiguration,
kubeAPIServerOverride string,
k8sClient clientset.Interface,
podInformer cache.SharedIndexInformer,
podUpdateSubscriber channel.Subscriber,
secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB,
) (*Controller, error) {
ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb)
if err != nil {
return nil, err
}

// Create the NetworkAttachmentDefinition client, which handles access to secondary network object
// definition from the API Server.
netAttachDefClient, err := createNetworkAttachDefClient(clientConnectionConfig, kubeAPIServerOverride)
if err != nil {
return nil, fmt.Errorf("NetworkAttachmentDefinition client creation failed: %v", err)
}

// Create podController to handle secondary network configuration for Pods with
// k8s.v1.cni.cncf.io/networks Annotation defined.
podWatchController, err := podwatch.NewPodController(
k8sClient, netAttachDefClient, podInformer,
podUpdateSubscriber, ovsBridgeClient)
if err != nil {
return nil, err
}
return &Controller{
ovsBridgeClient: ovsBridgeClient,
secNetConfig: secNetConfig,
podController: podWatchController}, nil
}

// Run starts the Pod controller for secondary networks.
func (c *Controller) Run(stopCh <-chan struct{}) {
c.podController.Run(stopCh)
}

// CreateNetworkAttachDefClient creates net-attach-def client handle from the given config.
func createNetworkAttachDefClient(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) (netdefclient.K8sCniCncfIoV1Interface, error) {
kubeConfig, err := k8s.CreateRestConfig(config, kubeAPIServerOverride)
if err != nil {
return nil, err
}

netAttachDefClient, err := netdefclient.NewForConfig(kubeConfig)
if err != nil {
return nil, err
}
return netAttachDefClient, nil
}

func createOVSBridge(bridges []agentconfig.OVSBridgeConfig, ovsdb *ovsdb.OVSDB) (ovsconfig.OVSBridgeClient, error) {
if len(bridges) == 0 {
return nil, nil
}
// Only one OVS bridge is supported.
bridgeConfig := bridges[0]
ovsBridgeClient := newOVSBridgeFn(bridgeConfig.BridgeName, ovsconfig.OVSDatapathSystem, ovsdb)
if err := ovsBridgeClient.Create(); err != nil {
return nil, fmt.Errorf("failed to create OVS bridge %s: %v", bridgeConfig.BridgeName, err)
}
klog.InfoS("OVS bridge created", "bridge", bridgeConfig.BridgeName)
return ovsBridgeClient, nil
}
91 changes: 11 additions & 80 deletions pkg/agent/secondarynetwork/init_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,52 +21,28 @@ import (
"fmt"
"net"

"github.com/TomCodeLV/OVSDB-golang-lib/pkg/ovsdb"
netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
componentbaseconfig "k8s.io/component-base/config"
"k8s.io/klog/v2"

"antrea.io/antrea/pkg/agent/interfacestore"
"antrea.io/antrea/pkg/agent/secondarynetwork/podwatch"
"antrea.io/antrea/pkg/agent/util"
agentconfig "antrea.io/antrea/pkg/config/agent"
"antrea.io/antrea/pkg/ovs/ovsconfig"
"antrea.io/antrea/pkg/util/channel"
"antrea.io/antrea/pkg/util/k8s"
)

var (
// Funcs which will be overridden with mock funcs in tests.
interfaceByNameFn = net.InterfaceByName
newOVSBridgeFn = ovsconfig.NewOVSBridge
)

// Initialize sets up OVS bridges and starts the Pod controller for secondary networks.
func Initialize(
clientConnectionConfig componentbaseconfig.ClientConnectionConfiguration,
kubeAPIServerOverride string,
k8sClient clientset.Interface,
podInformer cache.SharedIndexInformer,
nodeName string,
podUpdateSubscriber channel.Subscriber,
stopCh <-chan struct{},
secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error {

ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb)
if err != nil {
return err
}

// Initialize sets up OVS bridges.
func (c *Controller) Initialize() error {
// We only support moving and restoring of interface configuration to OVS Bridge for the single physical interface case.
if len(secNetConfig.OVSBridges) != 0 {
phyInterfaces := make([]string, len(secNetConfig.OVSBridges[0].PhysicalInterfaces))
copy(phyInterfaces, secNetConfig.OVSBridges[0].PhysicalInterfaces)
if len(c.secNetConfig.OVSBridges) != 0 {
phyInterfaces := make([]string, len(c.secNetConfig.OVSBridges[0].PhysicalInterfaces))
copy(phyInterfaces, c.secNetConfig.OVSBridges[0].PhysicalInterfaces)
if len(phyInterfaces) == 1 {

bridgedName, _, err := util.PrepareHostInterfaceConnection(
ovsBridgeClient,
c.ovsBridgeClient,
phyInterfaces[0],
0,
map[string]interface{}{
Expand All @@ -78,49 +54,18 @@ func Initialize(
}
phyInterfaces[0] = bridgedName
}
if err = connectPhyInterfacesToOVSBridge(ovsBridgeClient, phyInterfaces); err != nil {
if err := connectPhyInterfacesToOVSBridge(c.ovsBridgeClient, phyInterfaces); err != nil {
return err
}
}

// Create the NetworkAttachmentDefinition client, which handles access to secondary network object
// definition from the API Server.
netAttachDefClient, err := createNetworkAttachDefClient(clientConnectionConfig, kubeAPIServerOverride)
if err != nil {
return fmt.Errorf("NetworkAttachmentDefinition client creation failed: %v", err)
}

// Create podController to handle secondary network configuration for Pods with
// k8s.v1.cni.cncf.io/networks Annotation defined.
if podWatchController, err := podwatch.NewPodController(
k8sClient, netAttachDefClient, podInformer,
podUpdateSubscriber, ovsBridgeClient); err != nil {
return err
} else {
go podWatchController.Run(stopCh)
}
return nil
}

// RestoreHostInterfaceConfiguration restores interface configuration from secondary-bridge back to host-interface.
func RestoreHostInterfaceConfiguration(secNetConfig *agentconfig.SecondaryNetworkConfig) {
if len(secNetConfig.OVSBridges) != 0 && len(secNetConfig.OVSBridges[0].PhysicalInterfaces) == 1 {
util.RestoreHostInterfaceConfiguration(secNetConfig.OVSBridges[0].BridgeName, secNetConfig.OVSBridges[0].PhysicalInterfaces[0])
}
}

func createOVSBridge(bridges []agentconfig.OVSBridgeConfig, ovsdb *ovsdb.OVSDB) (ovsconfig.OVSBridgeClient, error) {
if len(bridges) == 0 {
return nil, nil
// Restore restores interface configuration from secondary-bridge back to host-interface.
func (c *Controller) Restore() {
if len(c.secNetConfig.OVSBridges) != 0 && len(c.secNetConfig.OVSBridges[0].PhysicalInterfaces) == 1 {
util.RestoreHostInterfaceConfiguration(c.secNetConfig.OVSBridges[0].BridgeName, c.secNetConfig.OVSBridges[0].PhysicalInterfaces[0])
}
// Only one OVS bridge is supported.
bridgeConfig := bridges[0]
ovsBridgeClient := newOVSBridgeFn(bridgeConfig.BridgeName, ovsconfig.OVSDatapathSystem, ovsdb)
if err := ovsBridgeClient.Create(); err != nil {
return nil, fmt.Errorf("failed to create OVS bridge %s: %v", bridgeConfig.BridgeName, err)
}
klog.InfoS("OVS bridge created", "bridge", bridgeConfig.BridgeName)
return ovsBridgeClient, nil
}

func connectPhyInterfacesToOVSBridge(ovsBridgeClient ovsconfig.OVSBridgeClient, phyInterfaces []string) error {
Expand All @@ -146,17 +91,3 @@ func connectPhyInterfacesToOVSBridge(ovsBridgeClient ovsconfig.OVSBridgeClient,
}
return nil
}

// CreateNetworkAttachDefClient creates net-attach-def client handle from the given config.
func createNetworkAttachDefClient(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) (netdefclient.K8sCniCncfIoV1Interface, error) {
kubeConfig, err := k8s.CreateRestConfig(config, kubeAPIServerOverride)
if err != nil {
return nil, err
}

netAttachDefClient, err := netdefclient.NewForConfig(kubeConfig)
if err != nil {
return nil, err
}
return netAttachDefClient, nil
}
26 changes: 3 additions & 23 deletions pkg/agent/secondarynetwork/init_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,30 +17,10 @@

package secondarynetwork

import (
"errors"

"github.com/TomCodeLV/OVSDB-golang-lib/pkg/ovsdb"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
componentbaseconfig "k8s.io/component-base/config"

agentconfig "antrea.io/antrea/pkg/config/agent"
"antrea.io/antrea/pkg/util/channel"
)

func Initialize(
clientConnectionConfig componentbaseconfig.ClientConnectionConfiguration,
kubeAPIServerOverride string,
k8sClient clientset.Interface,
podInformer cache.SharedIndexInformer,
nodeName string,
podUpdateSubscriber channel.Subscriber,
stopCh <-chan struct{},
secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error {
return errors.New("not supported on Windows")
func (c *Controller) Initialize() error {
return nil
}

func RestoreHostInterfaceConfiguration(secNetConfig *agentconfig.SecondaryNetworkConfig) {
func (c *Controller) Restore() {
// Not supported on Windows.
}
Loading

0 comments on commit ad15933

Please sign in to comment.