Merge pull request #163 from l1b0k/master

BSWANG · web-flow · commit f275c0f3a191 · 2021-01-05T23:20:49.000+08:00
check nic status
diff --git a/daemon/daemon.go b/daemon/daemon.go
@@ -18,6 +18,8 @@ import (
 	"github.com/AliyunContainerService/terway/pkg/tracing"
 	"github.com/AliyunContainerService/terway/rpc"
 	"github.com/AliyunContainerService/terway/types"
+	"github.com/containernetworking/cni/libcni"
+	containertypes "github.com/containernetworking/cni/pkg/types"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -45,6 +47,11 @@ const (
 	tracingKeyPendingPodsCount = "pending_pods_count"
 
 	commandMapping = "mapping"
+
+	cniDefaultPath = "/opt/cni/bin"
+	// this file is generated from configmap
+	terwayCNIConf  = "/etc/eni/10-terway.conf"
+	cniExecTimeout = 10 * time.Second
 )
 
 type networkService struct {
@@ -64,6 +71,8 @@ type networkService struct {
 	pendingPodsLock sync.RWMutex
 	sync.RWMutex
 
+	cniBinPath string
+
 	rpc.UnimplementedTerwayBackendServer
 }
 
@@ -261,6 +270,9 @@ func (networkService *networkService) AllocIP(ctx context.Context, r *rpc.AllocI
 					Type: eniMultiIP.GetType(),
 				},
 			},
+			NetNs: func(s string) *string {
+				return &s
+			}(r.Netns),
 		}
 		networkContext.resources = append(networkContext.resources, newRes.Resources...)
 		if networkService.eipResMgr != nil && podinfo.EipInfo.PodEip {
@@ -319,6 +331,9 @@ func (networkService *networkService) AllocIP(ctx context.Context, r *rpc.AllocI
 					Type: vpcEni.GetType(),
 				},
 			},
+			NetNs: func(s string) *string {
+				return &s
+			}(r.Netns),
 		}
 		networkContext.resources = append(networkContext.resources, newRes.Resources...)
 		if networkService.eipResMgr != nil && podinfo.EipInfo.PodEip {
@@ -381,6 +396,9 @@ func (networkService *networkService) AllocIP(ctx context.Context, r *rpc.AllocI
 					Type: vpcVeth.GetType(),
 				},
 			},
+			NetNs: func(s string) *string {
+				return &s
+			}(r.Netns),
 		}
 		networkContext.resources = append(networkContext.resources, newRes.Resources...)
 		err = networkService.resourceDB.Put(podInfoKey(podinfo.Namespace, podinfo.Name), newRes)
@@ -679,18 +697,73 @@ func (networkService *networkService) startGarbageCollectionLoop() {
 }
 
 func (networkService *networkService) startPeriodCheck() {
-	log.Debugf("compare poll with metadata")
-	podMapping, err := networkService.GetResourceMapping()
-	if err != nil {
-		log.Error(err)
-		return
-	}
-	for _, res := range podMapping {
-		if res.Valid {
-			continue
+	// check pool
+	func() {
+		log.Debugf("compare poll with metadata")
+		podMapping, err := networkService.GetResourceMapping()
+		if err != nil {
+			log.Error(err)
+			return
 		}
-		_ = tracing.RecordPodEvent(res.Name, res.Namespace, corev1.EventTypeWarning, "ResourceInvalid", fmt.Sprintf("resource %s", res.LocalResID))
-	}
+		for _, res := range podMapping {
+			if res.Valid {
+				continue
+			}
+			_ = tracing.RecordPodEvent(res.Name, res.Namespace, corev1.EventTypeWarning, "ResourceInvalid", fmt.Sprintf("resource %s", res.LocalResID))
+		}
+	}()
+	// call CNI CHECK, make sure all dev is ok
+	func() {
+		log.Debugf("call CNI CHECK")
+		defer func() {
+			log.Debugf("call CNI CHECK end")
+		}()
+		networkService.RLock()
+		podResList, err := networkService.resourceDB.List()
+		networkService.RUnlock()
+		if err != nil {
+			log.Error(err)
+			return
+		}
+		ff, err := ioutil.ReadFile(terwayCNIConf)
+		if err != nil {
+			log.Error(err)
+			return
+		}
+		for _, v := range podResList {
+			res := v.(PodResources)
+			if res.NetNs == nil {
+				continue
+			}
+			log.Debugf("checking pod name %s", res.PodInfo.Name)
+			cniCfg := libcni.NewCNIConfig([]string{networkService.cniBinPath}, nil)
+			func() {
+				ctx, cancel := context.WithTimeout(context.Background(), cniExecTimeout)
+				defer cancel()
+				err := cniCfg.CheckNetwork(ctx, &libcni.NetworkConfig{
+					Network: &containertypes.NetConf{
+						CNIVersion: "0.4.0",
+						Name:       "terway",
+						Type:       "terway",
+					},
+					Bytes: ff,
+				}, &libcni.RuntimeConf{
+					ContainerID: "fake", // must provide
+					NetNS:       *res.NetNs,
+					IfName:      "eth0",
+					Args: [][2]string{
+						{"K8S_POD_NAME", res.PodInfo.Name},
+						{"K8S_POD_NAMESPACE", res.PodInfo.Namespace},
+					},
+				})
+				if err != nil {
+					log.Error(err)
+					return
+				}
+			}()
+		}
+	}()
+
 }
 
 // tracing
@@ -806,12 +879,17 @@ func (networkService *networkService) GetResourceMapping() ([]tracing.PodMapping
 
 func newNetworkService(configFilePath, kubeconfig, master, daemonMode string) (rpc.TerwayBackendServer, error) {
 	log.Debugf("start network service with: %s, %s", configFilePath, daemonMode)
+	cniBinPath := os.Getenv("CNI_PATH")
+	if cniBinPath == "" {
+		cniBinPath = cniDefaultPath
+	}
 	netSrv := &networkService{
 		configFilePath:  configFilePath,
 		kubeConfig:      kubeconfig,
 		master:          master,
 		pendingPods:     map[string]interface{}{},
 		pendingPodsLock: sync.RWMutex{},
+		cniBinPath:      cniBinPath,
 	}
 	if daemonMode == daemonModeENIMultiIP || daemonMode == daemonModeVPC || daemonMode == daemonModeENIOnly {
 		netSrv.daemonMode = daemonMode
diff --git a/daemon/resource_manager.go b/daemon/resource_manager.go
@@ -32,6 +32,7 @@ type ResourceItem struct {
 type PodResources struct {
 	Resources []ResourceItem
 	PodInfo   *podInfo
+	NetNs     *string
 }
 
 type resourceManagerInitItem struct {
diff --git a/go.mod b/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/vishvananda/netlink v1.1.1-0.20201206203632-88079d98e65d
 	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae
 	golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
-	golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e
+	golang.org/x/sys v0.0.0-20201221093633-bc327ba9c2f0
 	google.golang.org/grpc v1.34.0
 	google.golang.org/protobuf v1.25.0
 	gopkg.in/yaml.v2 v2.4.0
diff --git a/go.sum b/go.sum
@@ -291,8 +291,8 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
-golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201221093633-bc327ba9c2f0 h1:n+DPcgTwkgWzIFpLmoimYR2K2b0Ga5+Os4kayIN0vGo=
+golang.org/x/sys v0.0.0-20201221093633-bc327ba9c2f0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/pkg/link/interface.go b/pkg/link/interface.go
@@ -19,7 +19,7 @@ func GetDeviceNumber(mac string) (int32, error) {
 			return int32(link.Attrs().Index), nil
 		}
 	}
-	return 0, errors.Errorf("cannot found mac address: %s", mac)
+	return 0, errors.Wrapf(ErrNotFound, "can't found dev by mac %s", mac)
 }
 
 // GetDeviceName get interface device name by mac address
@@ -34,5 +34,5 @@ func GetDeviceName(mac string) (string, error) {
 			return link.Attrs().Name, nil
 		}
 	}
-	return "", errors.Errorf("cannot found mac address: %s", mac)
+	return "", errors.Wrapf(ErrNotFound, "can't found dev by mac %s", mac)
 }
diff --git a/pkg/link/interface_unsupport.go b/pkg/link/interface_unsupport.go
@@ -2,16 +2,12 @@
 
 package link
 
-import (
-	"github.com/pkg/errors"
-)
-
 // GetDeviceNumber get interface device number by mac address
 func GetDeviceNumber(mac string) (int32, error) {
-	return 0, errors.Errorf("not supported arch")
+	return 0, ErrUnsupported
 }
 
 // GetDeviceName get interface device name by mac address
 func GetDeviceName(mac string) (string, error) {
-	return "", errors.Errorf("not supported arch")
+	return "", ErrUnsupported
 }
diff --git a/pkg/link/type.go b/pkg/link/type.go
@@ -0,0 +1,10 @@
+package link
+
+import (
+	"errors"
+)
+
+var (
+	ErrUnsupported = errors.New("not supported arch")
+	ErrNotFound    = errors.New("not found")
+)
diff --git a/pkg/sysctl/sysctl.go b/pkg/sysctl/sysctl.go
@@ -0,0 +1,81 @@
+// Copyright 2019 Authors of Cilium
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sysctl
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	prefixDir = "/proc/sys"
+)
+
+// Setting represents a sysctl setting. Its purpose it to be able to iterate
+// over a slice of settings.
+type Setting struct {
+	Name      string
+	Val       string
+	IgnoreErr bool
+}
+
+func fullPath(name string) string {
+	return filepath.Join(prefixDir, strings.Replace(name, ".", "/", -1))
+}
+
+func writeSysctl(name string, value string) error {
+	fPath := fullPath(name)
+	f, err := os.OpenFile(fPath, os.O_RDWR, 0644)
+	if err != nil {
+		return fmt.Errorf("could not open the sysctl file %s: %s",
+			fPath, err)
+	}
+	defer f.Close()
+	if _, err := io.WriteString(f, value); err != nil {
+		return fmt.Errorf("could not write to the systctl file %s: %s",
+			fPath, err)
+	}
+	return nil
+}
+
+// Disable disables the given sysctl parameter.
+func Disable(name string) error {
+	return writeSysctl(name, "0")
+}
+
+// Enable enables the given sysctl parameter.
+func Enable(name string) error {
+	return writeSysctl(name, "1")
+}
+
+// Write writes the given sysctl parameter.
+func Write(name string, val string) error {
+	return writeSysctl(name, val)
+}
+
+// Read reads the given sysctl parameter.
+func Read(name string) (string, error) {
+	fPath := fullPath(name)
+	val, err := ioutil.ReadFile(fPath)
+	if err != nil {
+		return "", fmt.Errorf("Failed to read %s: %s", fPath, val)
+	}
+
+	return strings.TrimRight(string(val), "\n"), nil
+}
diff --git a/plugin/driver/drivers.go b/plugin/driver/drivers.go
@@ -21,6 +21,22 @@ var (
 	IPVlanDriver NetnsDriver = newIPVlanDriver()
 )
 
+type RecordPodEvent func(msg string)
+
+type CheckConfig struct {
+	RecordPodEvent
+
+	NetNS ns.NetNS
+
+	HostVethName string
+	DeviceID     int32
+
+	ContainerIFName string
+	// for pod
+	IPv4Addr *net.IPNet
+	Gateway  net.IP
+}
+
 // NetnsDriver to config container netns interface and routes
 type NetnsDriver interface {
 	Setup(hostVeth string,
@@ -40,6 +56,8 @@ type NetnsDriver interface {
 		containerVeth string,
 		netNS ns.NetNS,
 		containerIP net.IP) error
+
+	Check(cfg *CheckConfig) error
 }
 
 type vethDriver struct {
@@ -430,6 +448,10 @@ func (d *vethDriver) Teardown(hostIfName string,
 	return netlink.LinkDel(hostVeth)
 }
 
+func (d *vethDriver) Check(cfg *CheckConfig) error {
+	return nil
+}
+
 func setupVethPair(contVethName, pairName string, mtu int, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
 	contVeth, err := makeVethPair(contVethName, pairName, mtu)
 	if err != nil {
diff --git a/plugin/driver/ipvlan.go b/plugin/driver/ipvlan.go
diff --git a/plugin/driver/raw_nic.go b/plugin/driver/raw_nic.go
diff --git a/plugin/driver/utils.go b/plugin/driver/utils.go
diff --git a/plugin/terway/cni.go b/plugin/terway/cni.go

Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@ type ResourceItem struct {`
`32`	`32`	`type PodResources struct {`
`33`	`33`	`Resources []ResourceItem`
`34`	`34`	`PodInfo *podInfo`
	`35`	`+ NetNs *string`
`35`	`36`	`}`
`36`	`37`
`37`	`38`	`type resourceManagerInitItem struct {`
Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ func GetDeviceNumber(mac string) (int32, error) {`
`19`	`19`	`return int32(link.Attrs().Index), nil`
`20`	`20`	`}`
`21`	`21`	`}`
`22`		`- return 0, errors.Errorf("cannot found mac address: %s", mac)`
	`22`	`+ return 0, errors.Wrapf(ErrNotFound, "can't found dev by mac %s", mac)`
`23`	`23`	`}`
`24`	`24`
`25`	`25`	`// GetDeviceName get interface device name by mac address`
`@@ -34,5 +34,5 @@ func GetDeviceName(mac string) (string, error) {`
`34`	`34`	`return link.Attrs().Name, nil`
`35`	`35`	`}`
`36`	`36`	`}`
`37`		`- return "", errors.Errorf("cannot found mac address: %s", mac)`
	`37`	`+ return "", errors.Wrapf(ErrNotFound, "can't found dev by mac %s", mac)`
`38`	`38`	`}`