Skip to content

Commit

Permalink
nic status check
Browse files Browse the repository at this point in the history
Signed-off-by: l1b0k <[email protected]>
  • Loading branch information
l1b0k committed Jan 5, 2021
1 parent 224ee8c commit 4f342e3
Show file tree
Hide file tree
Showing 13 changed files with 723 additions and 99 deletions.
100 changes: 89 additions & 11 deletions daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"github.com/AliyunContainerService/terway/pkg/tracing"
"github.com/AliyunContainerService/terway/rpc"
"github.com/AliyunContainerService/terway/types"
"github.com/containernetworking/cni/libcni"
containertypes "github.com/containernetworking/cni/pkg/types"

"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -45,6 +47,11 @@ const (
tracingKeyPendingPodsCount = "pending_pods_count"

commandMapping = "mapping"

cniDefaultPath = "/opt/cni/bin"
// this file is generated from configmap
terwayCNIConf = "/etc/eni/10-terway.conf"
cniExecTimeout = 10 * time.Second
)

type networkService struct {
Expand All @@ -64,6 +71,8 @@ type networkService struct {
pendingPodsLock sync.RWMutex
sync.RWMutex

cniBinPath string

rpc.UnimplementedTerwayBackendServer
}

Expand Down Expand Up @@ -261,6 +270,9 @@ func (networkService *networkService) AllocIP(ctx context.Context, r *rpc.AllocI
Type: eniMultiIP.GetType(),
},
},
NetNs: func(s string) *string {
return &s
}(r.Netns),
}
networkContext.resources = append(networkContext.resources, newRes.Resources...)
if networkService.eipResMgr != nil && podinfo.EipInfo.PodEip {
Expand Down Expand Up @@ -319,6 +331,9 @@ func (networkService *networkService) AllocIP(ctx context.Context, r *rpc.AllocI
Type: vpcEni.GetType(),
},
},
NetNs: func(s string) *string {
return &s
}(r.Netns),
}
networkContext.resources = append(networkContext.resources, newRes.Resources...)
if networkService.eipResMgr != nil && podinfo.EipInfo.PodEip {
Expand Down Expand Up @@ -381,6 +396,9 @@ func (networkService *networkService) AllocIP(ctx context.Context, r *rpc.AllocI
Type: vpcVeth.GetType(),
},
},
NetNs: func(s string) *string {
return &s
}(r.Netns),
}
networkContext.resources = append(networkContext.resources, newRes.Resources...)
err = networkService.resourceDB.Put(podInfoKey(podinfo.Namespace, podinfo.Name), newRes)
Expand Down Expand Up @@ -679,18 +697,73 @@ func (networkService *networkService) startGarbageCollectionLoop() {
}

func (networkService *networkService) startPeriodCheck() {
log.Debugf("compare poll with metadata")
podMapping, err := networkService.GetResourceMapping()
if err != nil {
log.Error(err)
return
}
for _, res := range podMapping {
if res.Valid {
continue
// check pool
func() {
log.Debugf("compare poll with metadata")
podMapping, err := networkService.GetResourceMapping()
if err != nil {
log.Error(err)
return
}
_ = tracing.RecordPodEvent(res.Name, res.Namespace, corev1.EventTypeWarning, "ResourceInvalid", fmt.Sprintf("resource %s", res.LocalResID))
}
for _, res := range podMapping {
if res.Valid {
continue
}
_ = tracing.RecordPodEvent(res.Name, res.Namespace, corev1.EventTypeWarning, "ResourceInvalid", fmt.Sprintf("resource %s", res.LocalResID))
}
}()
// call CNI CHECK, make sure all dev is ok
func() {
log.Debugf("call CNI CHECK")
defer func() {
log.Debugf("call CNI CHECK end")
}()
networkService.RLock()
podResList, err := networkService.resourceDB.List()
networkService.RUnlock()
if err != nil {
log.Error(err)
return
}
ff, err := ioutil.ReadFile(terwayCNIConf)
if err != nil {
log.Error(err)
return
}
for _, v := range podResList {
res := v.(PodResources)
if res.NetNs == nil {
continue
}
log.Debugf("checking pod name %s", res.PodInfo.Name)
cniCfg := libcni.NewCNIConfig([]string{networkService.cniBinPath}, nil)
func() {
ctx, cancel := context.WithTimeout(context.Background(), cniExecTimeout)
defer cancel()
err := cniCfg.CheckNetwork(ctx, &libcni.NetworkConfig{
Network: &containertypes.NetConf{
CNIVersion: "0.4.0",
Name: "terway",
Type: "terway",
},
Bytes: ff,
}, &libcni.RuntimeConf{
ContainerID: "fake", // must provide
NetNS: *res.NetNs,
IfName: "eth0",
Args: [][2]string{
{"K8S_POD_NAME", res.PodInfo.Name},
{"K8S_POD_NAMESPACE", res.PodInfo.Namespace},
},
})
if err != nil {
log.Error(err)
return
}
}()
}
}()

}

// tracing
Expand Down Expand Up @@ -806,12 +879,17 @@ func (networkService *networkService) GetResourceMapping() ([]tracing.PodMapping

func newNetworkService(configFilePath, kubeconfig, master, daemonMode string) (rpc.TerwayBackendServer, error) {
log.Debugf("start network service with: %s, %s", configFilePath, daemonMode)
cniBinPath := os.Getenv("CNI_PATH")
if cniBinPath == "" {
cniBinPath = cniDefaultPath
}
netSrv := &networkService{
configFilePath: configFilePath,
kubeConfig: kubeconfig,
master: master,
pendingPods: map[string]interface{}{},
pendingPodsLock: sync.RWMutex{},
cniBinPath: cniBinPath,
}
if daemonMode == daemonModeENIMultiIP || daemonMode == daemonModeVPC || daemonMode == daemonModeENIOnly {
netSrv.daemonMode = daemonMode
Expand Down
1 change: 1 addition & 0 deletions daemon/resource_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ type ResourceItem struct {
type PodResources struct {
Resources []ResourceItem
PodInfo *podInfo
NetNs *string
}

type resourceManagerInitItem struct {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
github.com/vishvananda/netlink v1.1.1-0.20201206203632-88079d98e65d
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e
golang.org/x/sys v0.0.0-20201221093633-bc327ba9c2f0
google.golang.org/grpc v1.34.0
google.golang.org/protobuf v1.25.0
gopkg.in/yaml.v2 v2.4.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -291,8 +291,8 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201221093633-bc327ba9c2f0 h1:n+DPcgTwkgWzIFpLmoimYR2K2b0Ga5+Os4kayIN0vGo=
golang.org/x/sys v0.0.0-20201221093633-bc327ba9c2f0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
4 changes: 2 additions & 2 deletions pkg/link/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func GetDeviceNumber(mac string) (int32, error) {
return int32(link.Attrs().Index), nil
}
}
return 0, errors.Errorf("cannot found mac address: %s", mac)
return 0, errors.Wrapf(ErrNotFound, "can't found dev by mac %s", mac)
}

// GetDeviceName get interface device name by mac address
Expand All @@ -34,5 +34,5 @@ func GetDeviceName(mac string) (string, error) {
return link.Attrs().Name, nil
}
}
return "", errors.Errorf("cannot found mac address: %s", mac)
return "", errors.Wrapf(ErrNotFound, "can't found dev by mac %s", mac)
}
8 changes: 2 additions & 6 deletions pkg/link/interface_unsupport.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,12 @@

package link

import (
"github.com/pkg/errors"
)

// GetDeviceNumber get interface device number by mac address
func GetDeviceNumber(mac string) (int32, error) {
return 0, errors.Errorf("not supported arch")
return 0, ErrUnsupported
}

// GetDeviceName get interface device name by mac address
func GetDeviceName(mac string) (string, error) {
return "", errors.Errorf("not supported arch")
return "", ErrUnsupported
}
10 changes: 10 additions & 0 deletions pkg/link/type.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package link

import (
"errors"
)

var (
ErrUnsupported = errors.New("not supported arch")
ErrNotFound = errors.New("not found")
)
81 changes: 81 additions & 0 deletions pkg/sysctl/sysctl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sysctl

import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
)

const (
prefixDir = "/proc/sys"
)

// Setting represents a sysctl setting. Its purpose it to be able to iterate
// over a slice of settings.
type Setting struct {
Name string
Val string
IgnoreErr bool
}

func fullPath(name string) string {
return filepath.Join(prefixDir, strings.Replace(name, ".", "/", -1))
}

func writeSysctl(name string, value string) error {
fPath := fullPath(name)
f, err := os.OpenFile(fPath, os.O_RDWR, 0644)
if err != nil {
return fmt.Errorf("could not open the sysctl file %s: %s",
fPath, err)
}
defer f.Close()
if _, err := io.WriteString(f, value); err != nil {
return fmt.Errorf("could not write to the systctl file %s: %s",
fPath, err)
}
return nil
}

// Disable disables the given sysctl parameter.
func Disable(name string) error {
return writeSysctl(name, "0")
}

// Enable enables the given sysctl parameter.
func Enable(name string) error {
return writeSysctl(name, "1")
}

// Write writes the given sysctl parameter.
func Write(name string, val string) error {
return writeSysctl(name, val)
}

// Read reads the given sysctl parameter.
func Read(name string) (string, error) {
fPath := fullPath(name)
val, err := ioutil.ReadFile(fPath)
if err != nil {
return "", fmt.Errorf("Failed to read %s: %s", fPath, val)
}

return strings.TrimRight(string(val), "\n"), nil
}
22 changes: 22 additions & 0 deletions plugin/driver/drivers.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,22 @@ var (
IPVlanDriver NetnsDriver = newIPVlanDriver()
)

type RecordPodEvent func(msg string)

type CheckConfig struct {
RecordPodEvent

NetNS ns.NetNS

HostVethName string
DeviceID int32

ContainerIFName string
// for pod
IPv4Addr *net.IPNet
Gateway net.IP
}

// NetnsDriver to config container netns interface and routes
type NetnsDriver interface {
Setup(hostVeth string,
Expand All @@ -40,6 +56,8 @@ type NetnsDriver interface {
containerVeth string,
netNS ns.NetNS,
containerIP net.IP) error

Check(cfg *CheckConfig) error
}

type vethDriver struct {
Expand Down Expand Up @@ -430,6 +448,10 @@ func (d *vethDriver) Teardown(hostIfName string,
return netlink.LinkDel(hostVeth)
}

func (d *vethDriver) Check(cfg *CheckConfig) error {
return nil
}

func setupVethPair(contVethName, pairName string, mtu int, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
contVeth, err := makeVethPair(contVethName, pairName, mtu)
if err != nil {
Expand Down
Loading

0 comments on commit 4f342e3

Please sign in to comment.