Skip to content

Commit 2f9c682

Browse files
mye956mye956
committed
Checking if require tc show command works before advertising fault injection capability
1 parent 2c19f03 commit 2f9c682

File tree

3 files changed

+128
-4
lines changed

3 files changed

+128
-4
lines changed

‎agent/app/agent_capability.go

+1
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,7 @@ func (agent *ecsAgent) appendFaultInjectionCapabilities(capabilities []*ecs.Attr
553553

554554
if isFaultInjectionToolingAvailable() {
555555
capabilities = appendNameOnlyAttribute(capabilities, attributePrefix+capabilityFaultInjection)
556+
seelog.Debug("Fault injection capability is enabled.")
556557
} else {
557558
seelog.Warn("Fault injection capability not enabled: Required network tools are missing")
558559
}

‎agent/app/agent_capability_unix.go

+23-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package app
1818

1919
import (
2020
"context"
21+
"fmt"
2122
"os/exec"
2223
"path/filepath"
2324
"strings"
@@ -30,6 +31,7 @@ import (
3031
"github.com/aws/amazon-ecs-agent/agent/taskresource/volume"
3132
"github.com/aws/amazon-ecs-agent/agent/utils"
3233
"github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/model/ecs"
34+
"github.com/aws/amazon-ecs-agent/ecs-agent/tmds/utils/netconfig"
3335
"github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper"
3436
"github.com/aws/aws-sdk-go/aws"
3537
"github.com/cihub/seelog"
@@ -45,6 +47,7 @@ const (
4547
modInfoCmd = "modinfo"
4648
faultInjectionKernelModules = "sch_netem"
4749
ctxTimeoutDuration = 60 * time.Second
50+
tcShowCmdString = "tc -j q show dev %s parent 1:1"
4851
)
4952

5053
var (
@@ -250,6 +253,7 @@ var isFaultInjectionToolingAvailable = checkFaultInjectionTooling
250253
// wrapper around exec.LookPath
251254
var lookPathFunc = exec.LookPath
252255
var osExecWrapper = execwrapper.NewExec()
256+
var networkConfigClient = netconfig.NewNetworkConfigClient()
253257

254258
// checkFaultInjectionTooling checks for the required network packages like iptables, tc
255259
// to be available on the host before ecs.capability.fault-injection can be advertised
@@ -263,7 +267,7 @@ func checkFaultInjectionTooling() bool {
263267
return false
264268
}
265269
}
266-
return checkFaultInjectionModules()
270+
return checkFaultInjectionModules() && checkTCShowTooling()
267271
}
268272

269273
// checkFaultInjectionModules checks for the required kernel modules such as sch_netem to be installed
@@ -278,3 +282,21 @@ func checkFaultInjectionModules() bool {
278282
}
279283
return true
280284
}
285+
286+
func checkTCShowTooling() bool {
287+
ctxWithTimeout, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
288+
defer cancel()
289+
hostDeviceName, netErr := netconfig.DefaultNetInterfaceName(networkConfigClient.NetlinkClient)
290+
if netErr != nil {
291+
seelog.Warnf("Failed to obtain the network interface device name on the host: %v", netErr)
292+
return false
293+
}
294+
tcShowCmd := fmt.Sprintf(tcShowCmdString, hostDeviceName)
295+
cmdList := strings.Split(tcShowCmd, " ")
296+
_, err := osExecWrapper.CommandContext(ctxWithTimeout, cmdList[0], cmdList[1:]...).CombinedOutput()
297+
if err != nil {
298+
seelog.Warnf("Failed to call %s which is needed for fault-injection feature: %v", tcShowCmd, err)
299+
return false
300+
}
301+
return true
302+
}

‎agent/app/agent_capability_unix_test.go

+104-3
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@ package app
1919
import (
2020
"context"
2121
"errors"
22+
"fmt"
23+
"net"
2224
"os"
2325
"os/exec"
2426
"path/filepath"
27+
"strings"
2528
"testing"
2629

2730
app_mocks "github.com/aws/amazon-ecs-agent/agent/app/mocks"
@@ -40,12 +43,36 @@ import (
4043
mock_mobypkgwrapper "github.com/aws/amazon-ecs-agent/agent/utils/mobypkgwrapper/mocks"
4144
"github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/model/ecs"
4245
md "github.com/aws/amazon-ecs-agent/ecs-agent/manageddaemon"
46+
"github.com/aws/amazon-ecs-agent/ecs-agent/tmds/utils/netconfig"
4347
"github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper"
4448
mock_execwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks"
49+
mock_netlinkwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/netlinkwrapper/mocks"
4550
"github.com/aws/aws-sdk-go/aws"
4651
aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
4752
"github.com/golang/mock/gomock"
4853
"github.com/stretchr/testify/assert"
54+
"github.com/vishvananda/netlink"
55+
)
56+
57+
const (
58+
deviceName = "eth0"
59+
internalError = "internal error"
60+
)
61+
62+
var (
63+
routes = []netlink.Route{
64+
netlink.Route{
65+
Gw: net.ParseIP("10.194.20.1"),
66+
Dst: nil,
67+
LinkIndex: 0,
68+
},
69+
}
70+
link = &netlink.Device{
71+
LinkAttrs: netlink.LinkAttrs{
72+
Index: 0,
73+
Name: deviceName,
74+
},
75+
}
4976
)
5077

5178
func init() {
@@ -982,21 +1009,34 @@ func TestCheckFaultInjectionTooling(t *testing.T) {
9821009
lookPathFunc = originalLookPath
9831010
}()
9841011
originalOSExecWrapper := execwrapper.NewExec()
1012+
originalNetConfig := netconfig.NewNetworkConfigClient()
9851013
defer func() {
9861014
osExecWrapper = originalOSExecWrapper
1015+
networkConfigClient = originalNetConfig
9871016
}()
9881017

9891018
t.Run("all tools and kernel modules available", func(t *testing.T) {
9901019
lookPathFunc = func(file string) (string, error) {
991-
return "/usr/bin" + file, nil
1020+
return "/usr/bin/" + file, nil
9921021
}
9931022
ctrl := gomock.NewController(t)
9941023
defer ctrl.Finish()
9951024
mockExec := mock_execwrapper.NewMockExec(ctrl)
9961025
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
1026+
mock_netlinkwrapper := mock_netlinkwrapper.NewMockNetLink(ctrl)
1027+
cmdList := convertToInterfaceList(strings.Split(fmt.Sprintf(tcShowCmdString, deviceName), " "))
1028+
1029+
gomock.InOrder(
1030+
mock_netlinkwrapper.EXPECT().RouteList(nil, netlink.FAMILY_ALL).Return(routes, nil).AnyTimes(),
1031+
mock_netlinkwrapper.EXPECT().LinkByIndex(link.Attrs().Index).Return(link, nil).AnyTimes(),
1032+
)
1033+
networkConfigClient.NetlinkClient = mock_netlinkwrapper
9971034
gomock.InOrder(
9981035
mockExec.EXPECT().CommandContext(gomock.Any(), modInfoCmd, faultInjectionKernelModules).Times(1).Return(cmdExec),
9991036
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
1037+
1038+
mockExec.EXPECT().CommandContext(gomock.Any(), cmdList[0], cmdList[1:]...).Times(1).Return(cmdExec),
1039+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
10001040
)
10011041
osExecWrapper = mockExec
10021042
assert.True(t,
@@ -1006,7 +1046,7 @@ func TestCheckFaultInjectionTooling(t *testing.T) {
10061046

10071047
t.Run("missing kernel modules", func(t *testing.T) {
10081048
lookPathFunc = func(file string) (string, error) {
1009-
return "/usr/bin" + file, nil
1049+
return "/usr/bin/" + file, nil
10101050
}
10111051
ctrl := gomock.NewController(t)
10121052
defer ctrl.Finish()
@@ -1022,18 +1062,79 @@ func TestCheckFaultInjectionTooling(t *testing.T) {
10221062
"Expected checkFaultInjectionTooling to return false when kernel modules are not available")
10231063
})
10241064

1065+
t.Run("failed to obtain default host device name", func(t *testing.T) {
1066+
lookPathFunc = func(file string) (string, error) {
1067+
return "/usr/bin/" + file, nil
1068+
}
1069+
ctrl := gomock.NewController(t)
1070+
defer ctrl.Finish()
1071+
mockExec := mock_execwrapper.NewMockExec(ctrl)
1072+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
1073+
mock_netlinkwrapper := mock_netlinkwrapper.NewMockNetLink(ctrl)
1074+
1075+
gomock.InOrder(
1076+
mock_netlinkwrapper.EXPECT().RouteList(nil, netlink.FAMILY_ALL).Return(routes, errors.New(internalError)).AnyTimes(),
1077+
)
1078+
networkConfigClient.NetlinkClient = mock_netlinkwrapper
1079+
gomock.InOrder(
1080+
mockExec.EXPECT().CommandContext(gomock.Any(), modInfoCmd, faultInjectionKernelModules).Times(1).Return(cmdExec),
1081+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
1082+
)
1083+
osExecWrapper = mockExec
1084+
assert.False(t,
1085+
checkFaultInjectionTooling(),
1086+
"Expected checkFaultInjectionTooling to return false when unable to find default host interface name")
1087+
})
1088+
1089+
t.Run("failed tc show command", func(t *testing.T) {
1090+
lookPathFunc = func(file string) (string, error) {
1091+
return "/usr/bin/" + file, nil
1092+
}
1093+
ctrl := gomock.NewController(t)
1094+
defer ctrl.Finish()
1095+
mockExec := mock_execwrapper.NewMockExec(ctrl)
1096+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
1097+
mock_netlinkwrapper := mock_netlinkwrapper.NewMockNetLink(ctrl)
1098+
cmdList := convertToInterfaceList(strings.Split(fmt.Sprintf(tcShowCmdString, deviceName), " "))
1099+
1100+
gomock.InOrder(
1101+
mock_netlinkwrapper.EXPECT().RouteList(nil, netlink.FAMILY_ALL).Return(routes, nil).AnyTimes(),
1102+
mock_netlinkwrapper.EXPECT().LinkByIndex(link.Attrs().Index).Return(link, nil).AnyTimes(),
1103+
)
1104+
networkConfigClient.NetlinkClient = mock_netlinkwrapper
1105+
gomock.InOrder(
1106+
mockExec.EXPECT().CommandContext(gomock.Any(), modInfoCmd, faultInjectionKernelModules).Times(1).Return(cmdExec),
1107+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
1108+
1109+
mockExec.EXPECT().CommandContext(gomock.Any(), cmdList[0], cmdList[1:]...).Times(1).Return(cmdExec),
1110+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, errors.New("What is \"parent\"? Try \"tc qdisc help\".")),
1111+
)
1112+
osExecWrapper = mockExec
1113+
assert.False(t,
1114+
checkFaultInjectionTooling(),
1115+
"Expected checkFaultInjectionTooling to return false when required tc show command failed")
1116+
})
1117+
10251118
tools := []string{"iptables", "tc", "nsenter"}
10261119
for _, tool := range tools {
10271120
t.Run(tool+" missing", func(t *testing.T) {
10281121
lookPathFunc = func(file string) (string, error) {
10291122
if file == tool {
10301123
return "", exec.ErrNotFound
10311124
}
1032-
return "/usr/bin" + file, nil
1125+
return "/usr/bin/" + file, nil
10331126
}
10341127
assert.False(t,
10351128
checkFaultInjectionTooling(),
10361129
"Expected checkFaultInjectionTooling to return false when a tool is missing")
10371130
})
10381131
}
10391132
}
1133+
1134+
func convertToInterfaceList(strings []string) []interface{} {
1135+
interfaces := make([]interface{}, len(strings))
1136+
for i, s := range strings {
1137+
interfaces[i] = s
1138+
}
1139+
return interfaces
1140+
}

0 commit comments

Comments
 (0)