Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support more forms of failure: execute command in containers #169

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# kube-monkey [![Build Status](https://travis-ci.org/asobti/kube-monkey.svg?branch=master)](https://travis-ci.org/asobti/kube-monkey) [![Go Report](https://goreportcard.com/badge/github.com/asobti/kube-monkey)](https://goreportcard.com/report/github.com/asobti/kube-monkey)

kube-monkey is an implementation of [Netflix's Chaos Monkey](https://github.com/Netflix/chaosmonkey) for [Kubernetes](http://kubernetes.io/) clusters. It randomly deletes Kubernetes (k8s) pods in the cluster encouraging and validating the development of failure-resilient services.
kube-monkey is an implementation of [Netflix's Chaos Monkey](https://github.com/Netflix/chaosmonkey) for [Kubernetes](http://kubernetes.io/) clusters. It randomly deletes Kubernetes (k8s) pods or introduces failure by executing command in containers in the cluster encouraging and validating the development of failure-resilient services.

Join us at [#kube-monkey](https://kubernetes.slack.com/messages/kube-monkey) on Kubernetes Slack.

@@ -37,6 +37,11 @@ that belong to a k8s app as Pods inherit labels from their k8s app. So, if kube-
* if `random-max-percent`, provide a number from 0-100 to specify the max % of pods kube-monkey can kill
* if `fixed-percent`, provide a number from 0-100 to specify the % of pods to kill

**`kube-monkey/container-name`**: Specify container for monkey to execute command in, default value is the first container

**`kube-monkey/container-name`**:When harm_type is 'exec_pod', command to execute is set on this pod annotation


#### Example of opted-in Deployment killing one pod per purge

```yaml
@@ -131,6 +136,7 @@ Configuration keys and descriptions can be found in [`config/param/param.go`](ht
```toml
[kubemonkey]
dry_run = true # Terminations are only logged
harm_type = "exec_pod" # Monkey's behavior is to execute command in containers. Default value is "delete_pod", which represents deleting pods
run_hour = 8 # Run scheduling at 8am on weekdays
start_hour = 10 # Don't schedule any pod deaths before 10am
end_hour = 16 # Don't schedule any pod deaths after 4pm
8 changes: 4 additions & 4 deletions chaos/chaos.go
Original file line number Diff line number Diff line change
@@ -115,25 +115,25 @@ func (c *Chaos) terminate(clientset kube.Interface) error {
// Validate killtype
switch killType {
case config.KillFixedLabelValue:
return c.Victim().DeleteRandomPods(clientset, killValue)
return c.Victim().HarmRandomPods(clientset, killValue)
case config.KillAllLabelValue:
killNum, err := c.Victim().KillNumberForKillingAll(clientset)
if err != nil {
return err
}
return c.Victim().DeleteRandomPods(clientset, killNum)
return c.Victim().HarmRandomPods(clientset, killNum)
case config.KillRandomMaxLabelValue:
killNum, err := c.Victim().KillNumberForMaxPercentage(clientset, killValue)
if err != nil {
return err
}
return c.Victim().DeleteRandomPods(clientset, killNum)
return c.Victim().HarmRandomPods(clientset, killNum)
case config.KillFixedPercentageLabelValue:
killNum, err := c.Victim().KillNumberForFixedPercentage(clientset, killValue)
if err != nil {
return err
}
return c.Victim().DeleteRandomPods(clientset, killNum)
return c.Victim().HarmRandomPods(clientset, killNum)
default:
return fmt.Errorf("failed to recognize KillType label for %s %s", c.Victim().Kind(), c.Victim().Name())
}
8 changes: 4 additions & 4 deletions chaos/chaos_test.go
Original file line number Diff line number Diff line change
@@ -82,7 +82,7 @@ func (s *ChaosTestSuite) TestTerminateKillFixed() {
killValue := 1
v.On("KillType", s.client).Return(config.KillFixedLabelValue, nil)
v.On("KillValue", s.client).Return(killValue, nil)
v.On("DeleteRandomPods", s.client, killValue).Return(nil)
v.On("HarmRandomPods", s.client, killValue).Return(nil)
_ = s.chaos.terminate(s.client)
v.AssertExpectations(s.T())
}
@@ -92,7 +92,7 @@ func (s *ChaosTestSuite) TestTerminateAllPods() {
v.On("KillType", s.client).Return(config.KillAllLabelValue, nil)
v.On("KillValue", s.client).Return(0, nil)
v.On("KillNumberForKillingAll", s.client).Return(0, nil)
v.On("DeleteRandomPods", s.client, 0).Return(nil)
v.On("HarmRandomPods", s.client, 0).Return(nil)
_ = s.chaos.terminate(s.client)
v.AssertExpectations(s.T())
}
@@ -103,7 +103,7 @@ func (s *ChaosTestSuite) TestTerminateKillRandomMaxPercentage() {
v.On("KillType", s.client).Return(config.KillRandomMaxLabelValue, nil)
v.On("KillValue", s.client).Return(killValue, nil)
v.On("KillNumberForMaxPercentage", s.client, mock.AnythingOfType("int")).Return(0, nil)
v.On("DeleteRandomPods", s.client, 0).Return(nil)
v.On("HarmRandomPods", s.client, 0).Return(nil)
_ = s.chaos.terminate(s.client)
v.AssertExpectations(s.T())
}
@@ -114,7 +114,7 @@ func (s *ChaosTestSuite) TestTerminateKillFixedPercentage() {
v.On("KillType", s.client).Return(config.KillFixedPercentageLabelValue, nil)
v.On("KillValue", s.client).Return(killValue, nil)
v.On("KillNumberForFixedPercentage", s.client, mock.AnythingOfType("int")).Return(0, nil)
v.On("DeleteRandomPods", s.client, 0).Return(nil)
v.On("HarmRandomPods", s.client, 0).Return(nil)
_ = s.chaos.terminate(s.client)
v.AssertExpectations(s.T())
}
2 changes: 1 addition & 1 deletion chaos/chaosmock.go
Original file line number Diff line number Diff line change
@@ -41,7 +41,7 @@ func (vm *victimMock) DeleteRandomPod(clientset kube.Interface) error {
return args.Error(0)
}

func (vm *victimMock) DeleteRandomPods(clientset kube.Interface, killValue int) error {
func (vm *victimMock) HarmRandomPods(clientset kube.Interface, killValue int) error {
args := vm.Called(clientset, killValue)
return args.Error(0)
}
7 changes: 7 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
@@ -33,6 +33,8 @@ const (
KillFixedPercentageLabelValue = "fixed-percent"
KillFixedLabelValue = "fixed"
KillAllLabelValue = "kill-all"
ContainerNameKey = "kube-monkey/container-name"
ExecCmdKey = "kube-monkey/exec-cmd"
)

func SetDefaults() {
@@ -47,6 +49,7 @@ func SetDefaults() {
viper.SetDefault(param.GracePeriodSec, 5)
viper.SetDefault(param.BlacklistedNamespaces, []string{metav1.NamespaceSystem})
viper.SetDefault(param.WhitelistedNamespaces, []string{metav1.NamespaceAll})
viper.SetDefault(param.HarmType, "delete_pod")

viper.SetDefault(param.DebugEnabled, false)
viper.SetDefault(param.DebugScheduleDelay, 30)
@@ -97,6 +100,10 @@ func Timezone() *time.Location {
return location
}

func HarmType() string {
return viper.GetString(param.HarmType)
}

func RunHour() int {
return viper.GetInt(param.RunHour)
}
8 changes: 8 additions & 0 deletions config/param/param.go
Original file line number Diff line number Diff line change
@@ -6,6 +6,14 @@ const (
// Default: true
DryRun = "kubemonkey.dry_run"

// HarmMode specifies how monkey do harmful work in the cluster
// Type: string
// Default: delete_pod
// Available values: delete_pod, exec_pod
// delete_pod: monkey will kill pod in cluster
// exec_pod: monkey will exec command specified in harm_value in pod
HarmType = "kubemonkey.harm_type"

// Timezone specifies the timezone to use when
// scheduling Pod terminations
// Type: string
19 changes: 15 additions & 4 deletions glide.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions glide.yaml
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ import:
- package: github.com/fsnotify/fsnotify
- package: github.com/golang/glog
- package: github.com/spf13/viper
- package: github.com/docker/spdystream
- package: github.com/stretchr/testify
subpackages:
- mock
102 changes: 101 additions & 1 deletion kubernetes/kubernetes.go
Original file line number Diff line number Diff line change
@@ -7,9 +7,15 @@ It's recommended to create a new clientset after a period of inactivity
package kubernetes

import (
"bytes"
"fmt"

"github.com/golang/glog"
"io"
"k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/remotecommand"
"net/url"
"strings"

cfg "github.com/asobti/kube-monkey/config"

@@ -18,6 +24,21 @@ import (
"k8s.io/client-go/rest"
)

// ExecOptions passed to ExecWithOptions
type ExecOptions struct {
Command []string

Namespace string
PodName string
ContainerName string

Stdin io.Reader
CaptureStdout bool
CaptureStderr bool
// If false, whitespace in std{err,out} will be removed.
PreserveWhitespace bool
}

// CreateClient creates, verifes and returns an instance of k8 clientset
func CreateClient() (*kube.Clientset, error) {
client, err := NewInClusterClient()
@@ -52,7 +73,86 @@ func NewInClusterClient() (*kube.Clientset, error) {
return clientset, nil
}

func GetRestConfig() (*rest.Config, error) {
config, err := rest.InClusterConfig()
if err != nil {
glog.Errorf("failed to obtain config from InClusterConfig: %v", err)
return nil, err
}

if apiserverHost, override := cfg.ClusterAPIServerHost(); override {
glog.V(5).Infof("API server host overriden to: %s\n", apiserverHost)
config.Host = apiserverHost
}
return config, err
}

func VerifyClient(client discovery.DiscoveryInterface) bool {
_, err := client.ServerVersion()
return err == nil
}

// ExecCommandInContainerWithFullOutput executes a command in the
// specified container and return stdout, stderr and error
func ExecCommandInContainerWithFullOutput(clientset kube.Interface, podName, containerName, namespace string, cmd ...string) (string, string, error) {
return ExecWithOptions(ExecOptions{
Command: cmd,
Namespace: namespace,
PodName: podName,
ContainerName: containerName,

Stdin: nil,
CaptureStdout: true,
CaptureStderr: true,
PreserveWhitespace: false,
}, clientset)
}

// ExecWithOptions executes a command in the specified container,
// returning stdout, stderr and error. `options` allowed for
// additional parameters to be passed.
func ExecWithOptions(options ExecOptions, clientset kube.Interface) (string, string, error) {
glog.Infof("ExecWithOptions: %v", options)

restconfig, err := GetRestConfig()
if err != nil {
panic(err)
}

req := clientset.CoreV1().RESTClient().Post().
Resource("pods").
Name(options.PodName).
Namespace(options.Namespace).
SubResource("exec").
Param("container", options.ContainerName)

req.VersionedParams(&v1.PodExecOptions{
Container: options.ContainerName,
Command: options.Command,
Stdin: options.Stdin != nil,
Stdout: options.CaptureStdout,
Stderr: options.CaptureStderr,
TTY: false,
}, scheme.ParameterCodec)

var stdout, stderr bytes.Buffer
err = execute("POST", req.URL(), restconfig, options.Stdin, &stdout, &stderr, false)

if options.PreserveWhitespace {
return stdout.String(), stderr.String(), err
}
return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err
}

func execute(method string, url *url.URL, config *rest.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool) error {
exec, err := remotecommand.NewSPDYExecutor(config, method, url)
if err != nil {
return err
}
return exec.Stream(remotecommand.StreamOptions{
Stdin: stdin,
Stdout: stdout,
Stderr: stderr,
Tty: tty,
})
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading