diff --git a/README.md b/README.md index 0fa6d2552b..5dedb76b7b 100644 --- a/README.md +++ b/README.md @@ -65,12 +65,12 @@ For help, please consider the following venues (in order): ## Recommended Version -For all Kubernetes releases, we recommend installing the latest VPC CNI release. The following table denotes our minimum recommended +For all Kubernetes releases, *we recommend installing the latest VPC CNI release*. The following table denotes our *oldest* recommended VPC CNI version for each actively supported Kubernetes release. -| Kubernetes Release | 1.27 | 1.26 | 1.25 | 1.24 | 1.23 | -| ------------------ | -------- | -------- | -------- | ------- | ------- | -| VPC CNI Version | v1.12.5+ | v1.12.0+ | v1.11.4+ | v1.9.3+ | v1.8.0+ | +| Kubernetes Release | 1.29 | 1.28 | 1.27 | 1.26 | 1.25 | 1.24 | +| ------------------ | -------- | -------- | -------- | -------- | -------- | ------- | +| VPC CNI Version | v1.14.1+ | v1.13.4+ | v1.12.5+ | v1.12.0+ | v1.11.4+ | v1.9.3+ | ## Version Upgrade @@ -282,7 +282,8 @@ in the *Amazon EC2 User Guide for Linux Instances*. For example, an `m4.4xlarge` launches with 1 network interface and 30 IP addresses\. If 5 pods are placed on the node and 5 free IP addresses are removed from the IP address warm pool, then `ipamd` attempts to allocate more interfaces until `WARM_ENI_TARGET` free interfaces are available on the node. -If `WARM_IP_TARGET` is set, then this environment variable is ignored and the `WARM_IP_TARGET` behavior is used instead. + +**NOTE!** If `WARM_IP_TARGET` is set, then this environment variable is ignored and the `WARM_IP_TARGET` behavior is used instead. #### `WARM_IP_TARGET` @@ -333,6 +334,10 @@ elasticity, but uses roughly half as many IPs as using WARM_IP_TARGET alone (32 This also improves the reliability of the EKS cluster by reducing the number of calls necessary to allocate or deallocate private IPs, which may be throttled, especially at scaling-related times. +**NOTE!** +1. If `MINIMUM_IP_TARGET` is set, `WARM_ENI_TARGET` will be ignored. Please utilize `WARM_IP_TARGET` instead. +2. If `MINIMUM_IP_TARGET` is set and `WARM_IP_TARGET` is not set, `WARM_IP_TARGET` is assumed to be 0, which leads to the number of IPs attached to the node will be the value of `MINIMUM_IP_TARGET`. This configuration will prevent future ENIs/IPs from being allocated. It is strongly recommended that `WARM_IP_TARGET` should be set greater than 0 when `MINIMUM_IP_TARGET` is set. + #### `MAX_ENI` Type: Integer diff --git a/charts/aws-vpc-cni/templates/daemonset.yaml b/charts/aws-vpc-cni/templates/daemonset.yaml index d9f85db5d1..3b47d880a5 100644 --- a/charts/aws-vpc-cni/templates/daemonset.yaml +++ b/charts/aws-vpc-cni/templates/daemonset.yaml @@ -146,8 +146,10 @@ spec: volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir + {{- if .Values.nodeAgent.enabled }} - mountPath: /sys/fs/bpf name: bpf-pin-path + {{- end }} - mountPath: /var/log/aws-routed-eni name: log-dir - mountPath: /var/run/aws-node diff --git a/go.mod b/go.mod index 03cac8312b..b9a229e897 100644 --- a/go.mod +++ b/go.mod @@ -6,16 +6,16 @@ require ( github.com/apparentlymart/go-cidr v1.1.0 github.com/aws/amazon-vpc-cni-k8s/test/agent v0.0.0-20231212223725-21c4bd73015b github.com/aws/amazon-vpc-resource-controller-k8s v1.4.1 - github.com/aws/aws-sdk-go v1.48.10 + github.com/aws/aws-sdk-go v1.49.13 github.com/containernetworking/cni v1.1.2 - github.com/containernetworking/plugins v1.2.0 + github.com/containernetworking/plugins v1.4.0 github.com/coreos/go-iptables v0.7.0 - github.com/go-logr/logr v1.3.0 + github.com/go-logr/logr v1.4.1 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.6.0 github.com/onsi/ginkgo/v2 v2.13.2 - github.com/onsi/gomega v1.29.0 + github.com/onsi/gomega v1.30.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 @@ -28,15 +28,15 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/net v0.17.0 golang.org/x/sys v0.15.0 - google.golang.org/grpc v1.59.0 + google.golang.org/grpc v1.60.1 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 helm.sh/helm/v3 v3.13.2 - k8s.io/api v0.28.4 - k8s.io/apimachinery v0.28.4 - k8s.io/cli-runtime v0.28.4 - k8s.io/client-go v0.28.4 + k8s.io/api v0.29.0 + k8s.io/apimachinery v0.29.0 + k8s.io/cli-runtime v0.29.0 + k8s.io/client-go v0.29.0 sigs.k8s.io/controller-runtime v0.16.3 ) @@ -86,10 +86,11 @@ require ( github.com/google/btree v1.0.1 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/mux v1.8.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -122,6 +123,7 @@ require ( github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect @@ -133,7 +135,7 @@ require ( github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cobra v1.7.0 // indirect - github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect + github.com/vishvananda/netns v0.0.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -144,31 +146,31 @@ require ( go.opentelemetry.io/otel/trace v1.19.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sync v0.4.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.14.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.28.3 // indirect k8s.io/apiserver v0.28.3 // indirect k8s.io/component-base v0.28.3 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect k8s.io/kubectl v0.28.2 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect oras.land/oras-go v1.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index a8bd1090e7..2cf4212f75 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,8 @@ github.com/aws/amazon-vpc-cni-k8s/test/agent v0.0.0-20231212223725-21c4bd73015b github.com/aws/amazon-vpc-cni-k8s/test/agent v0.0.0-20231212223725-21c4bd73015b/go.mod h1:NvS1b2fBgkUvAWgBF8h0aRaVVoUeIlpUMnlTW2wIqik= github.com/aws/amazon-vpc-resource-controller-k8s v1.4.1 h1:43uJXFNTjk5Gzi2Qpqk30ycaaE7DOVvBDKi35wzsrsQ= github.com/aws/amazon-vpc-resource-controller-k8s v1.4.1/go.mod h1:tXPJP0SFdkVa7ALghDjThtavyYnP0MKO8V0ZHlDNCU8= -github.com/aws/aws-sdk-go v1.48.10 h1:0LIFG3wp2Dt6PsxKWCg1Y1xRrn2vZnW5/gWdgaBalKg= -github.com/aws/aws-sdk-go v1.48.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y= +github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -71,8 +71,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= -github.com/containernetworking/plugins v1.2.0 h1:SWgg3dQG1yzUo4d9iD8cwSVh1VqI+bP7mkPDoSfP9VU= -github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+r/s9qIfPdqlLF4TW8c4= +github.com/containernetworking/plugins v1.4.0 h1:+w22VPYgk7nQHw7KT92lsRmuToHvb7wwSv9iTbXzzic= +github.com/containernetworking/plugins v1.4.0/go.mod h1:UYhcOyjefnrQvKvmmyEKsUA+M9Nfn7tqULPpH0Pkcj0= github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8= github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -133,11 +133,11 @@ github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpj github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= @@ -198,14 +198,15 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -216,6 +217,8 @@ github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= @@ -332,22 +335,21 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= @@ -426,8 +428,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -488,8 +490,8 @@ golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -535,8 +537,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -565,18 +567,18 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -600,7 +602,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -619,28 +620,28 @@ helm.sh/helm/v3 v3.13.2 h1:IcO9NgmmpetJODLZhR3f3q+6zzyXVKlRizKFwbi7K8w= helm.sh/helm/v3 v3.13.2/go.mod h1:GIHDwZggaTGbedevTlrQ6DB++LBN6yuQdeGj0HNaDx0= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= -k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= +k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= +k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= -k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= -k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= +k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= +k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= k8s.io/apiserver v0.28.3 h1:8Ov47O1cMyeDzTXz0rwcfIIGAP/dP7L8rWbEljRcg5w= k8s.io/apiserver v0.28.3/go.mod h1:YIpM+9wngNAv8Ctt0rHG4vQuX/I5rvkEMtZtsxW2rNM= -k8s.io/cli-runtime v0.28.4 h1:IW3aqSNFXiGDllJF4KVYM90YX4cXPGxuCxCVqCD8X+Q= -k8s.io/cli-runtime v0.28.4/go.mod h1:MLGRB7LWTIYyYR3d/DOgtUC8ihsAPA3P8K8FDNIqJ0k= -k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY= -k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4= +k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4= +k8s.io/cli-runtime v0.29.0/go.mod h1:VKudXp3X7wR45L+nER85YUzOQIru28HQpXr0mTdeCrk= +k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= +k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/kubectl v0.28.2 h1:fOWOtU6S0smdNjG1PB9WFbqEIMlkzU5ahyHkc7ESHgM= k8s.io/kubectl v0.28.2/go.mod h1:6EQWTPySF1fn7yKoQZHYf9TPwIl2AygHEcJoxFekr64= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= @@ -651,7 +652,7 @@ sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKU sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 4bb740b078..7f958647e8 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -395,6 +395,11 @@ func (c *IPAMContext) nodeInit() error { if err != nil { return errors.Wrap(err, "ipamd init: failed to set up host network") } + err = c.networkClient.CleanUpStaleAWSChains(c.enableIPv4, c.enableIPv6) + if err != nil { + // We should not error if clean up fails since these chains don't affect the rules + log.Debugf("Failed to clean up stale AWS chains: %v", err) + } metadataResult, err := c.awsClient.DescribeAllENIs() if err != nil { diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 1cf0386b66..cd6bee4a70 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -150,6 +150,7 @@ func TestNodeInit(t *testing.T) { m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) m.awsutils.EXPECT().GetPrimaryENImac().Return("") m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, true, false).Return(nil) + m.network.EXPECT().CleanUpStaleAWSChains(true, false).Return(nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().RefreshSGIDs(gomock.Any()).AnyTimes().Return(nil) @@ -234,6 +235,7 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) m.awsutils.EXPECT().GetPrimaryENImac().Return("") m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, true, false).Return(nil) + m.network.EXPECT().CleanUpStaleAWSChains(true, false).Return(nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().RefreshSGIDs(gomock.Any()).AnyTimes().Return(nil) @@ -308,6 +310,7 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { primaryIP := net.ParseIP(ipaddr01) m.network.EXPECT().SetupHostNetwork(cidrs, eni1.MAC, &primaryIP, false, false, true).Return(nil) + m.network.EXPECT().CleanUpStaleAWSChains(false, true).Return(nil) m.awsutils.EXPECT().GetIPv6PrefixesFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv6Prefixes, nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().GetPrimaryENImac().Return(eni1.MAC) diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index 16c73ec9fd..e878c53a63 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -229,7 +229,7 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp ParentIfIndex: int32(trunkENILinkIndex), } - log.Infof("Send AddNetworkReply: IPv4Addr %s, IPv6Addr: %s, DeviceNumber: %d, err: %v", ipv4Addr, ipv6Addr, deviceNumber, err) + log.Infof("Send AddNetworkReply: IPv4Addr: %s, IPv6Addr: %s, DeviceNumber: %d, err: %v", ipv4Addr, ipv6Addr, deviceNumber, err) return &resp, nil } @@ -314,7 +314,7 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp } } - log.Infof("Send DelNetworkReply: IPv4Addr %s, DeviceNumber: %d, err: %v", ip, deviceNumber, err) + log.Infof("Send DelNetworkReply: IPv4Addr: %s, IPv6Addr: %s, DeviceNumber: %d, err: %v", ipv4Addr, ipv6Addr, deviceNumber, err) return &rpc.DelNetworkReply{Success: err == nil, IPv4Addr: ipv4Addr, IPv6Addr: ipv6Addr, DeviceNumber: int32(deviceNumber)}, err } diff --git a/pkg/iptableswrapper/iptables.go b/pkg/iptableswrapper/iptables.go index 7f78e1e6fe..ce1ab24343 100644 --- a/pkg/iptableswrapper/iptables.go +++ b/pkg/iptableswrapper/iptables.go @@ -29,6 +29,7 @@ type IPTablesIface interface { ClearChain(table, chain string) error DeleteChain(table, chain string) error ListChains(table string) ([]string, error) + ChainExists(table, chain string) (bool, error) HasRandomFully() bool } @@ -98,6 +99,11 @@ func (i ipTables) ListChains(table string) ([]string, error) { return i.ipt.ListChains(table) } +// ChainExists implements IPTablesIface interface by calling iptables package +func (i ipTables) ChainExists(table, chain string) (bool, error) { + return i.ipt.ChainExists(table, chain) +} + // HasRandomFully implements IPTablesIface interface by calling iptables package func (i ipTables) HasRandomFully() bool { return i.ipt.HasRandomFully() diff --git a/pkg/iptableswrapper/mocks/iptables_maps.go b/pkg/iptableswrapper/mocks/iptables_maps.go index e72a99c8d4..843b8dffac 100644 --- a/pkg/iptableswrapper/mocks/iptables_maps.go +++ b/pkg/iptableswrapper/mocks/iptables_maps.go @@ -16,6 +16,7 @@ package mock_iptableswrapper import ( "fmt" "reflect" + "slices" "strings" "github.com/pkg/errors" @@ -44,7 +45,12 @@ func (ipt *MockIptables) Insert(table, chain string, pos int, rulespec ...string if ipt.DataplaneState[table] == nil { ipt.DataplaneState[table] = map[string][][]string{} } - ipt.DataplaneState[table][chain] = append(ipt.DataplaneState[table][chain], rulespec) + if len(ipt.DataplaneState[table][chain]) == pos-1 { + ipt.DataplaneState[table][chain] = append(ipt.DataplaneState[table][chain], rulespec) + } else { + ipt.DataplaneState[table][chain] = append(ipt.DataplaneState[table][chain][:pos], ipt.DataplaneState[table][chain][pos-1:]...) + ipt.DataplaneState[table][chain][pos] = rulespec + } return nil } @@ -91,6 +97,10 @@ func (ipt *MockIptables) List(table, chain string) ([]string, error) { var chains []string chainContents := ipt.DataplaneState[table][chain] for _, ruleSpec := range chainContents { + if slices.Contains(ruleSpec, "-N") { + chains = append(chains, strings.Join(ruleSpec, " ")) + continue + } sanitizedRuleSpec := []string{"-A", chain} for _, item := range ruleSpec { if strings.Contains(item, " ") { @@ -101,10 +111,15 @@ func (ipt *MockIptables) List(table, chain string) ([]string, error) { chains = append(chains, strings.Join(sanitizedRuleSpec, " ")) } return chains, nil - } func (ipt *MockIptables) NewChain(table, chain string) error { + exists, _ := ipt.ChainExists(table, chain) + if exists { + return errors.New("Chain already exists") + } + // Creating a new chain adds a -N chain rule to iptables + ipt.Append(table, chain, "-N", chain) return nil } @@ -113,6 +128,12 @@ func (ipt *MockIptables) ClearChain(table, chain string) error { } func (ipt *MockIptables) DeleteChain(table, chain string) error { + // More than just the create chain rule + if len(ipt.DataplaneState[table][chain]) > 1 { + err := fmt.Sprintf("Chain %s is not empty", chain) + return errors.New(err) + } + delete(ipt.DataplaneState[table], chain) return nil } @@ -124,6 +145,14 @@ func (ipt *MockIptables) ListChains(table string) ([]string, error) { return chains, nil } +func (ipt *MockIptables) ChainExists(table, chain string) (bool, error) { + _, ok := ipt.DataplaneState[table][chain] + if ok { + return true, nil + } + return false, nil +} + func (ipt *MockIptables) HasRandomFully() bool { // TODO: Work out how to write a test case for this return true diff --git a/pkg/iptableswrapper/mocks/iptables_mocks.go b/pkg/iptableswrapper/mocks/iptables_mocks.go index 480fc6783d..f4a40df6d5 100644 --- a/pkg/iptableswrapper/mocks/iptables_mocks.go +++ b/pkg/iptableswrapper/mocks/iptables_mocks.go @@ -85,6 +85,21 @@ func (mr *MockIPTablesIfaceMockRecorder) AppendUnique(arg0, arg1 interface{}, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendUnique", reflect.TypeOf((*MockIPTablesIface)(nil).AppendUnique), varargs...) } +// ChainExists mocks base method. +func (m *MockIPTablesIface) ChainExists(arg0, arg1 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChainExists", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ChainExists indicates an expected call of ChainExists. +func (mr *MockIPTablesIfaceMockRecorder) ChainExists(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExists", reflect.TypeOf((*MockIPTablesIface)(nil).ChainExists), arg0, arg1) +} + // ClearChain mocks base method. func (m *MockIPTablesIface) ClearChain(arg0, arg1 string) error { m.ctrl.T.Helper() diff --git a/pkg/networkutils/mocks/network_mocks.go b/pkg/networkutils/mocks/network_mocks.go index a4c9016777..e68b213f44 100644 --- a/pkg/networkutils/mocks/network_mocks.go +++ b/pkg/networkutils/mocks/network_mocks.go @@ -50,6 +50,20 @@ func (m *MockNetworkAPIs) EXPECT() *MockNetworkAPIsMockRecorder { return m.recorder } +// CleanUpStaleAWSChains mocks base method. +func (m *MockNetworkAPIs) CleanUpStaleAWSChains(arg0, arg1 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanUpStaleAWSChains", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanUpStaleAWSChains indicates an expected call of CleanUpStaleAWSChains. +func (mr *MockNetworkAPIsMockRecorder) CleanUpStaleAWSChains(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanUpStaleAWSChains", reflect.TypeOf((*MockNetworkAPIs)(nil).CleanUpStaleAWSChains), arg0, arg1) +} + // GetExcludeSNATCIDRs mocks base method. func (m *MockNetworkAPIs) GetExcludeSNATCIDRs() []string { m.ctrl.T.Helper() diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go index c003eab987..4189c9cbe0 100644 --- a/pkg/networkutils/network.go +++ b/pkg/networkutils/network.go @@ -152,6 +152,7 @@ type NetworkAPIs interface { SetupENINetwork(eniIP string, mac string, deviceNumber int, subnetCIDR string) error // UpdateHostIptablesRules updates the nat table iptables rules on the host UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled bool, v6Enabled bool) error + CleanUpStaleAWSChains(v4Enabled, v6Enabled bool) error UseExternalSNAT() bool GetExcludeSNATCIDRs() []string GetExternalServiceCIDRs() []string @@ -375,6 +376,51 @@ func (n *linuxNetwork) UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC str return n.updateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled) } +func (n *linuxNetwork) CleanUpStaleAWSChains(v4Enabled, v6Enabled bool) error { + ipProtocol := iptables.ProtocolIPv4 + if v6Enabled { + ipProtocol = iptables.ProtocolIPv6 + } + + ipt, err := n.newIptables(ipProtocol) + if err != nil { + return errors.Wrap(err, "stale chain cleanup: failed to create iptables") + } + + exists, err := ipt.ChainExists("nat", "AWS-SNAT-CHAIN-1") + if err != nil { + return errors.Wrap(err, "stale chain cleanup: failed to check if AWS-SNAT-CHAIN-1 exists") + } + + if exists { + existingChains, err := ipt.ListChains("nat") + if err != nil { + return errors.Wrap(err, "stale chain cleanup: failed to list iptables nat chains") + } + + for _, chain := range existingChains { + if !strings.HasPrefix(chain, "AWS-CONNMARK-CHAIN") && !strings.HasPrefix(chain, "AWS-SNAT-CHAIN") { + continue + } + parsedChain := strings.Split(chain, "-") + chainNum, err := strconv.Atoi(parsedChain[len(parsedChain)-1]) + if err != nil { + return errors.Wrap(err, "stale chain cleanup: failed to convert string to int") + } + // Chains 1 --> x (0 indexed) will be stale + if chainNum > 0 { + // No need to clear the chain since computeStaleIptablesRules cleans up all rules already + log.Infof("Deleting stale chain: %s", chain) + err := ipt.DeleteChain("nat", chain) + if err != nil { + return errors.Wrapf(err, "stale chain cleanup: failed to delete chain %s", chain) + } + } + } + } + return nil +} + func (n *linuxNetwork) updateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled bool, v6Enabled bool) error { primaryIntf, err := findPrimaryInterfaceName(primaryMAC) @@ -434,15 +480,13 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne log.Debugf("Total CIDRs to program - %d", len(allCIDRs)) // build IPTABLES chain for SNAT of non-VPC outbound traffic and excluded CIDRs var chains []string - for i := 0; i <= len(allCIDRs); i++ { - chain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", i) - log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) - if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { - log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) - return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") - } - chains = append(chains, chain) + chain := "AWS-SNAT-CHAIN-0" + log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) + if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { + log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) + return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") } + chains = append(chains, chain) // build SNAT rules for outbound non-VPC traffic var iptableRules []iptablesRule @@ -456,23 +500,20 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0", }}) - for i, cidr := range allCIDRs { - curChain := chains[i] - curName := fmt.Sprintf("[%d] AWS-SNAT-CHAIN", i) - nextChain := chains[i+1] + for _, cidr := range allCIDRs { comment := "AWS SNAT CHAIN" if cidr.isExclusion { comment += " EXCLUSION" } - log.Debugf("Setup Host Network: iptables -A %s ! -d %s -t nat -j %s", curChain, cidr, nextChain) + log.Debugf("Setup Host Network: iptables -A %s -d %s -t nat -j %s", chain, cidr, "RETURN") iptableRules = append(iptableRules, iptablesRule{ - name: curName, + name: chain, shouldExist: !n.useExternalSNAT, table: "nat", - chain: curChain, + chain: chain, rule: []string{ - "!", "-d", cidr.cidr, "-m", "comment", "--comment", comment, "-j", nextChain, + "-d", cidr.cidr, "-m", "comment", "--comment", comment, "-j", "RETURN", }}) } @@ -494,15 +535,6 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne } } - lastChain := chains[len(chains)-1] - iptableRules = append(iptableRules, iptablesRule{ - name: "last SNAT rule for non-VPC outbound traffic", - shouldExist: !n.useExternalSNAT, - table: "nat", - chain: lastChain, - rule: snatRule, - }) - snatStaleRules, err := computeStaleIptablesRules(ipt, "nat", "AWS-SNAT-CHAIN", iptableRules, chains) if err != nil { return []iptablesRule{}, err @@ -510,6 +542,14 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne iptableRules = append(iptableRules, snatStaleRules...) + iptableRules = append(iptableRules, iptablesRule{ + name: "last SNAT rule for non-VPC outbound traffic", + shouldExist: !n.useExternalSNAT, + table: "nat", + chain: chain, + rule: snatRule, + }) + iptableRules = append(iptableRules, iptablesRule{ name: "connmark for primary ENI", shouldExist: n.nodePortSupportEnabled, @@ -556,16 +596,15 @@ func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptable excludeCIDRs := sets.NewString(n.excludeSNATCIDRs...) log.Debugf("Total CIDRs to exempt from connmark rules - %d", len(allCIDRs)) + var chains []string - for i := 0; i <= len(allCIDRs); i++ { - chain := fmt.Sprintf("AWS-CONNMARK-CHAIN-%d", i) - log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) - if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { - log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) - return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") - } - chains = append(chains, chain) + chain := "AWS-CONNMARK-CHAIN-0" + log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) + if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { + log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) + return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") } + chains = append(chains, chain) var iptableRules []iptablesRule log.Debugf("Setup Host Network: iptables -t nat -A PREROUTING -i %s+ -m comment --comment \"AWS, outbound connections\" -j AWS-CONNMARK-CHAIN-0", n.vethPrefix) @@ -590,37 +629,23 @@ func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptable "-j", "AWS-CONNMARK-CHAIN-0", }}) - for i, cidr := range allCIDRs { - curChain := chains[i] - curName := fmt.Sprintf("[%d] AWS-SNAT-CHAIN", i) - nextChain := chains[i+1] + for _, cidr := range allCIDRs { comment := "AWS CONNMARK CHAIN, VPC CIDR" if excludeCIDRs.Has(cidr) { comment = "AWS CONNMARK CHAIN, EXCLUDED CIDR" } - log.Debugf("Setup Host Network: iptables -A %s ! -d %s -t nat -j %s", curChain, cidr, nextChain) + log.Debugf("Setup Host Network: iptables -A %s -d %s -t nat -j %s", chain, cidr, "RETURN") iptableRules = append(iptableRules, iptablesRule{ - name: curName, + name: chain, shouldExist: !n.useExternalSNAT, table: "nat", - chain: curChain, + chain: chain, rule: []string{ - "!", "-d", cidr, "-m", "comment", "--comment", comment, "-j", nextChain, + "-d", cidr, "-m", "comment", "--comment", comment, "-j", "RETURN", }}) } - iptableRules = append(iptableRules, iptablesRule{ - name: "connmark rule for external outbound traffic", - shouldExist: !n.useExternalSNAT, - table: "nat", - chain: chains[len(chains)-1], - rule: []string{ - "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", - "--set-xmark", fmt.Sprintf("%#x/%#x", n.mainENIMark, n.mainENIMark), - }, - }) - // Force delete existing restore mark rule so that the subsequent rule gets added to the end iptableRules = append(iptableRules, iptablesRule{ name: "connmark to fwmark copy", @@ -652,6 +677,17 @@ func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptable } iptableRules = append(iptableRules, connmarkStaleRules...) + iptableRules = append(iptableRules, iptablesRule{ + name: "connmark rule for external outbound traffic", + shouldExist: !n.useExternalSNAT, + table: "nat", + chain: chain, + rule: []string{ + "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", + "--set-xmark", fmt.Sprintf("%#x/%#x", n.mainENIMark, n.mainENIMark), + }, + }) + log.Debugf("iptableRules: %v", iptableRules) return iptableRules, nil } @@ -659,7 +695,6 @@ func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptable func (n *linuxNetwork) updateIptablesRules(iptableRules []iptablesRule, ipt iptableswrapper.IPTablesIface) error { for _, rule := range iptableRules { log.Debugf("execute iptable rule : %s", rule.name) - exists, err := ipt.Exists(rule.table, rule.chain, rule.rule...) log.Debugf("rule %v exists %v, err %v", rule, exists, err) if err != nil { @@ -668,10 +703,19 @@ func (n *linuxNetwork) updateIptablesRules(iptableRules []iptablesRule, ipt ipta } if !exists && rule.shouldExist { - err = ipt.Append(rule.table, rule.chain, rule.rule...) - if err != nil { - log.Errorf("host network setup: failed to add %v, %v", rule, err) - return errors.Wrapf(err, "host network setup: failed to add %v", rule) + if rule.name == "AWS-CONNMARK-CHAIN-0" || rule.name == "AWS-SNAT-CHAIN-0" { + // All CIDR rules must go before the SNAT/Mark rule + err = ipt.Insert(rule.table, rule.chain, 1, rule.rule...) + if err != nil { + log.Errorf("host network setup: failed to insert %v, %v", rule, err) + return errors.Wrapf(err, "host network setup: failed to add %v", rule) + } + } else { + err = ipt.Append(rule.table, rule.chain, rule.rule...) + if err != nil { + log.Errorf("host network setup: failed to add %v, %v", rule, err) + return errors.Wrapf(err, "host network setup: failed to add %v", rule) + } } } else if exists && !rule.shouldExist { err = ipt.Delete(rule.table, rule.chain, rule.rule...) @@ -726,7 +770,7 @@ func computeStaleIptablesRules(ipt iptableswrapper.IPTablesIface, table, chainPr return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to list rules from table %s with chain prefix %s", table, chainPrefix) } activeChains := sets.NewString(chains...) - log.Debugf("Setup Host Network: computing stale iptables rules for %s table with chain prefix %s") + log.Debugf("Setup Host Network: computing stale iptables rules for %s table with chain prefix %s", table, chainPrefix) for _, staleRule := range existingRules { if len(staleRule.rule) == 0 && activeChains.Has(staleRule.chain) { log.Debugf("Setup Host Network: active chain found: %s", staleRule.chain) diff --git a/pkg/networkutils/network_test.go b/pkg/networkutils/network_test.go index b16b018ca9..906b060a4b 100644 --- a/pkg/networkutils/network_test.go +++ b/pkg/networkutils/network_test.go @@ -363,6 +363,10 @@ func TestSetupHostNetworkNodePortEnabledAndSNATDisabled(t *testing.T) { }, }, }, + "nat": { + "AWS-SNAT-CHAIN-0": [][]string{{"-N", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{{"-N", "AWS-CONNMARK-CHAIN-0"}}, + }, }, mockIptables.(*mock_iptables.MockIptables).DataplaneState) } @@ -396,9 +400,9 @@ func TestSetupHostNetworkNodePortDisabledAndSNATEnabled(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, + "AWS-SNAT-CHAIN-0": [][]string{{"-N", "AWS-SNAT-CHAIN-0"}, {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{{"-N", "AWS-CONNMARK-CHAIN-0"}, {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, @@ -416,23 +420,23 @@ func TestSetupHostNetworkNodePortDisabledAndSNATEnabled(t *testing.T) { } func TestLoadMTUFromEnvTooLow(t *testing.T) { - _ = os.Setenv(envMTU, "1") + os.Setenv(envMTU, "1") assert.Equal(t, GetEthernetMTU(""), minimumMTU) } func TestLoadMTUFromEnv1500(t *testing.T) { - _ = os.Setenv(envMTU, "1500") + os.Setenv(envMTU, "1500") assert.Equal(t, GetEthernetMTU(""), 1500) } func TestLoadMTUFromEnvTooHigh(t *testing.T) { - _ = os.Setenv(envMTU, "65536") + os.Setenv(envMTU, "65536") assert.Equal(t, GetEthernetMTU(""), maximumMTU) } func TestLoadExcludeSNATCIDRsFromEnv(t *testing.T) { - _ = os.Setenv(envExternalSNAT, "false") - _ = os.Setenv(envExcludeSNATCIDRs, "10.12.0.0/16,10.13.0.0/16") + os.Setenv(envExternalSNAT, "false") + os.Setenv(envExcludeSNATCIDRs, "10.12.0.0/16,10.13.0.0/16") expected := []string{"10.12.0.0/16", "10.13.0.0/16"} assert.Equal(t, parseCIDRString(envExcludeSNATCIDRs), expected) @@ -465,17 +469,23 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, - "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, - "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, - "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, - "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-N", "AWS-SNAT-CHAIN-0"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-N", "AWS-CONNMARK-CHAIN-0"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, @@ -512,20 +522,22 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "AWS-SNAT-CHAIN-1") //AWS SNAT CHAN proves backwards compatibility - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") - _ = mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-5") - _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") - _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") - _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-0") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-0") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "RETURN") //AWS SNAT CHAN proves backwards compatibility + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-1") + mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") + mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testEniIPNet, false, true, false) @@ -534,17 +546,22 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "AWS-SNAT-CHAIN-3": [][]string{}, - "AWS-SNAT-CHAIN-4": [][]string{}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, - "AWS-CONNMARK-CHAIN-3": [][]string{}, - "AWS-CONNMARK-CHAIN-4": [][]string{}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-N", "AWS-SNAT-CHAIN-0"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "AWS-SNAT-CHAIN-1": [][]string{ + {"-N", "AWS-SNAT-CHAIN-1"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-N", "AWS-CONNMARK-CHAIN-0"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, @@ -581,20 +598,22 @@ func TestSetupHostNetworkWithDifferentVethPrefix(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "AWS-SNAT-CHAIN-1") //AWS SNAT CHAN proves backwards compatibility - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") - _ = mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-5") - _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") - _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") - _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-0") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-0") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "RETURN") //AWS SNAT CHAN proves backwards compatibility + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-1") + mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") + mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testEniIPNet, false, true, false) @@ -602,17 +621,26 @@ func TestSetupHostNetworkWithDifferentVethPrefix(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, - "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, - "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, - "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, - "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-N", "AWS-SNAT-CHAIN-0"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "AWS-SNAT-CHAIN-1": [][]string{ + {"-N", "AWS-SNAT-CHAIN-1"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-N", "AWS-CONNMARK-CHAIN-0"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-i", "veth+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, @@ -649,19 +677,21 @@ func TestSetupHostNetworkExternalNATCleanupConnmark(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") - _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") - _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") - _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-0") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-0") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") + mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") // remove exclusions vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} @@ -671,17 +701,9 @@ func TestSetupHostNetworkExternalNATCleanupConnmark(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{}, - "AWS-SNAT-CHAIN-1": [][]string{}, - "AWS-SNAT-CHAIN-2": [][]string{}, - "AWS-SNAT-CHAIN-3": [][]string{}, - "AWS-SNAT-CHAIN-4": [][]string{}, + "AWS-SNAT-CHAIN-0": [][]string{{"-N", "AWS-SNAT-CHAIN-0"}}, "POSTROUTING": [][]string{}, - "AWS-CONNMARK-CHAIN-0": [][]string{}, - "AWS-CONNMARK-CHAIN-1": [][]string{}, - "AWS-CONNMARK-CHAIN-2": [][]string{}, - "AWS-CONNMARK-CHAIN-3": [][]string{}, - "AWS-CONNMARK-CHAIN-4": [][]string{}, + "AWS-CONNMARK-CHAIN-0": [][]string{{"-N", "AWS-CONNMARK-CHAIN-0"}}, "PREROUTING": [][]string{}, }, "mangle": { @@ -714,19 +736,21 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") - _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") - _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") - _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-0") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-0") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") + mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") // remove exclusions vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} @@ -736,17 +760,23 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, - "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, - "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, - "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, - "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-N", "AWS-SNAT-CHAIN-0"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-N", "AWS-CONNMARK-CHAIN-0"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, @@ -782,16 +812,18 @@ func TestUpdateHostIptablesRules(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "AWS-SNAT-CHAIN-1") //AWS SNAT CHAN proves backwards compatibility - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") - _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") - _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") - _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") - _ = mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80") - _ = mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") - _ = mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-0") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-0") + mockIptables.Insert("nat", "AWS-SNAT-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "RETURN") //AWS SNAT CHAN proves backwards compatibility + mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + mockIptables.Insert("nat", "AWS-CONNMARK-CHAIN-0", 1, "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") + mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80") + mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testEniIPNet, false, true, false) @@ -799,13 +831,19 @@ func TestUpdateHostIptablesRules(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-N", "AWS-SNAT-CHAIN-0"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-N", "AWS-CONNMARK-CHAIN-0"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-i", "veth+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, @@ -830,6 +868,82 @@ func TestUpdateHostIptablesRules(t *testing.T) { }, }, mockIptables.(*mock_iptables.MockIptables).DataplaneState) } + +func TestCleanUpStaleAWSChains(t *testing.T) { + ctrl, mockNetLink, _, mockNS, mockIptables := setup(t) + defer ctrl.Finish() + + ln := &linuxNetwork{ + useExternalSNAT: false, + ipv6EgressEnabled: true, + nodePortSupportEnabled: true, + mainENIMark: defaultConnmark, + mtu: testMTU, + vethPrefix: eniPrefix, + + netLink: mockNetLink, + ns: mockNS, + newIptables: func(iptables.Protocol) (iptableswrapper.IPTablesIface, error) { + return mockIptables, nil + }, + } + setupNetLinkMocks(ctrl, mockNetLink) + + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-1") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-2") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-3") + mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-4") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-1") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-2") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-3") + mockIptables.NewChain("nat", "AWS-CONNMARK-CHAIN-4") + + vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testEniIPNet, false, true, false) + assert.NoError(t, err) + + err = ln.CleanUpStaleAWSChains(true, false) + assert.NoError(t, err) + + assert.Equal(t, + map[string]map[string][][]string{ + "nat": { + "AWS-SNAT-CHAIN-0": [][]string{ + {"-N", "AWS-SNAT-CHAIN-0"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-N", "AWS-CONNMARK-CHAIN-0"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, + "PREROUTING": [][]string{ + {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, + "filter": { + "FORWARD": [][]string{ + { + "-d", "fd00::ac:00/118", "-m", "conntrack", "--ctstate", "NEW", "-m", "comment", + "--comment", "Block Node Local Pod access via IPv6", "-j", "REJECT", + }, + }, + }, + "mangle": { + "PREROUTING": [][]string{ + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, + }, mockIptables.(*mock_iptables.MockIptables).DataplaneState) +} + func TestSetupHostNetworkMultipleCIDRs(t *testing.T) { ctrl, mockNetLink, _, mockNS, mockIptables := setup(t) defer ctrl.Finish() @@ -987,7 +1101,7 @@ func TestSetupHostNetworkDeleteOldConnmarkRuleForNonVpcOutboundTraffic(t *testin setupNetLinkMocks(ctrl, mockNetLink) // add the "old" rule used in an ealier version of the CNI - _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0") + mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0") var vpcCIDRs []string err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testEniIPNet, false, true, false) @@ -1044,7 +1158,7 @@ func TestGetExternalServiceCIDRs(t *testing.T) { } for _, tc := range testCases { - _ = os.Setenv(envExternalServiceCIDRs, tc.cidrString) + os.Setenv(envExternalServiceCIDRs, tc.cidrString) cidrs := ln.GetExternalServiceCIDRs() assert.Equal(t, cidrs, tc.expectedCidrs) } diff --git a/scripts/lib/performance_tests.sh b/scripts/lib/performance_tests.sh index 50233a84a9..3e340468c9 100644 --- a/scripts/lib/performance_tests.sh +++ b/scripts/lib/performance_tests.sh @@ -173,7 +173,7 @@ function install_cw_agent(){ echo "Install Cloudwatch Agent DS" $KUBECTL_PATH apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-serviceaccount.yaml - echo '{ "logs": { "metrics_collected": { "kubernetes": { "metrics_collection_interval": 30, "cluster_name": "eks-net-perf" }},"force_flush_interval": 5 }}' | jq > cwagentconfig.json + echo '{ "logs": { "metrics_collected": { "kubernetes": { "metrics_collection_interval": 30, "cluster_name": "eks-net-perf" }},"force_flush_interval": 5 }}' | jq '.' > cwagentconfig.json $KUBECTL_PATH create cm -n $CW_NAMESPACE cwagentconfig --from-file cwagentconfig.json $KUBECTL_PATH apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml diff --git a/scripts/run-static-canary.sh b/scripts/run-static-canary.sh index 91087b06cd..59621474fd 100755 --- a/scripts/run-static-canary.sh +++ b/scripts/run-static-canary.sh @@ -10,6 +10,10 @@ GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build" # TEST_IMAGE_REGISTRY is the registry in test-infra-* accounts where e2e test images are stored TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazonaws.com"} +# If $ENDPOINT is set, as in it is for beta clusters then $ENDPOINT_OPTION, +# defined in lib/cluster.sh will add --eks-endpoint=$ENDPOINT to the ginkgo +# test command + source "$SCRIPT_DIR"/lib/cluster.sh source "$SCRIPT_DIR"/lib/canary.sh @@ -24,11 +28,12 @@ function run_ginkgo_test() { --aws-vpc-id="$VPC_ID" \ --ng-name-label-key="kubernetes.io/os" \ --ng-name-label-val="linux" \ - --test-image-registry=$TEST_IMAGE_REGISTRY) + --test-image-registry=$TEST_IMAGE_REGISTRY \ + $ENDPOINT_OPTION) } load_cluster_details run_ginkgo_test "STATIC_CANARY" -echo "all tests ran successfully in $(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds" \ No newline at end of file +echo "all tests ran successfully in $(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds" diff --git a/test/agent/cmd/snat-utils/main.go b/test/agent/cmd/snat-utils/main.go index e0096538b4..cf59127013 100644 --- a/test/agent/cmd/snat-utils/main.go +++ b/test/agent/cmd/snat-utils/main.go @@ -76,25 +76,43 @@ func validateIPTableRules(randomizedSNATValue string, numOfCidrs int) error { currChain := "AWS-SNAT-CHAIN-0" lastChain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", numOfCidrs) - i := 0 - for i < numOfCidrs { + + exists, err := iptables.ChainExists("nat", "AWS-SNAT-CHAIN-1") + if err != nil { + return err + } + // If AWS-SNAT-CHAIN-1 exists, we run the old logic + if exists { + i := 0 + for i < numOfCidrs { + rules, err := iptables.List("nat", currChain) + if err != nil { + return err + } + i = i + 1 + nextChain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", i) + foundNextChain := false + for _, rule := range rules { + target := fmt.Sprintf("-j %s", nextChain) + if strings.Contains(rule, target) { + currChain = nextChain + foundNextChain = true + break + } + } + if !foundNextChain { + return fmt.Errorf("failed: AWS-SNAT chain broken for %s", currChain) + } + } + } else { + lastChain = "AWS-SNAT-CHAIN-0" rules, err := iptables.List("nat", currChain) if err != nil { return err } - i = i + 1 - nextChain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", i) - foundNextChain := false - for _, rule := range rules { - target := fmt.Sprintf("-j %s", nextChain) - if strings.Contains(rule, target) { - currChain = nextChain - foundNextChain = true - break - } - } - if foundNextChain == false { - return fmt.Errorf("failed: AWS-SNAT chain broken for %s", currChain) + // One rule per cidr + SNAT rule + chain creation rule + if len(rules) != numOfCidrs+2 { + return fmt.Errorf("failed: AWS-SNAT chain does not contain the correct amount of rules") } } @@ -107,7 +125,6 @@ func validateIPTableRules(randomizedSNATValue string, numOfCidrs int) error { // Check for rule with following pattern match := fmt.Sprintf(".*-j SNAT.*%s", expectedString) r, _ := regexp.Compile(match) - for _, rule := range rules { if r.Match([]byte(rule)) { containsExpectedString = true diff --git a/test/integration/snat/snat_test.go b/test/integration/snat/snat_test.go index f73fe6faf7..f95b6854f6 100644 --- a/test/integration/snat/snat_test.go +++ b/test/integration/snat/snat_test.go @@ -51,7 +51,12 @@ var _ = Describe("SNAT tests", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(vpcOutput.Vpcs)).To(BeNumerically(">", 0)) - numOfCidrs := len(vpcOutput.Vpcs[0].CidrBlockAssociationSet) + numOfCidrs := 0 + for _, vpc := range vpcOutput.Vpcs[0].CidrBlockAssociationSet { + if *vpc.CidrBlockState.State == "associated" { + numOfCidrs = numOfCidrs + 1 + } + } By("Check whether SNAT IP table has random-fully with AWS_VPC_K8S_CNI_RANDOMIZESNAT set to default value of prng") ValidateIPTableRules("prng", numOfCidrs)