Merge pull request #6 from BSWANG/master

* felix upgrade to 3.5.2 and use build in Dockerfile using multistage build
* fix eni autoconfig by eni.service
* add eni-multi-ip documents
* support reserve ip on statefulset release

Dockerfile

@@ -4,10 +4,32 @@ COPY . .
 RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X \"main.gitVer=`git rev-parse --short HEAD 2>/dev/null`\" " -o terwayd .
 RUN cd plugin/terway && CGO_ENABLED=0 GOOS=linux go build -o terway .
 
+FROM calico/go-build:v0.20 as felix-builder
+RUN apk --no-cache add ip6tables tini ipset iputils iproute2 conntrack-tools file git
+ENV GIT_BRANCH=v3.5.2
+ENV GIT_COMMIT=1e59bb818c35b96e5de6e882fcb07510f81b50da
+#ENV http_proxy=1.1.1.1:1080
+#ENV https_proxy=1.1.1.1:1080
+RUN mkdir -p /go/src/github.com/projectcalico/ && cd /go/src/github.com/projectcalico/ && \
+    git clone -b ${GIT_BRANCH} https://github.com/projectcalico/felix.git && \
+    cd felix && [ "`git rev-parse HEAD`" = "${GIT_COMMIT}" ]
+RUN cd /go/src/github.com/projectcalico/felix && glide up --strip-vendor || glide up --strip-vendor || glide up --strip-vendor # retry 3 times
+COPY policy /terway_patch
+RUN cd /go/src/github.com/projectcalico/felix && git apply /terway_patch/*.patch && \
+    go build -v -i -o bin/calico-felix-amd64 -v -ldflags \
+    "-X github.com/projectcalico/felix/buildinfo.GitVersion=${GIT_BRANCH} \
+    -X github.com/projectcalico/felix/buildinfo.BuildDate=$(date -u +'%FT%T%z') \
+    -X github.com/projectcalico/felix/buildinfo.GitRevision=${GIT_COMMIT} \
+    -B 0x${GIT_COMMIT}" "github.com/projectcalico/felix/cmd/calico-felix" && \
+    ( ldd bin/calico-felix-amd64 2>&1 | grep -q -e "Not a valid dynamic program" \
+    -e "not a dynamic executable" || \
+    ( echo "Error: bin/calico-felix-amd64 was not statically linked"; false ) )
+
 FROM alpine:3.8
-COPY script/ /bin/
+COPY policy/policyinit.sh /bin/
 RUN apk --update add curl ipset bash iproute2 ethtool bridge-utils && chmod +x /bin/policyinit.sh && rm -f /var/cache/apk/*
-RUN curl -sSL -o /bin/calico-felix https://docker-plugin.oss-cn-shanghai.aliyuncs.com/calico-felix && chmod +x /bin/calico-felix
+COPY --from=felix-builder /go/src/github.com/projectcalico/felix/bin/calico-felix-amd64 /bin/calico-felix
+RUN chmod +x /bin/calico-felix
 COPY --from=builder /go/src/github.com/AliyunContainerService/terway/terwayd /usr/bin/terwayd
 COPY --from=builder /go/src/github.com/AliyunContainerService/terway/plugin/terway/terway /usr/bin/terway
 ENTRYPOINT ["/usr/bin/terwayd"]

README-zh_CN.md

@@ -8,20 +8,33 @@ CNI plugin for alibaba cloud VPC/ENI
 ## 安装Kubernetes
 使用kubeadm的指导文档 https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/ 来创建集群
 
-安装好了之后要将iptables的policy换成ACCEPT，`iptables -P FORWARD ACCEPT`
+安装好了之后要:
+* 将iptables的policy换成ACCEPT，`iptables -P FORWARD ACCEPT`。
+* 检查节点上的"rp_filter"内核参数，并在每个节点上将其设置为"0"。
 
 通过`kubectl get cs`验证集群安装完成
 
 ## 安装terway插件
 
-修改[terway.yml](./terway.yml)文件中的eni.conf的配置中的授权和网段配置，以及Network的网段配置，然后通过`kubectl apply -f terway.conf`来安装terway插件。
+Terway有两种安装模式：
+
+* VPC模式
+
+	VPC模式，使用Aliyun VPC路由来打通网络，可以使用独立ENI给Pod，安装方式：<br />
+	修改[terway.yml](./terway.yml)文件中的eni.conf的配置中的授权和网段配置，以及Network的网段配置，然后通过`kubectl apply -f terway.conf`来安装terway插件。
+
+* ENI多IP模式
+
+	ENI多IP模式，使用Aliyun ENI的辅助IP来打通网络，不受VPC的路由条目限制，安装方式：<br />
+	修改[terway-multiip.yml](./terway-multiip.yml)文件中的eni.conf的配置中的授权和网段配置，以及Network的网段配置，然后通过`kubectl apply -f terway.conf`来安装terway插件。
+
 
 使用`kubectl get ds terway`看到插件在每个节点上都运行起来后，表明插件安装成功。
 
 ## 验证terway的功能
 
 ### 一般VPC网络的容器
-在容器没有做任何特殊配置时，terway会通过在节点上的podCidr中去分配地址然后配置给容器。
+在VPC安装模式下，在容器没有做任何特殊配置时，terway会通过在节点上的podCidr中去分配地址然后配置给容器。
 例如：
 
 ```
@@ -47,9 +60,9 @@ If you don't see a command prompt, try pressing enter.
        valid_lft forever preferred_lft forever
 ```   
 
-### 使用ENI弹性网卡获得等同于底层网络的性能
+#### 使用ENI弹性网卡获得等同于底层网络的性能
 
-在Pod的其中一个container的`requests`中增加对eni的需求： `aliyun/eni: 1`， 下面的例子将创建一个Nginx Pod，并分配一个ENI
+在VPC安装模式下，在Pod的其中一个container的`requests`中增加对eni的需求： `aliyun/eni: 1`， 下面的例子将创建一个Nginx Pod，并分配一个ENI
 
 ```
 apiVersion: v1
@@ -88,6 +101,29 @@ spec:
        valid_lft forever preferred_lft forever
 ```
 
+#### ENI辅助IP的容器：
+
+在ENI多IP安装模式下，Terway会通过创建和分配ENI和ENI网卡上的辅助IP地址给Pod使用，Pod上的IP地址将和VPC和VSwitch的IP地址相同段，例如：
+
+```
+[root@iZj6c86lmr8k9rk78ju0ncZ ~]# kubectl get pod -o wide
+NAME                     READY   STATUS    RESTARTS   AGE   IP              NODE                                 NOMINATED NODE
+nginx-64f497f8fd-ckpdm   1/1     Running   0          4d    192.168.0.191   cn-hangzhou.i-j6c86lmr8k9rk78ju0nc   <none>
+[root@iZj6c86lmr8k9rk78ju0ncZ ~]# kubectl get node -o wide cn-hangzhou.i-j6c86lmr8k9rk78ju0nc
+NAME                                 STATUS   ROLES    AGE   VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION              CONTAINER-RUNTIME
+cn-hangzhou.i-j6c86lmr8k9rk78ju0nc   Ready    <none>   12d   v1.11.5   192.168.0.154   <none>        CentOS Linux 7 (Core)   3.10.0-693.2.2.el7.x86_64   docker://17.6.2
+[root@iZj6c86lmr8k9rk78ju0ncZ ~]# kubectl exec -it nginx-64f497f8fd-ckpdm bash
+root@nginx-64f497f8fd-ckpdm:/# ip addr show
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
+    inet 127.0.0.1/8 scope host lo
+       valid_lft forever preferred_lft forever
+3: eth0@if106: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
+    link/ether 4a:60:eb:97:f4:07 brd ff:ff:ff:ff:ff:ff link-netnsid 0
+    inet 192.168.0.191/32 brd 192.168.0.191 scope global eth0
+       valid_lft forever preferred_lft forever
+```
+
 ### 使用NetworkPolicy来限制容器间访问
 
 Terway插件兼容标准的K8S中的NetworkPolicy来控制容器间的访问，例如：

README.md

@@ -9,21 +9,35 @@ English | [简体中文](./README-zh_CN.md)
 ### Install Kubernetes
 Install Kubernetes via Kubeadm: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
 
-After setup kubernetes cluster. Change `iptables` `Forward` default policy to `ACCEPT` on every node of cluster: `iptables -P FORWARD ACCEPT`.
+After setup kubernetes cluster.
+* Change `iptables` `Forward` default policy to `ACCEPT` on every node of cluster: `iptables -P FORWARD ACCEPT`.
+* Check the `rp_filter` in sysctl parameters, set them to "0" on every node of cluster.
 
 Make sure cluster up and healthy by `kubectl get cs`.
 
 ### Install Terway network plugin
 
-Replace `Network` and `AccessKey/AccessKeySecret` in [terway.yml](./terway.yml) with your cluster pod subnet and aliyun openapi credentials. Then use `kubectl apply -f terway.yml` to install Terway into kubernetes cluster.
+<br />
+Terway plugin have two installation modes
+
+* VPC Mode
+
+	VPC Mode, Using `Aliyun VPC` route table to connect the pods. Can assign dedicated ENI to Pod. Install method: <br />
+	Replace `Network` and `AccessKey/AccessKeySecret` in [terway.yml](./terway.yml) with your cluster pod subnet and aliyun openapi credentials. Then use `kubectl apply -f terway.yml` to install Terway into kubernetes cluster.
+
+* ENI Secondary IP Mode
+
+	ENI Secondary IP Mode, Using `Aliyun ENI's secondary ip` to connect the pods. This mode not limited by VPC route tables quotation. Install method: <br />
+	Replace `Network` and `AccessKey/AccessKeySecret` in [terway-multiip.yml](./terway-multiip.yml) with your cluster pod subnet and aliyun openapi credentials. Then use `kubectl apply -f terway.yml` to install Terway into kubernetes cluster.
+
 
 Using `kubectl get ds terway -n kube-system` to watch plugin launching. Plugin install completed while terway daemonset available pods equal to nodes.
 
 ### Terway network plugin usage:
 
 #### Vpc network container:
 
-Terway will config pod's address using node's `podCidr` when pod not have any especial config. eg:
+On VPC installation mode, Terway will config pod's address using node's `podCidr` when pod not have any especial config. eg:
 
 ```
 [root@iZj6c86lmr8k9rk78ju0ncZ ~]# kubectl run -it --rm --image busybox busybox
@@ -49,7 +63,7 @@ If you don't see a command prompt, try pressing enter.
 ```   
 
 #### Using ENI network interface to get the performance equivalent to the underlying network.
-Config `eni` request `aliyun/eni: 1` in one container of pod. The following example will create an Nginx Pod and assign an ENI:
+On VPC installation mode, Config `eni` request `aliyun/eni: 1` in one container of pod. The following example will create an Nginx Pod and assign an ENI:
 
 ```
 apiVersion: v1
@@ -86,6 +100,29 @@ spec:
        valid_lft forever preferred_lft forever
 ```
 
+#### ENI Secondary IP Pod：
+
+On ENI secondary IP installation mode, Terway will create & allocate ENI secondary IP for pod. The IP of pod will in same IP Range:
+
+```
+[root@iZj6c86lmr8k9rk78ju0ncZ ~]# kubectl get pod -o wide
+NAME                     READY   STATUS    RESTARTS   AGE   IP              NODE                                 NOMINATED NODE
+nginx-64f497f8fd-ckpdm   1/1     Running   0          4d    192.168.0.191   cn-hangzhou.i-j6c86lmr8k9rk78ju0nc   <none>
+[root@iZj6c86lmr8k9rk78ju0ncZ ~]# kubectl get node -o wide cn-hangzhou.i-j6c86lmr8k9rk78ju0nc
+NAME                                 STATUS   ROLES    AGE   VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION              CONTAINER-RUNTIME
+cn-hangzhou.i-j6c86lmr8k9rk78ju0nc   Ready    <none>   12d   v1.11.5   192.168.0.154   <none>        CentOS Linux 7 (Core)   3.10.0-693.2.2.el7.x86_64   docker://17.6.2
+[root@iZj6c86lmr8k9rk78ju0ncZ ~]# kubectl exec -it nginx-64f497f8fd-ckpdm bash
+root@nginx-64f497f8fd-ckpdm:/# ip addr show
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
+    inet 127.0.0.1/8 scope host lo
+       valid_lft forever preferred_lft forever
+3: eth0@if106: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
+    link/ether 4a:60:eb:97:f4:07 brd ff:ff:ff:ff:ff:ff link-netnsid 0
+    inet 192.168.0.191/32 brd 192.168.0.191 scope global eth0
+       valid_lft forever preferred_lft forever
+```
+
 #### Using network policy to limit accessible between containers.
 
 The Terway plugin is compatible with NetworkPolicy in the standard K8S to control access between containers, for example:

daemon/eni-multi-ip.go

@@ -23,8 +23,8 @@ type ENIIP struct {
 type ENI struct {
 	lock sync.Mutex
 	*types.ENI
-	ips  []*ENIIP
-	ecs  aliyun.ECS
+	ips []*ENIIP
+	ecs aliyun.ECS
 }
 
 // 尝试绑一个
@@ -41,7 +41,7 @@ func (e *ENI) allocateExistENIsIP() *ENIIP {
 		}
 		ipNew := &ENIIP{
 			ENIIP: &types.ENIIP{
-				Eni: e.ENI,
+				Eni:        e.ENI,
 				SecAddress: ip,
 			},
 		}
@@ -51,8 +51,15 @@ func (e *ENI) allocateExistENIsIP() *ENIIP {
 	return nil
 }
 
-func (f *ENIIPFactory) Create() (types.NetworkResource, error) {
+func (f *ENIIPFactory) Create() (ip types.NetworkResource, err error) {
 	f.RLock()
+	defer func() {
+		if ip == nil {
+			logrus.Debugf("create result: %v, error: %v", ip, err != nil)
+		} else {
+			logrus.Debugf("create result: %v, error: %v", ip.GetResourceId(), err != nil)
+		}
+	}()
 	for _, eni := range f.enis {
 		ip := eni.allocateExistENIsIP()
 		if ip != nil {
@@ -78,7 +85,7 @@ func (f *ENIIPFactory) Create() (types.NetworkResource, error) {
 	}
 
 	mainENIIP := &types.ENIIP{
-		Eni: eni.ENI,
+		Eni:        eni.ENI,
 		SecAddress: eni.ENI.Address.IP,
 	}
 
@@ -93,10 +100,13 @@ func (f *ENIIPFactory) Create() (types.NetworkResource, error) {
 	return mainENIIP, nil
 }
 
-func (f *ENIIPFactory) Dispose(res types.NetworkResource) error {
+func (f *ENIIPFactory) Dispose(res types.NetworkResource) (err error) {
+	defer func() {
+		logrus.Debugf("dispose result: %v, error: %v", res.GetResourceId(), err != nil)
+	}()
 	ip := res.(*types.ENIIP)
 	var (
-		eni *ENI
+		eni   *ENI
 		eniip *ENIIP
 	)
 	f.RLock()
@@ -141,7 +151,7 @@ func (f *ENIIPFactory) Dispose(res types.NetworkResource) error {
 
 	// main ip of eni, raise put_it_back error
 	if ip.Eni.Address.IP.Equal(ip.SecAddress) {
-		return fmt.Errorf("ip tobe release is primary ip of eni")
+		return fmt.Errorf("ip to be release is primary ip of eni")
 	}
 
 	err = f.eniFactory.ecs.UnAssignIPForENI(ip.Eni.ID, ip.SecAddress)
@@ -164,7 +174,6 @@ func newENIIPFactory(config *types.PoolConfig) (*ENIIPFactory, error) {
 	return &ENIIPFactory{}, nil
 }
 
-
 type ENIIPResourceManager struct {
 	pool pool.ObjectPool
 }
@@ -177,7 +186,7 @@ func NewENIIPResourceManager(poolConfig *types.PoolConfig, ecs aliyun.ECS, alloc
 
 	factory := &ENIIPFactory{
 		eniFactory: eniFactory,
-		enis: []*ENI{},
+		enis:       []*ENI{},
 	}
 
 	capacity, err := ecs.GetInstanceMaxPrivateIP(poolConfig.InstanceID)
@@ -187,12 +196,13 @@ func NewENIIPResourceManager(poolConfig *types.PoolConfig, ecs aliyun.ECS, alloc
 
 	if poolConfig.MaxPoolSize > capacity {
 		logrus.Infof("max pool size bigger than node capacity, set max pool size to capacity")
+		poolConfig.MaxPoolSize = capacity
 	}
 
 	poolCfg := pool.PoolConfig{
-		MaxIdle: poolConfig.MaxPoolSize,
-		MinIdle: poolConfig.MinPoolSize,
-		Factory: factory,
+		MaxIdle:  poolConfig.MaxPoolSize,
+		MinIdle:  poolConfig.MinPoolSize,
+		Factory:  factory,
 		Capacity: capacity,
 		Initializer: func(holder pool.ResourceHolder) error {
 			// not use main eni for eni multiple ip allocate
@@ -212,14 +222,14 @@ func NewENIIPResourceManager(poolConfig *types.PoolConfig, ecs aliyun.ECS, alloc
 				}
 				poolENI := &ENI{
 					lock: sync.Mutex{},
-					ENI: eni,
-					ips: []*ENIIP{},
-					ecs: ecs,
+					ENI:  eni,
+					ips:  []*ENIIP{},
+					ecs:  ecs,
 				}
 				factory.enis = append(factory.enis, poolENI)
 				for _, ip := range ips {
 					eniIP := &types.ENIIP{
-						Eni: eni,
+						Eni:        eni,
 						SecAddress: ip,
 					}
 					_, ok := stubMap[eniIP.GetResourceId()]
@@ -251,6 +261,9 @@ func (m *ENIIPResourceManager) Allocate(ctx *NetworkContext, prefer string) (typ
 }
 
 func (m *ENIIPResourceManager) Release(context *NetworkContext, resId string) error {
+	if context != nil && context.pod != nil {
+		return m.pool.ReleaseWithReverse(resId, context.pod.IpStickTime)
+	}
 	return m.pool.Release(resId)
 }
 
@@ -264,4 +277,4 @@ func (m *ENIIPResourceManager) GarbageCollection(inUseSet map[string]interface{}
 		}
 	}
 	return nil
-}
+}

daemon/eni.go

@@ -27,7 +27,7 @@ func NewENIResourceManager(poolConfig *types.PoolConfig, ecs aliyun.ECS, allocat
 		return nil, errors.Wrapf(err, "error get eni max capacity for eni factory")
 	}
 
-	capacity = int(float64(capacity) * poolConfig.EniCapRatio) + poolConfig.EniCapShift - 1
+	capacity = int(float64(capacity)*poolConfig.EniCapRatio) + poolConfig.EniCapShift - 1
 	if poolConfig.MaxPoolSize > capacity {
 		poolConfig.MaxPoolSize = capacity
 	}
@@ -78,6 +78,9 @@ func (m *ENIResourceManager) Allocate(ctx *NetworkContext, prefer string) (types
 }
 
 func (m *ENIResourceManager) Release(context *NetworkContext, resId string) error {
+	if context != nil && context.pod != nil {
+		return m.pool.ReleaseWithReverse(resId, context.pod.IpStickTime)
+	}
 	return m.pool.Release(resId)
 }
 
@@ -117,4 +120,4 @@ func (f *ENIFactory) Create() (types.NetworkResource, error) {
 func (f *ENIFactory) Dispose(resource types.NetworkResource) error {
 	eni := resource.(*types.ENI)
 	return f.ecs.FreeENI(eni.ID, f.instanceId)
-}
+}