Route flannel via existing wireguard interface

cmd/cluster_add_external_worker.go

@@ -163,6 +163,10 @@ An external server must meet the following requirements:
 		FatalOnError(err)
 		saveCluster(cluster)
 
+		// restart flannel on all nodes due to wireguard restart
+		err = clusterManager.RestartFlannel()
+		FatalOnError(err)
+
 		// all work on the already existing nodes is completed by now
 		for _, node := range existingNodes {
 			coordinator.CompleteProgress(node.Name)

cmd/cluster_add_worker.go

@@ -123,6 +123,10 @@ You can specify the worker server type as in cluster create.`,
 		FatalOnError(err)
 		saveCluster(cluster)
 
+		// restart flannel on all nodes due to wireguard restart
+		err = clusterManager.RestartFlannel()
+		FatalOnError(err)
+
 		// all work on the already existing nodes is completed by now
 		for _, node := range existingNodes {
 			coordinator.CompleteProgress(node.Name)

pkg/clustermanager/cluster.go

@@ -145,6 +145,24 @@ func (manager *Manager) SetupEncryptedNetwork() error {
 	return nil
 }
 
+//RestartFlannel restarts flannel on all nodes after wireguard restart
+func (manager *Manager) RestartFlannel() error {
+	cmdRestartFlannel :=
+		`kubectl -n kube-system delete pod -l 'app=flannel'`
+
+	for _, node := range manager.nodes {
+		if node.IsMaster {
+			_, err := manager.nodeCommunicator.RunCmd(node, cmdRestartFlannel)
+			if err != nil {
+				return err
+			}
+			break
+		}
+	}
+
+	return nil
+}
+
 //InstallMasters installs the kubernetes control plane to master nodes
 func (manager *Manager) InstallMasters() error {
 
@@ -153,6 +171,7 @@ func (manager *Manager) InstallMasters() error {
 		{"configure kubectl", "rm -rf $HOME/.kube && mkdir -p $HOME/.kube && cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config"},
 		{"install flannel", "kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml"},
 		{"configure flannel", "kubectl -n kube-system patch ds kube-flannel-ds --type json -p '[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}}]'"},
+		{"configure flannel to work via wireguard interface", "kubectl -n kube-system patch ds kube-flannel-ds -p '{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"args\":[\"--ip-masq\",\"--kube-subnet-mgr\",\"--iface\",\"wg0\"],\"name\":\"kube-flannel\"}]}}}}'"},
 		//{"install hcloud integration", fmt.Sprintf("kubectl -n kube-system create secret generic hcloud --from-literal=token=%s", AppConf.CurrentContext.Token)},
 		//{"deploy cloud controller manager", "kubectl apply -f  https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/master/deploy/v1.0.0.yaml"},
 	}

pkg/clustermanager/provision_node.go

@@ -76,12 +76,12 @@ func (provisioner *NodeProvisioner) prepareAndInstall() error {
 	if err != nil {
 		return err
 	}
-	err = provisioner.updateAndInstall()
+	err = provisioner.prepareNetwork()
 	if err != nil {
 		return err
 	}
-
-	return nil
+	err = provisioner.updateAndInstall()
+	return err
 }
 
 func (provisioner *NodeProvisioner) installTransportTools() error {
@@ -112,14 +112,9 @@ func (provisioner *NodeProvisioner) preparePackages() error {
 		return err
 	}
 
-	// wireguard
-	_, err = provisioner.communicator.RunCmd(provisioner.node, "add-apt-repository ppa:wireguard/wireguard -y")
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
+
 func (provisioner *NodeProvisioner) prepareKubernetes() error {
 	// kubernetes
 	_, err := provisioner.communicator.RunCmd(provisioner.node, "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -")
@@ -160,6 +155,64 @@ Pin-Priority: 1000
 	return nil
 }
 
+func (provisioner *NodeProvisioner) prepareNetwork() error {
+
+	provisioner.eventService.AddEvent(provisioner.node.Name, "prepare network")
+
+	err := provisioner.prepareFlannel()
+	if err != nil {
+		return err
+	}
+
+	err = provisioner.prepareWireguard()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (provisioner *NodeProvisioner) prepareFlannel() error {
+	// udev action to run systemd service on each flannel interface add
+	flannelUdevRules :=
+		`SUBSYSTEM=="net", ACTION=="add", KERNEL=="flannel.*", TAG+="systemd", ENV{SYSTEMD_WANTS}="flannel-created@%k.service"
+`
+	// systemd oneshot unit to run ethtool on corresponding interface
+	flannelSystemd :=
+		`[Unit]
+Description=Disable TX checksum offload on flannel interface
+[Service]
+Type=oneshot
+ExecStart=/sbin/ethtool -K %I tx off
+`
+	err := provisioner.communicator.WriteFile(provisioner.node, "/etc/udev/rules.d/71-flannel.rules", flannelUdevRules, false)
+	if err != nil {
+		return err
+	}
+
+	err = provisioner.communicator.WriteFile(provisioner.node, "/etc/systemd/system/[email protected]", flannelSystemd, false)
+	if err != nil {
+		return err
+	}
+
+	_, err = provisioner.communicator.RunCmd(provisioner.node, "systemctl daemon-reload; systemctl restart systemd-udevd.service")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (provisioner *NodeProvisioner) prepareWireguard() error {
+
+	_, err := provisioner.communicator.RunCmd(provisioner.node, "add-apt-repository ppa:wireguard/wireguard -y")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (provisioner *NodeProvisioner) updateAndInstall() error {
 	provisioner.eventService.AddEvent(provisioner.node.Name, "updating packages")
 	_, err := provisioner.communicator.RunCmd(provisioner.node, "apt-get update")
@@ -168,7 +221,7 @@ func (provisioner *NodeProvisioner) updateAndInstall() error {
 	}
 
 	provisioner.eventService.AddEvent(provisioner.node.Name, "installing packages")
-	command := fmt.Sprintf("apt-get install -y docker-ce kubelet=%s kubeadm=%s kubectl=%s wireguard linux-headers-$(uname -r) linux-headers-virtual",
+	command := fmt.Sprintf("apt-get install -y docker-ce kubelet=%s kubeadm=%s kubectl=%s wireguard ethtool linux-headers-$(uname -r) linux-headers-virtual",
 		*K8sVersion, *K8sVersion, *K8sVersion)
 	_, err = provisioner.communicator.RunCmd(provisioner.node, command)
 	if err != nil {