diff --git a/cmd/cluster_add_external_worker.go b/cmd/cluster_add_external_worker.go index 2075f128..bf72415b 100644 --- a/cmd/cluster_add_external_worker.go +++ b/cmd/cluster_add_external_worker.go @@ -163,6 +163,10 @@ An external server must meet the following requirements: FatalOnError(err) saveCluster(cluster) + // restart flannel on all nodes due to wireguard restart + err = clusterManager.RestartFlannel() + FatalOnError(err) + // all work on the already existing nodes is completed by now for _, node := range existingNodes { coordinator.CompleteProgress(node.Name) diff --git a/cmd/cluster_add_worker.go b/cmd/cluster_add_worker.go index 442cdca4..19934947 100644 --- a/cmd/cluster_add_worker.go +++ b/cmd/cluster_add_worker.go @@ -123,6 +123,10 @@ You can specify the worker server type as in cluster create.`, FatalOnError(err) saveCluster(cluster) + // restart flannel on all nodes due to wireguard restart + err = clusterManager.RestartFlannel() + FatalOnError(err) + // all work on the already existing nodes is completed by now for _, node := range existingNodes { coordinator.CompleteProgress(node.Name) diff --git a/pkg/clustermanager/cluster.go b/pkg/clustermanager/cluster.go index d511d34a..0d653e42 100644 --- a/pkg/clustermanager/cluster.go +++ b/pkg/clustermanager/cluster.go @@ -145,6 +145,24 @@ func (manager *Manager) SetupEncryptedNetwork() error { return nil } +//RestartFlannel restarts flannel on all nodes after wireguard restart +func (manager *Manager) RestartFlannel() error { + cmdRestartFlannel := + `kubectl -n kube-system delete pod -l 'app=flannel'` + + for _, node := range manager.nodes { + if node.IsMaster { + _, err := manager.nodeCommunicator.RunCmd(node, cmdRestartFlannel) + if err != nil { + return err + } + break + } + } + + return nil +} + //InstallMasters installs the kubernetes control plane to master nodes func (manager *Manager) InstallMasters() error { @@ -153,6 +171,7 @@ func (manager *Manager) InstallMasters() error { {"configure kubectl", "rm -rf $HOME/.kube && mkdir -p $HOME/.kube && cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config"}, {"install flannel", "kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml"}, {"configure flannel", "kubectl -n kube-system patch ds kube-flannel-ds --type json -p '[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}}]'"}, + {"configure flannel to work via wireguard interface", "kubectl -n kube-system patch ds kube-flannel-ds -p '{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"args\":[\"--ip-masq\",\"--kube-subnet-mgr\",\"--iface\",\"wg0\"],\"name\":\"kube-flannel\"}]}}}}'"}, //{"install hcloud integration", fmt.Sprintf("kubectl -n kube-system create secret generic hcloud --from-literal=token=%s", AppConf.CurrentContext.Token)}, //{"deploy cloud controller manager", "kubectl apply -f https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/master/deploy/v1.0.0.yaml"}, } diff --git a/pkg/clustermanager/provision_node.go b/pkg/clustermanager/provision_node.go index 39f245f7..9dfd8198 100644 --- a/pkg/clustermanager/provision_node.go +++ b/pkg/clustermanager/provision_node.go @@ -76,12 +76,12 @@ func (provisioner *NodeProvisioner) prepareAndInstall() error { if err != nil { return err } - err = provisioner.updateAndInstall() + err = provisioner.prepareNetwork() if err != nil { return err } - - return nil + err = provisioner.updateAndInstall() + return err } func (provisioner *NodeProvisioner) installTransportTools() error { @@ -112,14 +112,9 @@ func (provisioner *NodeProvisioner) preparePackages() error { return err } - // wireguard - _, err = provisioner.communicator.RunCmd(provisioner.node, "add-apt-repository ppa:wireguard/wireguard -y") - if err != nil { - return err - } - return nil } + func (provisioner *NodeProvisioner) prepareKubernetes() error { // kubernetes _, err := provisioner.communicator.RunCmd(provisioner.node, "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -") @@ -160,6 +155,64 @@ Pin-Priority: 1000 return nil } +func (provisioner *NodeProvisioner) prepareNetwork() error { + + provisioner.eventService.AddEvent(provisioner.node.Name, "prepare network") + + err := provisioner.prepareFlannel() + if err != nil { + return err + } + + err = provisioner.prepareWireguard() + if err != nil { + return err + } + + return nil +} + +func (provisioner *NodeProvisioner) prepareFlannel() error { + // udev action to run systemd service on each flannel interface add + flannelUdevRules := + `SUBSYSTEM=="net", ACTION=="add", KERNEL=="flannel.*", TAG+="systemd", ENV{SYSTEMD_WANTS}="flannel-created@%k.service" +` + // systemd oneshot unit to run ethtool on corresponding interface + flannelSystemd := + `[Unit] +Description=Disable TX checksum offload on flannel interface +[Service] +Type=oneshot +ExecStart=/sbin/ethtool -K %I tx off +` + err := provisioner.communicator.WriteFile(provisioner.node, "/etc/udev/rules.d/71-flannel.rules", flannelUdevRules, false) + if err != nil { + return err + } + + err = provisioner.communicator.WriteFile(provisioner.node, "/etc/systemd/system/flannel-created@.service", flannelSystemd, false) + if err != nil { + return err + } + + _, err = provisioner.communicator.RunCmd(provisioner.node, "systemctl daemon-reload; systemctl restart systemd-udevd.service") + if err != nil { + return err + } + + return nil +} + +func (provisioner *NodeProvisioner) prepareWireguard() error { + + _, err := provisioner.communicator.RunCmd(provisioner.node, "add-apt-repository ppa:wireguard/wireguard -y") + if err != nil { + return err + } + + return nil +} + func (provisioner *NodeProvisioner) updateAndInstall() error { provisioner.eventService.AddEvent(provisioner.node.Name, "updating packages") _, err := provisioner.communicator.RunCmd(provisioner.node, "apt-get update") @@ -168,7 +221,7 @@ func (provisioner *NodeProvisioner) updateAndInstall() error { } provisioner.eventService.AddEvent(provisioner.node.Name, "installing packages") - command := fmt.Sprintf("apt-get install -y docker-ce kubelet=%s kubeadm=%s kubectl=%s wireguard linux-headers-$(uname -r) linux-headers-virtual", + command := fmt.Sprintf("apt-get install -y docker-ce kubelet=%s kubeadm=%s kubectl=%s wireguard ethtool linux-headers-$(uname -r) linux-headers-virtual", *K8sVersion, *K8sVersion, *K8sVersion) _, err = provisioner.communicator.RunCmd(provisioner.node, command) if err != nil {