diff --git a/.golangci.yml b/.golangci.yml index b6d964f4..c0890458 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,3 +1,6 @@ +run: + skip-dirs: + - cmd linters-settings: govet: check-shadowing: true @@ -14,6 +17,8 @@ linters-settings: min-occurrences: 2 misspell: locale: US + funlen: + lines: 100 linters: enable-all: true @@ -23,3 +28,10 @@ linters: - gocyclo - govet - lll + +issues: + exclude-rules: + - path: pkg/clustermanager/ssh_communicator.go + text: "G106: Use of ssh InsecureIgnoreHostKey should be audited" + linters: + - gosec diff --git a/cmd/cluster_add_external_worker.go b/cmd/cluster_add_external_worker.go index 14e83bb2..82da3bea 100644 --- a/cmd/cluster_add_external_worker.go +++ b/cmd/cluster_add_external_worker.go @@ -128,7 +128,7 @@ An external server must meet the following requirements: break } externalNode.PrivateIPAddress = fmt.Sprintf("%s.%d", cidrPrefix, nextNode) - coordinator := pkg.NewProgressCoordinator() + coordinator := pkg.NewProgressCoordinator(DebugMode) hetznerProvider := hetzner.NewHetznerProvider(AppConf.Context, AppConf.Client, *cluster, AppConf.CurrentContext.Token) clusterManager := clustermanager.NewClusterManagerFromCluster(*cluster, hetznerProvider, sshClient, coordinator) diff --git a/cmd/cluster_add_worker.go b/cmd/cluster_add_worker.go index 4879747d..597077f9 100644 --- a/cmd/cluster_add_worker.go +++ b/cmd/cluster_add_worker.go @@ -91,7 +91,7 @@ You can specify the worker server type as in cluster create.`, } } - coordinator := pkg.NewProgressCoordinator() + coordinator := pkg.NewProgressCoordinator(DebugMode) hetznerProvider := hetzner.NewHetznerProvider(AppConf.Context, AppConf.Client, *cluster, AppConf.CurrentContext.Token) clusterManager := clustermanager.NewClusterManagerFromCluster(*cluster, hetznerProvider, AppConf.SSHClient, coordinator) err := AppConf.SSHClient.(*clustermanager.SSHCommunicator).CapturePassphrase(sshKeyName) diff --git a/cmd/cluster_create.go b/cmd/cluster_create.go index 4f5bb003..8687c4d1 100644 --- a/cmd/cluster_create.go +++ b/cmd/cluster_create.go @@ -3,13 +3,14 @@ package cmd import ( "errors" "fmt" - "github.com/hetznercloud/hcloud-go/hcloud" - "github.com/xetys/hetzner-kube/pkg/phases" "log" "net" "os" "time" + "github.com/hetznercloud/hcloud-go/hcloud" + "github.com/xetys/hetzner-kube/pkg/phases" + "github.com/spf13/cobra" "github.com/xetys/hetzner-kube/pkg" "github.com/xetys/hetzner-kube/pkg/clustermanager" @@ -101,7 +102,7 @@ func RunClusterCreate(cmd *cobra.Command, args []string) { time.Sleep(10 * time.Second) } - coordinator := pkg.NewProgressCoordinator() + coordinator := pkg.NewProgressCoordinator(DebugMode) clusterManager := clustermanager.NewClusterManager(hetznerProvider, sshClient, coordinator, clusterName, haEnabled, isolatedEtcd, cloudInit) cluster := clusterManager.Cluster() diff --git a/cmd/cluster_phase.go b/cmd/cluster_phase.go index 0bbc9621..22e71a2e 100644 --- a/cmd/cluster_phase.go +++ b/cmd/cluster_phase.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "github.com/spf13/cobra" "github.com/xetys/hetzner-kube/pkg" "github.com/xetys/hetzner-kube/pkg/clustermanager" @@ -38,7 +39,7 @@ func getCommonPhaseDependencies(steps int, cmd *cobra.Command, args []string) (c FatalOnError(err) err = AppConf.SSHClient.(*clustermanager.SSHCommunicator).CapturePassphrase(masterNode.SSHKeyName) FatalOnError(err) - coordinator := pkg.NewProgressCoordinator() + coordinator := pkg.NewProgressCoordinator(DebugMode) for _, node := range provider.GetAllNodes() { coordinator.StartProgress(node.Name, steps) diff --git a/cmd/cluster_phase_install_masters.go b/cmd/cluster_phase_install_masters.go index 07d50105..8d6e9571 100644 --- a/cmd/cluster_phase_install_masters.go +++ b/cmd/cluster_phase_install_masters.go @@ -31,7 +31,7 @@ var installMastersPhaseCommand = &cobra.Command{ if err != nil { return err } - coordinator := pkg.NewProgressCoordinator() + coordinator := pkg.NewProgressCoordinator(DebugMode) for _, node := range provider.GetAllNodes() { steps := 3 diff --git a/cmd/cluster_phase_install_workers.go b/cmd/cluster_phase_install_workers.go index 042bc924..17add38e 100644 --- a/cmd/cluster_phase_install_workers.go +++ b/cmd/cluster_phase_install_workers.go @@ -26,7 +26,7 @@ var installWorkersCommand = &cobra.Command{ if err != nil { return err } - coordinator := pkg.NewProgressCoordinator() + coordinator := pkg.NewProgressCoordinator(DebugMode) for _, node := range provider.GetAllNodes() { steps := 2 diff --git a/cmd/cluster_phase_setup_ha.go b/cmd/cluster_phase_setup_ha.go index 3573819d..28d4e0b3 100644 --- a/cmd/cluster_phase_setup_ha.go +++ b/cmd/cluster_phase_setup_ha.go @@ -26,7 +26,7 @@ var setupHAPhaseCommand = &cobra.Command{ if err != nil { return err } - coordinator := pkg.NewProgressCoordinator() + coordinator := pkg.NewProgressCoordinator(DebugMode) for _, node := range provider.GetAllNodes() { steps := 2 diff --git a/cmd/root.go b/cmd/root.go index d7efbd30..81bba8b1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -7,7 +7,6 @@ import ( "github.com/mitchellh/go-homedir" "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/xetys/hetzner-kube/pkg" ) var cfgFile string @@ -21,10 +20,8 @@ var rootCmd = &cobra.Command{ `, PersistentPreRun: func(cmd *cobra.Command, args []string) { - pkg.RenderProgressBars = false if DebugMode { fmt.Println("Running in Debug Mode!") - pkg.RenderProgressBars = true } AppConf = NewAppConfig(DebugMode) }, diff --git a/pkg/addons/addon_cert_manager.go b/pkg/addons/addon_cert_manager.go index 3bf812bd..5f9605f9 100644 --- a/pkg/addons/addon_cert_manager.go +++ b/pkg/addons/addon_cert_manager.go @@ -15,12 +15,10 @@ type CertmanagerAddon struct { // NewCertmanagerAddon creates an addon installing cert-manager func NewCertmanagerAddon(cluster clustermanager.ClusterProvider, communicator clustermanager.NodeCommunicator) ClusterAddon { masterNode, err := cluster.GetMasterNode() + FatalOnError(err) - return &CertmanagerAddon{masterNode: masterNode, communicator: communicator} -} -func init() { - addAddon(NewCertmanagerAddon) + return &CertmanagerAddon{masterNode: masterNode, communicator: communicator} } // Name returns the addons name diff --git a/pkg/addons/addon_dashboard.go b/pkg/addons/addon_dashboard.go index c8641d22..a5a246a5 100644 --- a/pkg/addons/addon_dashboard.go +++ b/pkg/addons/addon_dashboard.go @@ -18,10 +18,6 @@ func NewDashboardAddon(provider clustermanager.ClusterProvider, communicator clu return DashboardAddon{masterNode: masterNode, communicator: communicator} } -func init() { - addAddon(NewDashboardAddon) -} - // Name returns the addons name func (addon DashboardAddon) Name() string { return "dashboard" diff --git a/pkg/addons/addon_docker_registry.go b/pkg/addons/addon_docker_registry.go index a8abe3a7..de1293b3 100644 --- a/pkg/addons/addon_docker_registry.go +++ b/pkg/addons/addon_docker_registry.go @@ -15,12 +15,10 @@ type DockerregistryAddon struct { // NewDockerregistryAddon creates an addon providing a private docker registry func NewDockerregistryAddon(provider clustermanager.ClusterProvider, communicator clustermanager.NodeCommunicator) ClusterAddon { masterNode, err := provider.GetMasterNode() + FatalOnError(err) - return &DockerregistryAddon{masterNode: masterNode, communicator: communicator} -} -func init() { - addAddon(NewDockerregistryAddon) + return &DockerregistryAddon{masterNode: masterNode, communicator: communicator} } // Name returns the addons name @@ -47,6 +45,7 @@ func (addon *DockerregistryAddon) URL() string { func (addon *DockerregistryAddon) Install(args ...string) { node := *addon.masterNode _, err := addon.communicator.RunCmd(node, "helm install --set persistence.enabled=true stable/docker-registry") + FatalOnError(err) log.Println("docker-registry installed") } @@ -55,6 +54,7 @@ func (addon *DockerregistryAddon) Install(args ...string) { func (addon DockerregistryAddon) Uninstall() { node := *addon.masterNode _, err := addon.communicator.RunCmd(node, "helm delete --purge `helm list | grep docker-registry | awk '{print $1;}'`") + FatalOnError(err) log.Println("docker-registry uninstalled") } diff --git a/pkg/addons/addon_hcloud_controller_manager.go b/pkg/addons/addon_hcloud_controller_manager.go index c7076ff2..9e8310ff 100644 --- a/pkg/addons/addon_hcloud_controller_manager.go +++ b/pkg/addons/addon_hcloud_controller_manager.go @@ -18,7 +18,9 @@ type HCloudControllerManagerAddon struct { // NewHCloudControllerManagerAddon returns a CloudProvider instance with type HCloudControllerManagerAddon func NewHCloudControllerManagerAddon(provider clustermanager.ClusterProvider, communicator clustermanager.NodeCommunicator) ClusterAddon { masterNode, err := provider.GetMasterNode() + FatalOnError(err) + return &HCloudControllerManagerAddon{ masterNode: masterNode, communicator: communicator, @@ -27,10 +29,6 @@ func NewHCloudControllerManagerAddon(provider clustermanager.ClusterProvider, co } } -func init() { - addAddon(NewHCloudControllerManagerAddon) -} - // Name returns the addons name func (addon *HCloudControllerManagerAddon) Name() string { return "hcloud-controller-manager" @@ -58,6 +56,7 @@ func (addon *HCloudControllerManagerAddon) Install(args ...string) { [Service] Environment="KUBELET_EXTRA_ARGS=--cloud-provider=external" ` + for _, node := range addon.nodes { err := addon.communicator.WriteFile(node, "/etc/systemd/system/kubelet.service.d/20-hcloud.conf", config, clustermanager.AllRead) FatalOnError(err) diff --git a/pkg/addons/addon_helm.go b/pkg/addons/addon_helm.go index 3c716cac..ffbcb4c6 100644 --- a/pkg/addons/addon_helm.go +++ b/pkg/addons/addon_helm.go @@ -18,10 +18,6 @@ func NewHelmAddon(provider clustermanager.ClusterProvider, communicator clusterm return HelmAddon{masterNode: masterNode, communicator: communicator} } -func init() { - addAddon(NewHelmAddon) -} - // Name returns the addons name func (addon HelmAddon) Name() string { return "helm" @@ -44,10 +40,11 @@ func (addon HelmAddon) URL() string { // Install performs all steps to install the addon func (addon HelmAddon) Install(args ...string) { - node := *addon.masterNode _, err := addon.communicator.RunCmd(node, "curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash") + FatalOnError(err) + serviceAccount := `apiVersion: v1 kind: ServiceAccount metadata: diff --git a/pkg/addons/addon_hetzner_csi.go b/pkg/addons/addon_hetzner_csi.go index b27ae069..cb7e01da 100644 --- a/pkg/addons/addon_hetzner_csi.go +++ b/pkg/addons/addon_hetzner_csi.go @@ -45,6 +45,7 @@ func (addon *HetznerCSIAddon) Install(args ...string) { if err != nil { FatalOnError(err) } + _, err = addon.communicator.RunCmd(*addon.masterNode, "kubectl apply -f https://raw.githubusercontent.com/kubernetes/csi-api/release-1.13/pkg/crd/manifests/csinodeinfo.yaml") if err != nil { FatalOnError(err) @@ -70,6 +71,7 @@ func (addon *HetznerCSIAddon) Uninstall() { if err != nil { FatalOnError(err) } + _, err = addon.communicator.RunCmd(*addon.masterNode, "kubectl delete -f https://raw.githubusercontent.com/kubernetes/csi-api/release-1.13/pkg/crd/manifests/csinodeinfo.yaml --ignore-not-found") if err != nil { FatalOnError(err) @@ -80,20 +82,15 @@ func (addon *HetznerCSIAddon) Uninstall() { if err != nil { FatalOnError(err) } - } // NewHetznerCSIAddon creates an instance of HetznerCSIAddon func NewHetznerCSIAddon(provider clustermanager.ClusterProvider, communicator clustermanager.NodeCommunicator) ClusterAddon { masterNode, _ := provider.GetMasterNode() + return &HetznerCSIAddon{ masterNode: masterNode, communicator: communicator, provider: provider.(*hetzner.Provider), } } - -// adding the addon to the global list -func init() { - addAddon(NewHetznerCSIAddon) -} diff --git a/pkg/addons/addon_ingress.go b/pkg/addons/addon_ingress.go index 2ba4c25b..9c8af3f0 100644 --- a/pkg/addons/addon_ingress.go +++ b/pkg/addons/addon_ingress.go @@ -16,11 +16,8 @@ type IngressAddon struct { func NewIngressAddon(provider clustermanager.ClusterProvider, communicator clustermanager.NodeCommunicator) ClusterAddon { masterNode, err := provider.GetMasterNode() FatalOnError(err) - return &IngressAddon{masterNode: masterNode, communicator: communicator} -} -func init() { - addAddon(NewIngressAddon) + return &IngressAddon{masterNode: masterNode, communicator: communicator} } // Name returns the addons name diff --git a/pkg/addons/addon_openebs.go b/pkg/addons/addon_openebs.go index 0a399c54..ed61a89e 100644 --- a/pkg/addons/addon_openebs.go +++ b/pkg/addons/addon_openebs.go @@ -18,10 +18,6 @@ func NewOpenEBSAddon(provider clustermanager.ClusterProvider, communicator clust return &OpenEBSAddon{masterNode: masterNode, communicator: communicator} } -func init() { - addAddon(NewOpenEBSAddon) -} - // Name returns the addons name func (addon OpenEBSAddon) Name() string { return "openebs" diff --git a/pkg/addons/addon_prometheus.go b/pkg/addons/addon_prometheus.go index 4f196a1d..32a8a5ca 100644 --- a/pkg/addons/addon_prometheus.go +++ b/pkg/addons/addon_prometheus.go @@ -19,6 +19,7 @@ type PrometheusAddon struct { func NewPrometheusAddon(provider clustermanager.ClusterProvider, communicator clustermanager.NodeCommunicator) ClusterAddon { masterNode, err := provider.GetMasterNode() FatalOnError(err) + return &PrometheusAddon{ masterNode: masterNode, communicator: communicator, @@ -27,10 +28,6 @@ func NewPrometheusAddon(provider clustermanager.ClusterProvider, communicator cl } } -func init() { - addAddon(NewPrometheusAddon) -} - // Name returns the addons name func (addon *PrometheusAddon) Name() string { return "kube-prometheus" diff --git a/pkg/addons/addon_rook.go b/pkg/addons/addon_rook.go index 0fab84a0..629ff62c 100644 --- a/pkg/addons/addon_rook.go +++ b/pkg/addons/addon_rook.go @@ -7,6 +7,8 @@ import ( "github.com/xetys/hetzner-kube/pkg/clustermanager" ) +const rookSleepTime = 20 * time.Second + // RookAddon installs rook type RookAddon struct { masterNode *clustermanager.Node @@ -20,10 +22,6 @@ func NewRookAddon(provider clustermanager.ClusterProvider, communicator clusterm return &RookAddon{masterNode: masterNode, communicator: communicator, nodes: provider.GetAllNodes()} } -func init() { - addAddon(NewRookAddon) -} - // Name returns the addons name func (addon RookAddon) Name() string { return "rook" @@ -51,6 +49,7 @@ func (addon RookAddon) Install(args ...string) { _, err := addon.communicator.RunCmd(node, "kubectl apply -f https://raw.githubusercontent.com/rook/rook/v0.7.1/cluster/examples/kubernetes/rook-operator.yaml") FatalOnError(err) fmt.Println("waiting until rook is installed") + for { _, err := addon.communicator.RunCmd(node, "kubectl get cluster") @@ -58,6 +57,7 @@ func (addon RookAddon) Install(args ...string) { break } } + _, err = addon.communicator.RunCmd(node, "kubectl apply -f https://raw.github.com/rook/rook/v0.7.1/cluster/examples/kubernetes/rook-cluster.yaml") FatalOnError(err) _, err = addon.communicator.RunCmd(node, "kubectl apply -f https://raw.github.com/rook/rook/v0.7.1/cluster/examples/kubernetes/rook-storageclass.yaml") @@ -75,6 +75,7 @@ func (addon RookAddon) Install(args ...string) { // Uninstall performs all steps to remove the addon func (addon RookAddon) Uninstall() { node := *addon.masterNode + addon.communicator.RunCmd(node, "kubectl delete -n rook pool replicapool") addon.communicator.RunCmd(node, "kubectl delete storageclass rook-block") addon.communicator.RunCmd(node, "kubectl delete crd clusters.rook.io pools.rook.io objectstores.rook.io filesystems.rook.io volumeattachments.rook.io # ignore errors if on K8s 1.5 and 1.6") @@ -82,13 +83,14 @@ func (addon RookAddon) Uninstall() { addon.communicator.RunCmd(node, "kubectl delete -f https://raw.githubusercontent.com/rook/rook/master/cluster/examples/kubernetes/ceph/operator.yaml") addon.communicator.RunCmd(node, "kubectl delete clusterroles rook-agent") addon.communicator.RunCmd(node, "kubectl delete clusterrolebindings rook-agent") - time.Sleep(20 * time.Second) + time.Sleep(rookSleepTime) addon.communicator.RunCmd(node, "kubectl delete namespace rook") for _, node := range addon.nodes { if node.IsEtcd || node.IsMaster { continue } + fmt.Printf("deleting rook on node %s\n", node.Name) addon.communicator.RunCmd(node, "rm -rf /var/lib/rook") } diff --git a/pkg/addons/cluster_addon.go b/pkg/addons/cluster_addon.go index 9e56eef4..160c2fd7 100644 --- a/pkg/addons/cluster_addon.go +++ b/pkg/addons/cluster_addon.go @@ -15,12 +15,6 @@ type ClusterAddon interface { // ClusterAddonInitializer is a function creating ClusterAddon instances from given parameters type ClusterAddonInitializer func(provider clustermanager.ClusterProvider, communicator clustermanager.NodeCommunicator) ClusterAddon -var addonInitializers = make([]ClusterAddonInitializer, 0) - -func addAddon(clusterAddon ClusterAddonInitializer) { - addonInitializers = append(addonInitializers, clusterAddon) -} - // ClusterAddonService provide the addon service type ClusterAddonService struct { provider clustermanager.ClusterProvider @@ -30,11 +24,24 @@ type ClusterAddonService struct { // NewClusterAddonService creates an instance of the cluster addon service func NewClusterAddonService(provider clustermanager.ClusterProvider, nodeComm clustermanager.NodeCommunicator) *ClusterAddonService { - clusterAddons := []ClusterAddon{} - for _, initializer := range addonInitializers { - clusterAddons = append(clusterAddons, initializer(provider, nodeComm)) + clusterAddons := []ClusterAddon{ + NewCertmanagerAddon(provider, nodeComm), + NewDashboardAddon(provider, nodeComm), + NewDockerregistryAddon(provider, nodeComm), + NewHCloudControllerManagerAddon(provider, nodeComm), + NewHelmAddon(provider, nodeComm), + NewHetznerCSIAddon(provider, nodeComm), + NewIngressAddon(provider, nodeComm), + NewOpenEBSAddon(provider, nodeComm), + NewPrometheusAddon(provider, nodeComm), + NewRookAddon(provider, nodeComm), + } + + return &ClusterAddonService{ + provider: provider, + nodeCommunicator: nodeComm, + addons: clusterAddons, } - return &ClusterAddonService{provider: provider, nodeCommunicator: nodeComm, addons: clusterAddons} } // AddonExists return true, if an addon with the requested name exists @@ -44,6 +51,7 @@ func (addonService *ClusterAddonService) AddonExists(addonName string) bool { return true } } + return false } diff --git a/pkg/clustermanager/cluster.go b/pkg/clustermanager/cluster.go index f7ba2eca..1184370e 100644 --- a/pkg/clustermanager/cluster.go +++ b/pkg/clustermanager/cluster.go @@ -88,12 +88,15 @@ func (manager *Manager) ProvisionNodes(nodes []Node) error { errChan := make(chan error) trueChan := make(chan bool) numProcs := 0 + for _, node := range nodes { numProcs++ + go func(node Node) { manager.eventService.AddEvent(node.Name, "install packages") //_, err := manager.nodeCommunicator.RunCmd(node, "wget -cO- https://raw.githubusercontent.com/xetys/hetzner-kube/master/install-docker-kubeadm.sh | bash -") provisioner := NewNodeProvisioner(node, manager) + err := provisioner.Provision(node, manager.nodeCommunicator, manager.eventService) if err != nil { errChan <- err @@ -112,6 +115,7 @@ func (manager *Manager) ProvisionNodes(nodes []Node) error { // modifies the state of manager.Nodes func (manager *Manager) SetupEncryptedNetwork() error { var err error + var keyPair WgKeyPair for i := range manager.nodes { @@ -129,17 +133,21 @@ func (manager *Manager) SetupEncryptedNetwork() error { errChan := make(chan error) trueChan := make(chan bool) numProc := 0 + for _, node := range nodes { numProc++ + go func(node Node) { manager.eventService.AddEvent(node.Name, "configure wireguard") wireGuardConf := GenerateWireguardConf(node, manager.nodes) + err := manager.nodeCommunicator.WriteFile(node, "/etc/wireguard/wg0.conf", wireGuardConf, OwnerRead) if err != nil { errChan <- err } overlayRouteConf := GenerateOverlayRouteSystemdService(node) + err = manager.nodeCommunicator.WriteFile(node, "/etc/systemd/system/overlay-route.service", overlayRouteConf, AllRead) if err != nil { errChan <- err @@ -162,7 +170,9 @@ func (manager *Manager) SetupEncryptedNetwork() error { if err != nil { return err } + manager.clusterProvider.SetNodes(manager.nodes) + return nil } @@ -186,7 +196,6 @@ func (manager *Manager) InstallMasters(keepCerts KeepCerts) error { for _, node := range manager.nodes { if node.IsMaster { - var resetCommand string switch keepCerts { @@ -212,6 +221,7 @@ func (manager *Manager) InstallMasters(keepCerts KeepCerts) error { } numProc++ + go func(node Node) { manager.installMasterStep(node, numMaster, masterNode, commands, trueChan, errChan) }(node) @@ -236,6 +246,7 @@ func (manager *Manager) InstallMasters(keepCerts KeepCerts) error { func (manager *Manager) installMasterStep(node Node, numMaster int, masterNode Node, commands []NodeCommand, trueChan chan bool, errChan chan error) { // create master-configuration var etcdNodes []Node + if manager.haEnabled { if manager.isolatedEtcd { etcdNodes = manager.clusterProvider.GetEtcdNodes() @@ -243,8 +254,10 @@ func (manager *Manager) installMasterStep(node Node, numMaster int, masterNode N etcdNodes = manager.clusterProvider.GetMasterNodes() } } + masterNodes := manager.clusterProvider.GetMasterNodes() masterConfig := GenerateMasterConfiguration(node, masterNodes, etcdNodes, manager.Cluster().KubernetesVersion) + if err := manager.nodeCommunicator.WriteFile(node, "/root/master-config.yaml", masterConfig, AllRead); err != nil { errChan <- err } @@ -277,6 +290,7 @@ func (manager *Manager) installMasterStep(node Node, numMaster int, masterNode N for i, command := range commands { manager.eventService.AddEvent(node.Name, command.EventName) + _, err := manager.nodeCommunicator.RunCmd(node, command.Command) if err != nil { errChan <- err @@ -296,10 +310,10 @@ func (manager *Manager) installMasterStep(node Node, numMaster int, masterNode N // InstallEtcdNodes installs the etcd cluster func (manager *Manager) InstallEtcdNodes(nodes []Node, keepData bool) error { - errChan := make(chan error) trueChan := make(chan bool) numProcs := 0 + for _, node := range nodes { numProcs++ @@ -321,6 +335,7 @@ func (manager *Manager) etcdInstallStep(node Node, nodes []Node, errChan chan er } // set systemd service etcdSystemdService := GenerateEtcdSystemdService(node, nodes) + err := manager.nodeCommunicator.WriteFile(node, "/etc/systemd/system/etcd.service", etcdSystemdService, AllRead) if err != nil { errChan <- err @@ -328,6 +343,7 @@ func (manager *Manager) etcdInstallStep(node Node, nodes []Node, errChan chan er // install etcd for _, command := range commands { manager.eventService.AddEvent(node.Name, command.EventName) + _, err := manager.nodeCommunicator.RunCmd(node, command.Command) if err != nil { errChan <- err @@ -335,14 +351,18 @@ func (manager *Manager) etcdInstallStep(node Node, nodes []Node, errChan chan er } // configure etcd configureCommand := "systemctl enable etcd.service && systemctl stop etcd.service && rm -rf /var/lib/etcd && systemctl start etcd.service" + if keepData { configureCommand = "systemctl enable etcd.service && systemctl stop etcd.service && systemctl start etcd.service" } + manager.eventService.AddEvent(node.Name, "configure etcd") + _, err = manager.nodeCommunicator.RunCmd(node, configureCommand) if err != nil { errChan <- err } + if manager.isolatedEtcd { manager.eventService.AddEvent(node.Name, pkg.CompletedEvent) } else { @@ -370,8 +390,10 @@ func (manager *Manager) InstallWorkers(nodes []Node) error { for _, node := range nodes { if !node.IsMaster && !node.IsEtcd { numProcs++ + go func(node Node) { manager.eventService.AddEvent(node.Name, "registering node") + _, err := manager.nodeCommunicator.RunCmd( node, "for i in ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack_ipv4; do modprobe $i; done"+ @@ -379,28 +401,33 @@ func (manager *Manager) InstallWorkers(nodes []Node) error { if err != nil { errChan <- err } + if manager.haEnabled { - time.Sleep(10 * time.Second) // we need some time until the kubelet.conf appears + time.Sleep(timeBetweenTentative) // we need some time until the kubelet.conf appears kubeConfigs := []string{"kubelet.conf", "bootstrap-kubelet.conf"} manager.eventService.AddEvent(node.Name, "rewrite kubeconfigs") + for _, conf := range kubeConfigs { _, err := manager.nodeCommunicator.RunCmd(node, fmt.Sprintf(rewriteTpl, conf, conf)) if err != nil { errChan <- err } } + _, err = manager.nodeCommunicator.RunCmd(node, "systemctl restart docker && systemctl restart kubelet") if err != nil { errChan <- err } } + manager.eventService.AddEvent(node.Name, pkg.CompletedEvent) trueChan <- true }(node) } } + return waitOrError(trueChan, errChan, &numProcs) } @@ -417,6 +444,7 @@ func (manager *Manager) SetupHA() error { numProcs := 0 // deploy load balancer masterNodes := manager.clusterProvider.GetMasterNodes() + err = manager.DeployLoadBalancer(manager.nodes) if err != nil { return err @@ -424,6 +452,7 @@ func (manager *Manager) SetupHA() error { // set apiserver-count to number of masters apiServerCount := fmt.Sprintf("- --apiserver-count=%d\n image: gcr.io/", len(masterNodes)) + for _, node := range masterNodes { manager.eventService.AddEvent(node.Name, "set api-server count") manager.nodeCommunicator.TransformFileOverNode(node, node, "/etc/kubernetes/manifests/kube-apiserver.yaml", func(in string) string { @@ -448,12 +477,14 @@ func (manager *Manager) SetupHA() error { go func(node Node) { manager.eventService.AddEvent(node.Name, "rewrite kubeconfigs") + for _, conf := range kubeConfigs { _, err := manager.nodeCommunicator.RunCmd(node, fmt.Sprintf(rewriteTpl, conf, conf)) if err != nil { errChan <- err } } + _, err = manager.nodeCommunicator.RunCmd(node, "systemctl restart docker && systemctl restart kubelet") if err != nil { errChan <- err @@ -473,28 +504,33 @@ func (manager *Manager) SetupHA() error { // DeployLoadBalancer installs a client based load balancer for the master nodes to given nodes func (manager *Manager) DeployLoadBalancer(nodes []Node) error { - errChan := make(chan error) trueChan := make(chan bool) numProcs := 0 masterNodesIP := []string{} + for _, node := range manager.clusterProvider.GetMasterNodes() { masterNodesIP = append(masterNodesIP, node.IPAddress) } masterIps := strings.Join(masterNodesIP, " ") + for _, node := range nodes { if !node.IsMaster && node.IsEtcd { continue } + numProcs++ + go func(node Node) { manager.eventService.AddEvent(node.Name, "deploy load balancer") // delete old if exists _, err := manager.nodeCommunicator.RunCmd(node, `docker ps | grep master-lb | awk '{print "docker stop "$1" && docker rm "$1}' | sh`) + if err != nil { errChan <- err } + _, err = manager.nodeCommunicator.RunCmd(node, fmt.Sprintf("docker run -d --name=master-lb --restart=always -p 16443:16443 xetys/k8s-master-lb %s", masterIps)) if err != nil { errChan <- err diff --git a/pkg/clustermanager/configs.go b/pkg/clustermanager/configs.go index d3eded09..b46ccad8 100644 --- a/pkg/clustermanager/configs.go +++ b/pkg/clustermanager/configs.go @@ -97,6 +97,7 @@ WantedBy=multi-user.target for i, node := range etcdNodes { ips[i] = fmt.Sprintf("%s=http://%s:2380", node.Name, node.PrivateIPAddress) } + initialCluster := strings.Join(ips, ",") service := fmt.Sprintf( diff --git a/pkg/clustermanager/etcd.go b/pkg/clustermanager/etcd.go index 8e8b9184..9aff73d1 100644 --- a/pkg/clustermanager/etcd.go +++ b/pkg/clustermanager/etcd.go @@ -30,6 +30,7 @@ func (manager *EtcdManager) CreateSnapshot(name string) error { firstEtcdNode := etcdNodes[0] snapshotName := name + if snapshotName == "" { snapshotName = generateName() } @@ -73,7 +74,6 @@ func (manager *EtcdManager) RestoreSnapshot(name string, skipCopy bool) (bool, e // copySnapshot copies a snapshot to a node func (manager *EtcdManager) copySnapshot(firstEtcdNode, node Node, snapshotPath string) error { - _, err := manager.nodeCommunicator.RunCmd(node, "mkdir -p ~/etcd-snapshots") if err != nil { return err @@ -83,6 +83,7 @@ func (manager *EtcdManager) copySnapshot(firstEtcdNode, node Node, snapshotPath if err != nil { return err } + fmt.Printf("copied '%s' to node '%s'\n", snapshotPath, node.Name) return nil @@ -126,6 +127,7 @@ func (manager *EtcdManager) copyAndRestore(firstEtcdNode Node, snapshotPath stri // distribute snapshots to all etcd nodes fmt.Println("distributing snapshots across all nodes") + for _, node := range etcdNodes { initialCluster += fmt.Sprintf(",%s=http://%s:2380", node.Name, node.PrivateIPAddress) @@ -145,6 +147,7 @@ func (manager *EtcdManager) copyAndRestore(firstEtcdNode Node, snapshotPath stri // actual restore the cluster fmt.Println("begin restore process") + for _, node := range etcdNodes { err := manager.restoreNode(node, snapshotPath, initialCluster) if err != nil { diff --git a/pkg/clustermanager/provision_node.go b/pkg/clustermanager/provision_node.go index 8d058628..72a2fec1 100644 --- a/pkg/clustermanager/provision_node.go +++ b/pkg/clustermanager/provision_node.go @@ -9,6 +9,9 @@ import ( const maxErrors = 3 +const maxTentative = 10 +const timeBetweenTentative = 3 * time.Second + // NodeProvisioner provisions all basic packages to install docker, kubernetes and wireguard type NodeProvisioner struct { clusterName string @@ -32,10 +35,10 @@ func NewNodeProvisioner(node Node, manager *Manager) *NodeProvisioner { // Provision performs all steps to provision a node func (provisioner *NodeProvisioner) Provision(node Node, communicator NodeCommunicator, eventService EventService) error { var err error + errorCount := 0 for !provisioner.packagesAreInstalled(node, communicator) { - for err := provisioner.prepareAndInstall(); err != nil; { errorCount++ @@ -43,7 +46,6 @@ func (provisioner *NodeProvisioner) Provision(node Node, communicator NodeCommun return err } } - } if err != nil { @@ -64,27 +66,31 @@ func (provisioner *NodeProvisioner) packagesAreInstalled(node Node, communicator if strings.TrimSpace(out) == "0" { return true } + return false } func (provisioner *NodeProvisioner) prepareAndInstall() error { - err := provisioner.waitForCloudInitCompletion() if err != nil { return err } + err = provisioner.installTransportTools() if err != nil { return err } + err = provisioner.preparePackages() if err != nil { return err } + err = provisioner.updateAndInstall() if err != nil { return err } + err = provisioner.setSystemWideEnvironment() if err != nil { return err @@ -102,13 +108,14 @@ func (provisioner *NodeProvisioner) disableSwap() error { } _, err = provisioner.communicator.RunCmd(provisioner.node, "sed -i '/ swap / s/^/#/' /etc/fstab") + return err } func (provisioner *NodeProvisioner) waitForCloudInitCompletion() error { + var err error provisioner.eventService.AddEvent(provisioner.node.Name, "waiting for cloud-init completion") - var err error // define smal bash script to check if /var/lib/cloud/instance/boot-finished exist // this file created only when cloud-init finished its tasks @@ -132,10 +139,12 @@ exit 127 return err } - for i := 0; i < 10; i++ { - time.Sleep(3 * time.Second) + for i := 0; i < maxTentative; i++ { + time.Sleep(timeBetweenTentative) + _, err = provisioner.communicator.RunCmd(provisioner.node, "/root/cloud-init-status-check.sh") } + if err != nil { return err } @@ -150,13 +159,16 @@ exit 127 } func (provisioner *NodeProvisioner) installTransportTools() error { + var err error provisioner.eventService.AddEvent(provisioner.node.Name, "installing transport tools") - var err error - for i := 0; i < 10; i++ { - time.Sleep(3 * time.Second) + + for i := 0; i < maxTentative; i++ { + time.Sleep(timeBetweenTentative) + _, err = provisioner.communicator.RunCmd(provisioner.node, "apt-get update && apt-get install -y apt-transport-https ca-certificates curl software-properties-common") } + if err != nil { return err } @@ -207,6 +219,7 @@ Package: docker-ce Pin: version 18.09.2~3-0~ubuntu-bionic Pin-Priority: 1000 ` + err := provisioner.communicator.WriteFile(provisioner.node, "/etc/apt/preferences.d/docker-ce", aptPreferencesDocker, AllRead) if err != nil { return err @@ -227,6 +240,7 @@ Pin-Priority: 1000 func (provisioner *NodeProvisioner) updateAndInstall() error { provisioner.eventService.AddEvent(provisioner.node.Name, "updating packages") + _, err := provisioner.communicator.RunCmd(provisioner.node, "apt-get update") if err != nil { return err @@ -235,6 +249,7 @@ func (provisioner *NodeProvisioner) updateAndInstall() error { provisioner.eventService.AddEvent(provisioner.node.Name, "installing packages") command := fmt.Sprintf("apt-get install -y docker-ce kubelet=%s-00 kubeadm=%s-00 kubectl=%s-00 kubernetes-cni=0.7.5-00 wireguard linux-headers-$(uname -r) linux-headers-virtual", provisioner.kubernetesVersion, provisioner.kubernetesVersion, provisioner.kubernetesVersion) + _, err = provisioner.communicator.RunCmd(provisioner.node, command) if err != nil { return err @@ -247,10 +262,9 @@ func (provisioner *NodeProvisioner) updateAndInstall() error { // As soon as it is last step we are ok to set them in basic way func (provisioner *NodeProvisioner) setSystemWideEnvironment() error { provisioner.eventService.AddEvent(provisioner.node.Name, "set environment variables") - var err error // set HETZNER_KUBE_MASTER - _, err = provisioner.communicator.RunCmd(provisioner.node, fmt.Sprintf("echo \"HETZNER_KUBE_MASTER=%s\" >> /etc/environment", strconv.FormatBool(provisioner.node.IsMaster))) + _, err := provisioner.communicator.RunCmd(provisioner.node, fmt.Sprintf("echo \"HETZNER_KUBE_MASTER=%s\" >> /etc/environment", strconv.FormatBool(provisioner.node.IsMaster))) if err != nil { return err } diff --git a/pkg/clustermanager/ssh_communicator.go b/pkg/clustermanager/ssh_communicator.go index fc5a64d2..feabfca9 100644 --- a/pkg/clustermanager/ssh_communicator.go +++ b/pkg/clustermanager/ssh_communicator.go @@ -16,6 +16,9 @@ import ( "golang.org/x/crypto/ssh/terminal" ) +const sshTimeout = 10 * time.Second +const sshConnectionTentative = 10 + // SSHKey represents a keypair with the paths to the keys type SSHKey struct { Name string `json:"name"` @@ -40,10 +43,12 @@ func NewSSHCommunicator(sshKeys []SSHKey, debug bool) NodeCommunicator { passPhrases: make(map[string][]byte), debug: debug, } + if debug { outfile, _ := os.Create("hetzner-kube-debug.log") sshComm.log = log.New(outfile, "", 0) } + return sshComm } @@ -52,6 +57,7 @@ func (sshComm *SSHCommunicator) Log(msg ...string) { if !sshComm.debug { return } + sshComm.log.Println(msg) } @@ -61,6 +67,7 @@ func (sshComm *SSHCommunicator) RunCmd(node Node, command string) (output string if err != nil { return output, err } + defer connection.Close() defer session.Close() @@ -88,23 +95,27 @@ func (sshComm *SSHCommunicator) newSession(node Node) (*ssh.Session, *ssh.Client User: "root", Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, HostKeyCallback: ssh.InsecureIgnoreHostKey(), - Timeout: 10 * time.Second, + Timeout: sshTimeout, } + var connection *ssh.Client + for try := 0; ; try++ { connection, err = ssh.Dial("tcp", node.IPAddress+":22", config) if err != nil { sshComm.Log(node.Name+": dial failed: ", err.Error()) sshComm.Log(node.Name + ": retrying..") - if try > 10 { + + if try > sshConnectionTentative { return nil, nil, err } } else { break } + time.Sleep(1 * time.Second) } - // log.Println("Connected succeeded!") + session, err := connection.NewSession() if err != nil { return nil, nil, fmt.Errorf("session failed:%v", err) @@ -115,8 +126,9 @@ func (sshComm *SSHCommunicator) newSession(node Node) (*ssh.Session, *ssh.Client // WriteFile places a file at a given part from string. Permissions are 0644, or 0755 if executable true func (sshComm *SSHCommunicator) WriteFile(node Node, filePath string, content string, permission FilePermission) error { - signer, err := sshComm.getPrivateSSHKey(node.SSHKeyName) + var connection *ssh.Client + signer, err := sshComm.getPrivateSSHKey(node.SSHKeyName) if err != nil { return err } @@ -125,27 +137,31 @@ func (sshComm *SSHCommunicator) WriteFile(node Node, filePath string, content st User: "root", Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, HostKeyCallback: ssh.InsecureIgnoreHostKey(), - Timeout: 10 * time.Second, + Timeout: sshTimeout, } - var connection *ssh.Client + for try := 0; ; try++ { connection, err = ssh.Dial("tcp", node.IPAddress+":22", config) if err != nil { sshComm.Log(node.Name+": dial failed:", err.Error()) - if try > 10 { + + if try > sshConnectionTentative { return err } } else { break } + time.Sleep(1 * time.Second) } + defer connection.Close() - // log.Println("Connected succeeded!") + session, err := connection.NewSession() if err != nil { log.Fatalf(node.Name+": session failed:%v", err) } + defer session.Close() fileName := path.Base(filePath) @@ -190,18 +206,22 @@ func (sshComm *SSHCommunicator) TransformFileOverNode(sourceNode Node, targetNod // write file err = sshComm.WriteFile(targetNode, filePath, fileContent, AllRead) + return err } // findPrivateKeyByName returns a SSH key from its store func (sshComm *SSHCommunicator) findPrivateKeyByName(name string) (int, *SSHKey) { index := -1 + for i, v := range sshComm.sshKeys { if v.Name == name { index = i + return index, &v } } + return index, nil } @@ -233,6 +253,7 @@ func (sshComm *SSHCommunicator) CapturePassphrase(sshKeyName string) error { } fmt.Print("\n") + sshComm.passPhrases[privateKey.PrivateKeyPath] = text // check that the captured password is correct @@ -261,7 +282,6 @@ func (sshComm *SSHCommunicator) isEncrypted(privateKey *SSHKey) (bool, error) { block, _ := pem.Decode(pemBytes) if block == nil { return false, errors.New("SSH: no key found") - } return strings.Contains(block.Headers["Proc-Type"], "ENCRYPTED"), nil @@ -313,8 +333,10 @@ func (sshComm *SSHCommunicator) getSignerFromEncrypthedPrivateSSHKey(sshKeyName if err != nil { return nil, fmt.Errorf("error capturing passphrase:%v", err) } + passPhrase, err = sshComm.getPassphrase(privateKey.PrivateKeyPath) } + if err != nil { return nil, fmt.Errorf("parse key failed:%v", err) } diff --git a/pkg/clustermanager/wireguard.go b/pkg/clustermanager/wireguard.go index 51eee7d7..f33c7e6f 100644 --- a/pkg/clustermanager/wireguard.go +++ b/pkg/clustermanager/wireguard.go @@ -53,6 +53,7 @@ func PrivateIPPrefix(cidr string) (string, error) { if err != nil { return "", fmt.Errorf("unable to parse cidr %q", cidr) } + ipAddress = ipAddress.To4() if ipAddress == nil { return "", fmt.Errorf("unable to convert ip %q to IPv4s", ipAddress) @@ -65,7 +66,9 @@ func PrivateIPPrefix(cidr string) (string, error) { // Code is redacted from https://github.com/WireGuard/wireguard-go/blob/1c025570139f614f2083b935e2c58d5dbf199c2f/noise-helpers.go func GenerateKeyPair() (WgKeyPair, error) { var publicKey [32]byte + var privateKey [32]byte + _, err := rand.Reader.Read(privateKey[:]) if err != nil { return WgKeyPair{}, fmt.Errorf("unable to generate a private key: %v", err) diff --git a/pkg/clustermanager/wireguard_test.go b/pkg/clustermanager/wireguard_test.go index c9c5367f..178f5534 100644 --- a/pkg/clustermanager/wireguard_test.go +++ b/pkg/clustermanager/wireguard_test.go @@ -31,7 +31,6 @@ Endpoint = 1.1.1.1:51820 if generatedConf != expectedConf { t.Errorf("The file was not rendered as expected\n%s\n\n", generatedConf) } - } func TestGenerateKeyPair(t *testing.T) { @@ -85,15 +84,17 @@ func TestPrivateIPPrefix(t *testing.T) { expected: "250.251.252", }, } - for _, tC := range testCases { - t.Run(fmt.Sprintf("testing IP: %s", tC.source), func(t *testing.T) { - generated, err := clustermanager.PrivateIPPrefix(tC.source) + for _, tt := range testCases { + tt := tt + + t.Run(fmt.Sprintf("testing IP: %s", tt.source), func(t *testing.T) { + generated, err := clustermanager.PrivateIPPrefix(tt.source) if err != nil { - t.Errorf("Unexpected error on parsing valid IP\nParsing IP: %s\nExpected: %s\nGenerated: %s\n", tC.source, tC.expected, generated) + t.Errorf("Unexpected error on parsing valid IP\nParsing IP: %s\nExpected: %s\nGenerated: %s\n", tt.source, tt.expected, generated) } - if tC.expected != generated { - t.Errorf("\nParsing IP: %s\nExpected: %s\nGenerated: %s\n", tC.source, tC.expected, generated) + if tt.expected != generated { + t.Errorf("\nParsing IP: %s\nExpected: %s\nGenerated: %s\n", tt.source, tt.expected, generated) } }) } @@ -107,11 +108,13 @@ func TestPrivateIPPrefixWithWrongIpAddress(t *testing.T) { {source: "10.0.1.100"}, {source: "10.0.1.100/33"}, } - for _, tC := range testCases { - t.Run(fmt.Sprintf("testing IP: %s", tC.source), func(t *testing.T) { - _, err := clustermanager.PrivateIPPrefix(tC.source) + + for _, tt := range testCases { + tt := tt + t.Run(fmt.Sprintf("testing IP: %s", tt.source), func(t *testing.T) { + _, err := clustermanager.PrivateIPPrefix(tt.source) if err == nil { - t.Errorf("we expect an error on parsing invalid IP %q", tC.source) + t.Errorf("we expect an error on parsing invalid IP %q", tt.source) } }) } diff --git a/pkg/hetzner/hetzner_provider.go b/pkg/hetzner/hetzner_provider.go index 21d109c4..13370978 100644 --- a/pkg/hetzner/hetzner_provider.go +++ b/pkg/hetzner/hetzner_provider.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/kit/log/term" "github.com/gosuri/uiprogress" "github.com/hetznercloud/hcloud-go/hcloud" + "github.com/xetys/hetzner-kube/pkg" "github.com/xetys/hetzner-kube/pkg/clustermanager" ) @@ -70,7 +71,6 @@ func (provider *Provider) CreateNodes(suffix string, template clustermanager.Nod if err == nil { serverOptsTemplate.UserData = string(buf) } - } serverOptsTemplate.SSHKeys = append(serverOptsTemplate.SSHKeys, sshKey) @@ -82,6 +82,7 @@ func (provider *Provider) CreateNodes(suffix string, template clustermanager.Nod rand.Shuffle(datacentersCount, func(i, j int) { datacenters[i], datacenters[j] = datacenters[j], datacenters[i] }) var nodes []clustermanager.Node + for i := 1; i <= count; i++ { serverOpts := serverOptsTemplate nodeNumber := i + offset @@ -108,6 +109,7 @@ func (provider *Provider) CreateNodes(suffix string, template clustermanager.Nod privateIPLastBlock += 10 } } + cidrPrefix, err := clustermanager.PrivateIPPrefix(provider.nodeCidr) if err != nil { return nil, err @@ -202,7 +204,6 @@ func (provider *Provider) GetCluster() clustermanager.Cluster { // GetAdditionalMasterInstallCommands return the list of node command to execute on the cluster func (provider *Provider) GetAdditionalMasterInstallCommands() []clustermanager.NodeCommand { - return []clustermanager.NodeCommand{} } @@ -236,10 +237,12 @@ func (provider *Provider) filterNodes(filter nodeFilter) []clustermanager.Node { func (provider *Provider) runCreateServer(opts *hcloud.ServerCreateOpts) (*hcloud.ServerCreateResult, error) { log.Printf("creating server '%s'...", opts.Name) + server, _, err := provider.client.Server.GetByName(provider.context, opts.Name) if err != nil { return nil, err } + if server == nil { result, _, err := provider.client.Server.Create(provider.context, *opts) if err != nil { @@ -266,6 +269,7 @@ func (provider *Provider) runCreateServer(opts *hcloud.ServerCreateOpts) (*hclou } log.Printf("loading server '%s'...", opts.Name) + return &hcloud.ServerCreateResult{Server: server}, nil } @@ -276,7 +280,7 @@ func (provider *Provider) actionProgress(action *hcloud.Action) error { progress := uiprogress.New() progress.Start() - bar := progress.AddBar(100).AppendCompleted().PrependElapsed() + bar := progress.AddBar(pkg.ProgressCompleted).AppendCompleted().PrependElapsed() bar.Width = 40 bar.Empty = ' ' @@ -284,9 +288,11 @@ func (provider *Provider) actionProgress(action *hcloud.Action) error { select { case err := <-errCh: if err == nil { - bar.Set(100) + bar.Set(pkg.ProgressCompleted) } + progress.Stop() + return err case p := <-progressCh: bar.Set(p) diff --git a/pkg/hetzner/hetzner_provider_test.go b/pkg/hetzner/hetzner_provider_test.go index f00173db..44138d6e 100644 --- a/pkg/hetzner/hetzner_provider_test.go +++ b/pkg/hetzner/hetzner_provider_test.go @@ -75,6 +75,7 @@ func TestProviderGetMasterNodes(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.Name, func(t *testing.T) { provider := getProviderWithNodes(tt.Nodes) nodes := provider.GetMasterNodes() @@ -126,6 +127,7 @@ func TestProviderGetEtcdNodes(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.Name, func(t *testing.T) { provider := getProviderWithNodes(tt.Nodes) nodes := provider.GetEtcdNodes() @@ -172,6 +174,7 @@ func TestProviderGetWorkerNodes(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.Name, func(t *testing.T) { provider := getProviderWithNodes(tt.Nodes) nodes := provider.GetWorkerNodes() @@ -224,6 +227,7 @@ func TestProviderGetAllNodes(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.Name, func(t *testing.T) { provider := getProviderWithNodes(tt.Nodes) nodes := provider.GetAllNodes() @@ -267,6 +271,7 @@ func TestProviderGetMasterNode(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.Name, func(t *testing.T) { provider := getProviderWithNodes(tt.Nodes) node, _ := provider.GetMasterNode() @@ -294,6 +299,7 @@ func TestProviderGetMasterNodeIsMissing(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.Name, func(t *testing.T) { provider := getProviderWithNodes(tt.Nodes) _, err := provider.GetMasterNode() diff --git a/pkg/phases/phase_etcd.go b/pkg/phases/phase_etcd.go index a31204b7..f773332e 100644 --- a/pkg/phases/phase_etcd.go +++ b/pkg/phases/phase_etcd.go @@ -30,9 +30,10 @@ func (phase *EtcdSetupPhase) ShouldRun() bool { // Run runs the phase func (phase *EtcdSetupPhase) Run() error { - var etcdNodes []clustermanager.Node cluster := phase.clusterManager.Cluster() + var etcdNodes []clustermanager.Node + if cluster.IsolatedEtcd { etcdNodes = phase.provider.GetEtcdNodes() } else { diff --git a/pkg/phases/phase_install_masters.go b/pkg/phases/phase_install_masters.go index fbf9e425..6fec0afc 100644 --- a/pkg/phases/phase_install_masters.go +++ b/pkg/phases/phase_install_masters.go @@ -30,10 +30,12 @@ func (phase *InstallMastersPhase) ShouldRun() bool { // Run runs the phase func (phase *InstallMastersPhase) Run() error { keepCerts := clustermanager.NONE + if phase.options.KeepAllCerts { keepCerts = clustermanager.ALL } else if phase.options.KeepCaCerts { keepCerts = clustermanager.CA } + return phase.clusterManager.InstallMasters(keepCerts) } diff --git a/pkg/phases/phase_kube_restart.go b/pkg/phases/phase_kube_restart.go index c7476483..2e37485d 100644 --- a/pkg/phases/phase_kube_restart.go +++ b/pkg/phases/phase_kube_restart.go @@ -2,6 +2,7 @@ package phases import ( "fmt" + "github.com/xetys/hetzner-kube/pkg/clustermanager" ) @@ -26,16 +27,19 @@ func (phase *KubeRestartPhase) ShouldRun() bool { // Run runs the phase func (phase *KubeRestartPhase) Run() error { - fmt.Println("restarting") errChan := make(chan error) trueChan := make(chan bool) numProcs := 0 + + fmt.Println("restarting") + for _, node := range phase.provider.GetAllNodes() { numProcs++ + go func(node clustermanager.Node) { fmt.Printf("restarting docker+kubelet on node '%s'\n", node.Name) - _, err := phase.ssh.RunCmd(node, "systemctl restart docker && systemctl restart kubelet") + _, err := phase.ssh.RunCmd(node, "systemctl restart docker && systemctl restart kubelet") if err != nil { errChan <- err } diff --git a/pkg/phases/phase_provision.go b/pkg/phases/phase_provision.go index 2198b331..0d1eaa58 100644 --- a/pkg/phases/phase_provision.go +++ b/pkg/phases/phase_provision.go @@ -2,10 +2,13 @@ package phases import ( "fmt" - "github.com/xetys/hetzner-kube/pkg/clustermanager" "log" + + "github.com/xetys/hetzner-kube/pkg/clustermanager" ) +const maxAllowedTries = 3 + // ProvisionNodesPhase defines the phase which install all the tools for each node type ProvisionNodesPhase struct { clusterManager *clustermanager.Manager @@ -29,7 +32,7 @@ func (phase *ProvisionNodesPhase) Run() error { tries := 0 for err := phase.clusterManager.ProvisionNodes(cluster.Nodes); err != nil; { - if tries < 3 { + if tries < maxAllowedTries { fmt.Print(err) tries++ } else { diff --git a/pkg/progress.go b/pkg/progress.go index 4e2c309b..c84cfced 100644 --- a/pkg/progress.go +++ b/pkg/progress.go @@ -2,6 +2,9 @@ package pkg import "github.com/gosuri/uiprogress" +// ProgressCompleted indicate the value for progress bar when completed +const ProgressCompleted = 100 + // Progress define the progress on command execution type Progress struct { Name string diff --git a/pkg/progress_coordinator.go b/pkg/progress_coordinator.go index da9e991f..b9e011eb 100644 --- a/pkg/progress_coordinator.go +++ b/pkg/progress_coordinator.go @@ -15,28 +15,30 @@ const CompletedEvent = "complete!" // UIProgressCoordinator coortinate display of progress in UI type UIProgressCoordinator struct { + debug bool group sync.WaitGroup progresses map[string]*Progress } -// RenderProgressBars indicate if we need to display progress in UI -var RenderProgressBars bool - // NewProgressCoordinator create a new progress coordinator UI -func NewProgressCoordinator() *UIProgressCoordinator { - if isUIEnabled() { +func NewProgressCoordinator(debug bool) *UIProgressCoordinator { + pc := UIProgressCoordinator{ + progresses: make(map[string]*Progress), + debug: debug, + } + + if pc.isUIEnabled() { uiprogress.Start() } - pc := new(UIProgressCoordinator) - pc.progresses = make(map[string]*Progress) - return pc + return &pc } -func isUIEnabled() bool { - if RenderProgressBars { +func (c *UIProgressCoordinator) isUIEnabled() bool { + if c.debug { return term.IsTerminal(os.Stdout) } + return false } @@ -45,6 +47,7 @@ func shortLeftPadRight(s string, padWidth int) string { l := len(s) return "..." + s[(l-(padWidth-2)):(l-1)] } + return strutil.PadRight(s, padWidth, ' ') } @@ -56,6 +59,7 @@ func (c *UIProgressCoordinator) StartProgress(name string, steps int) { channel: make(chan string), Name: name, } + progress.Bar.Width = 16 progress.Bar.PrependFunc(func(b *uiprogress.Bar) string { percent := strutil.PadLeft(fmt.Sprintf("%.01f%%", b.CompletedPercent()), 6, ' ') @@ -65,22 +69,29 @@ func (c *UIProgressCoordinator) StartProgress(name string, steps int) { percent, ) }) + c.progresses[name] = progress + c.group.Add(1) + go func(progress *Progress) { for { event := <-progress.channel - if !isUIEnabled() { + + if !c.isUIEnabled() { fmt.Printf("%s: %s (%d)", progress.Name, event, progress.Bar.Current()+1) fmt.Println() } + if event == CompletedEvent { progress.Bar.Set(progress.Bar.Total) progress.SetText(event) + break } progress.SetText(event) + if done := progress.Bar.Incr(); !done { break } @@ -106,7 +117,8 @@ func (c *UIProgressCoordinator) CompleteProgress(nodeName string) { // Wait temporary stop the progress UI func (c *UIProgressCoordinator) Wait() { c.group.Wait() - if isUIEnabled() { + + if c.isUIEnabled() { uiprogress.Stop() } }