Upgrade to kubernetes 1.11

lib/pharos/configuration/host.rb

@@ -56,6 +56,8 @@ def kubelet_args(local_only: false, cloud_provider: nil)
           args << "--hostname-override=#{hostname}"
         end
 
+        args += configurer(nil).kubelet_args
+
         args
       end
 

lib/pharos/host/configurer.rb

@@ -28,6 +28,11 @@ def configure_cfssl
         abstract_method!
       end
 
+      # @return [Array<String>]
+      def kubelet_args
+        []
+      end
+
       # @param args [Hash]
       def ensure_kubelet(args) # rubocop:disable Lint/UnusedMethodArgument
         abstract_method!

lib/pharos/host/el7/el7.rb

@@ -37,6 +37,11 @@ def configure_cfssl
         )
       end
 
+      # @return [Array<String>]
+      def kubelet_args
+        ['--cgroup-driver=systemd']
+      end
+
       def configure_container_runtime
         if docker?
           exec_script(

lib/pharos/host/el7/scripts/ensure-kubelet.sh

@@ -10,7 +10,7 @@ mkdir -p /etc/systemd/system/kubelet.service.d
 [Service]
 ExecStartPre=-/sbin/swapoff -a
 ExecStart=
-ExecStart=/usr/bin/kubelet ${KUBELET_ARGS} --cgroup-driver=systemd --pod-infra-container-image=${IMAGE_REPO}/pause-${ARCH}:3.1
+ExecStart=/usr/bin/kubelet ${KUBELET_ARGS} --pod-infra-container-image=${IMAGE_REPO}/pause-${ARCH}:3.1
 EOF
 
 yum_install_with_lock "kubelet" $KUBE_VERSION

lib/pharos/host/el7/scripts/install-kube-packages.sh

@@ -9,6 +9,9 @@ yum_install_with_lock "kubelet" $KUBE_VERSION
 yum_install_with_lock "kubectl" $KUBE_VERSION
 yum_install_with_lock "kubeadm" $KUBE_VERSION
 
+# use KUBELET_EXTRA_ARGS from /etc/systemd/system/kubelet.service.d/11-pharos.conf instead
+sed -i 's/^KUBELET_EXTRA_ARGS=/#\0/' /etc/sysconfig/kubelet
+
 if systemctl is-active --quiet rpcbind; then
     systemctl stop rpcbind
     systemctl disable rpcbind

lib/pharos/host/el7/scripts/reset.sh

@@ -17,7 +17,7 @@ if systemctl is-active --quiet crio ; then
     systemctl disable crio
 fi
 
-kubeadm reset
+kubeadm reset --force
 
 yum remove -y kubeadm kubelet kubectl docker
 

lib/pharos/host/el7/scripts/upgrade-kubeadm.sh

@@ -4,11 +4,15 @@
 
 set -ex
 
-if [ $(kubeadm version -o short) = "v${VERSION}" ]; then
+if [ -x /usr/local/bin/pharos-kubeadm-${VERSION} ]; then
     exit
 fi
 
-versionlock="/etc/yum/pluginconf.d/versionlock.list"
-linefromfile "^0:kubeadm-" $versionlock
-yum install -y "kubeadm-${VERSION}"
-lineinfile "^0:kubeadm-" "0:kubeadm-${VERSION}-0.*" $versionlock
+BIN_URL="https://dl.bintray.com/kontena/pharos-bin/kube/${VERSION}/kubeadm-${ARCH}.gz"
+
+curl -fsSL $BIN_URL -o /tmp/kubeadm.gz
+curl -fsSL "${BIN_URL}.asc" -o tmp/kubeadm.gz.asc
+gpg --verify /tmp/kubeadm.gz.asc /tmp/kubeadm.gz
+gunzip /tmp/kubeadm.gz
+install -o root -g root -m 0755 -T /tmp/kubeadm /usr/local/bin/pharos-kubeadm-${VERSION}
+rm /tmp/kubeadm /tmp/kubeadm.gz.asc

lib/pharos/host/ubuntu/scripts/install-kube-packages.sh

@@ -7,6 +7,9 @@ apt-mark unhold kubelet kubectl kubeadm
 apt-get install -y kubelet=${KUBE_VERSION}-00 kubectl=${KUBE_VERSION}-00 kubeadm=${KUBEADM_VERSION}-00
 apt-mark hold kubelet kubectl kubeadm
 
+# use KUBELET_EXTRA_ARGS from /etc/systemd/system/kubelet.service.d/11-pharos.conf instead
+sed -i 's/^KUBELET_EXTRA_ARGS=/#\0/' /etc/default/kubelet
+
 if ! dpkg -s nfs-common > /dev/null; then
     systemctl mask rpcbind
     apt-get install -y nfs-common

lib/pharos/host/ubuntu/scripts/reset.sh

@@ -17,7 +17,7 @@ if systemctl is-active --quiet crio ; then
     systemctl disable crio
 fi
 
-kubeadm reset
+kubeadm reset --force
 
 apt-get purge -y --allow-change-held-packages --purge kubeadm kubelet kubectl docker.io
 apt-get autoremove -y

lib/pharos/host/ubuntu/scripts/upgrade-kubeadm.sh

@@ -2,12 +2,15 @@
 
 set -ex
 
-if [ $(kubeadm version -o short) = "v${VERSION}" ]; then
+if [ -x /usr/local/bin/pharos-kubeadm-${VERSION} ]; then
     exit
 fi
 
-cd /tmp
-export DEBIAN_FRONTEND=noninteractive
-apt-get download kubeadm=${VERSION}-00
-dpkg -i --ignore-depends=kubelet kubeadm_${VERSION}*.deb
-rm -f kubeadm_${VERSION}*.deb
+BIN_URL="https://dl.bintray.com/kontena/pharos-bin/kube/${VERSION}/kubeadm-${ARCH}.gz"
+
+curl -fsSL $BIN_URL -o /tmp/kubeadm.gz
+curl -fsSL "${BIN_URL}.asc" -o tmp/kubeadm.gz.asc
+gpg --verify /tmp/kubeadm.gz.asc /tmp/kubeadm.gz
+gunzip /tmp/kubeadm.gz
+install -o root -g root -m 0755 -T /tmp/kubeadm /usr/local/bin/pharos-kubeadm-${VERSION}
+rm /tmp/kubeadm /tmp/kubeadm.gz.asc

lib/pharos/kubeadm.rb

@@ -21,9 +21,12 @@ def initialize(config, host)
       # @return [Hash]
       def generate_config
         config = {
-          'apiVersion' => 'kubeadm.k8s.io/v1alpha1',
+          'apiVersion' => 'kubeadm.k8s.io/v1alpha2',
           'kind' => 'MasterConfiguration',
-          'nodeName' => @host.hostname,
+
+          'nodeRegistration' => {
+            'name' => @host.hostname
+          },
           'kubernetesVersion' => Pharos::KUBE_VERSION,
           'imageRepository' => @config.image_repository,
           'api' => {
@@ -35,24 +38,25 @@ def generate_config
             'serviceSubnet' => @config.network.service_cidr,
             'podSubnet' => @config.network.pod_network_cidr
           },
+          'apiServerExtraArgs' => {},
           'controllerManagerExtraArgs' => {
             'horizontal-pod-autoscaler-use-rest-clients' => 'true'
-          },
-          'noTaintMaster' => !master_taint?
+          }
         }
 
-        if @host.container_runtime == 'cri-o'
-          config['criSocket'] = '/var/run/crio/crio.sock'
+        unless master_taint?
+          config['nodeRegistration']['taints'] = []
         end
 
-        config['apiServerExtraArgs'] = {
-          'apiserver-count' => @config.master_hosts.size.to_s
-        }
+        if @host.container_runtime == 'cri-o'
+          config['nodeRegistration']['criSocket'] = '/var/run/crio/crio.sock'
+        end
 
         if @config.cloud && @config.cloud.provider != 'external'
-          config['cloudProvider'] = @config.cloud.provider
           if @config.cloud.config
+            config['apiServerExtraArgs']['cloud-provider'] = @config.cloud.provider
             config['apiServerExtraArgs']['cloud-config'] = CLOUD_CFG_FILE
+            config['controllerManagerExtraArgs']['cloud-provider'] = @config.cloud.provider
             config['controllerManagerExtraArgs']['cloud-config'] = CLOUD_CFG_FILE
           end
         end
@@ -121,23 +125,27 @@ def configure_internal_etcd(config)
           "https://#{h.peer_address}:2379"
         }
         config['etcd'] = {
-          'endpoints' => endpoints
+          'external' => {
+            'endpoints' => endpoints
+          }
         }
 
-        config['etcd']['certFile'] = '/etc/pharos/pki/etcd/client.pem'
-        config['etcd']['caFile'] = '/etc/pharos/pki/ca.pem'
-        config['etcd']['keyFile'] = '/etc/pharos/pki/etcd/client-key.pem'
+        config['etcd']['external']['certFile'] = '/etc/pharos/pki/etcd/client.pem'
+        config['etcd']['external']['caFile'] = '/etc/pharos/pki/ca.pem'
+        config['etcd']['external']['keyFile'] = '/etc/pharos/pki/etcd/client-key.pem'
       end
 
       # @param config [Hash]
       def configure_external_etcd(config)
         config['etcd'] = {
-          'endpoints' => @config.etcd.endpoints
+          'external' => {
+            'endpoints' => @config.etcd.endpoints
+          }
         }
 
-        config['etcd']['certFile'] = '/etc/pharos/etcd/certificate.pem' if @config.etcd.certificate
-        config['etcd']['caFile'] = '/etc/pharos/etcd/ca-certificate.pem' if @config.etcd.ca_certificate
-        config['etcd']['keyFile'] = '/etc/pharos/etcd/certificate-key.pem' if @config.etcd.key
+        config['etcd']['external']['certFile'] = '/etc/pharos/etcd/certificate.pem' if @config.etcd.certificate
+        config['etcd']['external']['caFile'] = '/etc/pharos/etcd/ca-certificate.pem' if @config.etcd.ca_certificate
+        config['etcd']['external']['keyFile'] = '/etc/pharos/etcd/certificate-key.pem' if @config.etcd.key
       end
 
       # @param config [Hash]
@@ -189,19 +197,13 @@ def volume_mounts_for_authentication_token_webhook
       # @param config [Hash]
       def configure_kube_proxy(config)
         config['kubeProxy'] = {
-          'config' => {
-            'featureGates' => {}
-          }
+          'config' => {}
         }
 
         if @config.kube_proxy.mode
           config['kubeProxy']['config']['mode'] = @config.kube_proxy.mode
         end
 
-        if @config.kube_proxy.mode == 'ipvs'
-          config['kubeProxy']['config']['featureGates']['SupportIPVSProxyMode'] = true
-        end
-
         config
       end
 

lib/pharos/phases/configure_dns.rb

@@ -6,7 +6,8 @@ class ConfigureDNS < Pharos::Phase
       title "Configure DNS"
 
       def call
-        patch_kubedns(
+        patch_deployment(
+          'coredns',
           replicas: @config.dns_replicas,
           max_surge: max_surge,
           max_unavailable: max_unavailable
@@ -45,15 +46,15 @@ def max_unavailable
 
       # @param replicas [Integer]
       # @param nodes [Integer]
-      def patch_kubedns(replicas:, max_surge:, max_unavailable:)
-        logger.info { "Patching kube-dns addon with #{replicas} replicas (max-surge #{max_surge}, max-unavailable #{max_unavailable})..." }
+      def patch_deployment(name, replicas:, max_surge:, max_unavailable:)
+        logger.info { "Patching #{name} deployment with #{replicas} replicas (max-surge #{max_surge}, max-unavailable #{max_unavailable})..." }
 
         resource = Pharos::Kube.session(@master.api_address).resource(
           apiVersion: 'extensions/v1beta1',
           kind: 'Deployment',
           metadata: {
             namespace: 'kube-system',
-            name: 'kube-dns'
+            name: name
           },
           spec: {
             replicas: replicas,
@@ -75,9 +76,7 @@ def patch_kubedns(replicas:, max_surge:, max_unavailable:)
                             {
                               key: "k8s-app",
                               operator: "In",
-                              values: [
-                                "kube-dns"
-                              ]
+                              values: [name]
                             }
                           ]
                         },

lib/pharos/phases/configure_kubelet.rb

@@ -89,37 +89,27 @@ def existing_dropin
       def build_systemd_dropin
         options = []
         options << "Environment='KUBELET_EXTRA_ARGS=#{kubelet_extra_args.join(' ')}'"
-        options << "Environment='KUBELET_DNS_ARGS=#{kubelet_dns_args.join(' ')}'"
         options << "ExecStartPre=-/sbin/swapoff -a"
 
         "[Service]\n#{options.join("\n")}\n"
       end
 
       # @return [Array<String>]
-      def kubelet_dns_args
-        args = [
-          "--cluster-dns=#{@config.network.dns_service_ip}",
-          "--cluster-domain=cluster.local"
-        ]
+      def kubelet_extra_args
+        args = []
+        if @config.kubelet&.read_only_port
+          args << "--read-only-port=10255"
+        end
+        args += @host.kubelet_args(local_only: false, cloud_provider: @config.cloud&.provider)
 
         if @host.resolvconf.systemd_resolved_stub
           # use usptream resolvers instead of systemd stub resolver at localhost for `dnsPolicy: Default` pods
+          # XXX: kubeadm also handles this?
           args << '--resolv-conf=/run/systemd/resolve/resolv.conf'
         elsif @host.resolvconf.nameserver_localhost
           fail "Host has /etc/resolv.conf configured with localhost as a resolver"
         end
 
-        args
-      end
-
-      # @return [Array<String>]
-      def kubelet_extra_args
-        args = []
-        unless @config.kubelet&.read_only_port
-          args << "--read-only-port=0"
-        end
-        args += @host.kubelet_args(local_only: false, cloud_provider: @config.cloud&.provider)
-
         args << "--authentication-token-webhook=true"
         args << "--pod-infra-container-image=#{@config.image_repository}/pause-#{@host.cpu_arch.name}:3.1"
         args << "--cloud-provider=#{@config.cloud.provider}" if @config.cloud

lib/pharos/phases/configure_master.rb

@@ -35,6 +35,7 @@ def install
         cfg = kubeadm.generate_config
 
         logger.info { "Initializing control plane ..." }
+        logger.debug { cfg.to_yaml }
 
         @ssh.tempfile(content: cfg.to_yaml, prefix: "kubeadm.cfg") do |tmp_file|
           exec_script(
@@ -55,6 +56,7 @@ def reconfigure
         cfg = kubeadm.generate_config
 
         logger.info { "Configuring control plane ..." }
+        logger.debug { cfg.to_yaml }
 
         @ssh.tempfile(content: cfg.to_yaml, prefix: "kubeadm.cfg") do |tmp_file|
           exec_script(

lib/pharos/phases/migrate_worker.rb

@@ -8,6 +8,8 @@ class MigrateWorker < Pharos::Phase
       def call
         if migrate_1_1_to_1_2?
           migrate_1_1_to_1_2
+        elsif migrate_1_3?
+          migrate_1_3
         else
           logger.info { 'Nothing to migrate.' }
         end
@@ -21,6 +23,23 @@ def migrate_1_1_to_1_2
         logger.info { 'Migrating from 1.1 to 1.2 ...' }
         @ssh.file('/etc/systemd/system/kubelet.service.d/5-pharos.conf').unlink
       end
+
+      def migrate_1_3?
+        return false unless @ssh.file('/etc/kubernetes/kubelet.conf').exist?
+        return false if @ssh.file('/var/lib/kubelet/config.yaml').exist?
+
+        true
+      end
+
+      def migrate_1_3
+        logger.info { 'Upgrade kubelet config for 1.10' }
+
+        # use the new version of kubeadm to write out /var/lib/kubelet/config.yaml for new kubelet version to be installed
+        # the kube master must be running, which is the case for upgrades
+        host_configurer.upgrade_kubeadm(Pharos::KUBEADM_VERSION)
+
+        @ssh.exec!("sudo /usr/local/bin/pharos-kubeadm-#{Pharos::KUBEADM_VERSION} upgrade node config --kubelet-version=v#{Pharos::KUBE_VERSION}")
+      end
     end
   end
 end

lib/pharos/phases/upgrade_master.rb

@@ -31,12 +31,13 @@ def upgrade_kubeadm
       end
 
       def upgrade
-        logger.info { "Upgrading control plane ..." }
-
         cfg = kubeadm.generate_config
 
+        logger.info { "Upgrading control plane ..." }
+        logger.debug { cfg.to_yaml }
+
         @ssh.tempfile(content: cfg.to_yaml, prefix: "kubeadm.cfg") do |tmp_file|
-          @ssh.exec!("sudo kubeadm upgrade apply #{Pharos::KUBE_VERSION} -y --ignore-preflight-errors=all --allow-experimental-upgrades --config #{tmp_file}")
+          @ssh.exec!("sudo /usr/local/bin/pharos-kubeadm-#{Pharos::KUBEADM_VERSION} upgrade apply #{Pharos::KUBE_VERSION} -y --ignore-preflight-errors=all --allow-experimental-upgrades --config #{tmp_file}")
         end
         logger.info { "Control plane upgrade succeeded!" }
       end

lib/pharos/scripts/configure-etcd.sh

@@ -8,7 +8,7 @@ etcd_version_matches() {
 
 mkdir -p /etc/kubernetes/manifests
 mkdir -p /etc/kubernetes/tmp
-if [ ! -e /etc/kubernetes/manifests/pharos-etcd.yaml ] || [ ! etcd_version_matches ] || [ ! etcd_healthy ]; then
+if [ ! -e /etc/kubernetes/manifests/pharos-etcd.yaml ] || ! etcd_version_matches; then
   cat  >/etc/kubernetes/tmp/pharos-etcd.yaml <<EOF && mv /etc/kubernetes/tmp/pharos-etcd.yaml /etc/kubernetes/manifests/pharos-etcd.yaml
 apiVersion: v1
 kind: Pod

lib/pharos_cluster.rb

@@ -9,7 +9,7 @@
 
 module Pharos
   CRIO_VERSION = '1.10.6'
-  KUBE_VERSION = ENV.fetch('KUBE_VERSION') { '1.10.5' }
+  KUBE_VERSION = ENV.fetch('KUBE_VERSION') { '1.11.0' }
   KUBEADM_VERSION = ENV.fetch('KUBEADM_VERSION') { KUBE_VERSION }
   ETCD_VERSION = ENV.fetch('ETCD_VERSION') { '3.1.12' }
   KUBELET_PROXY_VERSION = '0.3.6'