From 845efb05ccc16e4f1766a3703b5bb549eaea7092 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Tue, 26 Jan 2021 10:45:42 -0600 Subject: [PATCH] Issue #187: Convert cluster to K8s 1.19 and containerd instead of Docker. --- README.md | 2 + ansible.cfg | 2 +- example.config.yml | 4 +- k8s-manifests/docker-registry/manifest.yml | 15 +++--- k8s-manifests/drupal/drupal/manifest.yml | 11 ++-- k8s-manifests/drupal/mysql/manifest.yml | 2 +- .../ingress/deployment/manifest.yml | 6 ++- .../metrics-server-deployment.yaml | 52 ------------------- k8s-manifests/nfs/deployment/manifest.yml | 5 +- main.yml | 6 +-- requirements.yml | 2 +- setup/networking/example.vars.yml | 2 +- tasks/iptables-legacy.yml | 6 --- tasks/k8s-services.yml | 35 ++++--------- tasks/net-setup.yml | 27 ++++++++++ tasks/test-setup.yml | 2 - testing/presentation/main.yml | 1 - vars/main.yml | 28 +++++----- 18 files changed, 84 insertions(+), 124 deletions(-) delete mode 100644 k8s-manifests/kube-system/metrics-server-deployment.yaml delete mode 100644 tasks/iptables-legacy.yml create mode 100644 tasks/net-setup.yml diff --git a/README.md b/README.md index b1dd328..a205507 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ The process for setting up all the Raspberry Pis is outlined in the Wiki: 1. [Provision the Raspberry Pis](http://www.pidramble.com/wiki/setup/provision) 1. [Deploy Drupal to the Raspberry Pis](http://www.pidramble.com/wiki/setup/deploy-drupal) +> It is recommended you use the 64-bit version of Raspberry Pi OS, as some of this project's dependencies require it (and may not install on 32-bit Pi OS). + #### Adding more nodes You can add more than four nodes, if you desire; add additional hosts in the same sequence in the following files: diff --git a/ansible.cfg b/ansible.cfg index 441c4db..a625aa9 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -4,8 +4,8 @@ nocows = 1 forks = 10 retry_files_enabled = False stdout_callback = yaml -callback_whitelist = tones bin_ansible_callbacks = False +interpreter_python=auto_silent [ssh_connection] pipelining = True diff --git a/example.config.yml b/example.config.yml index 0f84ef5..d21b9b8 100644 --- a/example.config.yml +++ b/example.config.yml @@ -1,7 +1,7 @@ --- # Drupal configuration. drupal_domain: cluster.pidramble.test -drupal_docker_image: geerlingguy/drupal:latest-arm32v7 +drupal_docker_image: geerlingguy/drupal:latest-arm64 # Generate a salt with: `php -r "echo bin2hex(random_bytes(25));"` drupal_hash_salt: ef4e7eb18bd889e2c89720e71ea98beaae5f563d8685638d6e drupal_files_dir: /var/www/html/sites/default/files @@ -19,7 +19,7 @@ security_sudoers_passwordless: - pi # Docker configuration. -docker_registry_image: vsellier/docker-registry-arm:2.7.0 +docker_registry_image: vsellier/docker-registry-arm:2.7.1 docker_registry_domain: registry.pidramble.test docker_registry_ingress_host: 'kube3' docker_registry_http_secret: fdb19409c851605cd6c46615888d4c0e37858121df7c diff --git a/k8s-manifests/docker-registry/manifest.yml b/k8s-manifests/docker-registry/manifest.yml index 9571029..16dda76 100644 --- a/k8s-manifests/docker-registry/manifest.yml +++ b/k8s-manifests/docker-registry/manifest.yml @@ -18,7 +18,7 @@ spec: # Docker registry Deployment definition. --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: docker-registry @@ -86,7 +86,7 @@ data: # Docker registry Ingress definition. --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: @@ -98,9 +98,12 @@ spec: - host: {{ docker_registry_domain }} http: paths: - - backend: - serviceName: docker-registry - servicePort: 5000 - path: / + - path: / + pathType: Prefix + backend: + service: + name: docker-registry + port: + number: 5000 tls: - secretName: {{ manifest_namespace }}-tls-cert diff --git a/k8s-manifests/drupal/drupal/manifest.yml b/k8s-manifests/drupal/drupal/manifest.yml index e49da4b..1074a34 100644 --- a/k8s-manifests/drupal/drupal/manifest.yml +++ b/k8s-manifests/drupal/drupal/manifest.yml @@ -33,7 +33,7 @@ spec: # Drupal Deployment definition. --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: drupal @@ -113,7 +113,7 @@ spec: # Drupal Ingress. --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: drupal-ingress @@ -125,9 +125,12 @@ spec: http: paths: - path: / + pathType: Prefix backend: - serviceName: drupal - servicePort: 80 + service: + name: drupal + port: + number: 80 # Drupal Horizontal Pod Autoscaler. --- diff --git a/k8s-manifests/drupal/mysql/manifest.yml b/k8s-manifests/drupal/mysql/manifest.yml index e40c728..30a767a 100644 --- a/k8s-manifests/drupal/mysql/manifest.yml +++ b/k8s-manifests/drupal/mysql/manifest.yml @@ -39,7 +39,7 @@ spec: # MySQL Deployment. --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: mysql diff --git a/k8s-manifests/kube-system/ingress/deployment/manifest.yml b/k8s-manifests/kube-system/ingress/deployment/manifest.yml index 677d641..4860514 100644 --- a/k8s-manifests/kube-system/ingress/deployment/manifest.yml +++ b/k8s-manifests/kube-system/ingress/deployment/manifest.yml @@ -7,13 +7,17 @@ metadata: namespace: kube-system --- kind: DaemonSet -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 metadata: name: traefik-ingress-controller namespace: kube-system labels: k8s-app: traefik-ingress-lb spec: + selector: + matchLabels: + k8s-app: traefik-ingress-lb + name: traefik-ingress-lb template: metadata: labels: diff --git a/k8s-manifests/kube-system/metrics-server-deployment.yaml b/k8s-manifests/kube-system/metrics-server-deployment.yaml deleted file mode 100644 index 0d25853..0000000 --- a/k8s-manifests/kube-system/metrics-server-deployment.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Overridden version of deploy/1.8+/metrics-server-deployment.yaml from -# kubernetes-incubator/metrics-server so we can control the container image and -# command. See: https://github.com/geerlingguy/raspberry-pi-dramble/issues/128 ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metrics-server - namespace: kube-system ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: metrics-server - namespace: kube-system - labels: - k8s-app: metrics-server -spec: - selector: - matchLabels: - k8s-app: metrics-server - template: - metadata: - name: metrics-server - labels: - k8s-app: metrics-server - spec: - serviceAccountName: metrics-server - volumes: - # mount in tmp so we can safely use from-scratch images and/or read-only containers - - name: tmp-dir - emptyDir: {} - containers: - - name: metrics-server - image: {{ metrics_server_image }} - imagePullPolicy: IfNotPresent - volumeMounts: - - name: tmp-dir - mountPath: /tmp - command: - - /metrics-server - - --kubelet-insecure-tls - - --kubelet-preferred-address-types=InternalIP - tolerations: - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - tolerationSeconds: {{ kubernetes_toleration_not_ready }} - - effect: NoExecute - key: node.kubernetes.io/unreachable - operator: Exists - tolerationSeconds: {{ kubernetes_toleration_not_ready }} diff --git a/k8s-manifests/nfs/deployment/manifest.yml b/k8s-manifests/nfs/deployment/manifest.yml index 4a491b3..77a3599 100755 --- a/k8s-manifests/nfs/deployment/manifest.yml +++ b/k8s-manifests/nfs/deployment/manifest.yml @@ -1,7 +1,7 @@ # NFS Deployment. --- kind: Deployment -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 metadata: name: nfs-client-provisioner namespace: default @@ -9,6 +9,9 @@ spec: replicas: 1 strategy: type: Recreate + selector: + matchLabels: + app: nfs-client-provisioner template: metadata: labels: diff --git a/main.yml b/main.yml index db694e3..7164401 100644 --- a/main.yml +++ b/main.yml @@ -22,7 +22,7 @@ - import_tasks: tasks/disable-swap.yml tags: ['always'] - - import_tasks: tasks/iptables-legacy.yml + - import_tasks: tasks/net-setup.yml when: - ansible_distribution_version == "10" - deploy_target != 'docker' @@ -47,8 +47,8 @@ - role: geerlingguy.pip tags: ['pip', 'docker'] - - role: geerlingguy.docker_arm - tags: ['docker'] + - role: geerlingguy.containerd + tags: ['containerd'] - role: geerlingguy.kubernetes tags: ['kubernetes'] diff --git a/requirements.yml b/requirements.yml index 5b42f30..3acaf3b 100644 --- a/requirements.yml +++ b/requirements.yml @@ -3,6 +3,6 @@ - src: geerlingguy.swap - src: geerlingguy.nfs - src: geerlingguy.pip -- src: geerlingguy.docker_arm +- src: geerlingguy.containerd - src: geerlingguy.kubernetes - src: geerlingguy.k8s_manifests diff --git a/setup/networking/example.vars.yml b/setup/networking/example.vars.yml index f93f7ef..4402916 100644 --- a/setup/networking/example.vars.yml +++ b/setup/networking/example.vars.yml @@ -19,4 +19,4 @@ dns_nameservers: - "8.8.8.8" - "8.8.4.4" -dramble_ip_gateway: 10.0.100.1 \ No newline at end of file +dramble_ip_gateway: "10.0.100.1" \ No newline at end of file diff --git a/tasks/iptables-legacy.yml b/tasks/iptables-legacy.yml deleted file mode 100644 index c7cafb2..0000000 --- a/tasks/iptables-legacy.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# See: https://github.com/kubernetes/kubernetes/issues/71305 -- name: Use iptables-legacy instead of nftables. - alternatives: - name: iptables - path: /usr/sbin/iptables-legacy diff --git a/tasks/k8s-services.yml b/tasks/k8s-services.yml index 7102ee0..ea17fe2 100644 --- a/tasks/k8s-services.yml +++ b/tasks/k8s-services.yml @@ -7,32 +7,15 @@ delegate_to: "{{ groups.dramble[0] }}" run_once: True -- name: Ensure metrics-server directory exists. - file: - path: "~/metrics-server" - state: directory +- name: Download metrics-server manifest to the cluster. + get_url: + url: https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + dest: ~/metrics-server.yaml + mode: '0664' -- name: Download the metrics-server project from GitHub. - unarchive: - src: https://github.com/kubernetes-sigs/metrics-server/archive/{{ metrics_server_version }}.tar.gz - dest: "~/metrics-server" - remote_src: yes - extra_opts: [ "--strip-components=1" ] - creates: "~/metrics-server/README.md" - register: metrics_server_download - run_once: True - delegate_to: "{{ groups.dramble[0] }}" - -- name: Overwrite metrics-server deployment manifest. - template: - src: k8s-manifests/kube-system/metrics-server-deployment.yaml - dest: "~/metrics-server/deploy/1.8+/metrics-server-deployment.yaml" - run_once: True - delegate_to: "{{ groups.dramble[0] }}" - -# TODO: This task should do an *apply* if the deployment task above was changed. -- name: Apply metrics-server services to the cluster. - command: kubectl create -f ~/metrics-server/deploy/1.8+/ - when: metrics_server_download is changed +- name: Apply metrics-server manifest to the cluster. + k8s: + state: present + src: ~/metrics-server.yaml run_once: True delegate_to: "{{ groups.dramble[0] }}" diff --git a/tasks/net-setup.yml b/tasks/net-setup.yml new file mode 100644 index 0000000..e0fc1c0 --- /dev/null +++ b/tasks/net-setup.yml @@ -0,0 +1,27 @@ +--- +# See: https://github.com/kubernetes/kubernetes/issues/71305 +- name: Use iptables-legacy instead of nftables. + alternatives: + name: iptables + path: /usr/sbin/iptables-legacy + +# See: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic +- name: Ensure procps is installed. + package: + name: procps + state: present + +- name: Enable the br_netfilter module. + modprobe: + name: br_netfilter + state: present + +- name: Let iptables see bridged traffic. + sysctl: + name: "{{ item }}" + value: '1' + state: present + loop: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-ip6tables + - net.ipv4.ip_forward diff --git a/tasks/test-setup.yml b/tasks/test-setup.yml index 80fc362..bf25f68 100644 --- a/tasks/test-setup.yml +++ b/tasks/test-setup.yml @@ -11,9 +11,7 @@ - name: Override vars for non-ARM test environments. set_fact: docker_version: 5:19.03.1~3-0~debian-buster - mysql_container_image: 'mysql:5.7' docker_registry_image: registry:2 - metrics_server_image: k8s.gcr.io/metrics-server-amd64:v0.3.2 nfs_client_image: quay.io/external_storage/nfs-client-provisioner:latest when: deploy_target != 'pi' diff --git a/testing/presentation/main.yml b/testing/presentation/main.yml index ef05f89..458212f 100644 --- a/testing/presentation/main.yml +++ b/testing/presentation/main.yml @@ -15,6 +15,5 @@ - "{{ drupal_docker_image }}" - "{{ docker_registry_image }}" - "{{ mysql_container_image }}" - - "{{ metrics_server_image }}" - "{{ traefik_image }}" - "{{ nfs_client_image }}" diff --git a/vars/main.yml b/vars/main.yml index 5db9d0b..15d0c93 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,18 +1,16 @@ --- -# Docker configuration. -docker_install_compose: false -docker_users: - - pi +# Containerd configuration. +docker_apt_arch: arm64 # Kubernetes configuration. kubernetes_allow_pods_on_master: false -kubernetes_version: '1.15' +kubernetes_version: '1.19' kubernetes_packages: - - name: kubelet=1.15.0-00 + - name: kubelet=1.19.7-00 state: present - - name: kubectl=1.15.0-00 + - name: kubectl=1.19.7-00 state: present - - name: kubeadm=1.15.0-00 + - name: kubeadm=1.19.7-00 state: present - name: kubernetes-cni state: present @@ -25,8 +23,8 @@ k8s_manifests: - nfs/deployment - kube-system/ingress/rbac - kube-system/ingress/deployment - - dir: docker-registry - namespace: registry + # - dir: docker-registry + # namespace: registry - dir: drupal/mysql namespace: drupal - dir: drupal/drupal @@ -44,16 +42,14 @@ pip_install_packages: swap_file_state: absent swap_file_path: /dev/mapper/packer--debian--10--amd64--vg-swap_1 -# See: https://github.com/docker-library/mysql/issues/318 -mysql_container_image: hypriot/rpi-mysql:5.5 +# Image to use for MySQL. +mysql_container_image: mariadb:10 # Image to use for the traefik-ingress-controller daemonset. traefik_image: traefik:1.7 -# Version and image to use for the metrics-server deployment. -metrics_server_version: v0.3.6 -# See: https://console.cloud.google.com/gcr/images/google-containers/GLOBAL/metrics-server-arm64?gcrImageListsize=30 -metrics_server_image: gcr.io/google_containers/metrics-server-arm:v0.3.6 +# Manifest to use for the metrics-server deployment. +metrics_server_manifest: https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml # Image to use for nfs-client deployment. nfs_client_image: quay.io/external_storage/nfs-client-provisioner-arm:latest