Issue #187: Convert cluster to K8s 1.19 and containerd instead of Doc…

…ker.

README.md

@@ -51,6 +51,8 @@ The process for setting up all the Raspberry Pis is outlined in the Wiki:
   1. [Provision the Raspberry Pis](http://www.pidramble.com/wiki/setup/provision)
   1. [Deploy Drupal to the Raspberry Pis](http://www.pidramble.com/wiki/setup/deploy-drupal)
 
+> It is recommended you use the 64-bit version of Raspberry Pi OS, as some of this project's dependencies require it (and may not install on 32-bit Pi OS).
+
 #### Adding more nodes
 
 You can add more than four nodes, if you desire; add additional hosts in the same sequence in the following files:

ansible.cfg

@@ -4,8 +4,8 @@ nocows = 1
 forks = 10
 retry_files_enabled = False
 stdout_callback = yaml
-callback_whitelist = tones
 bin_ansible_callbacks = False
+interpreter_python=auto_silent
 
 [ssh_connection]
 pipelining = True

example.config.yml

@@ -1,7 +1,7 @@
 ---
 # Drupal configuration.
 drupal_domain: cluster.pidramble.test
-drupal_docker_image: geerlingguy/drupal:latest-arm32v7
+drupal_docker_image: geerlingguy/drupal:latest-arm64
 # Generate a salt with: `php -r "echo bin2hex(random_bytes(25));"`
 drupal_hash_salt: ef4e7eb18bd889e2c89720e71ea98beaae5f563d8685638d6e
 drupal_files_dir: /var/www/html/sites/default/files
@@ -19,7 +19,7 @@ security_sudoers_passwordless:
   - pi
 
 # Docker configuration.
-docker_registry_image: vsellier/docker-registry-arm:2.7.0
+docker_registry_image: vsellier/docker-registry-arm:2.7.1
 docker_registry_domain: registry.pidramble.test
 docker_registry_ingress_host: 'kube3'
 docker_registry_http_secret: fdb19409c851605cd6c46615888d4c0e37858121df7c

k8s-manifests/docker-registry/manifest.yml

@@ -18,7 +18,7 @@ spec:
 
 # Docker registry Deployment definition.
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: docker-registry
@@ -86,7 +86,7 @@ data:
 
 # Docker registry Ingress definition.
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   annotations:
@@ -98,9 +98,12 @@ spec:
   - host: {{ docker_registry_domain }}
     http:
       paths:
-      - backend:
-          serviceName: docker-registry
-          servicePort: 5000
-        path: /
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: docker-registry
+            port:
+              number: 5000
   tls:
     - secretName: {{ manifest_namespace }}-tls-cert

k8s-manifests/drupal/drupal/manifest.yml

@@ -33,7 +33,7 @@ spec:
 
 # Drupal Deployment definition.
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: drupal
@@ -113,7 +113,7 @@ spec:
 
 # Drupal Ingress.
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: drupal-ingress
@@ -125,9 +125,12 @@ spec:
     http:
       paths:
       - path: /
+        pathType: Prefix
         backend:
-          serviceName: drupal
-          servicePort: 80
+          service:
+            name: drupal
+            port:
+              number: 80
 
 # Drupal Horizontal Pod Autoscaler.
 ---

k8s-manifests/drupal/mysql/manifest.yml

@@ -39,7 +39,7 @@ spec:
 
 # MySQL Deployment.
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: mysql

k8s-manifests/kube-system/ingress/deployment/manifest.yml

@@ -7,13 +7,17 @@ metadata:
   namespace: kube-system
 ---
 kind: DaemonSet
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 metadata:
   name: traefik-ingress-controller
   namespace: kube-system
   labels:
     k8s-app: traefik-ingress-lb
 spec:
+  selector:
+    matchLabels:
+      k8s-app: traefik-ingress-lb
+      name: traefik-ingress-lb
   template:
     metadata:
       labels:

k8s-manifests/nfs/deployment/manifest.yml

@@ -1,14 +1,17 @@
 # NFS Deployment.
 ---
 kind: Deployment
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 metadata:
   name: nfs-client-provisioner
   namespace: default
 spec:
   replicas: 1
   strategy:
     type: Recreate
+  selector:
+    matchLabels:
+      app: nfs-client-provisioner
   template:
     metadata:
       labels:

main.yml

@@ -22,7 +22,7 @@
     - import_tasks: tasks/disable-swap.yml
       tags: ['always']
 
-    - import_tasks: tasks/iptables-legacy.yml
+    - import_tasks: tasks/net-setup.yml
       when:
         - ansible_distribution_version == "10"
         - deploy_target != 'docker'
@@ -47,8 +47,8 @@
     - role: geerlingguy.pip
       tags: ['pip', 'docker']
 
-    - role: geerlingguy.docker_arm
-      tags: ['docker']
+    - role: geerlingguy.containerd
+      tags: ['containerd']
 
     - role: geerlingguy.kubernetes
       tags: ['kubernetes']

requirements.yml

@@ -3,6 +3,6 @@
 - src: geerlingguy.swap
 - src: geerlingguy.nfs
 - src: geerlingguy.pip
-- src: geerlingguy.docker_arm
+- src: geerlingguy.containerd
 - src: geerlingguy.kubernetes
 - src: geerlingguy.k8s_manifests

setup/networking/example.vars.yml

@@ -19,4 +19,4 @@ dns_nameservers:
   - "8.8.8.8"
   - "8.8.4.4"
 
-dramble_ip_gateway: 10.0.100.1
+dramble_ip_gateway: "10.0.100.1"

tasks/k8s-services.yml

@@ -7,32 +7,15 @@
   delegate_to: "{{ groups.dramble[0] }}"
   run_once: True
 
-- name: Ensure metrics-server directory exists.
-  file:
-    path: "~/metrics-server"
-    state: directory
+- name: Download metrics-server manifest to the cluster.
+  get_url:
+    url: https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
+    dest: ~/metrics-server.yaml
+    mode: '0664'
 
-- name: Download the metrics-server project from GitHub.
-  unarchive:
-    src: https://github.com/kubernetes-sigs/metrics-server/archive/{{ metrics_server_version }}.tar.gz
-    dest: "~/metrics-server"
-    remote_src: yes
-    extra_opts: [ "--strip-components=1" ]
-    creates: "~/metrics-server/README.md"
-  register: metrics_server_download
-  run_once: True
-  delegate_to: "{{ groups.dramble[0] }}"
-
-- name: Overwrite metrics-server deployment manifest.
-  template:
-    src: k8s-manifests/kube-system/metrics-server-deployment.yaml
-    dest: "~/metrics-server/deploy/1.8+/metrics-server-deployment.yaml"
-  run_once: True
-  delegate_to: "{{ groups.dramble[0] }}"
-
-# TODO: This task should do an *apply* if the deployment task above was changed.
-- name: Apply metrics-server services to the cluster.
-  command: kubectl create -f ~/metrics-server/deploy/1.8+/
-  when: metrics_server_download is changed
+- name: Apply metrics-server manifest to the cluster.
+  k8s:
+    state: present
+    src: ~/metrics-server.yaml
   run_once: True
   delegate_to: "{{ groups.dramble[0] }}"

tasks/net-setup.yml

@@ -0,0 +1,27 @@
+---
+# See: https://github.com/kubernetes/kubernetes/issues/71305
+- name: Use iptables-legacy instead of nftables.
+  alternatives:
+    name: iptables
+    path: /usr/sbin/iptables-legacy
+
+# See: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic
+- name: Ensure procps is installed.
+  package:
+    name: procps
+    state: present
+
+- name: Enable the br_netfilter module.
+  modprobe:
+    name: br_netfilter
+    state: present
+
+- name: Let iptables see bridged traffic.
+  sysctl:
+    name: "{{ item }}"
+    value: '1'
+    state: present
+  loop:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.ipv4.ip_forward

tasks/test-setup.yml

@@ -11,9 +11,7 @@
 - name: Override vars for non-ARM test environments.
   set_fact:
     docker_version: 5:19.03.1~3-0~debian-buster
-    mysql_container_image: 'mysql:5.7'
     docker_registry_image: registry:2
-    metrics_server_image: k8s.gcr.io/metrics-server-amd64:v0.3.2
     nfs_client_image: quay.io/external_storage/nfs-client-provisioner:latest
   when: deploy_target != 'pi'
 

testing/presentation/main.yml

@@ -15,6 +15,5 @@
         - "{{ drupal_docker_image }}"
         - "{{ docker_registry_image }}"
         - "{{ mysql_container_image }}"
-        - "{{ metrics_server_image }}"
         - "{{ traefik_image }}"
         - "{{ nfs_client_image }}"

vars/main.yml

@@ -1,18 +1,16 @@
 ---
-# Docker configuration.
-docker_install_compose: false
-docker_users:
-  - pi
+# Containerd configuration.
+docker_apt_arch: arm64
 
 # Kubernetes configuration.
 kubernetes_allow_pods_on_master: false
-kubernetes_version: '1.15'
+kubernetes_version: '1.19'
 kubernetes_packages:
-  - name: kubelet=1.15.0-00
+  - name: kubelet=1.19.7-00
     state: present
-  - name: kubectl=1.15.0-00
+  - name: kubectl=1.19.7-00
     state: present
-  - name: kubeadm=1.15.0-00
+  - name: kubeadm=1.19.7-00
     state: present
   - name: kubernetes-cni
     state: present
@@ -25,8 +23,8 @@ k8s_manifests:
   - nfs/deployment
   - kube-system/ingress/rbac
   - kube-system/ingress/deployment
-  - dir: docker-registry
-    namespace: registry
+  # - dir: docker-registry
+  #   namespace: registry
   - dir: drupal/mysql
     namespace: drupal
   - dir: drupal/drupal
@@ -44,16 +42,14 @@ pip_install_packages:
 swap_file_state: absent
 swap_file_path: /dev/mapper/packer--debian--10--amd64--vg-swap_1
 
-# See: https://github.com/docker-library/mysql/issues/318
-mysql_container_image: hypriot/rpi-mysql:5.5
+# Image to use for MySQL.
+mysql_container_image: mariadb:10
 
 # Image to use for the traefik-ingress-controller daemonset.
 traefik_image: traefik:1.7
 
-# Version and image to use for the metrics-server deployment.
-metrics_server_version: v0.3.6
-# See: https://console.cloud.google.com/gcr/images/google-containers/GLOBAL/metrics-server-arm64?gcrImageListsize=30
-metrics_server_image: gcr.io/google_containers/metrics-server-arm:v0.3.6
+# Manifest to use for the metrics-server deployment.
+metrics_server_manifest: https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
 
 # Image to use for nfs-client deployment.
 nfs_client_image: quay.io/external_storage/nfs-client-provisioner-arm:latest