nixos/kubernetes: replace docker shim with CRI-O

nixos/modules/services/cluster/kubernetes/addons/dashboard.nix

@@ -46,7 +46,7 @@ in {
     };
 
     image = mkOption {
-      description = "Docker image to seed for the kubernetes dashboard container.";
+      description = "Container image to be used the kubernetes dashboard.";
       type = types.attrs;
       default = {
         imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
@@ -58,8 +58,6 @@ in {
   };
 
   config = mkIf cfg.enable {
-    services.kubernetes.kubelet.seedDockerImages = [(pkgs.dockerTools.pullImage cfg.image)];
-
     services.kubernetes.addonManager.addons = {
       kubernetes-dashboard-deployment = {
         kind = "Deployment";

nixos/modules/services/cluster/kubernetes/addons/dns.nix

@@ -51,7 +51,7 @@ in {
     };
 
     coredns = mkOption {
-      description = "Docker image to seed for the CoreDNS container.";
+      description = "Container image used for the CoreDNS container.";
       type = types.attrs;
       default = {
         imageName = "coredns/coredns";
@@ -63,9 +63,6 @@ in {
   };
 
   config = mkIf cfg.enable {
-    services.kubernetes.kubelet.seedDockerImages =
-      singleton (pkgs.dockerTools.pullImage cfg.coredns);
-
     services.kubernetes.addonManager.bootstrapAddons = {
       coredns-cr = {
         apiVersion = "rbac.authorization.k8s.io/v1";
@@ -202,7 +199,7 @@ in {
                 {
                   args = [ "-conf" "/etc/coredns/Corefile" ];
                   image = with cfg.coredns; "${imageName}:${finalImageTag}";
-                  imagePullPolicy = "Never";
+                  imagePullPolicy = "Always";
                   livenessProbe = {
                     failureThreshold = 5;
                     httpGet = {

nixos/modules/services/cluster/kubernetes/default.nix

@@ -87,7 +87,7 @@ in {
 
         Master role will enable etcd, apiserver, scheduler, controller manager
         addon manager, flannel and proxy services.
-        Node role will enable flannel, docker, kubelet and proxy services.
+        Node role will enable flannel, cri-o, kubelet and proxy services.
       '';
       default = [];
       type = types.listOf (types.enum ["master" "node"]);
@@ -222,14 +222,8 @@ in {
     })
 
     (mkIf cfg.kubelet.enable {
-      virtualisation.docker = {
+      virtualisation.cri-o = {
         enable = mkDefault true;
-
-        # kubernetes needs access to logs
-        logDriver = mkDefault "json-file";
-
-        # iptables must be disabled for kubernetes
-        extraOptions = "--iptables=false --ip-masq=false";
       };
     })
 
@@ -269,7 +263,6 @@ in {
       users.users.kubernetes = {
         uid = config.ids.uids.kubernetes;
         description = "Kubernetes user";
-        extraGroups = [ "docker" ];
         group = "kubernetes";
         home = cfg.dataDir;
         createHome = true;

nixos/modules/services/cluster/kubernetes/flannel.nix

@@ -8,16 +8,6 @@ let
 
   # we want flannel to use kubernetes itself as configuration backend, not direct etcd
   storageBackend = "kubernetes";
-
-  # needed for flannel to pass options to docker
-  mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
-    buildInputs = [ pkgs.makeWrapper ];
-  } ''
-    mkdir -p $out
-
-    # bashInteractive needed for `compgen`
-    makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "${pkgs.kubernetes}/bin/mk-docker-opts.sh"
-  '';
 in
 {
   ###### interface
@@ -43,43 +33,16 @@ in
         cniVersion = "0.3.1";
         delegate = {
           isDefaultGateway = true;
-          bridge = "docker0";
         };
       }];
     };
 
-    systemd.services.mk-docker-opts = {
-      description = "Pre-Docker Actions";
-      path = with pkgs; [ gawk gnugrep ];
-      script = ''
-        ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
-        systemctl restart docker
-      '';
-      serviceConfig.Type = "oneshot";
-    };
-
-    systemd.paths.flannel-subnet-env = {
-      wantedBy = [ "flannel.service" ];
-      pathConfig = {
-        PathModified = "/run/flannel/subnet.env";
-        Unit = "mk-docker-opts.service";
-      };
-    };
-
-    systemd.services.docker = {
-      environment.DOCKER_OPTS = "-b none";
-      serviceConfig.EnvironmentFile = "-/run/flannel/docker";
-    };
-
-    # read environment variables generated by mk-docker-opts
-    virtualisation.docker.extraOptions = "$DOCKER_OPTS";
-
     networking = {
       firewall.allowedUDPPorts = [
         8285  # flannel udp
         8472  # flannel vxlan
       ];
-      dhcpcd.denyInterfaces = [ "docker*" "flannel*" ];
+      dhcpcd.denyInterfaces = [ "flannel*" ];
     };
 
     services.kubernetes.pki.certs = {

nixos/modules/services/cluster/kubernetes/kubelet.nix

@@ -7,24 +7,21 @@ let
   cfg = top.kubelet;
 
   cniConfig =
-    if cfg.cni.config != [] && cfg.cni.configDir != null then
+    if cfg.cni.config != [ ] && cfg.cni.configDir != null then
       throw "Verbatim CNI-config and CNI configDir cannot both be set."
     else if cfg.cni.configDir != null then
       cfg.cni.configDir
-    else
+    else if cfg.cni.config != [ ] then
       (pkgs.buildEnv {
         name = "kubernetes-cni-config";
-        paths = imap (i: entry:
-          pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
-        ) cfg.cni.config;
-      });
-
-  infraContainer = pkgs.dockerTools.buildImage {
-    name = "pause";
-    tag = "latest";
-    contents = top.package.pause;
-    config.Cmd = "/bin/pause";
-  };
+        paths = imap
+          (i: entry:
+            pkgs.writeTextDir "${toString (10 + i)}-${entry.type}.conf" (builtins.toJSON entry)
+          )
+          cfg.cni.config;
+        # fallback to CRI-O's default network dir if the user has not
+        # specified another one
+      }) else "/etc/cni/net.d";
 
   kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
 
@@ -191,12 +188,6 @@ in
       type = int;
     };
 
-    seedDockerImages = mkOption {
-      description = "List of docker images to preload on system";
-      default = [];
-      type = listOf package;
-    };
-
     taints = mkOption {
       description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
       default = {};
@@ -235,19 +226,22 @@ in
   ###### implementation
   config = mkMerge [
     (mkIf cfg.enable {
-      services.kubernetes.kubelet.seedDockerImages = [infraContainer];
-
       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
         wantedBy = [ "kubernetes.target" ];
-        after = [ "network.target" "docker.service" "kube-apiserver.service" ];
-        path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
+        after = [ "network.target" "cri-o.service" "kube-apiserver.service" ];
+        path = with pkgs; [
+          gitMinimal
+          openssh
+          utillinux
+          iproute
+          ethtool
+          thin-provisioning-tools
+          iptables
+          socat
+          config.virtualisation.cri-o.package
+        ] ++ top.path;
         preStart = ''
-          ${concatMapStrings (img: ''
-            echo "Seeding docker image: ${img}"
-            docker load <${img}
-          '') cfg.seedDockerImages}
-
           rm /opt/cni/bin/* || true
           ${concatMapStrings (package: ''
             echo "Linking cni package: ${package}"
@@ -283,7 +277,6 @@ in
               "--network-plugin=${cfg.networkPlugin}"} \
             ${optionalString (cfg.nodeIp != null)
               "--node-ip=${cfg.nodeIp}"} \
-            --pod-infra-container-image=pause \
             ${optionalString (cfg.manifests != {})
               "--pod-manifest-path=/etc/${manifestPath}"} \
             --port=${toString cfg.port} \
@@ -296,12 +289,17 @@ in
             ${optionalString (cfg.tlsKeyFile != null)
               "--tls-private-key-file=${cfg.tlsKeyFile}"} \
             ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+            --container-runtime=remote \
+            --container-runtime-endpoint=/var/run/crio/crio.sock \
+            --cgroup-driver=systemd \
             ${cfg.extraOpts}
           '';
           WorkingDirectory = top.dataDir;
         };
       };
 
+      virtualisation.cri-o.networkDir = cniConfig;
+
       # Allways include cni plugins
       services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];