Add separate Calico etcd

roles/calico_master/defaults/main.yaml

@@ -7,3 +7,4 @@ calico_node_image: "quay.io/calico/node:v3.1.3"
 calico_cni_image: "quay.io/calico/cni:v3.1.3"
 calico_upgrade_image: "quay.io/calico/upgrade:v1.0.5"
 calico_ipv4pool_ipip: "always"
+use_calico_etcd: False

roles/calico_master/tasks/certs.yml

@@ -10,6 +10,29 @@
   - calico_certs_provided
   - not (calico_etcd_ca_cert_file is defined and calico_etcd_cert_file is defined and calico_etcd_key_file is defined and calico_etcd_endpoints is defined)
 
+- name: Calico Node | Set separate Calico etcd flag
+  set_fact:
+    use_calico_etcd: "{{ calico_etcd_initial_cluster is defined or calico_etcd_generate_certs is defined or calico_etcd_service_ip is defined or calico_etcd_clients_port is defined or calico_etcd_peers_port is defined or calico_etcd_cert_dir is defined or calico_etcd_mount is defined | bool }}"
+
+- name: Calico Node | Error if using separate etcd with invalid arguments
+  fail:
+    msg: "Must provide all or none of the following etcd params: calico_etcd_initial_cluster, calico_etcd_generate_certs, calico_etcd_service_ip, calico_etcd_clients_port, calico_etcd_peers_port, calico_etcd_cert_dir, and calico_etcd_mount"
+  when:
+  - use_calico_etcd
+  - not (calico_certs_provided and calico_etcd_initial_cluster is defined and calico_etcd_generate_certs is defined and calico_etcd_service_ip is defined and calico_etcd_clients_port is defined and calico_etcd_peers_port is defined and calico_etcd_cert_dir is defined and calico_etcd_mount is defined)
+
+- name: Calico Node | Configure separate Calico etcd and certs
+  when: use_calico_etcd
+  become: yes
+  include_role:
+    name: etcd
+    tasks_from: server_certificates
+  vars:
+    etcd_cert_prefix: calico-etcd-
+    etcd_cert_config_dir: "{{ calico_etcd_cert_dir }}"
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+    etcd_cert_subdir: "calico-etcd-{{ openshift.common.hostname }}"
+
 - name: Calico Node | Set etcd cert location facts
   when: not calico_certs_provided
   set_fact:

roles/calico_master/tasks/main.yml

@@ -1,6 +1,12 @@
 ---
 - include_tasks: certs.yml
 
+- name: Calico Master | Clean Calico etcd data
+  when: calico_cleanup_path is defined and calico_cleanup_path != ""
+  file:
+    state: absent
+    path: "{{ calico_cleanup_path }}"
+
 - name: Calico Master | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-node
   oc_adm_policy_user:
     user: system:serviceaccount:kube-system:calico-node
@@ -33,6 +39,22 @@
   register: mktemp
   changed_when: False
 
+- name: Calico Master | Write separate Calico etcd manifest
+  when: use_calico_etcd
+  template:
+    dest: "{{ mktemp.stdout }}/calico-etcd.yml"
+    src: calico-etcd.yml.j2
+
+- name: Calico Master | Launch separate Calico etcd
+  when: use_calico_etcd
+  command: >
+    {{ openshift_client_binary }} apply
+    -f {{ mktemp.stdout }}/calico-etcd.yml
+    --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+  register: calico_etcd_create_output
+  failed_when: "calico_etcd_create_output.rc != 0"
+  changed_when: "('created' in calico_etcd_create_output.stdout) or ('configured' in calico_etcd_create_output.stdout)"
+
 - name: Calico Master | Parse node version
   set_fact:
     node_version: "{{ calico_node_image | regex_replace('^.*node:v?(.*)$', '\\1') }}"

roles/calico_master/templates/calico-etcd.yml.j2

@@ -0,0 +1,88 @@
+# This manifest installs the Calico etcd on the master.  This uses a DaemonSet
+# to force it to run on the master even when the master isn't schedulable, and uses
+# nodeSelector to ensure it only runs on the master.
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: calico-etcd
+  namespace: kube-system
+  labels:
+    k8s-app: calico-etcd
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-etcd
+      annotations:
+        # Mark this pod as a critical add-on; when enabled, the critical add-on scheduler
+        # reserves resources for critical add-on pods so that they can be rescheduled after
+        # a failure.  This annotation works in tandem with the toleration below.
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      tolerations:
+      # this taint is set by all kubelets running `--cloud-provider=external`
+      # so we should tolerate it to schedule the calico pods
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      # Toleration allows the pod to run on master
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode.
+      # This, along with the annotation above marks this pod as a critical add-on.
+      - key: CriticalAddonsOnly
+        operator: Exists
+      # Only run this pod on configure nodes with calico-etcd true in /etc/ansible/hosts.
+      nodeSelector:
+        calico-etcd: "true"
+      hostNetwork: true
+      serviceAccountName: calico-node
+      containers:
+        - name: calico-etcd
+          image: quay.io/coreos/etcd:v3.2.5
+          env:
+            - name: CALICO_ETCD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CALICO_ETCD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          command: ["/bin/sh","-c"]
+          args: ["/usr/local/bin/etcd --name=$CALICO_ETCD_NAME --data-dir={{ calico_etcd_mount }}/calico-data --advertise-client-urls=https://$CALICO_ETCD_IP:{{ calico_etcd_clients_port }} --listen-client-urls=https://0.0.0.0:{{ calico_etcd_clients_port }} --listen-peer-urls=https://$CALICO_ETCD_IP:{{ calico_etcd_peers_port }} --cert-file={{ calico_etcd_cert_file }} --key-file={{ calico_etcd_key_file }} --trusted-ca-file={{ calico_etcd_ca_cert_file }} --initial-cluster-token=calico-cluster-1 --initial-cluster={{ calico_etcd_initial_cluster }} --initial-advertise-peer-urls=https://$CALICO_ETCD_IP:{{ calico_etcd_peers_port }} --peer-client-cert-auth --peer-trusted-ca-file={{ calico_etcd_ca_cert_file }} --peer-cert-file={{ calico_etcd_cert_file }} --peer-key-file={{ calico_etcd_key_file }}"]
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: var-etcd
+              mountPath: {{ calico_etcd_mount }}
+            - name: etcd-certs
+              mountPath: {{ calico_etcd_cert_dir }}
+      volumes:
+        - name: var-etcd
+          hostPath:
+            path: {{ calico_etcd_mount }}
+        - name: etcd-certs
+          hostPath:
+            path: {{ calico_etcd_cert_dir }}
+
+---
+
+# This manifest installs the Service which gets traffic to the Calico
+# etcd.
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: calico-etcd
+  name: calico-etcd
+  namespace: kube-system
+spec:
+  # Select the calico-etcd pod running on the master.
+  selector:
+    k8s-app: calico-etcd
+  # This ClusterIP needs to be known in advance, since we cannot rely
+  # on DNS to get access to etcd.
+  clusterIP: {{ calico_etcd_service_ip }}
+  ports:
+    - port: {{ calico_etcd_clients_port }}