3.11 - update calico setup

playbooks/openshift-master/private/config.yml

@@ -79,7 +79,7 @@
     when: openshift_use_nuage | default(false) | bool
   - role: nuage_master
     when: openshift_use_nuage | default(false) | bool
-  - role: calico_master
+  - role: calico
     when: openshift_use_calico | default(false) | bool
   tasks:
   - import_role:

playbooks/openshift-node/private/join.yml

@@ -57,6 +57,19 @@
     openshift_master_host: "{{ groups.oo_first_master.0 }}"
     openshift_manage_node_is_master: "{{ ('oo_masters_to_config' in group_names) | bool }}"
 
+- name: Create additional node network plugin groups
+  hosts: "{{ openshift_node_scale_up_group | default('oo_nodes_to_config') }}"
+  tasks:
+  - group_by:
+      key: oo_nodes_use_{{ (openshift_use_calico | default(False)) | ternary('calico','nothing') }}
+    changed_when: False
+
+- name: Additional calico node config
+  hosts: oo_nodes_use_calico
+  roles:
+  - role: calico_node
+    when: openshift_use_calico | default(false) | bool
+
 - name: Node Join Checkpoint End
   hosts: all
   gather_facts: false

roles/calico/README.md

@@ -1,3 +1,48 @@
 # Calico
 
-Please see [calico_master](../calico_master/README.md)
+Configure Calico components for the Master host.
+
+## Requirements
+
+* Ansible 2.2
+
+## Installation
+
+To install, set the following inventory configuration parameters:
+
+* `openshift_use_calico=True`
+* `openshift_use_openshift_sdn=False`
+* `os_sdn_network_plugin_name='cni'`
+
+By default, Calico will share the etcd used by OpenShift.
+To configure Calico to use a separate instance of etcd, place etcd SSL client certs on your master,
+then set the following variables in your inventory.ini:
+
+* `calico_etcd_ca_cert_file=/path/to/etcd-ca.crt`
+* `calico_etcd_cert_file=/path/to/etcd-client.crt`
+* `calico_etcd_key_file=/path/to/etcd-client.key`
+* `calico_etcd_endpoints=https://etcd:2379`
+
+## Upgrading
+
+OpenShift-Ansible installs Calico as a self-hosted install. Previously, Calico ran as a systemd service. Running Calico
+in this manner is now deprecated, and must be upgraded to a hosted cluster. Please run the Legacy Upgrade playbook to
+upgrade your existing Calico deployment to a hosted deployment:
+
+        ansible-playbook -i inventory.ini playbooks/byo/calico/legacy_upgrade.yml
+
+## Additional Calico/Node and Felix Configuration Options
+
+Additional parameters that can be defined in the inventory are:
+
+
+| Environment | Description | Schema | Default |   
+|---------|----------------------|---------|---------|
+| CALICO_IPV4POOL_IPIP | IPIP Mode to use for the IPv4 POOL created at start up.	| off, always, cross-subnet	| always |
+| CALICO_LOG_DIR | Directory on the host machine where Calico Logs are written.| String	| /var/log/calico |
+
+### Contact Information
+
+Author: Dan Osborne <dan@projectcalico.org>
+
+For support, join the `#openshift` channel on the [calico users slack](calicousers.slack.com).

roles/calico_master/defaults/main.yaml → roles/calico/defaults/main.yaml

@@ -7,3 +7,4 @@ calico_node_image: "quay.io/calico/node:v3.1.3"
 calico_cni_image: "quay.io/calico/cni:v3.1.3"
 calico_upgrade_image: "quay.io/calico/upgrade:v1.0.5"
 calico_ipv4pool_ipip: "always"
+use_calico_etcd: False

roles/calico/meta/main.yml

@@ -13,5 +13,5 @@ galaxy_info:
   - cloud
   - system
 dependencies:
+- role: lib_utils
 - role: openshift_facts
-- role: container_runtime

roles/calico_master/tasks/certs.yml → roles/calico/tasks/certs.yml

@@ -10,6 +10,29 @@
   - calico_certs_provided
   - not (calico_etcd_ca_cert_file is defined and calico_etcd_cert_file is defined and calico_etcd_key_file is defined and calico_etcd_endpoints is defined)
 
+- name: Calico Node | Set separate Calico etcd flag
+  set_fact:
+    use_calico_etcd: "{{ calico_etcd_initial_cluster is defined or calico_etcd_generate_certs is defined or calico_etcd_service_ip is defined or calico_etcd_clients_port is defined or calico_etcd_peers_port is defined or calico_etcd_cert_dir is defined or calico_etcd_mount is defined | bool }}"
+
+- name: Calico Node | Error if using separate etcd with invalid arguments
+  fail:
+    msg: "Must provide all or none of the following etcd params: calico_etcd_initial_cluster, calico_etcd_generate_certs, calico_etcd_service_ip, calico_etcd_clients_port, calico_etcd_peers_port, calico_etcd_cert_dir, and calico_etcd_mount"
+  when:
+  - use_calico_etcd
+  - not (calico_certs_provided and calico_etcd_initial_cluster is defined and calico_etcd_generate_certs is defined and calico_etcd_service_ip is defined and calico_etcd_clients_port is defined and calico_etcd_peers_port is defined and calico_etcd_cert_dir is defined and calico_etcd_mount is defined)
+
+- name: Calico Node | Configure separate Calico etcd and certs
+  when: use_calico_etcd
+  become: yes
+  include_role:
+    name: etcd
+    tasks_from: server_certificates
+  vars:
+    etcd_cert_prefix: calico-etcd-
+    etcd_cert_config_dir: "{{ calico_etcd_cert_dir }}"
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+    etcd_cert_subdir: "calico-etcd-{{ openshift.common.hostname }}"
+
 - name: Calico Node | Set etcd cert location facts
   when: not calico_certs_provided
   set_fact:

roles/calico/tasks/main.yml

@@ -1,47 +1,129 @@
 ---
-- name: Check for legacy service
-  stat:
-    path: /lib/systemd/system/calico.service
-    get_checksum: false
-    get_attributes: false
-    get_mime: false
-  register: sym
-- fail:
-    msg: You are running a systemd based installation of Calico. Please run the calico upgrade playbook to upgrade to a self-hosted installation.
-  when: sym.stat.exists
-
-- name: Configure NetworkManager to ignore Calico interfaces
-  copy:
-    src: files/calico.conf
-    dest: /etc/NetworkManager/conf.d/
-  when: using_network_manager | default(true) | bool
-  register: nm
-
-- name: restart NetworkManager
-  systemd:
-    name: NetworkManager
-    state: restarted
-  when: nm.changed
-
-# TODO: Move into shared vars file
-- name: Load default node image
+- name: Calico | Run kube proxy
+  run_once: true
+  import_role:
+    name: kube_proxy_and_dns
+
+- include_tasks: certs.yml
+
+- name: Calico | Clean Calico etcd data
+  when: calico_cleanup_path is defined and calico_cleanup_path != ""
+  file:
+    state: absent
+    path: "{{ calico_cleanup_path }}"
+
+- name: Calico | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-node
+  oc_adm_policy_user:
+    user: system:serviceaccount:kube-system:calico-node
+    resource_kind: scc
+    resource_name: privileged
+    state: present
+
+- name: Calico | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-kube-controllers
+  oc_adm_policy_user:
+    user: system:serviceaccount:kube-system:calico-kube-controllers
+    resource_kind: scc
+    resource_name: privileged
+    state: present
+
+- name: Calico | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-upgrade-job
+  oc_adm_policy_user:
+    user: system:serviceaccount:kube-system:calico-upgrade-job
+    resource_kind: scc
+    resource_name: privileged
+    state: present
+
+- name: Calico | Set default selector for kube-system
+  command: >
+    {{ openshift_client_binary }}
+    --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+    annotate  ns kube-system openshift.io/node-selector="" --overwrite
+
+- name: Calico | Create temp directory
+  command: mktemp -d /tmp/openshift-ansible-XXXXXXX
+  register: mktemp
+  changed_when: False
+
+- name: Calico | Write separate Calico etcd manifest
+  when: use_calico_etcd
+  template:
+    dest: "{{ mktemp.stdout }}/calico-etcd.yml"
+    src: calico-etcd.yml.j2
+
+- name: Calico | Launch separate Calico etcd
+  when: use_calico_etcd
+  command: >
+    {{ openshift_client_binary }} apply
+    -f {{ mktemp.stdout }}/calico-etcd.yml
+    --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+  register: calico_etcd_create_output
+  failed_when: "calico_etcd_create_output.rc != 0"
+  changed_when: "('created' in calico_etcd_create_output.stdout) or ('configured' in calico_etcd_create_output.stdout)"
+
+- name: Calico | Parse node version
+  set_fact:
+    node_version: "{{ calico_node_image | regex_replace('^.*node:v?(.*)$', '\\1') }}"
+    cnx: "{{ calico_node_image | regex_replace('^.*/(.*)-node:.*$', '\\1') }}"
+    use_calico_credentials: "{{ calico_image_credentials is defined | bool }}"
+
+- name: Calico | Encode Docker Credentials
+  shell: >
+    cat {{ calico_image_credentials }} | openssl base64 -A
+  register: calico_encoded_credentials_output
+  failed_when: "calico_encoded_credentials_output.rc != 0 or calico_encoded_credentials_output.stdout == ''"
+  when: use_calico_credentials
+
+- name: Calico | Set Encoded Docker Credentials Fact
   set_fact:
-    calico_node_image: "quay.io/calico/node:v2.6.7"
-  when: calico_node_image is not defined
+    calico_encoded_credentials: "{{ calico_encoded_credentials_output.stdout }}"
+  when: use_calico_credentials
 
-- name: Prepull Images
-  command: "{{ openshift_container_cli }} pull {{ calico_node_image }}"
+- name: Calico | Write Calico Pull Secret
+  template:
+    dest: "{{ mktemp.stdout }}/calico-pull-secret.yml"
+    src: calico-pull-secret.yml.j2
+  when: use_calico_credentials
 
-- name: Apply node label
-  delegate_to: "{{ groups.oo_first_master.0 }}"
+- name: Calico | Create Calico Pull Secret
+  when: use_calico_credentials
   command: >
-    {{ openshift_client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig label node {{ openshift.node.nodename | lower }} --overwrite projectcalico.org/ds-ready=true
-
-- name: Wait for node running
-  uri:
-    url: http://localhost:9099/readiness
-    status_code: 204
-  delay: 3
-  retries: 10
-  register: result
-  until: result.status == 204
+    {{ openshift_client_binary }} apply
+    -f {{ mktemp.stdout }}/calico-pull-secret.yml
+    --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+  register: calico_pull_secret_create_output
+  failed_when: "calico_pull_secret_create_output.rc != 0"
+  changed_when: "('created' in calico_pull_secret_create_output.stdout) or ('configured' in calico_pull_secret_create_output.stdout)"
+
+- name: Calico | Set the correct liveness and readiness checks
+  set_fact:
+    calico_binary_checks: "{{ (node_version > '3.2.0' and cnx != 'cnx') or (node_version > '2.2.0' and cnx == 'cnx') | bool }}"
+
+- name: Calico | Write Calico v2
+  template:
+    dest: "{{ mktemp.stdout }}/calico.yml"
+    src: calico.yml.j2
+  when:
+    - node_version | regex_search('^[0-9]\.[0-9]\.[0-9]') and node_version < '3.0.0'
+    - cnx != "cnx"
+
+- name: Calico | Write Calico v3
+  template:
+    dest: "{{ mktemp.stdout }}/calico.yml"
+    src: calicov3.yml.j2
+  when: (node_version | regex_search('^[0-9]\.[0-9]\.[0-9]') and node_version >= '3.0.0') or (node_version == 'master') or (cnx == "cnx" and node_version >= '2.0.0')
+
+- name: Calico | Launch Calico
+  run_once: true
+  command: >
+    {{ openshift_client_binary }} apply
+    -f {{ mktemp.stdout }}/calico.yml
+    --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+  register: calico_create_output
+  failed_when: "calico_create_output.rc != 0"
+  changed_when: "('created' in calico_create_output.stdout) or ('configured' in calico_create_output.stdout)"
+
+- name: Calico | Delete temp directory
+  file:
+    name: "{{ mktemp.stdout }}"
+    state: absent
+  changed_when: False

roles/calico/templates/calico-etcd.yml.j2

@@ -0,0 +1,88 @@
+# This manifest installs the Calico etcd on the master.  This uses a DaemonSet
+# to force it to run on the master even when the master isn't schedulable, and uses
+# nodeSelector to ensure it only runs on the master.
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: calico-etcd
+  namespace: kube-system
+  labels:
+    k8s-app: calico-etcd
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-etcd
+      annotations:
+        # Mark this pod as a critical add-on; when enabled, the critical add-on scheduler
+        # reserves resources for critical add-on pods so that they can be rescheduled after
+        # a failure.  This annotation works in tandem with the toleration below.
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      tolerations:
+      # this taint is set by all kubelets running `--cloud-provider=external`
+      # so we should tolerate it to schedule the calico pods
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      # Toleration allows the pod to run on master
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode.
+      # This, along with the annotation above marks this pod as a critical add-on.
+      - key: CriticalAddonsOnly
+        operator: Exists
+      # Only run this pod on configure nodes with calico-etcd true in /etc/ansible/hosts.
+      nodeSelector:
+        calico-etcd: "true"
+      hostNetwork: true
+      serviceAccountName: calico-node
+      containers:
+        - name: calico-etcd
+          image: quay.io/coreos/etcd:v3.2.5
+          env:
+            - name: CALICO_ETCD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CALICO_ETCD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          command: ["/bin/sh","-c"]
+          args: ["/usr/local/bin/etcd --name=$CALICO_ETCD_NAME --data-dir={{ calico_etcd_mount }}/calico-data --advertise-client-urls=https://$CALICO_ETCD_IP:{{ calico_etcd_clients_port }} --listen-client-urls=https://0.0.0.0:{{ calico_etcd_clients_port }} --listen-peer-urls=https://$CALICO_ETCD_IP:{{ calico_etcd_peers_port }} --cert-file={{ calico_etcd_cert_file }} --key-file={{ calico_etcd_key_file }} --trusted-ca-file={{ calico_etcd_ca_cert_file }} --initial-cluster-token=calico-cluster-1 --initial-cluster={{ calico_etcd_initial_cluster }} --initial-advertise-peer-urls=https://$CALICO_ETCD_IP:{{ calico_etcd_peers_port }} --peer-client-cert-auth --peer-trusted-ca-file={{ calico_etcd_ca_cert_file }} --peer-cert-file={{ calico_etcd_cert_file }} --peer-key-file={{ calico_etcd_key_file }}"]
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: var-etcd
+              mountPath: {{ calico_etcd_mount }}
+            - name: etcd-certs
+              mountPath: {{ calico_etcd_cert_dir }}
+      volumes:
+        - name: var-etcd
+          hostPath:
+            path: {{ calico_etcd_mount }}
+        - name: etcd-certs
+          hostPath:
+            path: {{ calico_etcd_cert_dir }}
+
+---
+
+# This manifest installs the Service which gets traffic to the Calico
+# etcd.
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: calico-etcd
+  name: calico-etcd
+  namespace: kube-system
+spec:
+  # Select the calico-etcd pod running on the master.
+  selector:
+    k8s-app: calico-etcd
+  # This ClusterIP needs to be known in advance, since we cannot rely
+  # on DNS to get access to etcd.
+  clusterIP: {{ calico_etcd_service_ip }}
+  ports:
+    - port: {{ calico_etcd_clients_port }}

roles/calico/templates/calico-pull-secret.yml.j2

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: calico-pull-secret
+  namespace: kube-system
+data:
+  .dockerconfigjson: {{ calico_encoded_credentials }}
+type: kubernetes.io/dockerconfigjson

roles/calico_master/templates/calico.yml.j2 → roles/calico/templates/calico.yml.j2

@@ -126,8 +126,6 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
-      nodeSelector:
-        projectcalico.org/ds-ready: "true"
       hostNetwork: true
       tolerations:
         # Make sure calico/node gets scheduled on all nodes.