diff --git a/playbooks/openshift-master/private/config.yml b/playbooks/openshift-master/private/config.yml index 05eab628876..97a57554300 100644 --- a/playbooks/openshift-master/private/config.yml +++ b/playbooks/openshift-master/private/config.yml @@ -79,7 +79,7 @@ when: openshift_use_nuage | default(false) | bool - role: nuage_master when: openshift_use_nuage | default(false) | bool - - role: calico_master + - role: calico when: openshift_use_calico | default(false) | bool tasks: - import_role: diff --git a/playbooks/openshift-node/private/join.yml b/playbooks/openshift-node/private/join.yml index 9e7dcc27a04..0b93746c99c 100644 --- a/playbooks/openshift-node/private/join.yml +++ b/playbooks/openshift-node/private/join.yml @@ -57,6 +57,19 @@ openshift_master_host: "{{ groups.oo_first_master.0 }}" openshift_manage_node_is_master: "{{ ('oo_masters_to_config' in group_names) | bool }}" +- name: Create additional node network plugin groups + hosts: "{{ openshift_node_scale_up_group | default('oo_nodes_to_config') }}" + tasks: + - group_by: + key: oo_nodes_use_{{ (openshift_use_calico | default(False)) | ternary('calico','nothing') }} + changed_when: False + +- name: Additional calico node config + hosts: oo_nodes_use_calico + roles: + - role: calico_node + when: openshift_use_calico | default(false) | bool + - name: Node Join Checkpoint End hosts: all gather_facts: false diff --git a/roles/calico/README.md b/roles/calico/README.md index 5846abfbdd1..ada1509cb81 100644 --- a/roles/calico/README.md +++ b/roles/calico/README.md @@ -1,3 +1,48 @@ # Calico -Please see [calico_master](../calico_master/README.md) +Configure Calico components for the Master host. + +## Requirements + +* Ansible 2.2 + +## Installation + +To install, set the following inventory configuration parameters: + +* `openshift_use_calico=True` +* `openshift_use_openshift_sdn=False` +* `os_sdn_network_plugin_name='cni'` + +By default, Calico will share the etcd used by OpenShift. +To configure Calico to use a separate instance of etcd, place etcd SSL client certs on your master, +then set the following variables in your inventory.ini: + +* `calico_etcd_ca_cert_file=/path/to/etcd-ca.crt` +* `calico_etcd_cert_file=/path/to/etcd-client.crt` +* `calico_etcd_key_file=/path/to/etcd-client.key` +* `calico_etcd_endpoints=https://etcd:2379` + +## Upgrading + +OpenShift-Ansible installs Calico as a self-hosted install. Previously, Calico ran as a systemd service. Running Calico +in this manner is now deprecated, and must be upgraded to a hosted cluster. Please run the Legacy Upgrade playbook to +upgrade your existing Calico deployment to a hosted deployment: + + ansible-playbook -i inventory.ini playbooks/byo/calico/legacy_upgrade.yml + +## Additional Calico/Node and Felix Configuration Options + +Additional parameters that can be defined in the inventory are: + + +| Environment | Description | Schema | Default | +|---------|----------------------|---------|---------| +| CALICO_IPV4POOL_IPIP | IPIP Mode to use for the IPv4 POOL created at start up. | off, always, cross-subnet | always | +| CALICO_LOG_DIR | Directory on the host machine where Calico Logs are written.| String | /var/log/calico | + +### Contact Information + +Author: Dan Osborne + +For support, join the `#openshift` channel on the [calico users slack](calicousers.slack.com). diff --git a/roles/calico_master/defaults/main.yaml b/roles/calico/defaults/main.yaml similarity index 100% rename from roles/calico_master/defaults/main.yaml rename to roles/calico/defaults/main.yaml diff --git a/roles/calico/meta/main.yml b/roles/calico/meta/main.yml index eeb3909b2ce..2b36764c4a5 100644 --- a/roles/calico/meta/main.yml +++ b/roles/calico/meta/main.yml @@ -13,5 +13,5 @@ galaxy_info: - cloud - system dependencies: +- role: lib_utils - role: openshift_facts -- role: container_runtime diff --git a/roles/calico_master/tasks/certs.yml b/roles/calico/tasks/certs.yml similarity index 100% rename from roles/calico_master/tasks/certs.yml rename to roles/calico/tasks/certs.yml diff --git a/roles/calico/tasks/main.yml b/roles/calico/tasks/main.yml index 9a1a7954752..8de31929d3b 100644 --- a/roles/calico/tasks/main.yml +++ b/roles/calico/tasks/main.yml @@ -1,47 +1,129 @@ --- -- name: Check for legacy service - stat: - path: /lib/systemd/system/calico.service - get_checksum: false - get_attributes: false - get_mime: false - register: sym -- fail: - msg: You are running a systemd based installation of Calico. Please run the calico upgrade playbook to upgrade to a self-hosted installation. - when: sym.stat.exists - -- name: Configure NetworkManager to ignore Calico interfaces - copy: - src: files/calico.conf - dest: /etc/NetworkManager/conf.d/ - when: using_network_manager | default(true) | bool - register: nm - -- name: restart NetworkManager - systemd: - name: NetworkManager - state: restarted - when: nm.changed - -# TODO: Move into shared vars file -- name: Load default node image +- name: Calico | Run kube proxy + run_once: true + import_role: + name: kube_proxy_and_dns + +- include_tasks: certs.yml + +- name: Calico | Clean Calico etcd data + when: calico_cleanup_path is defined and calico_cleanup_path != "" + file: + state: absent + path: "{{ calico_cleanup_path }}" + +- name: Calico | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-node + oc_adm_policy_user: + user: system:serviceaccount:kube-system:calico-node + resource_kind: scc + resource_name: privileged + state: present + +- name: Calico | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-kube-controllers + oc_adm_policy_user: + user: system:serviceaccount:kube-system:calico-kube-controllers + resource_kind: scc + resource_name: privileged + state: present + +- name: Calico | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-upgrade-job + oc_adm_policy_user: + user: system:serviceaccount:kube-system:calico-upgrade-job + resource_kind: scc + resource_name: privileged + state: present + +- name: Calico | Set default selector for kube-system + command: > + {{ openshift_client_binary }} + --config={{ openshift.common.config_base }}/master/admin.kubeconfig + annotate ns kube-system openshift.io/node-selector="" --overwrite + +- name: Calico | Create temp directory + command: mktemp -d /tmp/openshift-ansible-XXXXXXX + register: mktemp + changed_when: False + +- name: Calico | Write separate Calico etcd manifest + when: use_calico_etcd + template: + dest: "{{ mktemp.stdout }}/calico-etcd.yml" + src: calico-etcd.yml.j2 + +- name: Calico | Launch separate Calico etcd + when: use_calico_etcd + command: > + {{ openshift_client_binary }} apply + -f {{ mktemp.stdout }}/calico-etcd.yml + --config={{ openshift.common.config_base }}/master/admin.kubeconfig + register: calico_etcd_create_output + failed_when: "calico_etcd_create_output.rc != 0" + changed_when: "('created' in calico_etcd_create_output.stdout) or ('configured' in calico_etcd_create_output.stdout)" + +- name: Calico | Parse node version + set_fact: + node_version: "{{ calico_node_image | regex_replace('^.*node:v?(.*)$', '\\1') }}" + cnx: "{{ calico_node_image | regex_replace('^.*/(.*)-node:.*$', '\\1') }}" + use_calico_credentials: "{{ calico_image_credentials is defined | bool }}" + +- name: Calico | Encode Docker Credentials + shell: > + cat {{ calico_image_credentials }} | openssl base64 -A + register: calico_encoded_credentials_output + failed_when: "calico_encoded_credentials_output.rc != 0 or calico_encoded_credentials_output.stdout == ''" + when: use_calico_credentials + +- name: Calico | Set Encoded Docker Credentials Fact set_fact: - calico_node_image: "quay.io/calico/node:v2.6.7" - when: calico_node_image is not defined + calico_encoded_credentials: "{{ calico_encoded_credentials_output.stdout }}" + when: use_calico_credentials -- name: Prepull Images - command: "{{ openshift_container_cli }} pull {{ calico_node_image }}" +- name: Calico | Write Calico Pull Secret + template: + dest: "{{ mktemp.stdout }}/calico-pull-secret.yml" + src: calico-pull-secret.yml.j2 + when: use_calico_credentials -- name: Apply node label - delegate_to: "{{ groups.oo_first_master.0 }}" +- name: Calico | Create Calico Pull Secret + when: use_calico_credentials command: > - {{ openshift_client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig label node {{ openshift.node.nodename | lower }} --overwrite projectcalico.org/ds-ready=true - -- name: Wait for node running - uri: - url: http://localhost:9099/readiness - status_code: 204 - delay: 3 - retries: 10 - register: result - until: result.status == 204 + {{ openshift_client_binary }} apply + -f {{ mktemp.stdout }}/calico-pull-secret.yml + --config={{ openshift.common.config_base }}/master/admin.kubeconfig + register: calico_pull_secret_create_output + failed_when: "calico_pull_secret_create_output.rc != 0" + changed_when: "('created' in calico_pull_secret_create_output.stdout) or ('configured' in calico_pull_secret_create_output.stdout)" + +- name: Calico | Set the correct liveness and readiness checks + set_fact: + calico_binary_checks: "{{ (node_version > '3.2.0' and cnx != 'cnx') or (node_version > '2.2.0' and cnx == 'cnx') | bool }}" + +- name: Calico | Write Calico v2 + template: + dest: "{{ mktemp.stdout }}/calico.yml" + src: calico.yml.j2 + when: + - node_version | regex_search('^[0-9]\.[0-9]\.[0-9]') and node_version < '3.0.0' + - cnx != "cnx" + +- name: Calico | Write Calico v3 + template: + dest: "{{ mktemp.stdout }}/calico.yml" + src: calicov3.yml.j2 + when: (node_version | regex_search('^[0-9]\.[0-9]\.[0-9]') and node_version >= '3.0.0') or (node_version == 'master') or (cnx == "cnx" and node_version >= '2.0.0') + +- name: Calico | Launch Calico + run_once: true + command: > + {{ openshift_client_binary }} apply + -f {{ mktemp.stdout }}/calico.yml + --config={{ openshift.common.config_base }}/master/admin.kubeconfig + register: calico_create_output + failed_when: "calico_create_output.rc != 0" + changed_when: "('created' in calico_create_output.stdout) or ('configured' in calico_create_output.stdout)" + +- name: Calico | Delete temp directory + file: + name: "{{ mktemp.stdout }}" + state: absent + changed_when: False diff --git a/roles/calico_master/templates/calico-etcd.yml.j2 b/roles/calico/templates/calico-etcd.yml.j2 similarity index 100% rename from roles/calico_master/templates/calico-etcd.yml.j2 rename to roles/calico/templates/calico-etcd.yml.j2 diff --git a/roles/calico_master/templates/calico-pull-secret.yml.j2 b/roles/calico/templates/calico-pull-secret.yml.j2 similarity index 100% rename from roles/calico_master/templates/calico-pull-secret.yml.j2 rename to roles/calico/templates/calico-pull-secret.yml.j2 diff --git a/roles/calico_master/templates/calico.yml.j2 b/roles/calico/templates/calico.yml.j2 similarity index 99% rename from roles/calico_master/templates/calico.yml.j2 rename to roles/calico/templates/calico.yml.j2 index f8934845dba..883c8dc6d7c 100644 --- a/roles/calico_master/templates/calico.yml.j2 +++ b/roles/calico/templates/calico.yml.j2 @@ -126,8 +126,6 @@ spec: annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: - nodeSelector: - projectcalico.org/ds-ready: "true" hostNetwork: true tolerations: # Make sure calico/node gets scheduled on all nodes. diff --git a/roles/calico_master/templates/calicov3.yml.j2 b/roles/calico/templates/calicov3.yml.j2 similarity index 99% rename from roles/calico_master/templates/calicov3.yml.j2 rename to roles/calico/templates/calicov3.yml.j2 index 021e5b48abc..53a36d4c01e 100644 --- a/roles/calico_master/templates/calicov3.yml.j2 +++ b/roles/calico/templates/calicov3.yml.j2 @@ -185,8 +185,6 @@ spec: imagePullSecrets: - name: calico-pull-secret {% endif %} - nodeSelector: - projectcalico.org/ds-ready: "true" hostNetwork: true tolerations: # Make sure calico/node gets scheduled on all nodes. @@ -294,7 +292,7 @@ spec: # chosen from this range. Changing this value after installation will have # no effect. This should fall within '--cluster-cidr'. - name: CALICO_IPV4POOL_CIDR - value: "{{ openshift.master.sdn_cluster_network_cidr }}" + value: "{{ openshift_cluster_network_cidr }}" - name: CALICO_IPV4POOL_IPIP value: "{{ calico_ipv4pool_ipip }}" # Disable IPv6 on Kubernetes. diff --git a/roles/calico_master/README.md b/roles/calico_master/README.md deleted file mode 100644 index 310eabeec3f..00000000000 --- a/roles/calico_master/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Calico (Master) - -Configure Calico components for the Master host. - -## Requirements - -* Ansible 2.2 - -## Installation - -To install, set the following inventory configuration parameters: - -* `openshift_use_calico=True` -* `openshift_use_openshift_sdn=False` -* `os_sdn_network_plugin_name='cni'` - -By default, Calico will share the etcd used by OpenShift. -To configure Calico to use a separate instance of etcd, place etcd SSL client certs on your master, -then set the following variables in your inventory.ini: - -* `calico_etcd_ca_cert_file=/path/to/etcd-ca.crt` -* `calico_etcd_cert_file=/path/to/etcd-client.crt` -* `calico_etcd_key_file=/path/to/etcd-client.key` -* `calico_etcd_endpoints=https://etcd:2379` - -## Upgrading - -OpenShift-Ansible installs Calico as a self-hosted install. Previously, Calico ran as a systemd service. Running Calico -in this manner is now deprecated, and must be upgraded to a hosted cluster. Please run the Legacy Upgrade playbook to -upgrade your existing Calico deployment to a hosted deployment: - - ansible-playbook -i inventory.ini playbooks/byo/calico/legacy_upgrade.yml - -## Additional Calico/Node and Felix Configuration Options - -Additional parameters that can be defined in the inventory are: - - -| Environment | Description | Schema | Default | -|---------|----------------------|---------|---------| -| CALICO_IPV4POOL_IPIP | IPIP Mode to use for the IPv4 POOL created at start up. | off, always, cross-subnet | always | -| CALICO_LOG_DIR | Directory on the host machine where Calico Logs are written.| String | /var/log/calico | - -### Contact Information - -Author: Dan Osborne - -For support, join the `#openshift` channel on the [calico users slack](calicousers.slack.com). diff --git a/roles/calico_master/tasks/main.yml b/roles/calico_master/tasks/main.yml deleted file mode 100644 index 8744be911bc..00000000000 --- a/roles/calico_master/tasks/main.yml +++ /dev/null @@ -1,129 +0,0 @@ ---- -- name: Calico | Run kube proxy - run_once: true - import_role: - name: kube_proxy_and_dns - -- include_tasks: certs.yml - -- name: Calico Master | Clean Calico etcd data - when: calico_cleanup_path is defined and calico_cleanup_path != "" - file: - state: absent - path: "{{ calico_cleanup_path }}" - -- name: Calico Master | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-node - oc_adm_policy_user: - user: system:serviceaccount:kube-system:calico-node - resource_kind: scc - resource_name: privileged - state: present - -- name: Calico Master | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-kube-controllers - oc_adm_policy_user: - user: system:serviceaccount:kube-system:calico-kube-controllers - resource_kind: scc - resource_name: privileged - state: present - -- name: Calico Master | oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:calico-upgrade-job - oc_adm_policy_user: - user: system:serviceaccount:kube-system:calico-upgrade-job - resource_kind: scc - resource_name: privileged - state: present - -- name: Set default selector for kube-system - command: > - {{ openshift_client_binary }} - --config={{ openshift.common.config_base }}/master/admin.kubeconfig - annotate ns kube-system openshift.io/node-selector="" --overwrite - -- name: Calico Master | Create temp directory - command: mktemp -d /tmp/openshift-ansible-XXXXXXX - register: mktemp - changed_when: False - -- name: Calico Master | Write separate Calico etcd manifest - when: use_calico_etcd - template: - dest: "{{ mktemp.stdout }}/calico-etcd.yml" - src: calico-etcd.yml.j2 - -- name: Calico Master | Launch separate Calico etcd - when: use_calico_etcd - command: > - {{ openshift_client_binary }} apply - -f {{ mktemp.stdout }}/calico-etcd.yml - --config={{ openshift.common.config_base }}/master/admin.kubeconfig - register: calico_etcd_create_output - failed_when: "calico_etcd_create_output.rc != 0" - changed_when: "('created' in calico_etcd_create_output.stdout) or ('configured' in calico_etcd_create_output.stdout)" - -- name: Calico Master | Parse node version - set_fact: - node_version: "{{ calico_node_image | regex_replace('^.*node:v?(.*)$', '\\1') }}" - cnx: "{{ calico_node_image | regex_replace('^.*/(.*)-node:.*$', '\\1') }}" - use_calico_credentials: "{{ calico_image_credentials is defined | bool }}" - -- name: Calico | Encode Docker Credentials - shell: > - cat {{ calico_image_credentials }} | openssl base64 -A - register: calico_encoded_credentials_output - failed_when: "calico_encoded_credentials_output.rc != 0 or calico_encoded_credentials_output.stdout == ''" - when: use_calico_credentials - -- name: Calico | Set Encoded Docker Credentials Fact - set_fact: - calico_encoded_credentials: "{{ calico_encoded_credentials_output.stdout }}" - when: use_calico_credentials - -- name: Calico | Write Calico Pull Secret - template: - dest: "{{ mktemp.stdout }}/calico-pull-secret.yml" - src: calico-pull-secret.yml.j2 - when: use_calico_credentials - -- name: Calico | Create Calico Pull Secret - when: use_calico_credentials - command: > - {{ openshift_client_binary }} apply - -f {{ mktemp.stdout }}/calico-pull-secret.yml - --config={{ openshift.common.config_base }}/master/admin.kubeconfig - register: calico_pull_secret_create_output - failed_when: "calico_pull_secret_create_output.rc != 0" - changed_when: "('created' in calico_pull_secret_create_output.stdout) or ('configured' in calico_pull_secret_create_output.stdout)" - -- name: Calico Master | Set the correct liveness and readiness checks - set_fact: - calico_binary_checks: "{{ (node_version > '3.2.0' and cnx != 'cnx') or (node_version > '2.2.0' and cnx == 'cnx') | bool }}" - -- name: Calico Master | Write Calico v2 - template: - dest: "{{ mktemp.stdout }}/calico.yml" - src: calico.yml.j2 - when: - - node_version | regex_search('^[0-9]\.[0-9]\.[0-9]') and node_version < '3.0.0' - - cnx != "cnx" - -- name: Calico Master | Write Calico v3 - template: - dest: "{{ mktemp.stdout }}/calico.yml" - src: calicov3.yml.j2 - when: (node_version | regex_search('^[0-9]\.[0-9]\.[0-9]') and node_version >= '3.0.0') or (node_version == 'master') or (cnx == "cnx" and node_version >= '2.0.0') - -- name: Calico Master | Launch Calico - run_once: true - command: > - {{ openshift_client_binary }} apply - -f {{ mktemp.stdout }}/calico.yml - --config={{ openshift.common.config_base }}/master/admin.kubeconfig - register: calico_create_output - failed_when: "calico_create_output.rc != 0" - changed_when: "('created' in calico_create_output.stdout) or ('configured' in calico_create_output.stdout)" - -- name: Calico Master | Delete temp directory - file: - name: "{{ mktemp.stdout }}" - state: absent - changed_when: False diff --git a/roles/calico_node/README.md b/roles/calico_node/README.md new file mode 100644 index 00000000000..c5c6d07347c --- /dev/null +++ b/roles/calico_node/README.md @@ -0,0 +1,3 @@ +# Calico Node + +Please see [calico](../calico/README.md) diff --git a/roles/calico/files/calico.conf b/roles/calico_node/files/calico.conf similarity index 100% rename from roles/calico/files/calico.conf rename to roles/calico_node/files/calico.conf diff --git a/roles/calico_master/meta/main.yml b/roles/calico_node/meta/main.yml similarity index 94% rename from roles/calico_master/meta/main.yml rename to roles/calico_node/meta/main.yml index 2b36764c4a5..102b82bde21 100644 --- a/roles/calico_master/meta/main.yml +++ b/roles/calico_node/meta/main.yml @@ -13,5 +13,4 @@ galaxy_info: - cloud - system dependencies: -- role: lib_utils - role: openshift_facts diff --git a/roles/calico_node/tasks/main.yml b/roles/calico_node/tasks/main.yml new file mode 100644 index 00000000000..0f6430b8dd9 --- /dev/null +++ b/roles/calico_node/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Calico Node | Configure NetworkManager to ignore Calico interfaces + copy: + src: files/calico.conf + dest: /etc/NetworkManager/conf.d/ + when: using_network_manager | default(true) | bool + register: nm + +- name: Calico Node | Restart NetworkManager + systemd: + name: NetworkManager + state: restarted + when: nm.changed