diff --git a/playbooks/openshift-etcd/private/remove-etcdv2-data.yml b/playbooks/openshift-etcd/private/remove-etcdv2-data.yml index 2dde36196ef..8e8147218ae 100644 --- a/playbooks/openshift-etcd/private/remove-etcdv2-data.yml +++ b/playbooks/openshift-etcd/private/remove-etcdv2-data.yml @@ -6,5 +6,3 @@ - import_role: name: etcd tasks_from: remove-etcd-v2-data.yml - vars: - etcd_peer: "{{ openshift.common.hostname }}" diff --git a/playbooks/openshift-etcd/private/scaleup.yml b/playbooks/openshift-etcd/private/scaleup.yml index f4a58f31937..41b795348ce 100644 --- a/playbooks/openshift-etcd/private/scaleup.yml +++ b/playbooks/openshift-etcd/private/scaleup.yml @@ -19,24 +19,31 @@ hosts: oo_new_etcd_to_config serial: 1 any_errors_fatal: true - pre_tasks: + tasks: - import_role: name: etcd tasks_from: add_new_member.yml + vars: + etcd_peer: "{{ hostvars[etcd_ca_host].etcd_ip }}" + - import_role: name: etcd tasks_from: server_certificates.yml vars: etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}" etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}" - tasks: + - import_role: name: os_firewall when: etcd_add_check.rc == 0 + # Setup etcd as a static pod if collocated with a master - import_role: name: etcd - when: etcd_add_check.rc == 0 + tasks_from: static.yml + when: + - etcd_add_check.rc == 0 + - inventory_hostname in groups['oo_masters'] vars: etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}" etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" @@ -44,24 +51,27 @@ etcd_initial_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}" etcd_ca_setup: False - # etcd_hostname fact is set in add_new_member.yml called above. - - name: Verify cluster is stable - command: > - {{ r_etcd_common_etcdctl_command }} - --cert-file {{ etcd_peer_cert_file }} - --key-file {{ etcd_peer_key_file }} - --ca-file {{ etcd_peer_ca_file }} - -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_hostname }}:{{ etcd_client_port }} - cluster-health - register: scaleup_health - retries: 3 - delay: 30 - until: scaleup_health.rc == 0 + - import_role: + name: etcd + tasks_from: rpm.yml + when: + - etcd_add_check.rc == 0 + - not inventory_hostname in groups['oo_masters'] + vars: + etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_initial_cluster_state: "existing" + etcd_initial_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}" + etcd_ca_setup: False + + - import_role: + name: etcd + tasks_from: verify_cluster_health.yml - name: Update master etcd client urls hosts: oo_masters_to_config serial: 1 - pre_tasks: + tasks: - set_fact: openshift_master_etcd_hosts: "{{ hostvars | lib_utils_oo_select_keys(groups['oo_etcd_to_config'] | union(groups['oo_new_etcd_to_config'] | default([]) )) @@ -78,7 +88,6 @@ vars: l_use_ssl: "{{ openshift_master_etcd_use_ssl | default(True) | bool}}" - post_tasks: - import_role: name: openshift_control_plane tasks_from: update_etcd_client_urls.yml diff --git a/playbooks/openshift-etcd/private/server_certificates.yml b/playbooks/openshift-etcd/private/server_certificates.yml index 24840d1740e..5e9d6bf1231 100644 --- a/playbooks/openshift-etcd/private/server_certificates.yml +++ b/playbooks/openshift-etcd/private/server_certificates.yml @@ -2,7 +2,7 @@ - name: Create etcd server certificates for etcd hosts hosts: oo_etcd_to_config any_errors_fatal: true - post_tasks: + tasks: - import_role: name: etcd tasks_from: server_certificates.yml diff --git a/playbooks/openshift-etcd/private/upgrade_rpm_members.yml b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml index 95cfc141e5e..fd29cd896f6 100644 --- a/playbooks/openshift-etcd/private/upgrade_rpm_members.yml +++ b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml @@ -10,7 +10,6 @@ tasks_from: upgrade_rpm.yml vars: r_etcd_upgrade_version: "{{ etcd_upgrade_version }}" - etcd_peer: "{{ openshift.common.hostname }}" when: - etcd_rpm_version.stdout | default('99') is version(etcd_upgrade_version, '<') - ansible_distribution == 'RedHat' diff --git a/playbooks/openshift-etcd/private/upgrade_static.yml b/playbooks/openshift-etcd/private/upgrade_static.yml index 88bfc46ac59..aeaec3cc998 100644 --- a/playbooks/openshift-etcd/private/upgrade_static.yml +++ b/playbooks/openshift-etcd/private/upgrade_static.yml @@ -6,6 +6,4 @@ - import_role: name: etcd tasks_from: upgrade_static.yml - vars: - etcd_peer: "{{ openshift.common.hostname }}" when: inventory_hostname in groups['oo_masters'] diff --git a/roles/etcd/defaults/main.yaml b/roles/etcd/defaults/main.yaml index 5f3178a498c..cd5b49c8160 100644 --- a/roles/etcd/defaults/main.yaml +++ b/roles/etcd/defaults/main.yaml @@ -83,9 +83,8 @@ etcd_listen_peer_urls: "{{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_ etcd_advertise_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}" etcd_listen_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}" -# required role variable -#etcd_peer: 127.0.0.1 -etcdctlv2: "{{ r_etcd_common_etcdctl_command }} --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://{{ etcd_peer }}:{{ etcd_client_port }}" +etcd_peer: "{{ openshift.common.hostname }}" +etcdctlv2: "{{ r_etcd_common_etcdctl_command }} --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoints {{ etcd_peer_url_scheme }}://{{ etcd_peer }}:{{ etcd_client_port }}" etcd_service: etcd # Location of the service file is fixed and not meant to be changed diff --git a/roles/etcd/etcdctl.sh b/roles/etcd/etcdctl.sh deleted file mode 100644 index 0e324a8a953..00000000000 --- a/roles/etcd/etcdctl.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Sets up handy aliases for etcd, need etcdctl2 and etcdctl3 because -# command flags are different between the two. Should work on stand -# alone etcd hosts and master + etcd hosts too because we use the peer keys. -etcdctl2() { - /usr/bin/etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://`hostname`:2379 ${@} -} - -etcdctl3() { - ETCDCTL_API=3 /usr/bin/etcdctl --cert /etc/etcd/peer.crt --key /etc/etcd/peer.key --cacert /etc/etcd/ca.crt --endpoints https://`hostname`:2379 ${@} -} diff --git a/roles/etcd/tasks/add_new_member.yml b/roles/etcd/tasks/add_new_member.yml index 930cf9e8b43..0d206a610b0 100644 --- a/roles/etcd/tasks/add_new_member.yml +++ b/roles/etcd/tasks/add_new_member.yml @@ -3,13 +3,7 @@ - import_tasks: set_facts.yml - name: Add new etcd members to cluster - command: > - {{ r_etcd_common_etcdctl_command }} - --cert-file {{ etcd_peer_cert_file }} - --key-file {{ etcd_peer_key_file }} - --ca-file {{ etcd_peer_ca_file }} - -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_ip }}:{{ etcd_client_port }} - member add {{ etcd_hostname }} {{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }} + command: "{{ etcdctlv2 }} member add {{ etcd_hostname }} {{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }}" delegate_to: "{{ etcd_ca_host }}" failed_when: - etcd_add_check.rc == 1 diff --git a/roles/etcd/tasks/remove-etcd-v2-data.yml b/roles/etcd/tasks/remove-etcd-v2-data.yml index 59cea54e78a..a1d1431ca88 100644 --- a/roles/etcd/tasks/remove-etcd-v2-data.yml +++ b/roles/etcd/tasks/remove-etcd-v2-data.yml @@ -1,10 +1,5 @@ --- -- name: Verify cluster is healthy pre-upgrade - command: "{{ etcdctlv2 }} cluster-health" - register: cluster_health - retries: 30 - delay: 6 - until: cluster_health.rc == 0 +- import_tasks: verify_cluster_health.yml - name: Check migrated status command: "{{ etcdctlv2 }} get /kubernetes.io" diff --git a/roles/etcd/tasks/static.yml b/roles/etcd/tasks/static.yml index cebe924174b..3dae634a953 100644 --- a/roles/etcd/tasks/static.yml +++ b/roles/etcd/tasks/static.yml @@ -84,7 +84,7 @@ - "{{ etcd_peer_key_file }}" - "--ca-file" - "{{ etcd_peer_ca_file }}" - - "-C" + - "--endpoints" - "{{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}" - "cluster-health" with_items: diff --git a/roles/etcd/tasks/upgrade_rpm.yml b/roles/etcd/tasks/upgrade_rpm.yml index c8c3fa4af71..b5c2ab191a5 100644 --- a/roles/etcd/tasks/upgrade_rpm.yml +++ b/roles/etcd/tasks/upgrade_rpm.yml @@ -8,12 +8,7 @@ # RHEL 7.3.3 with etcd-3.1.0-2.el7 # RHEL 7.3.2 with etcd-3.0.15-1.el7 -- name: Verify cluster is healthy pre-upgrade - command: "{{ etcdctlv2 }} cluster-health" - register: cluster_health - retries: 30 - delay: 6 - until: cluster_health.rc == 0 +- import_tasks: verify_cluster_health.yml - set_fact: l_etcd_target_package: "{{ 'etcd' if r_etcd_upgrade_version is not defined else 'etcd-'+r_etcd_upgrade_version+'*' }}" @@ -30,9 +25,4 @@ - name: restart etcd command: "{{ l_etcd_restart_command }}" -- name: Verify cluster is healthy - command: "{{ etcdctlv2 }} cluster-health" - register: etcdctl - until: etcdctl.rc == 0 - retries: 3 - delay: 10 +- import_tasks: verify_cluster_health.yml diff --git a/roles/etcd/tasks/upgrade_static.yml b/roles/etcd/tasks/upgrade_static.yml index 158de2b08b4..0b144251aaf 100644 --- a/roles/etcd/tasks/upgrade_static.yml +++ b/roles/etcd/tasks/upgrade_static.yml @@ -3,12 +3,7 @@ # INPUT r_etcd_upgrade_version -- name: Verify cluster is healthy pre-upgrade - command: "{{ etcdctlv2 }} cluster-health" - register: cluster_health - retries: 30 - delay: 6 - until: cluster_health.rc == 0 +- import_tasks: verify_cluster_health.yml - name: Check for old etcd service files stat: @@ -54,9 +49,4 @@ - set_fact: r_etcd_common_etcd_runtime: static_pod -- name: Verify cluster is healthy - command: "{{ etcdctlv2 }} cluster-health" - register: etcdctl - until: etcdctl.rc == 0 and 'stopped' not in etcdctl.stderr - retries: 30 - delay: 10 +- import_tasks: verify_cluster_health.yml diff --git a/roles/etcd/tasks/verify_cluster_health.yml b/roles/etcd/tasks/verify_cluster_health.yml new file mode 100644 index 00000000000..5c2ec8d46c1 --- /dev/null +++ b/roles/etcd/tasks/verify_cluster_health.yml @@ -0,0 +1,9 @@ +--- +- name: Verify cluster is healthy + command: "{{ etcdctlv2 }} cluster-health" + register: cluster_health + retries: 30 + delay: 6 + until: + - cluster_health.rc == 0 + - ('stopped' not in cluster_health.stderr) diff --git a/roles/etcd/templates/etcdctl.sh.j2 b/roles/etcd/templates/etcdctl.sh.j2 index b2860d3b39f..3f2ec113fad 100644 --- a/roles/etcd/templates/etcdctl.sh.j2 +++ b/roles/etcd/templates/etcdctl.sh.j2 @@ -4,7 +4,7 @@ # alone etcd hosts and master + etcd hosts too because we use the peer keys. etcdctl2() { - cmd="ETCDCTL_API=2 etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://`hostname`:2379 ${@}" + cmd="ETCDCTL_API=2 etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoints https://`hostname`:2379 ${@}" if [[ -f /usr/local/bin/master-exec ]]; then /usr/local/bin/master-exec etcd etcd /bin/sh -c "$cmd" else diff --git a/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py b/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py index c5ea29c1657..3a9e52b7e65 100644 --- a/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py +++ b/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py @@ -19,6 +19,10 @@ def __init__(self): def v2_playbook_on_stats(self, stats): + # Return if there are no custom stats to process + if stats.custom == {}: + return + phases = stats.custom['_run'] # Find the longest phase title