From 61af3e6341a6898ed759417cf875bb1fb6a9a3c6 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 27 May 2020 18:51:35 +0300 Subject: [PATCH 01/27] add support pgbackrest For bootstrap a patroni cluster and create replicas from backups. --- add_pgnode.yml | 27 ++++- deploy_pgcluster.yml | 23 ++++ inventory | 12 +- roles/patroni/tasks/main.yml | 6 +- roles/patroni/templates/patroni.yml.j2 | 5 + roles/pgbackrest/tasks/main.yml | 106 ++++++++++++++++++ roles/pgbackrest/tasks/ssh_keys.yml | 78 +++++++++++++ roles/pgbackrest/templates/pgbackrest.conf.j2 | 10 ++ .../templates/pgbackrest_bootstrap.sh.j2 | 17 +++ tags.md | 6 + vars/main.yml | 42 ++++++- 11 files changed, 322 insertions(+), 10 deletions(-) create mode 100644 roles/pgbackrest/tasks/main.yml create mode 100644 roles/pgbackrest/tasks/ssh_keys.yml create mode 100644 roles/pgbackrest/templates/pgbackrest.conf.j2 create mode 100644 roles/pgbackrest/templates/pgbackrest_bootstrap.sh.j2 diff --git a/add_pgnode.yml b/add_pgnode.yml index b31c75f09..f6ae2d4a8 100644 --- a/add_pgnode.yml +++ b/add_pgnode.yml @@ -11,8 +11,6 @@ - vars/main.yml - vars/system.yml - "vars/{{ ansible_os_family }}.yml" - vars: - existing_pgcluster: true pre_tasks: - name: Checking Linux distribution @@ -76,6 +74,31 @@ - role: ntp - role: ssh-keys +- hosts: pgbackrest:postgres_cluster + become: true + become_method: sudo + gather_facts: true + any_errors_fatal: true + vars_files: + - vars/main.yml + - "vars/{{ ansible_os_family }}.yml" + roles: + - role: pgbackrest + when: pgbackrest_install|bool + +- hosts: replica + become: true + become_method: sudo + gather_facts: true + any_errors_fatal: true + vars_files: + - vars/main.yml + - vars/system.yml + - "vars/{{ ansible_os_family }}.yml" + vars: + existing_pgcluster: true + + roles: - role: wal-g when: wal_g_install|bool diff --git a/deploy_pgcluster.yml b/deploy_pgcluster.yml index 143e9a164..d6ead578e 100644 --- a/deploy_pgcluster.yml +++ b/deploy_pgcluster.yml @@ -89,6 +89,29 @@ - role: ntp - role: ssh-keys +- hosts: pgbackrest:postgres_cluster + become: true + become_method: sudo + gather_facts: true + any_errors_fatal: true + vars_files: + - vars/main.yml + - "vars/{{ ansible_os_family }}.yml" + roles: + - role: pgbackrest + when: pgbackrest_install|bool + +- hosts: postgres_cluster + become: true + become_method: sudo + gather_facts: true + any_errors_fatal: true + vars_files: + - vars/main.yml + - vars/system.yml + - "vars/{{ ansible_os_family }}.yml" + + roles: - role: wal-g when: wal_g_install|bool diff --git a/inventory b/inventory index 85f6db077..6cba8ee29 100644 --- a/inventory +++ b/inventory @@ -36,12 +36,20 @@ replica # You can deploy the etcd cluster and the haproxy balancers on other dedicated servers. +# if pgbackrest_install: true and "repo_host" is set (in vars/main.yml) +[pgbackrest] # optional (Dedicated Repository Host) + + # Connection settings [all:vars] ansible_connection='ssh' ansible_ssh_port='22' ansible_user='root' -ansible_ssh_pass='testpas' # "sshpass" package is required for use "ansible_ssh_pass" -#ansible_ssh_private_key_file= +ansible_ssh_pass='secretpassword' # "sshpass" package is required for use "ansible_ssh_pass" +# ansible_ssh_private_key_file= # ansible_python_interpreter='/usr/bin/python3' # is required for use python3 +[pgbackrest:vars] +ansible_user='postgres' +ansible_ssh_pass='secretpassword' + diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 9ec914856..66f668838 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -544,8 +544,8 @@ command: "{{ postgresql_bin_dir }}/pg_isready -p {{ postgresql_port }}" register: pg_isready_result until: pg_isready_result.rc == 0 - retries: 30 - delay: 10 + retries: 1000 + delay: 30 changed_when: false when: is_master == "true" tags: patroni, patroni_start_master @@ -563,6 +563,8 @@ become: true become_user: postgres command: "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -c 'SELECT pg_reload_conf()'" + register: psql_reload_result + changed_when: psql_reload_result.rc == 0 failed_when: false # exec pg_reload_conf on all running postgres (to re-run with --tag pg_hba). when: existing_pgcluster is not defined or not existing_pgcluster|bool tags: patroni, pg_hba, pg_hba_generate diff --git a/roles/patroni/templates/patroni.yml.j2 b/roles/patroni/templates/patroni.yml.j2 index 4675a8197..eace23ddb 100644 --- a/roles/patroni/templates/patroni.yml.j2 +++ b/roles/patroni/templates/patroni.yml.j2 @@ -33,6 +33,11 @@ bootstrap: recovery_target_action: promote recovery_target_timeline: latest restore_command: wal-g wal-fetch %f %p +{% endif %} +{% if patroni_cluster_bootstrap_method == 'pgbackrest' %} + pgbackrest: + command: /etc/patroni/pgbackrest_bootstrap.sh + keep_existing_recovery_conf: True {% endif %} dcs: ttl: {{ patroni_ttl |d(30, true) |int }} diff --git a/roles/pgbackrest/tasks/main.yml b/roles/pgbackrest/tasks/main.yml new file mode 100644 index 000000000..340147017 --- /dev/null +++ b/roles/pgbackrest/tasks/main.yml @@ -0,0 +1,106 @@ +--- +# yamllint disable rule:line-length + +- block: # Debian pgdg repo + - name: Make sure pgdg apt key is installed + apt_key: + id: ACCC4CF8 + url: http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc + + - name: Make sure pgdg repository is installed + apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt/ {{ ansible_distribution_release }}-pgdg main" + + - name: Update apt cache + apt: + update_cache: true + environment: "{{ proxy_env | default({}) }}" + when: + - installation_method == "repo" + - ansible_os_family == "Debian" + - pgbackrest_install_from_pgdg_repo|bool + tags: pgbackrest, pgbackrest_repo, pgbackrest_install + +- block: # RedHat pgdg repo + - name: Get pgdg-redhat-repo-latest.noarch.rpm + get_url: + url: "https://download.postgresql.org/pub/repos/yum/reporpms/EL-{{ ansible_distribution_major_version }}-x86_64/pgdg-redhat-repo-latest.noarch.rpm" # noqa 204 + dest: /tmp/ + timeout: 30 + validate_certs: false + + - name: Make sure pgdg repository is installed + package: + name: /tmp/pgdg-redhat-repo-latest.noarch.rpm + state: present + + - name: Clean yum cache + command: yum clean all + args: + warn: false + when: ansible_distribution_major_version == '7' + + - name: Clean dnf cache + command: dnf clean all + args: + warn: false + when: ansible_distribution_major_version is version('8', '>=') + environment: "{{ proxy_env | default({}) }}" + when: + - installation_method == "repo" + - ansible_os_family == "RedHat" + - pgbackrest_install_from_pgdg_repo|bool + tags: pgbackrest, pgbackrest_repo, pgbackrest_install + +- name: Install pgbackrest + package: + name: pgbackrest + state: latest + tags: pgbackrest, pgbackrest_install + +- block: + - name: Ensure config directory exist + file: + path: "{{ pgbackrest_conf_file | dirname }}" + state: directory + + - name: "Generate conf file {{ pgbackrest_conf_file }}" + template: + src: pgbackrest.conf.j2 + dest: "{{ pgbackrest_conf_file }}" + owner: root + group: root + mode: 0644 + when: "'postgres_cluster' in group_names" + tags: pgbackrest, pgbackrest_conf + +# if pgbackrest_repo_type: "posix" and pgbackrest_repo_host is set +- import_tasks: ssh_keys.yml + when: + - pgbackrest_repo_type|lower != "s3" + - pgbackrest_repo_host is defined + - pgbackrest_repo_host | length > 0 + tags: pgbackrest, pgbackrest_ssh_keys + +- block: # if patroni_cluster_bootstrap_method: "pgbackrest" + - name: Make sure the pgbackrest bootstrap script directory exist + file: + dest: /etc/patroni + state: directory + owner: postgres + group: postgres + + - name: Create /etc/patroni/pgbackrest_bootstrap.sh script + template: + src: templates/pgbackrest_bootstrap.sh.j2 + dest: /etc/patroni/pgbackrest_bootstrap.sh + owner: postgres + group: postgres + mode: 0775 + when: + - patroni_cluster_bootstrap_method is defined + - patroni_cluster_bootstrap_method == "pgbackrest" + - "'postgres_cluster' in group_names" + tags: pgbackrest, pgbackrest_bootstrap_script + +... diff --git a/roles/pgbackrest/tasks/ssh_keys.yml b/roles/pgbackrest/tasks/ssh_keys.yml new file mode 100644 index 000000000..e3cb209d8 --- /dev/null +++ b/roles/pgbackrest/tasks/ssh_keys.yml @@ -0,0 +1,78 @@ +--- +# yamllint disable rule:line-length + +- name: ssh_keys | Ensure ssh key are created for "{{ pgbackrest_repo_user }}" user on pgbackrest server + user: + name: "{{ pgbackrest_repo_user }}" + generate_ssh_key: true + ssh_key_bits: 2048 + ssh_key_file: .ssh/id_rsa + when: "'pgbackrest' in group_names" + +- name: ssh_keys | Ensure ssh key are created for "postgres" user on database servers + user: + name: "postgres" + generate_ssh_key: true + ssh_key_bits: 2048 + ssh_key_file: .ssh/id_rsa + when: "'postgres_cluster' in group_names" + +- name: ssh_keys | Get public ssh key from pgbackrest server + slurp: + src: "~{{ pgbackrest_repo_user }}/.ssh/id_rsa.pub" + register: pgbackrest_sshkey + changed_when: false + when: "'pgbackrest' in group_names" + +- name: ssh_keys | Get public ssh key from database servers + slurp: + src: "~postgres/.ssh/id_rsa.pub" + register: postgres_cluster_sshkey + changed_when: false + when: "'postgres_cluster' in group_names" + +- name: ssh_keys | Add pgbackrest ssh key in "~postgres/.ssh/authorized_keys" on database servers + authorized_key: + user: postgres + state: present + key: "{{ hostvars[item].pgbackrest_sshkey['content'] | b64decode }}" + loop: "{{ groups['pgbackrest'] }}" + when: "'postgres_cluster' in group_names" + +- name: ssh_keys | Add database ssh keys in "~{{ pgbackrest_repo_user }}/.ssh/authorized_keys" on pgbackrest server + authorized_key: + user: "{{ pgbackrest_repo_user }}" + state: present + key: "{{ hostvars[item].postgres_cluster_sshkey['content'] | b64decode }}" + loop: "{{ groups['postgres_cluster'] }}" + when: "'pgbackrest' in group_names" + +- name: known_hosts | Get public ssh keys of hosts (ssh-keyscan) + command: "ssh-keyscan -trsa -p {{ ansible_ssh_port | default(22) }} {{ item }}" + loop: "{{ groups['all'] }}" + register: ssh_known_host_keyscan + changed_when: false + +- name: known_hosts | add ssh public keys in "~postgres/.ssh/known_hosts" on database servers + become: true + become_user: postgres + known_hosts: + host: "{{ item.item }}" + key: "{{ item.stdout }}" + path: "~postgres/.ssh/known_hosts" + no_log: true + loop: "{{ ssh_known_host_keyscan.results }}" + when: "'postgres_cluster' in group_names" + +- name: known_hosts | add ssh public keys in "~{{ pgbackrest_repo_user }}/.ssh/known_hosts" on pgbackrest server + become: true + become_user: "{{ pgbackrest_repo_user }}" + known_hosts: + host: "{{ item.item }}" + key: "{{ item.stdout }}" + path: "~{{ pgbackrest_repo_user }}/.ssh/known_hosts" + no_log: true + loop: "{{ ssh_known_host_keyscan.results }}" + when: "'pgbackrest' in group_names" + +... diff --git a/roles/pgbackrest/templates/pgbackrest.conf.j2 b/roles/pgbackrest/templates/pgbackrest.conf.j2 new file mode 100644 index 000000000..bba6c6180 --- /dev/null +++ b/roles/pgbackrest/templates/pgbackrest.conf.j2 @@ -0,0 +1,10 @@ +[global] +{% for global in pgbackrest_conf.global %} +{{ global.option }}={{ global.value }} +{% endfor %} + +[{{ pgbackrest_stanza }}] +{% for stanza in pgbackrest_conf.stanza %} +{{ stanza.option }}={{ stanza.value }} +{% endfor %} + diff --git a/roles/pgbackrest/templates/pgbackrest_bootstrap.sh.j2 b/roles/pgbackrest/templates/pgbackrest_bootstrap.sh.j2 new file mode 100644 index 000000000..4b58a30b7 --- /dev/null +++ b/roles/pgbackrest/templates/pgbackrest_bootstrap.sh.j2 @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +{% raw %} +while getopts ":-:" optchar; do + [[ "${optchar}" == "-" ]] || continue + case "${OPTARG}" in + datadir=* ) + DATA_DIR=${OPTARG#*=} + ;; + scope=* ) + SCOPE=${OPTARG#*=} + ;; + esac +done +{% endraw %} + +{{ pgbackrest_patroni_cluster_bootstrap_command }} + diff --git a/tags.md b/tags.md index 23f6a6b70..7a142d9ae 100644 --- a/tags.md +++ b/tags.md @@ -78,3 +78,9 @@ - wal_g - - wal_g_install - - wal_g_conf +- pgbackrest +- - pgbackrest_repo +- - pgbackrest_install +- - pgbackrest_conf +- - pgbackrest_ssh_keys +- - pgbackrest_bootstrap_script diff --git a/vars/main.yml b/vars/main.yml index c7296864f..9e091dfad 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -127,6 +127,7 @@ postgresql_parameters: - {option: "archive_timeout", value: "1800s"} - {option: "archive_command", value: "cd ."} # not doing anything yet with WAL-s # - {option: "archive_command", value: "wal-g wal-push %p"} # archive WAL-s using WAL-G +# - {option: "archive_command", value: "pgbackrest --stanza={{ pgbackrest_stanza }} archive-push %p"} # archive WAL-s using pgbackrest - {option: "wal_level", value: "replica"} # "replica" for PostgreSQL 9.6 and above (for 9.4, 9.5 use "hot_standby") - {option: "wal_keep_segments", value: "130"} - {option: "max_wal_senders", value: "10"} @@ -223,7 +224,7 @@ patroni_remove_data_directory_on_diverged_timelines: false # or 'true' # if it notices that timelines are diverging and the former master can not start streaming from the new master. # https://patroni.readthedocs.io/en/latest/replica_bootstrap.html#bootstrap -patroni_cluster_bootstrap_method: "initdb" # or "wal-g" +patroni_cluster_bootstrap_method: "initdb" # or "wal-g", "pgbackrest" # https://patroni.readthedocs.io/en/latest/replica_bootstrap.html#building-replicas patroni_create_replica_methods: @@ -233,7 +234,7 @@ patroni_create_replica_methods: - basebackup pgbackrest: - - {option: "command", value: "/usr/bin/pgbackrest --stanza= --delta restore"} + - {option: "command", value: "/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --delta restore"} - {option: "keep_data", value: "True"} - {option: "no_params", value: "True"} wal_e: @@ -249,8 +250,9 @@ basebackup: - {option: "checkpoint", value: "fast"} # "restore_command" written to recovery.conf when configuring follower (create replica) -postgresql_restore_command: [] -# postgresql_restore_command: 'wal-g wal-fetch %f %p' # restore WAL-s using WAL-G +postgresql_restore_command: "" +# postgresql_restore_command: "wal-g wal-fetch %f %p" # restore WAL-s using WAL-G +# postgresql_restore_command: "pgbackrest --stanza={{ pgbackrest_stanza }} archive-get %f %p" # restore WAL-s using pgbackrest # WAL-G @@ -269,4 +271,36 @@ wal_g_json: # more options see https://github.com/wal-g/wal-g#configuration # - {option: "WALG_S3_CA_CERT_FILE", value: "/path/to/custom/ca/file"} # - {option: "", value: ""} +# pgBackRest +pgbackrest_install: false # or 'true' +pgbackrest_install_from_pgdg_repo: true # or 'false' +pgbackrest_stanza: "stanza_name" # specify your --stanza +pgbackrest_repo_type: "posix" # or "s3" +pgbackrest_repo_host: "10.128.64.50" # dedicated repository host (if repo_type: "posix") +pgbackrest_repo_user: "postgres" # if "repo_host" is set +pgbackrest_conf_file: "/etc/pgbackrest.conf" +# see more options https://pgbackrest.org/configuration.html +pgbackrest_conf: + global: # [global] section + - {option: "log-level-file", value: "detail"} + - {option: "log-path", value: "/var/log/pgbackrest"} + - {option: "repo1-type", value: "{{ pgbackrest_repo_type |lower }}"} + - {option: "repo1-host", value: "{{ pgbackrest_repo_host }}"} + - {option: "repo1-host-user", value: "{{ pgbackrest_repo_user }}"} +# - {option: "repo1-path", value: "/repo"} +# - {option: "repo1-s3-endpoint", value: "http://172.26.9.200:9000"} +# - {option: "repo1-s3-bucket", value: "pgbackrest"} +# - {option: "repo1-s3-verify-tls", value: "n"} +# - {option: "repo1-s3-key", value: "accessKey"} +# - {option: "repo1-s3-key-secret", value: "superSECRETkey"} +# - {option: "repo1-s3-region", value: "us-east-1"} +# - {option: "", value: ""} + stanza: # [stanza_name] section + - {option: "pg1-path", value: "{{ postgresql_data_dir }}"} + - {option: "process-max", value: "2"} + - {option: "recovery-option", value: "recovery_target_action=promote"} +# - {option: "", value: ""} +pgbackrest_patroni_cluster_bootstrap_command: # if patroni_cluster_bootstrap_method: "pgbackrest" + "/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --delta restore" + ... From da7f8ccaf921ebc696eb75fb8eaad2bcbc67d51c Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 28 May 2020 15:15:53 +0300 Subject: [PATCH 02/27] Wait for the new cluster to initialize before generating the custom pg_hba.conf Before that, we generated the user pg_hba.conf file immediately after checking whether PostgreSQL is running and whether it accepts connections on the Master server. When using the custom bootstrap method, the patroni service can overwrite our pg_hba.conf file after the actual initialization of the new cluster is completed. Now we wait until the master writes the leader key in the DCS and when the Patroni node is running as the leader. --- roles/patroni/tasks/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 66f668838..29aa879bc 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -547,6 +547,15 @@ retries: 1000 delay: 30 changed_when: false + + - name: Wait for the new cluster to initialize before generating the custom pg_hba.conf + uri: + url: "http://{{ hostvars[inventory_hostname]['inventory_hostname'] }}:8008/leader" + status_code: 200 + register: result + until: result.status == 200 + retries: 10 + delay: 2 when: is_master == "true" tags: patroni, patroni_start_master From bd84600ba1c6a371c6c18c4796b5f031604dfc8e Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 28 May 2020 16:10:39 +0300 Subject: [PATCH 03/27] =?UTF-8?q?=D0=A1hange=20the=20owner=20of=20the=20pg?= =?UTF-8?q?backrest.conf=20file=20to=20postgres?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/pgbackrest/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/pgbackrest/tasks/main.yml b/roles/pgbackrest/tasks/main.yml index 340147017..4838b2eb7 100644 --- a/roles/pgbackrest/tasks/main.yml +++ b/roles/pgbackrest/tasks/main.yml @@ -68,8 +68,8 @@ template: src: pgbackrest.conf.j2 dest: "{{ pgbackrest_conf_file }}" - owner: root - group: root + owner: postgres + group: postgres mode: 0644 when: "'postgres_cluster' in group_names" tags: pgbackrest, pgbackrest_conf From fc53d67b99f8f3e90235d683818c5df3d48a5717 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 28 May 2020 17:06:55 +0300 Subject: [PATCH 04/27] Molecule_CI: update converge.yml Remove epel-release package does not help resolve the "Failed to download metadata for repo 'epel'" error in Github Actions for CentOS 8. --- molecule/default/converge.yml | 7 ------- molecule/postgrespro/converge.yml | 7 ------- 2 files changed, 14 deletions(-) diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index efc0741dd..6d178eb68 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -10,13 +10,6 @@ sysctl_set: false # Added to prevent test failures in CI. cacheable: true - - name: Prepare | Remove epel-release package - package: - name: epel-release - state: absent - when: - - ansible_os_family == "RedHat" - - name: Prepare | Clean yum cache command: yum clean all args: diff --git a/molecule/postgrespro/converge.yml b/molecule/postgrespro/converge.yml index d5247b165..266c272c4 100644 --- a/molecule/postgrespro/converge.yml +++ b/molecule/postgrespro/converge.yml @@ -10,13 +10,6 @@ sysctl_set: false # Added to prevent test failures in CI. cacheable: true - - name: Prepare | Remove epel-release package - package: - name: epel-release - state: absent - when: - - ansible_os_family == "RedHat" - - name: Prepare | Clean yum cache command: yum clean all args: From 874a074dd390a477c106aacca54c6a432b747a5c Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Fri, 29 May 2020 23:28:04 +0300 Subject: [PATCH 05/27] Add patroni_cluster_point_in_time_recovery (PITR) Variable: patroni_cluster_point_in_time_recovery # if true, the database cluster directory will be cleaned (for "wal-g") or overwritten (for "pgbackrest" --delta restore). # And also the patroni cluster "{{ patroni_cluster_name }}" will be removed from the DCS. # For pgbackrest (only) - Reinitialize cluster members (replicas) wil be run after successful recovery on the master server. # Do not use during initial deployment of a cluster. This mode is intended for the PITR of an existing patroni cluster only. requirements: pip3 install jmespath # (on ansible server and master server) pip3 install pexpect # (on master server) Specify in inventory file: ansible_python_interpreter='/usr/bin/python3' # is required for use python3 Run ansible with tag for PITR: ansible-playbook deploy_pgcluster.yml --tags point_in_time_recovery --- inventory | 2 +- roles/deploy-finish/tasks/main.yml | 4 +- roles/patroni/tasks/main.yml | 138 +++++++++++++++++- .../templates/pgbackrest_bootstrap.sh.j2 | 2 +- vars/main.yml | 8 +- 5 files changed, 141 insertions(+), 13 deletions(-) diff --git a/inventory b/inventory index 6cba8ee29..0dd249e47 100644 --- a/inventory +++ b/inventory @@ -2,7 +2,7 @@ # Please specify the ip addresses and connection settings for your environment # The specified ip addresses will be used to listen by the cluster components. -# "postgresql_exists='true'" if PostgreSQL is already exists and runing +# "postgresql_exists='true'" if PostgreSQL is already exists and running on master (for initial deployment only) # "hostname=" variable is optional (used to change the server name) # if dcs_exists: false and dcs_type: "etcd" (in vars/main.yml) diff --git a/roles/deploy-finish/tasks/main.yml b/roles/deploy-finish/tasks/main.yml index 84143a25e..ce7c78dbe 100644 --- a/roles/deploy-finish/tasks/main.yml +++ b/roles/deploy-finish/tasks/main.yml @@ -17,7 +17,7 @@ debug: var: patronictl_result.stdout_lines ignore_errors: true - tags: patroni_status, cluster_info, cluster_status + tags: patroni_status, cluster_info, cluster_status, point_in_time_recovery - block: - name: Get postgresql database list @@ -53,7 +53,7 @@ debug: var: dbs_result.stdout_lines ignore_errors: true - tags: databases, db_list, cluster_info, cluster_status + tags: databases, db_list, cluster_info, cluster_status, point_in_time_recovery - block: - name: PostgreSQL Cluster connection info diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 29aa879bc..a1f3806cb 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -463,7 +463,7 @@ when: is_master == "true" and postgresql_exists == "true" tags: patroni, patroni_start_master -- block: # wheh postgresql NOT exists +- block: # wheh postgresql NOT exists or PITR - name: Prepare PostgreSQL | make sure PostgreSQL data directory "{{ postgresql_data_dir }}" exists file: path: "{{ postgresql_data_dir }}" @@ -476,11 +476,13 @@ stat: path: "{{ postgresql_data_dir }}/PG_VERSION" register: pgdata_initialized + when: not patroni_cluster_point_in_time_recovery|bool - name: Prepare PostgreSQL | data directory check result fail: msg: "Whoops! data directory {{ postgresql_data_dir }} is already initialized" - when: pgdata_initialized.stat.exists + when: pgdata_initialized.stat.exists is defined and + pgdata_initialized.stat.exists tags: patroni, patroni_check_init # for Debian based distros only @@ -490,7 +492,8 @@ path: "{{ postgresql_conf_dir }}/postgresql.conf" register: postgresql_conf_file when: ansible_os_family == "Debian" and - postgresql_packages is not search("postgrespro") + postgresql_packages is not search("postgrespro") and + not patroni_cluster_point_in_time_recovery|bool - name: Prepare PostgreSQL | generate default postgresql config files become: true @@ -517,9 +520,85 @@ loop: - absent - directory - when: is_master == "true" + when: is_master == "true" and + patroni_cluster_bootstrap_method != "pgbackrest" # --delta restore when: postgresql_exists != "true" - tags: patroni + tags: patroni, point_in_time_recovery + +- block: # PITR +# Prepare (install pexpect) + - name: Prepare | Ensure the ansible required python library (jmespath) is exist + pip: + name: jmespath + state: present + executable: pip3 + extra_args: "--trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org" + umask: "0022" + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" + delegate_to: "{{ item }}" + loop: + - localhost + - "{{ groups.master[0] }}" + when: is_master == "true" and + (pgbackrest_install|bool or patroni_cluster_bootstrap_method == "pgbackrest") + + - name: Prepare | Ensure the ansible required python library (pexpect) is exist + pip: + name: pexpect + state: present + executable: pip3 + extra_args: "--trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org" + umask: "0022" + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" + when: is_master == "true" + +# Run PITR + - name: Stop patroni service on the Replica servers (if running) + systemd: + name: patroni + state: stopped + when: is_master != "true" + + - name: Stop patroni service on the Master server (if running) + systemd: + name: patroni + state: stopped + when: is_master == "true" + + - name: Remove patroni cluster "{{ patroni_cluster_name }}" from DCS (if exist) + become: true + become_user: postgres + expect: + command: "patronictl -c /etc/patroni/patroni.yml remove {{ patroni_cluster_name }}" + responses: + 'Please confirm the cluster name to remove': '{{ patroni_cluster_name }}' + 'You are about to remove all information in DCS': 'Yes I am aware' + register: patronictl_remove_result + changed_when: + patronictl_remove_result.rc == 0 and + patronictl_remove_result.stdout|lower is not search("key not found") + failed_when: + patronictl_remove_result.rc != 0 + environment: + PATH: "{{ ansible_env.PATH }}:/usr/bin:/usr/local/bin" + when: is_master == "true" + + - block: # for pgbackrest only (for use --delta restore) + - name: Run "{{ pgbackrest_patroni_cluster_restore_command }}" on master + debug: + msg: waiting for recovery to complete + + - name: "{{ pgbackrest_patroni_cluster_restore_command }}" + become: true + become_user: postgres + command: "{{ pgbackrest_patroni_cluster_restore_command }}" + when: is_master == "true" and + (pgbackrest_install|bool or patroni_cluster_bootstrap_method == "pgbackrest") + environment: "{{ proxy_env | default({}) }}" + when: patroni_cluster_point_in_time_recovery|bool + tags: patroni, point_in_time_recovery - block: # start patroni on master - name: Start patroni service on the Master server @@ -548,7 +627,7 @@ delay: 30 changed_when: false - - name: Wait for the new cluster to initialize before generating the custom pg_hba.conf + - name: Wait for the cluster to initialize (master is the leader with the lock) uri: url: "http://{{ hostvars[inventory_hostname]['inventory_hostname'] }}:8008/leader" status_code: 200 @@ -557,7 +636,7 @@ retries: 10 delay: 2 when: is_master == "true" - tags: patroni, patroni_start_master + tags: patroni, patroni_start_master, point_in_time_recovery - block: # pg_hba (using a templates/pg_hba.conf.j2) - name: Prepare PostgreSQL | generate pg_hba.conf @@ -609,6 +688,7 @@ loop: - absent - directory + when: not 'pgbackrest' in patroni_create_replica_methods # --delta restore - name: Start patroni service on Replica servers systemd: @@ -626,7 +706,49 @@ delay: 10 ignore_errors: false when: is_master != "true" - tags: patroni, patroni_start_replica + tags: patroni, patroni_start_replica, point_in_time_recovery + +# PITR +- block: # for pgbackrest only (for use --delta restore) + - name: List the Patroni members (get replicas) + become: true + become_user: postgres + command: "patronictl -c /etc/patroni/patroni.yml list {{ patroni_cluster_name }} -f json" + register: patronictl_list_result + changed_when: false + tags: patronictl_list + + - name: Reinitialize cluster members (replicas) + become: true + become_user: postgres + expect: + command: "patronictl -c /etc/patroni/patroni.yml reinit {{ patroni_cluster_name }} {{ item }}" + responses: + 'Are you sure you want to reinitialize members': 'y' + 'Do you want to cancel it and reinitialize anyway': 'y' + loop: "{{ patronictl_list_result.stdout |from_json |json_query('[?Role != `Leader`].Member') | list }}" + register: patronictl_reinit_result + changed_when: + patronictl_reinit_result.rc == 0 + failed_when: + patronictl_reinit_result.rc != 0 + + - name: Check that the patroni on the "{{ item }}" replica server is healthy + uri: + url: "http://{{ item }}:8008/health" + status_code: 200 + loop: "{{ patronictl_list_result.stdout |from_json |json_query('[?Role != `Leader`].Host') | list }}" + register: result + until: result.status == 200 + retries: 1000 + delay: 10 + environment: + PATH: "{{ ansible_env.PATH }}:/usr/bin:/usr/local/bin" + when: + - is_master == "true" + - patroni_cluster_point_in_time_recovery|bool + - (pgbackrest_install|bool or patroni_cluster_bootstrap_method == "pgbackrest") + tags: patroni, point_in_time_recovery # disable postgresql from autostart - block: # "Debian" diff --git a/roles/pgbackrest/templates/pgbackrest_bootstrap.sh.j2 b/roles/pgbackrest/templates/pgbackrest_bootstrap.sh.j2 index 4b58a30b7..f73a73dbe 100644 --- a/roles/pgbackrest/templates/pgbackrest_bootstrap.sh.j2 +++ b/roles/pgbackrest/templates/pgbackrest_bootstrap.sh.j2 @@ -13,5 +13,5 @@ while getopts ":-:" optchar; do done {% endraw %} -{{ pgbackrest_patroni_cluster_bootstrap_command }} +{{ pgbackrest_patroni_cluster_restore_command }} diff --git a/vars/main.yml b/vars/main.yml index 9e091dfad..60c491306 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -300,7 +300,13 @@ pgbackrest_conf: - {option: "process-max", value: "2"} - {option: "recovery-option", value: "recovery_target_action=promote"} # - {option: "", value: ""} -pgbackrest_patroni_cluster_bootstrap_command: # if patroni_cluster_bootstrap_method: "pgbackrest" +pgbackrest_patroni_cluster_restore_command: # if patroni_cluster_bootstrap_method: "pgbackrest" or PITR "/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --delta restore" +# PITR mode +patroni_cluster_point_in_time_recovery: false # or 'true' +# if true, the database cluster directory will be cleaned (for "wal-g") or overwritten (for "pgbackrest" --delta restore). +# And also the patroni cluster "{{ patroni_cluster_name }}" will be removed from the DCS. +# Do not use during initial deployment of a cluster. This mode is intended for the PITR of an existing patroni cluster only. + ... From fa087569952b3e86c837b60467646ab1149f8bb8 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Sat, 30 May 2020 22:44:10 +0300 Subject: [PATCH 06/27] Remove jmespath package dependency The json parsing conditions are rewritten to use ansible only (without json_query). --- roles/patroni/tasks/main.yml | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index a1f3806cb..40aabdc5a 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -527,22 +527,6 @@ - block: # PITR # Prepare (install pexpect) - - name: Prepare | Ensure the ansible required python library (jmespath) is exist - pip: - name: jmespath - state: present - executable: pip3 - extra_args: "--trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org" - umask: "0022" - environment: - PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" - delegate_to: "{{ item }}" - loop: - - localhost - - "{{ groups.master[0] }}" - when: is_master == "true" and - (pgbackrest_install|bool or patroni_cluster_bootstrap_method == "pgbackrest") - - name: Prepare | Ensure the ansible required python library (pexpect) is exist pip: name: pexpect @@ -710,38 +694,43 @@ # PITR - block: # for pgbackrest only (for use --delta restore) - - name: List the Patroni members (get replicas) + - name: List the patroni cluster members become: true become_user: postgres command: "patronictl -c /etc/patroni/patroni.yml list {{ patroni_cluster_name }} -f json" register: patronictl_list_result changed_when: false - tags: patronictl_list - name: Reinitialize cluster members (replicas) become: true become_user: postgres expect: - command: "patronictl -c /etc/patroni/patroni.yml reinit {{ patroni_cluster_name }} {{ item }}" + command: "patronictl -c /etc/patroni/patroni.yml reinit {{ patroni_cluster_name }} {{ item.Member }}" responses: 'Are you sure you want to reinitialize members': 'y' 'Do you want to cancel it and reinitialize anyway': 'y' - loop: "{{ patronictl_list_result.stdout |from_json |json_query('[?Role != `Leader`].Member') | list }}" + loop: "{{ patronictl_list_result.stdout |from_json }}" + loop_control: + label: "{{ item.Member, item.Host }}" register: patronictl_reinit_result changed_when: patronictl_reinit_result.rc == 0 failed_when: patronictl_reinit_result.rc != 0 + when: item.Role is not defined or item.Role != 'Leader' - - name: Check that the patroni on the "{{ item }}" replica server is healthy + - name: Check that the patroni on the replica server is healthy uri: - url: "http://{{ item }}:8008/health" + url: "http://{{ item.Host }}:8008/health" status_code: 200 - loop: "{{ patronictl_list_result.stdout |from_json |json_query('[?Role != `Leader`].Host') | list }}" + loop: "{{ patronictl_list_result.stdout |from_json }}" + loop_control: + label: "{{ item.Member, item.Host }}" register: result until: result.status == 200 retries: 1000 delay: 10 + when: item.Role is not defined or item.Role != 'Leader' environment: PATH: "{{ ansible_env.PATH }}:/usr/bin:/usr/local/bin" when: From 9735c2f904a0860efab9697b8bd84b73c374becd Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 1 Jun 2020 18:53:12 +0300 Subject: [PATCH 07/27] PITR mode for custom bootsrap The variable patroni_cluster_point_in_time_recovery has been deleted. Now there is no need to explicitly enable the PITR mode. In the default mode (initdb), there is protection against deleting data from the database directory (PGDATA), and also there will be no attempt to remove the cluster from DCS. But, if the variable "patroni_cluster_bootstrap_method" = "pgbackrest" or "wal-g": 1) the database cluster directory will be cleared (for "wal-g") or overwritten (for "pgbackrest" - delta restore). 2) and also the cluster cluster "{{patroni_cluster_name}}" will be deleted from DCS (if exist). --- roles/patroni/tasks/main.yml | 11 ++++++----- vars/main.yml | 15 +++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 40aabdc5a..e8e57eb7d 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -476,7 +476,7 @@ stat: path: "{{ postgresql_data_dir }}/PG_VERSION" register: pgdata_initialized - when: not patroni_cluster_point_in_time_recovery|bool + when: patroni_cluster_bootstrap_method == "initdb" - name: Prepare PostgreSQL | data directory check result fail: @@ -492,8 +492,9 @@ path: "{{ postgresql_conf_dir }}/postgresql.conf" register: postgresql_conf_file when: ansible_os_family == "Debian" and - postgresql_packages is not search("postgrespro") and - not patroni_cluster_point_in_time_recovery|bool + patroni_cluster_bootstrap_method == "initdb" and + postgresql_packages is not search("postgrespro") + - name: Prepare PostgreSQL | generate default postgresql config files become: true @@ -581,7 +582,8 @@ when: is_master == "true" and (pgbackrest_install|bool or patroni_cluster_bootstrap_method == "pgbackrest") environment: "{{ proxy_env | default({}) }}" - when: patroni_cluster_point_in_time_recovery|bool + when: patroni_cluster_bootstrap_method != "initdb" and + (pgbackrest_install|bool or wal_g_install|bool) tags: patroni, point_in_time_recovery - block: # start patroni on master @@ -735,7 +737,6 @@ PATH: "{{ ansible_env.PATH }}:/usr/bin:/usr/local/bin" when: - is_master == "true" - - patroni_cluster_point_in_time_recovery|bool - (pgbackrest_install|bool or patroni_cluster_bootstrap_method == "pgbackrest") tags: patroni, point_in_time_recovery diff --git a/vars/main.yml b/vars/main.yml index 60c491306..088e219ea 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -300,13 +300,12 @@ pgbackrest_conf: - {option: "process-max", value: "2"} - {option: "recovery-option", value: "recovery_target_action=promote"} # - {option: "", value: ""} -pgbackrest_patroni_cluster_restore_command: # if patroni_cluster_bootstrap_method: "pgbackrest" or PITR - "/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --delta restore" - -# PITR mode -patroni_cluster_point_in_time_recovery: false # or 'true' -# if true, the database cluster directory will be cleaned (for "wal-g") or overwritten (for "pgbackrest" --delta restore). -# And also the patroni cluster "{{ patroni_cluster_name }}" will be removed from the DCS. -# Do not use during initial deployment of a cluster. This mode is intended for the PITR of an existing patroni cluster only. +pgbackrest_patroni_cluster_restore_command: + '/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --delta restore' # restore from latest backup +# '/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --type=time "--target=2020-06-01 11:00:00+03" --delta restore' # Point-in-Time Recovery (example) + +# PITR mode (if patroni_cluster_bootstrap_method: "pgbackrest" or "wal-g"): +# 1) The database cluster directory will be cleaned (for "wal-g") or overwritten (for "pgbackrest" --delta restore). +# 2) And also the patroni cluster "{{ patroni_cluster_name }}" will be removed from the DCS (if exist) before recovery. ... From 603efbf646f00b174e80700a4151511c80a8c9f9 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 1 Jun 2020 18:57:58 +0300 Subject: [PATCH 08/27] patroni.yml fix trailing spaces --- roles/patroni/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index e8e57eb7d..3f3b8f28f 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -494,7 +494,6 @@ when: ansible_os_family == "Debian" and patroni_cluster_bootstrap_method == "initdb" and postgresql_packages is not search("postgrespro") - - name: Prepare PostgreSQL | generate default postgresql config files become: true From 8c4b06c10f6c3285aa24ee3789f49cbcb5a9044c Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 4 Jun 2020 19:29:03 +0300 Subject: [PATCH 09/27] pgbackrest: Start PostgreSQL for Recovery (WAL apply) instead of reinit the cluster members (replicas). We cannot reinitialize replicas for recovery at a specific point in time (--target), except for the latest backup. Because patroni will replace recovery conf the created earlier with pgbackrest. --- roles/patroni/tasks/main.yml | 143 ++++++++++++++++++++--------------- 1 file changed, 82 insertions(+), 61 deletions(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 3f3b8f28f..ac31f7802 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -492,7 +492,6 @@ path: "{{ postgresql_conf_dir }}/postgresql.conf" register: postgresql_conf_file when: ansible_os_family == "Debian" and - patroni_cluster_bootstrap_method == "initdb" and postgresql_packages is not search("postgrespro") - name: Prepare PostgreSQL | generate default postgresql config files @@ -506,9 +505,9 @@ --locale {{ postgresql_locale }} register: pg_createcluster_result failed_when: pg_createcluster_result.rc != 0 - when: ansible_os_family == "Debian" and - (postgresql_conf_file.stat.exists is defined) and - not postgresql_conf_file.stat.exists + when: not postgresql_conf_file.stat.exists and + (ansible_os_family == "Debian" and + postgresql_packages is not search("postgrespro")) - name: Prepare PostgreSQL | make sure the data directory "{{ postgresql_data_dir }}" is empty on Master file: @@ -525,7 +524,7 @@ when: postgresql_exists != "true" tags: patroni, point_in_time_recovery -- block: # PITR +- block: # PITR (custom bootstrap) # Prepare (install pexpect) - name: Prepare | Ensure the ansible required python library (pexpect) is exist pip: @@ -570,16 +569,84 @@ when: is_master == "true" - block: # for pgbackrest only (for use --delta restore) - - name: Run "{{ pgbackrest_patroni_cluster_restore_command }}" on master - debug: - msg: waiting for recovery to complete - - - name: "{{ pgbackrest_patroni_cluster_restore_command }}" - become: true - become_user: postgres - command: "{{ pgbackrest_patroni_cluster_restore_command }}" - when: is_master == "true" and - (pgbackrest_install|bool or patroni_cluster_bootstrap_method == "pgbackrest") + - name: Run "{{ pgbackrest_patroni_cluster_restore_command }}" on Master + command: "{{ pgbackrest_patroni_cluster_restore_command }} --target-action=promote" + async: 86400 # timeout 24 hours + poll: 0 + register: pgbackrest_restore_master + when: is_master == "true" + + # if patroni_create_replica_methods: "pgbackrest" + - name: Run "{{ pgbackrest_patroni_cluster_restore_command }}" on Replica + command: "{{ pgbackrest_patroni_cluster_restore_command }} --target-action=shutdown" # shutdown when recovery target is reached + async: 86400 # timeout 24 hours + poll: 0 + register: pgbackrest_restore_replica + when: is_master != "true" and 'pgbackrest' in patroni_create_replica_methods + + - name: Waiting for restore from backup + async_status: + jid: "{{ item.ansible_job_id }}" + loop: + - "{{ pgbackrest_restore_master }}" + - "{{ pgbackrest_restore_replica }}" + loop_control: + label: "{{ item.changed }}" + register: pgbackrest_restore_jobs_result + until: pgbackrest_restore_jobs_result.finished + retries: 2880 # timeout 24 hours + delay: 30 + when: item.ansible_job_id is defined + + - name: Start PostgreSQL for Recovery # Debian + command: "/usr/bin/pg_ctlcluster {{ postgresql_version }} {{ postgresql_cluster_name }} start" + when: ansible_os_family == "Debian" and + (is_master == "true" or + (is_master != "true" and 'pgbackrest' in patroni_create_replica_methods)) + + - name: Start PostgreSQL for Recovery # RedHat or PostgresPro + command: "{{ postgresql_bin_dir }}/pg_ctl start -D {{ postgresql_data_dir }}" + when: (ansible_os_family == "RedHat" or postgresql_packages is search("postgrespro")) and + (is_master == "true" or + (is_master != "true" and 'pgbackrest' in patroni_create_replica_methods)) + + - name: Waiting for PostgreSQL Recovery to complete (WAL apply) + command: "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -tAc 'SELECT pg_is_in_recovery()'" + register: pg_is_in_recovery + until: pg_is_in_recovery.stdout != "t" + retries: 1200 # timeout 10 hours + delay: 30 + changed_when: false + failed_when: false + when: is_master == "true" or + (is_master != "true" and 'pgbackrest' in patroni_create_replica_methods) + + - name: Check that PostgreSQL is stopped + command: "{{ postgresql_bin_dir }}/pg_ctl status -D {{ postgresql_data_dir }}" + register: pg_ctl_status_result + changed_when: false + failed_when: false + + - name: Stop PostgreSQL # "Debian" + command: "/usr/bin/pg_ctlcluster {{ postgresql_version }} {{ postgresql_cluster_name }} stop -m fast" + register: stop_result + until: stop_result.rc == 0 + retries: 10 + delay: 10 + when: ansible_os_family == "Debian" and + (pg_ctl_status_result.rc is defined and pg_ctl_status_result.rc != 3) + + - name: Stop PostgreSQL # "RedHat" or PostgresPro + command: "{{ postgresql_bin_dir }}/pg_ctl stop -D {{ postgresql_data_dir }} -m fast" + register: stop_result + until: stop_result.rc == 0 + retries: 10 + delay: 10 + when: (ansible_os_family == "RedHat" or postgresql_packages is search("postgrespro")) and + (pg_ctl_status_result.rc is defined and pg_ctl_status_result.rc != 3) + when: patroni_cluster_bootstrap_method == "pgbackrest" + become: true + become_user: postgres environment: "{{ proxy_env | default({}) }}" when: patroni_cluster_bootstrap_method != "initdb" and (pgbackrest_install|bool or wal_g_install|bool) @@ -693,52 +760,6 @@ when: is_master != "true" tags: patroni, patroni_start_replica, point_in_time_recovery -# PITR -- block: # for pgbackrest only (for use --delta restore) - - name: List the patroni cluster members - become: true - become_user: postgres - command: "patronictl -c /etc/patroni/patroni.yml list {{ patroni_cluster_name }} -f json" - register: patronictl_list_result - changed_when: false - - - name: Reinitialize cluster members (replicas) - become: true - become_user: postgres - expect: - command: "patronictl -c /etc/patroni/patroni.yml reinit {{ patroni_cluster_name }} {{ item.Member }}" - responses: - 'Are you sure you want to reinitialize members': 'y' - 'Do you want to cancel it and reinitialize anyway': 'y' - loop: "{{ patronictl_list_result.stdout |from_json }}" - loop_control: - label: "{{ item.Member, item.Host }}" - register: patronictl_reinit_result - changed_when: - patronictl_reinit_result.rc == 0 - failed_when: - patronictl_reinit_result.rc != 0 - when: item.Role is not defined or item.Role != 'Leader' - - - name: Check that the patroni on the replica server is healthy - uri: - url: "http://{{ item.Host }}:8008/health" - status_code: 200 - loop: "{{ patronictl_list_result.stdout |from_json }}" - loop_control: - label: "{{ item.Member, item.Host }}" - register: result - until: result.status == 200 - retries: 1000 - delay: 10 - when: item.Role is not defined or item.Role != 'Leader' - environment: - PATH: "{{ ansible_env.PATH }}:/usr/bin:/usr/local/bin" - when: - - is_master == "true" - - (pgbackrest_install|bool or patroni_cluster_bootstrap_method == "pgbackrest") - tags: patroni, point_in_time_recovery - # disable postgresql from autostart - block: # "Debian" - name: Turning off postgresql autostart from config "start.conf" (will be managed by patroni) From a2d088c870981edd64e722c653fde2e2f04b5de9 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 4 Jun 2020 20:30:35 +0300 Subject: [PATCH 10/27] patroni.yml: fix condition with wrong order I set the conditions in the wrong order, due to which an error occurred: TASK [patroni : Prepare PostgreSQL | generate default postgresql config files] *** fatal: [10.172.0.20]: FAILED! => {"msg": "The conditional check 'not postgresql_conf_file.stat.exists and (ansible_os_family == \"Debian\" and postgresql_packages is not search(\"postgrespro\"))' failed. The error was: error while evaluating conditional (not postgresql_conf_file.stat.exists and (ansible_os_family == \"Debian\" and postgresql_packages is not search(\"postgrespro\"))): 'dict object' has no attribute 'stat'\n\nThe error appears to be in '/home/runner/work/postgresql_cluster/postgresql_cluster/roles/patroni/tasks/main.yml': line 497, column 7, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\n - name: Prepare PostgreSQL | generate default postgresql config files\n ^ here\n"} --- roles/patroni/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index ac31f7802..70d3f5e85 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -505,9 +505,9 @@ --locale {{ postgresql_locale }} register: pg_createcluster_result failed_when: pg_createcluster_result.rc != 0 - when: not postgresql_conf_file.stat.exists and - (ansible_os_family == "Debian" and - postgresql_packages is not search("postgrespro")) + when: (ansible_os_family == "Debian" and + postgresql_packages is not search("postgrespro")) and + not postgresql_conf_file.stat.exists - name: Prepare PostgreSQL | make sure the data directory "{{ postgresql_data_dir }}" is empty on Master file: From 317896ef32d55f3a1028ccfd5bca6f4dbcb5eac7 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Fri, 5 Jun 2020 15:46:31 +0300 Subject: [PATCH 11/27] pgbackrest: add proxy_env for "install package" task. --- roles/pgbackrest/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/pgbackrest/tasks/main.yml b/roles/pgbackrest/tasks/main.yml index 4838b2eb7..b3a836d7c 100644 --- a/roles/pgbackrest/tasks/main.yml +++ b/roles/pgbackrest/tasks/main.yml @@ -56,6 +56,7 @@ package: name: pgbackrest state: latest + environment: "{{ proxy_env | default({}) }}" tags: pgbackrest, pgbackrest_install - block: From ceac03e6e0b4e1e69116146f3a28e12e61d70ac0 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Fri, 5 Jun 2020 16:08:26 +0300 Subject: [PATCH 12/27] PITR: Add the variable ansible_python_interpreter at the task level. Now you do not need to explicitly specify "ansible_python_interpreter=/usr/bin/python3" in the inventory file. --- roles/patroni/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 70d3f5e85..9896c2976 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -567,6 +567,8 @@ environment: PATH: "{{ ansible_env.PATH }}:/usr/bin:/usr/local/bin" when: is_master == "true" + vars: + ansible_python_interpreter: /usr/bin/python3 - block: # for pgbackrest only (for use --delta restore) - name: Run "{{ pgbackrest_patroni_cluster_restore_command }}" on Master From 284dc07169bb2933c3472d0c3695574627e668dc Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Fri, 5 Jun 2020 17:01:06 +0300 Subject: [PATCH 13/27] Check that the patroni is healthy on the replica server --- roles/patroni/tasks/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 9896c2976..ff32580c4 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -759,6 +759,15 @@ timeout: 120 delay: 10 ignore_errors: false + + - name: Check that the patroni is healthy on the replica server + uri: + url: "http://{{ hostvars[inventory_hostname]['inventory_hostname'] }}:8008/health" + status_code: 200 + register: replica_result + until: replica_result.status == 200 + retries: 100 + delay: 10 when: is_master != "true" tags: patroni, patroni_start_replica, point_in_time_recovery From 6a7574e7cee29d94d9919e1ecdf6ac27f244a1fe Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Fri, 5 Jun 2020 23:40:46 +0300 Subject: [PATCH 14/27] pgbackrest: Set "--target-action" if "--type" is specified only. Fixed: ERROR: [031]: option 'target-action' not valid without option 'type' in ('immediate', 'name', 'time', 'xid')"'time', 'xid')". --- roles/patroni/tasks/main.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index ff32580c4..d7a3972b0 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -572,7 +572,9 @@ - block: # for pgbackrest only (for use --delta restore) - name: Run "{{ pgbackrest_patroni_cluster_restore_command }}" on Master - command: "{{ pgbackrest_patroni_cluster_restore_command }} --target-action=promote" + command: > + {{ pgbackrest_patroni_cluster_restore_command }} + {{ '--target-action=promote' if pgbackrest_patroni_cluster_restore_command is search('--type=') else '' }} async: 86400 # timeout 24 hours poll: 0 register: pgbackrest_restore_master @@ -580,7 +582,9 @@ # if patroni_create_replica_methods: "pgbackrest" - name: Run "{{ pgbackrest_patroni_cluster_restore_command }}" on Replica - command: "{{ pgbackrest_patroni_cluster_restore_command }} --target-action=shutdown" # shutdown when recovery target is reached + command: > + {{ pgbackrest_patroni_cluster_restore_command }} + {{ '--target-action=shutdown' if pgbackrest_patroni_cluster_restore_command is search('--type=') else '' }} async: 86400 # timeout 24 hours poll: 0 register: pgbackrest_restore_replica From 6f0159004aa5fdca452ed6d76cd188cb7f1a12b3 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 10 Jun 2020 17:59:34 +0300 Subject: [PATCH 15/27] Add yedit module for patroni role ansible module for modifying yaml files. --- roles/patroni/library/yedit.py | 973 +++++++++++++++++++++++++++++++++ 1 file changed, 973 insertions(+) create mode 100644 roles/patroni/library/yedit.py diff --git a/roles/patroni/library/yedit.py b/roles/patroni/library/yedit.py new file mode 100644 index 000000000..9eb79e365 --- /dev/null +++ b/roles/patroni/library/yedit.py @@ -0,0 +1,973 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# pylint: disable=wrong-import-order,wrong-import-position,unused-import + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'core'} + +DOCUMENTATION = ''' +--- +module: yedit +version_added: "2.6" +short_description: Create, modify, and idempotently manage yaml files. +description: + - Modify yaml files programmatically. +options: + state: + description: + - State represents whether to create, modify, delete, or list yaml + required: true + default: present + choices: ["present", "absent", "list"] + aliases: [] + debug: + description: + - Turn on debug information. + required: false + default: false + type: bool + aliases: [] + src: + description: + - The file that is the target of the modifications. + required: false + aliases: [] + content: + description: + - Content represents the yaml content you desire to work with. This + - could be the file contents to write or the inmemory data to modify. + required: false + aliases: [] + content_type: + description: + - The python type of the content parameter. + required: false + choices: ['yaml', 'json'] + default: yaml + aliases: [] + key: + description: + - The path to the value you wish to modify. Emtpy string means the top of + - the document. + required: false + default: '' + aliases: [] + value: + description: + - The incoming value of parameter 'key'. + required: false + default: + aliases: [] + edits: + description: + - A list of edits to perform. These follow the same format as a single edit + required: false + aliases: [] + value_type: + description: + - The python type of the incoming value. + required: false + default: '' + aliases: [] + update: + description: + - Whether the update should be performed on a dict/hash or list/array + - object. + required: false + default: false + aliases: [] + type: bool + append: + description: + - Whether to append to an array/list. When the key does not exist or is + - null, a new array is created. When the key is of a non-list type, + - nothing is done. + required: false + default: false + aliases: [] + type: bool + index: + description: + - Used in conjunction with the update parameter. This will update a + - specific index in an array/list. + required: false + aliases: [] + curr_value: + description: + - Used in conjunction with the update parameter. This is the current + - value of 'key' in the yaml file. + required: false + default: None + aliases: [] + curr_value_format: + description: + - Format of the incoming current value. + choices: ["yaml", "json", "str"] + required: false + default: yaml + aliases: [] + backup_ext: + description: + - The backup file's appended string. + required: false + aliases: [] + backup: + description: + - Whether to make a backup copy of the current file when performing an + - edit. + required: false + default: false + aliases: [] + type: bool + separator: + description: + - The separator being used when parsing strings. + required: false + default: '.' + aliases: [] +author: +- "Kenny Woodson " +extends_documentation_fragment: [] +''' + +EXAMPLES = ''' +# Simple insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: test + value: somevalue + state: present +# Results: +# test: somevalue + +# Multilevel insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: a.b.c + value: d + state: present +# Results: +# a: +# b: +# c: d +# +# multiple edits at the same time +- name: perform multiple edits + yedit: + src: somefile.yml + edits: + - key: a.b.c + value: d + - key: a.b.c.d + value: e + state: present +# Results: +# a: +# b: +# c: +# d: e +''' + +import copy # noqa: F401 +import fcntl # noqa: F401 +import json # noqa: F401 +import os # noqa: F401 +import re # noqa: F401 +import shutil # noqa: F401 +import time # noqa: F401 + +try: + import ruamel.yaml as yaml # noqa: F401 +except ImportError: + import yaml # noqa: F401 + +from ansible.module_utils.basic import AnsibleModule + + +class YeditException(Exception): + ''' Exception class for Yedit ''' + pass + + +# pylint: disable=too-many-public-methods,too-many-instance-attributes +class Yedit(object): + ''' Class to modify yaml files ''' + re_valid_key = r"(((\[-?\d+\])|([0-9a-zA-Z%s/_-]+)).?)+$" + re_key = r"(?:\[(-?\d+)\])|([0-9a-zA-Z{}/_-]+)" + com_sep = set(['.', '#', '|', ':']) + + # pylint: disable=too-many-arguments + def __init__(self, + filename=None, + content=None, + content_type='yaml', + separator='.', + backup_ext=".{0}".format(time.strftime("%Y%m%dT%H%M%S")), + backup=False): + self.content = content + self._separator = separator + self.filename = filename + self.__yaml_dict = content + self.content_type = content_type + self.backup = backup + self.backup_ext = backup_ext + self.load(content_type=self.content_type) + if self.__yaml_dict is None: + self.__yaml_dict = {} + + @property + def separator(self): + ''' getter method for separator ''' + return self._separator + + @separator.setter + def separator(self, inc_sep): + ''' setter method for separator ''' + self._separator = inc_sep + + @property + def yaml_dict(self): + ''' getter method for yaml_dict ''' + return self.__yaml_dict + + @yaml_dict.setter + def yaml_dict(self, value): + ''' setter method for yaml_dict ''' + self.__yaml_dict = value + + @staticmethod + def parse_key(key, sep='.'): + '''parse the key allowing the appropriate separator''' + common_separators = list(Yedit.com_sep - set([sep])) + return re.findall(Yedit.re_key.format(''.join(common_separators)), key) + + @staticmethod + def valid_key(key, sep='.'): + '''validate the incoming key''' + common_separators = list(Yedit.com_sep - set([sep])) + if not re.match(Yedit.re_valid_key.format(''.join(common_separators)), key): + return False + + return True + + # pylint: disable=too-many-return-statements,too-many-branches + @staticmethod + def remove_entry(data, key, index=None, value=None, sep='.'): + ''' remove data at location key ''' + if key == '' and isinstance(data, dict): + if value is not None: + data.pop(value) + elif index is not None: + raise YeditException("remove_entry for a dictionary does not have an index {0}".format(index)) + else: + data.clear() + + return True + + elif key == '' and isinstance(data, list): + ind = None + if value is not None: + try: + ind = data.index(value) + except ValueError: + return False + elif index is not None: + ind = index + else: + del data[:] + + if ind is not None: + data.pop(ind) + + return True + + if not (key and Yedit.valid_key(key, sep)) and \ + isinstance(data, (list, dict)): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key and isinstance(data, dict): + data = data.get(dict_key) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + # process last index for remove + # expected list entry + if key_indexes[-1][0]: + if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + del data[int(key_indexes[-1][0])] + return True + + # expected dict entry + elif key_indexes[-1][1]: + if isinstance(data, dict): + del data[key_indexes[-1][1]] + return True + + @staticmethod + def add_entry(data, key, item=None, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a#b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key: + if isinstance(data, dict) and dict_key in data and data[dict_key]: # noqa: E501 + data = data[dict_key] + continue + + elif data and not isinstance(data, dict): + raise YeditException("Unexpected item type found while going through key " + + "path: {0} (at key: {1})".format(key, dict_key)) + + data[dict_key] = {} + data = data[dict_key] + + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + raise YeditException("Unexpected item type found while going through key path: {0}".format(key)) + + if key == '': + data = item + + # process last index for add + # expected list entry + elif key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data): # noqa: E501 + # key is next element in array so append + if int(key_indexes[-1][0]) > len(data)-1: + data.append(item) + else: + data[int(key_indexes[-1][0])] = item + + # expected dict entry + elif key_indexes[-1][1] and isinstance(data, dict): + data[key_indexes[-1][1]] = item + + # didn't add/update to an existing list, nor add/update key to a dict + # so we must have been provided some syntax like a.b.c[] = "data" for a + # non-existent array + else: + raise YeditException("Error adding to object at path: {0}".format(key)) + + return data + + @staticmethod + def get_entry(data, key, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a.b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes: + if dict_key and isinstance(data, dict): + data = data.get(dict_key) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + return data + + @staticmethod + def _write(filename, contents): + ''' Actually write the file contents to disk. This helps with mocking. ''' + + tmp_filename = filename + '.yedit' + + with open(tmp_filename, 'w') as yfd: + fcntl.flock(yfd, fcntl.LOCK_EX | fcntl.LOCK_NB) + yfd.write(contents) + yfd.flush() # flush internal buffers + os.fsync(yfd.fileno()) # ensure buffer content reached disk + fcntl.flock(yfd, fcntl.LOCK_UN) + + os.rename(tmp_filename, filename) + # While the rename is atomic, we also need to ensure, that the updated + # directory entry has reached the disk too. + # NOTE: this might fail on Windows systems. + dfd = None + try: + dfd = os.open(os.path.join(os.path.realpath('.'), os.path.dirname(filename)), os.O_DIRECTORY) + os.fsync(dfd) + finally: + if dfd: + os.close(dfd) + + def write(self): + ''' write to file ''' + if not self.filename: + raise YeditException('Please specify a filename.') + + if self.backup and self.file_exists(): + shutil.copy(self.filename, '{0}{1}'.format(self.filename, self.backup_ext)) + + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + # Try to use RoundTripDumper if supported. + if self.content_type == 'yaml': + try: + Yedit._write(self.filename, yaml.dump(self.yaml_dict, Dumper=yaml.RoundTripDumper)) + except AttributeError: + Yedit._write(self.filename, yaml.safe_dump(self.yaml_dict, default_flow_style=False)) + elif self.content_type == 'json': + Yedit._write(self.filename, json.dumps(self.yaml_dict, indent=4, sort_keys=True)) + else: + raise YeditException('Unsupported content_type: {0}.'.format(self.content_type) + + 'Please specify a content_type of yaml or json.') + + return (True, self.yaml_dict) + + def read(self): + ''' read from file ''' + # check if it exists + if self.filename is None or not self.file_exists(): + return None + + contents = None + with open(self.filename) as yfd: + contents = yfd.read() + + return contents + + def file_exists(self): + ''' return whether file exists ''' + if os.path.exists(self.filename): + return True + + return False + + def load(self, content_type='yaml'): + ''' return yaml file ''' + contents = self.read() + + if not contents and not self.content: + return None + + if self.content: + if isinstance(self.content, dict): + self.yaml_dict = self.content + return self.yaml_dict + elif isinstance(self.content, str): + contents = self.content + + # check if it is yaml + try: + if content_type == 'yaml' and contents: + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + # Try to use RoundTripLoader if supported. + try: + self.yaml_dict = yaml.load(contents, yaml.RoundTripLoader) + except AttributeError: + self.yaml_dict = yaml.safe_load(contents) + + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + elif content_type == 'json' and contents: + self.yaml_dict = json.loads(contents) + except yaml.YAMLError as err: + # Error loading yaml or json + raise YeditException('Problem with loading yaml file. {0}'.format(err)) + + return self.yaml_dict + + def get(self, key): + ''' get a specified key''' + try: + entry = Yedit.get_entry(self.yaml_dict, key, self.separator) + except KeyError: + entry = None + + return entry + + def pop(self, path, key_or_item): + ''' remove a key, value pair from a dict or an item for a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + if isinstance(entry, dict): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + if key_or_item in entry: + entry.pop(key_or_item) + return (True, self.yaml_dict) + return (False, self.yaml_dict) + + elif isinstance(entry, list): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + ind = None + try: + ind = entry.index(key_or_item) + except ValueError: + return (False, self.yaml_dict) + + entry.pop(ind) + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + def delete(self, path, index=None, value=None): + ''' remove path from a dict''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + result = Yedit.remove_entry(self.yaml_dict, path, index, value, self.separator) + if not result: + return (False, self.yaml_dict) + + return (True, self.yaml_dict) + + def exists(self, path, value): + ''' check if value exists at path''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, list): + if value in entry: + return True + return False + + elif isinstance(entry, dict): + if isinstance(value, dict): + rval = False + for key, val in value.items(): + if entry[key] != val: + rval = False + break + else: + rval = True + return rval + + return value in entry + + return entry == value + + def append(self, path, value): + '''append value to a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + self.put(path, []) + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + if not isinstance(entry, list): + return (False, self.yaml_dict) + + # AUDIT:maybe-no-member makes sense due to loading data from + # a serialized format. + # pylint: disable=maybe-no-member + entry.append(value) + return (True, self.yaml_dict) + + # pylint: disable=too-many-arguments + def update(self, path, value, index=None, curr_value=None): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, dict): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + if not isinstance(value, dict): + raise YeditException('Cannot replace key, value entry in dict with non-dict type. ' + + 'value=[{0}] type=[{1}]'.format(value, type(value))) + + entry.update(value) + return (True, self.yaml_dict) + + elif isinstance(entry, list): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + ind = None + if curr_value: + try: + ind = entry.index(curr_value) + except ValueError: + return (False, self.yaml_dict) + + elif index is not None: + ind = index + + if ind is not None and entry[ind] != value: + entry[ind] = value + return (True, self.yaml_dict) + + # see if it exists in the list + try: + ind = entry.index(value) + except ValueError: + # doesn't exist, append it + entry.append(value) + return (True, self.yaml_dict) + + # already exists, return + if ind is not None: + return (False, self.yaml_dict) + return (False, self.yaml_dict) + + def put(self, path, value): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry == value: + return (False, self.yaml_dict) + + # deepcopy didn't work + # Try to use ruamel.yaml and fallback to pyyaml + try: + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, + default_flow_style=False), + yaml.RoundTripLoader) + except AttributeError: + tmp_copy = copy.deepcopy(self.yaml_dict) + + # set the format attributes if available + try: + tmp_copy.fa.set_block_style() + except AttributeError: + pass + + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if result is None: + return (False, self.yaml_dict) + + # When path equals "" it is a special case. + # "" refers to the root of the document + # Only update the root path (entire document) when its a list or dict + if path == '': + if isinstance(result, list) or isinstance(result, dict): + self.yaml_dict = result + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + self.yaml_dict = tmp_copy + + return (True, self.yaml_dict) + + def create(self, path, value): + ''' create a yaml file ''' + if not self.file_exists(): + # deepcopy didn't work + # Try to use ruamel.yaml and fallback to pyyaml + try: + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, + default_flow_style=False), + yaml.RoundTripLoader) + except AttributeError: + tmp_copy = copy.deepcopy(self.yaml_dict) + + # set the format attributes if available + try: + tmp_copy.fa.set_block_style() + except AttributeError: + pass + + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if result is not None: + self.yaml_dict = tmp_copy + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + @staticmethod + def get_curr_value(invalue, val_type): + '''return the current value''' + if invalue is None: + return None + + curr_value = invalue + if val_type == 'yaml': + curr_value = yaml.safe_load(str(invalue)) + elif val_type == 'json': + curr_value = json.loads(invalue) + + return curr_value + + @staticmethod + def parse_value(inc_value, vtype=''): + '''determine value type passed''' + true_bools = ['y', 'Y', 'yes', 'Yes', 'YES', 'true', 'True', 'TRUE', + 'on', 'On', 'ON', ] + false_bools = ['n', 'N', 'no', 'No', 'NO', 'false', 'False', 'FALSE', + 'off', 'Off', 'OFF'] + + # It came in as a string but you didn't specify value_type as string + # we will convert to bool if it matches any of the above cases + if isinstance(inc_value, str) and 'bool' in vtype: + if inc_value not in true_bools and inc_value not in false_bools: + raise YeditException('Not a boolean type. str=[{0}] vtype=[{1}]'.format(inc_value, vtype)) + elif isinstance(inc_value, bool) and 'str' in vtype: + inc_value = str(inc_value) + + # There is a special case where '' will turn into None after yaml loading it so skip + if isinstance(inc_value, str) and inc_value == '': + pass + # If vtype is not str then go ahead and attempt to yaml load it. + elif isinstance(inc_value, str) and 'str' not in vtype: + try: + inc_value = yaml.safe_load(inc_value) + except Exception: + raise YeditException('Could not determine type of incoming value. ' + + 'value=[{0}] vtype=[{1}]'.format(type(inc_value), vtype)) + + return inc_value + + @staticmethod + def process_edits(edits, yamlfile): + '''run through a list of edits and process them one-by-one''' + results = [] + for edit in edits: + value = Yedit.parse_value(edit['value'], edit.get('value_type', '')) + if edit.get('action') == 'update': + # pylint: disable=line-too-long + curr_value = Yedit.get_curr_value( + Yedit.parse_value(edit.get('curr_value')), + edit.get('curr_value_format')) + + rval = yamlfile.update(edit['key'], + value, + edit.get('index'), + curr_value) + + elif edit.get('action') == 'append': + rval = yamlfile.append(edit['key'], value) + + else: + rval = yamlfile.put(edit['key'], value) + + if rval[0]: + results.append({'key': edit['key'], 'edit': rval[1]}) + + return {'changed': len(results) > 0, 'results': results} + + # pylint: disable=too-many-return-statements,too-many-branches + @staticmethod + def run_ansible(params): + '''perform the idempotent crud operations''' + yamlfile = Yedit(filename=params['src'], + backup=params['backup'], + content_type=params['content_type'], + backup_ext=params['backup_ext'], + separator=params['separator']) + + state = params['state'] + + if params['src']: + rval = yamlfile.load() + + if yamlfile.yaml_dict is None and state != 'present': + return {'failed': True, + 'msg': 'Error opening file [{0}]. Verify that the '.format(params['src']) + + 'file exists, that it is has correct permissions, and is valid yaml.'} + + if state == 'list': + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + yamlfile.yaml_dict = content + + if params['key']: + rval = yamlfile.get(params['key']) + + return {'changed': False, 'result': rval, 'state': state} + + elif state == 'absent': + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + yamlfile.yaml_dict = content + + if params['update']: + rval = yamlfile.pop(params['key'], params['value']) + else: + rval = yamlfile.delete(params['key'], params['index'], params['value']) + + if rval[0] and params['src']: + yamlfile.write() + + return {'changed': rval[0], 'result': rval[1], 'state': state} + + elif state == 'present': + # check if content is different than what is in the file + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + + # We had no edits to make and the contents are the same + if yamlfile.yaml_dict == content and \ + params['value'] is None: + return {'changed': False, 'result': yamlfile.yaml_dict, 'state': state} + + yamlfile.yaml_dict = content + + # If we were passed a key, value then + # we enapsulate it in a list and process it + # Key, Value passed to the module : Converted to Edits list # + edits = [] + _edit = {} + if params['value'] is not None: + _edit['value'] = params['value'] + _edit['value_type'] = params['value_type'] + _edit['key'] = params['key'] + + if params['update']: + _edit['action'] = 'update' + _edit['curr_value'] = params['curr_value'] + _edit['curr_value_format'] = params['curr_value_format'] + _edit['index'] = params['index'] + + elif params['append']: + _edit['action'] = 'append' + + edits.append(_edit) + + elif params['edits'] is not None: + edits = params['edits'] + + if edits: + results = Yedit.process_edits(edits, yamlfile) + + # if there were changes and a src provided to us we need to write + if results['changed'] and params['src']: + yamlfile.write() + + return {'changed': results['changed'], 'result': results['results'], 'state': state} + + # no edits to make + if params['src']: + rval = yamlfile.write() + return {'changed': rval[0], + 'result': rval[1], + 'state': state} + + # We were passed content but no src, key or value, or edits. Return contents in memory + return {'changed': False, 'result': yamlfile.yaml_dict, 'state': state} + return {'failed': True, 'msg': 'Unkown state passed'} + +def json_roundtrip_clean(js): + ''' Clean-up any non-string keys from a Python object, to ensure it can be serialized as JSON ''' + cleaned_json = json.dumps(js, skipkeys=True) + return json.loads(cleaned_json) + +# pylint: disable=too-many-branches +def main(): + ''' ansible oc module for secrets ''' + + module = AnsibleModule( + argument_spec=dict( + state=dict(default='present', type='str', + choices=['present', 'absent', 'list']), + debug=dict(default=False, type='bool'), + src=dict(default=None, type='str'), + content=dict(default=None), + content_type=dict(default='yaml', choices=['yaml', 'json']), + key=dict(default='', type='str'), + value=dict(), + value_type=dict(default='', type='str'), + update=dict(default=False, type='bool'), + append=dict(default=False, type='bool'), + index=dict(default=None, type='int'), + curr_value=dict(default=None, type='str'), + curr_value_format=dict(default='yaml', + choices=['yaml', 'json', 'str'], + type='str'), + backup=dict(default=False, type='bool'), + backup_ext=dict(default=".{0}".format(time.strftime("%Y%m%dT%H%M%S")), type='str'), + separator=dict(default='.', type='str'), + edits=dict(default=None, type='list'), + ), + mutually_exclusive=[["curr_value", "index"], ['update', "append"]], + required_one_of=[["content", "src"]], + ) + + # Verify we recieved either a valid key or edits with valid keys when receiving a src file. + # A valid key being not None or not ''. + if module.params['src'] is not None: + key_error = False + edit_error = False + + if module.params['key'] is None: + key_error = True + + if module.params['edits'] in [None, []]: + edit_error = True + + else: + for edit in module.params['edits']: + if edit.get('key') in [None, '']: + edit_error = True + break + + if key_error and edit_error: + return module.fail_json(failed=True, msg='Empty value for parameter key not allowed.') + + rval = json_roundtrip_clean(Yedit.run_ansible(module.params)) + if 'failed' in rval and rval['failed']: + return module.fail_json(**rval) + + return module.exit_json(**rval) + + +if __name__ == '__main__': + main() From c5e6604425f50c204b0db01e0485346f6c82e89f Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 10 Jun 2020 18:09:44 +0300 Subject: [PATCH 16/27] Update requirements.txt Added python modules: pexpect - required for PITR (for ansible module "expect") ruamel.yaml - required for PITR (for ansible module "yedit") Removed python requests module from explicit requirements (since patroni version 1.6.2) https://github.com/zalando/patroni/commit/90a4208390f22e797619ac33dc099b985e367cbd#diff-b4ef698db8ca845e5845c4618278f29a It wasn't used for anything critical, but causing a lot of problems when the new version of urllib3 is released. --- files/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/files/requirements.txt b/files/requirements.txt index bb93d2e04..ed0384b02 100644 --- a/files/requirements.txt +++ b/files/requirements.txt @@ -1,7 +1,6 @@ urllib3>=1.24.2,<1.25 boto PyYAML -requests six >= 1.7 kazoo>=1.3.1 python-etcd>=0.4.3,<0.5 @@ -12,3 +11,5 @@ tzlocal python-dateutil psutil>=2.0.0 cdiff +pexpect>=4.8.0 +ruamel.yaml>=0.16.10 From 00cbd2b8a0557d42d51cd43d0fd0055666076674 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 10 Jun 2020 19:16:56 +0300 Subject: [PATCH 17/27] PITR: Make sure the superuser and replication password does not differ from the specified. PITR: Make sure the superuser and replication password does not differ from the specified. If the password is different, it will be replaced with a new one. If there is no replication user, it will be created. See the variables (in vars/main.yml): patroni_superuser_username patroni_superuser_password patroni_replication_username patroni_replication_password You can also restore databases from another postgresql clusters. --- roles/patroni/tasks/main.yml | 57 ++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index d7a3972b0..04e6cb00c 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -525,18 +525,19 @@ tags: patroni, point_in_time_recovery - block: # PITR (custom bootstrap) -# Prepare (install pexpect) - - name: Prepare | Ensure the ansible required python library (pexpect) is exist +# Prepare (install pexpect, ruamel.yaml) + - name: Prepare | Ensure the ansible required python library is exist pip: - name: pexpect + name: "{{ item }}" state: present executable: pip3 extra_args: "--trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org" umask: "0022" + loop: + - pexpect + - ruamel.yaml environment: PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" - when: is_master == "true" - # Run PITR - name: Stop patroni service on the Replica servers (if running) systemd: @@ -715,6 +716,52 @@ when: existing_pgcluster is not defined or not existing_pgcluster|bool tags: patroni, pg_hba, pg_hba_generate +- block: # PITR (custom bootstrap) - superuser and replication + - name: Ensure that archive recovery complete + command: "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -tAc 'SELECT pg_is_in_recovery()'" + register: pg_is_in_recovery + until: pg_is_in_recovery.stdout != "t" + retries: 1200 # timeout 10 hours + delay: 30 + changed_when: false + when: is_master == "true" + + - name: Make sure the postgresql users are present, and password does not differ from the specified + postgresql_user: + db: postgres + name: "{{ item.role }}" + password: "{{ item.pass }}" + role_attr_flags: "{{ item.role_attr }}" + login_unix_socket: "{{ postgresql_unix_socket_dir }}" + port: "{{ postgresql_port }}" + loop: + - {role: '{{ patroni_superuser_username }}', pass: '{{ patroni_superuser_password }}', role_attr: 'SUPERUSER'} + - {role: '{{ patroni_replication_username }}', pass: '{{ patroni_replication_password }}', role_attr: 'LOGIN,REPLICATION'} + loop_control: + label: "{{ item.role }}" + when: is_master == "true" + + - name: Update postgresql authentication in patroni.yml + yedit: + src: /etc/patroni/patroni.yml + edits: + - key: postgresql.authentication.replication.username + value: "{{ patroni_replication_username }}" + - key: postgresql.authentication.replication.password + value: "{{ patroni_replication_password }}" + - key: postgresql.authentication.superuser.username + value: "{{ patroni_superuser_username }}" + - key: postgresql.authentication.superuser.password + value: "{{ patroni_superuser_password }}" + state: present + vars: + ansible_python_interpreter: /usr/bin/python3 + when: patroni_cluster_bootstrap_method != "initdb" and + (pgbackrest_install|bool or wal_g_install|bool) + become: true + become_user: postgres + tags: patroni, point_in_time_recovery + - block: # for add_pgnode.yml - name: Prepare PostgreSQL | fetch pg_hba.conf file from master run_once: true From 2fd6204f6db224bda85553e13727ee0c0b9643b7 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 11 Jun 2020 19:37:19 +0300 Subject: [PATCH 18/27] pgbackrest: fix libzstd dependency problem for CentOS 8.1 Workaround for CentOS 8.0/8.1. If you are using @pgBackRest on CentOS 8.1 (not RHEL), you will need to install libzstd RPM from an archived EPEL 8.1 release. The problem will be solved when CentOS 8.2 will be released. Fixed: TASK [pgbackrest : Install pgbackrest] ****************************************************************************************************************************************************************** fatal: [10.128.64.157]: FAILED! => {"changed": false, "failures": [], "msg": "Depsolve Error occured: \n Problem: cannot install the best candidate for the job\n - nothing provides libzstd.so.1()(64bit) needed by pgbackrest-2.27-2.rhel8.x86_64", "rc": 1, "results": []} --- roles/pgbackrest/tasks/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/roles/pgbackrest/tasks/main.yml b/roles/pgbackrest/tasks/main.yml index b3a836d7c..a5869426c 100644 --- a/roles/pgbackrest/tasks/main.yml +++ b/roles/pgbackrest/tasks/main.yml @@ -52,6 +52,30 @@ - pgbackrest_install_from_pgdg_repo|bool tags: pgbackrest, pgbackrest_repo, pgbackrest_install +# (workaround for CentOS 8.0/8.1) +# install libzstd RPM from an archived EPEL 8.1 release +# The problem will be solved when CentOS 8.2 will be released. +- block: + - name: Get libzstd rpm package from archived EPEL + get_url: + url: https://dl.fedoraproject.org/pub/archive/epel/8.1/Everything/x86_64/Packages/l/libzstd-1.4.4-1.el8.x86_64.rpm + dest: /tmp/ + timeout: 120 + validate_certs: false + register: get_libzstd_result + + - name: Install libzstd + package: + name: /tmp/libzstd-1.4.4-1.el8.x86_64.rpm + state: present + when: get_libzstd_result is changed + environment: "{{ proxy_env | default({}) }}" + when: + - ansible_distribution == "CentOS" + - ansible_distribution_major_version == '8' + - ansible_distribution_version is version('8.1', '<=') + tags: pgbackrest, pgbackrest_install + - name: Install pgbackrest package: name: pgbackrest From 2959084816660a9cdc25a37fca30b15187d66d00 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 16 Jun 2020 13:14:16 +0300 Subject: [PATCH 19/27] Patroni PITR: update postgresql authentication only when postgresql_user is changed. --- roles/patroni/tasks/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 04e6cb00c..2a18b36a5 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -526,7 +526,7 @@ - block: # PITR (custom bootstrap) # Prepare (install pexpect, ruamel.yaml) - - name: Prepare | Ensure the ansible required python library is exist + - name: Prepare | Make sure the ansible required python library is exist pip: name: "{{ item }}" state: present @@ -717,7 +717,7 @@ tags: patroni, pg_hba, pg_hba_generate - block: # PITR (custom bootstrap) - superuser and replication - - name: Ensure that archive recovery complete + - name: Make sure the Master is not in recovery mode command: "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -tAc 'SELECT pg_is_in_recovery()'" register: pg_is_in_recovery until: pg_is_in_recovery.stdout != "t" @@ -734,6 +734,7 @@ role_attr_flags: "{{ item.role_attr }}" login_unix_socket: "{{ postgresql_unix_socket_dir }}" port: "{{ postgresql_port }}" + register: postgresql_user_result loop: - {role: '{{ patroni_superuser_username }}', pass: '{{ patroni_superuser_password }}', role_attr: 'SUPERUSER'} - {role: '{{ patroni_replication_username }}', pass: '{{ patroni_replication_password }}', role_attr: 'LOGIN,REPLICATION'} @@ -756,6 +757,7 @@ state: present vars: ansible_python_interpreter: /usr/bin/python3 + when: hostvars[groups['master'][0]]['postgresql_user_result'] is changed when: patroni_cluster_bootstrap_method != "initdb" and (pgbackrest_install|bool or wal_g_install|bool) become: true From 590d8cbaa0e4a5624eb0a4ffb562ddaf49418830 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 16 Jun 2020 14:03:44 +0300 Subject: [PATCH 20/27] PITR: do not execute block "superuser and replication" for add_pgnode.yml --- roles/patroni/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 2a18b36a5..b9c46b536 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -759,7 +759,8 @@ ansible_python_interpreter: /usr/bin/python3 when: hostvars[groups['master'][0]]['postgresql_user_result'] is changed when: patroni_cluster_bootstrap_method != "initdb" and - (pgbackrest_install|bool or wal_g_install|bool) + (pgbackrest_install|bool or wal_g_install|bool) and + (existing_pgcluster is not defined or not existing_pgcluster|bool) become: true become_user: postgres tags: patroni, point_in_time_recovery From d94bfbc1d790e7c373d1919cc5904dca7794c660 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 16 Jun 2020 18:10:08 +0300 Subject: [PATCH 21/27] PITR: set hot_standby=off for TASK [patroni : Start PostgreSQL for Recovery] # for pgbackrest only (for use --delta restore) This is because pg_control on the standby remembers the previous primary server's max_connections. So you'll either have to have higher settings on the standby for at least one restart or simply start the standby for with hot_standby = off, and then re-enable it after it has replayed pending WAL. Fixed: TASK [patroni : Start PostgreSQL for Recovery] FATAL: hot standby is not possible because max_connections = 100 is a lower setting than on the master server (its value was 500) --- roles/patroni/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index b9c46b536..e32c0ffb7 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -606,13 +606,13 @@ when: item.ansible_job_id is defined - name: Start PostgreSQL for Recovery # Debian - command: "/usr/bin/pg_ctlcluster {{ postgresql_version }} {{ postgresql_cluster_name }} start" + command: "/usr/bin/pg_ctlcluster {{ postgresql_version }} {{ postgresql_cluster_name }} start -o '-c hot_standby=off'" when: ansible_os_family == "Debian" and (is_master == "true" or (is_master != "true" and 'pgbackrest' in patroni_create_replica_methods)) - name: Start PostgreSQL for Recovery # RedHat or PostgresPro - command: "{{ postgresql_bin_dir }}/pg_ctl start -D {{ postgresql_data_dir }}" + command: "{{ postgresql_bin_dir }}/pg_ctl start -D {{ postgresql_data_dir }} -o '-c hot_standby=off'" when: (ansible_os_family == "RedHat" or postgresql_packages is search("postgrespro")) and (is_master == "true" or (is_master != "true" and 'pgbackrest' in patroni_create_replica_methods)) From 7f01016697f56bd4c36d8407228fa6fd02521353 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 16 Jun 2020 19:23:05 +0300 Subject: [PATCH 22/27] Set become_user: postgres for TASK [deploy-finish : Get postgresql database list] Fixed: FATAL: Peer authentication failed for user "postgres". --- roles/deploy-finish/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/deploy-finish/tasks/main.yml b/roles/deploy-finish/tasks/main.yml index ce7c78dbe..1c5d60db9 100644 --- a/roles/deploy-finish/tasks/main.yml +++ b/roles/deploy-finish/tasks/main.yml @@ -22,6 +22,8 @@ - block: - name: Get postgresql database list run_once: true + become: true + become_user: postgres command: "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U postgres -c \" From 427f4e991884b71d670389e864df1d4dcaad0306 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Sun, 21 Jun 2020 18:26:29 +0300 Subject: [PATCH 23/27] Add role "etc_hosts" This role is optional. The lines specified in the "etc_hosts" variable will be added to the hosts file for postgresql_cluster nodes. --- add_pgnode.yml | 1 + deploy_pgcluster.yml | 1 + roles/etc_hosts/tasks/main.yml | 15 +++++++++++++++ tags.md | 1 + vars/system.yml | 5 +++++ 5 files changed, 23 insertions(+) create mode 100644 roles/etc_hosts/tasks/main.yml diff --git a/add_pgnode.yml b/add_pgnode.yml index f6ae2d4a8..d1a4874bb 100644 --- a/add_pgnode.yml +++ b/add_pgnode.yml @@ -62,6 +62,7 @@ - role: hostname - role: resolv_conf + - role: etc_hosts - role: add-repository - role: packages - role: sudo diff --git a/deploy_pgcluster.yml b/deploy_pgcluster.yml index d6ead578e..68ce167e0 100644 --- a/deploy_pgcluster.yml +++ b/deploy_pgcluster.yml @@ -77,6 +77,7 @@ - role: hostname - role: resolv_conf + - role: etc_hosts - role: add-repository - role: packages - role: sudo diff --git a/roles/etc_hosts/tasks/main.yml b/roles/etc_hosts/tasks/main.yml new file mode 100644 index 000000000..bdfd780d1 --- /dev/null +++ b/roles/etc_hosts/tasks/main.yml @@ -0,0 +1,15 @@ +--- + +- name: Add entries into /etc/hosts file + lineinfile: + path: /etc/hosts + regexp: "^{{ item }}" + line: "{{ item }}" + unsafe_writes: true # to prevent failures in CI + loop: "{{ etc_hosts }}" + when: + - etc_hosts is defined + - etc_hosts | length > 0 + tags: etc_hosts + +... diff --git a/tags.md b/tags.md index 7a142d9ae..6db97c490 100644 --- a/tags.md +++ b/tags.md @@ -12,6 +12,7 @@ - firewall - hostname - dns, nameservers +- etc_hosts - sysctl, kernel - disable_thp - limits diff --git a/vars/system.yml b/vars/system.yml index 570225184..a76dac826 100644 --- a/vars/system.yml +++ b/vars/system.yml @@ -7,6 +7,11 @@ nameservers: - "8.8.8.8" # example (Google Public DNS) - "9.9.9.9" # (Quad9 Public DNS) +# /etc/hosts (optional) +etc_hosts: [] +# - "10.128.64.143 pgbackrest.minio.local minio.local s3.eu-west-3.amazonaws.com" # example (MinIO) +# - "" + ntp_enabled: false # or 'true' if you want to install and configure the ntp service ntp_servers: [] # - "10.128.64.44" From 81c485b38864e6351a053cf1cab4d06e44df8eae Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 22 Jun 2020 18:32:14 +0300 Subject: [PATCH 24/27] pgbackrest: update repo1-s3-endpoint pgbackres accepts the DNS name format only. --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index 088e219ea..c4453c57a 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -288,7 +288,7 @@ pgbackrest_conf: - {option: "repo1-host", value: "{{ pgbackrest_repo_host }}"} - {option: "repo1-host-user", value: "{{ pgbackrest_repo_user }}"} # - {option: "repo1-path", value: "/repo"} -# - {option: "repo1-s3-endpoint", value: "http://172.26.9.200:9000"} +# - {option: "repo1-s3-endpoint", value: "minio.local"} # - {option: "repo1-s3-bucket", value: "pgbackrest"} # - {option: "repo1-s3-verify-tls", value: "n"} # - {option: "repo1-s3-key", value: "accessKey"} From f15e6d1665cb4125e3f6458f7235d09af4ef2fc3 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 23 Jun 2020 17:48:48 +0300 Subject: [PATCH 25/27] disable pgbackrest bootstrap script Not used. --- roles/pgbackrest/tasks/bootstrap_script.yml | 24 +++++++++++++++++++ roles/pgbackrest/tasks/main.yml | 26 +++++---------------- tags.md | 2 +- 3 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 roles/pgbackrest/tasks/bootstrap_script.yml diff --git a/roles/pgbackrest/tasks/bootstrap_script.yml b/roles/pgbackrest/tasks/bootstrap_script.yml new file mode 100644 index 000000000..b69c68873 --- /dev/null +++ b/roles/pgbackrest/tasks/bootstrap_script.yml @@ -0,0 +1,24 @@ +--- + +- block: # patroni cluster bootstrap script + - name: Make sure the pgbackrest bootstrap script directory exist + file: + dest: /etc/patroni + state: directory + owner: postgres + group: postgres + + - name: Create /etc/patroni/pgbackrest_bootstrap.sh script + template: + src: templates/pgbackrest_bootstrap.sh.j2 + dest: /etc/patroni/pgbackrest_bootstrap.sh + owner: postgres + group: postgres + mode: 0775 + when: + - patroni_cluster_bootstrap_method is defined + - patroni_cluster_bootstrap_method == "pgbackrest" + - "'postgres_cluster' in group_names" + tags: pgbackrest, pgbackrest_bootstrap_script + +... diff --git a/roles/pgbackrest/tasks/main.yml b/roles/pgbackrest/tasks/main.yml index a5869426c..b3236dc62 100644 --- a/roles/pgbackrest/tasks/main.yml +++ b/roles/pgbackrest/tasks/main.yml @@ -107,25 +107,11 @@ - pgbackrest_repo_host | length > 0 tags: pgbackrest, pgbackrest_ssh_keys -- block: # if patroni_cluster_bootstrap_method: "pgbackrest" - - name: Make sure the pgbackrest bootstrap script directory exist - file: - dest: /etc/patroni - state: directory - owner: postgres - group: postgres - - - name: Create /etc/patroni/pgbackrest_bootstrap.sh script - template: - src: templates/pgbackrest_bootstrap.sh.j2 - dest: /etc/patroni/pgbackrest_bootstrap.sh - owner: postgres - group: postgres - mode: 0775 - when: - - patroni_cluster_bootstrap_method is defined - - patroni_cluster_bootstrap_method == "pgbackrest" - - "'postgres_cluster' in group_names" - tags: pgbackrest, pgbackrest_bootstrap_script +# - import_tasks: bootstrap_script.yml +# when: +# - patroni_cluster_bootstrap_method is defined +# - patroni_cluster_bootstrap_method == "pgbackrest" +# - "'postgres_cluster' in group_names" +# tags: pgbackrest, pgbackrest_bootstrap_script ... diff --git a/tags.md b/tags.md index 6db97c490..4b71010a8 100644 --- a/tags.md +++ b/tags.md @@ -84,4 +84,4 @@ - - pgbackrest_install - - pgbackrest_conf - - pgbackrest_ssh_keys -- - pgbackrest_bootstrap_script + From 09f0dad85ff1d6b0991bfe5b8743e13d4d3cd1ad Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 23 Jun 2020 18:10:19 +0300 Subject: [PATCH 26/27] Update vars/main.yml. Delete exmaple pgbackrest conf for S3. An example configuration for S3 will be described separately. In order to reduce the number of lines in the variable file. --- vars/main.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/vars/main.yml b/vars/main.yml index c4453c57a..dddf1c696 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -287,13 +287,6 @@ pgbackrest_conf: - {option: "repo1-type", value: "{{ pgbackrest_repo_type |lower }}"} - {option: "repo1-host", value: "{{ pgbackrest_repo_host }}"} - {option: "repo1-host-user", value: "{{ pgbackrest_repo_user }}"} -# - {option: "repo1-path", value: "/repo"} -# - {option: "repo1-s3-endpoint", value: "minio.local"} -# - {option: "repo1-s3-bucket", value: "pgbackrest"} -# - {option: "repo1-s3-verify-tls", value: "n"} -# - {option: "repo1-s3-key", value: "accessKey"} -# - {option: "repo1-s3-key-secret", value: "superSECRETkey"} -# - {option: "repo1-s3-region", value: "us-east-1"} # - {option: "", value: ""} stanza: # [stanza_name] section - {option: "pg1-path", value: "{{ postgresql_data_dir }}"} From 5d894d22948df7ac5ff1dfcbf5b876f3fd4a227b Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 23 Jun 2020 18:45:40 +0300 Subject: [PATCH 27/27] Update the pgbackrest conf path In v2.02 the default location of the pgBackRest configuration file has changed from /etc/pgbackrest.conf to /etc/pgbackrest/pgbackrest.conf. If /etc/pgbackrest/pgbackrest.conf does not exist, the /etc/pgbackrest.conf file will be loaded instead, if it exists. --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index dddf1c696..7d7bc1623 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -278,7 +278,7 @@ pgbackrest_stanza: "stanza_name" # specify your --stanza pgbackrest_repo_type: "posix" # or "s3" pgbackrest_repo_host: "10.128.64.50" # dedicated repository host (if repo_type: "posix") pgbackrest_repo_user: "postgres" # if "repo_host" is set -pgbackrest_conf_file: "/etc/pgbackrest.conf" +pgbackrest_conf_file: "/etc/pgbackrest/pgbackrest.conf" # see more options https://pgbackrest.org/configuration.html pgbackrest_conf: global: # [global] section