From 97e60abc2bf2c6d70f2aecb3382d350ded69e4c0 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Wed, 21 Feb 2024 14:12:12 +0300 Subject: [PATCH] update_pgcluster.yml: Wait for caches to warm up after reboot (#580) --- config_pgcluster.yml | 4 +++ roles/patroni/tasks/custom_wal_dir.yml | 2 +- roles/update/README.md | 29 ++++++++++++++------ roles/update/tasks/extensions.yml | 2 +- roles/update/tasks/pre_checks.yml | 8 +++--- roles/update/tasks/start_traffic.yml | 10 +++++++ roles/update/tasks/stop_services.yml | 4 +-- roles/update/tasks/stop_traffic.yml | 2 +- roles/update/tasks/system.yml | 6 +++- roles/update/tasks/update_extensions.yml | 10 +++---- roles/update/vars/main.yml | 14 ++++++++++ roles/upgrade/tasks/extensions.yml | 2 +- roles/upgrade/tasks/initdb.yml | 4 +-- roles/upgrade/tasks/post_checks.yml | 8 +++--- roles/upgrade/tasks/post_upgrade.yml | 4 +-- roles/upgrade/tasks/pre_checks.yml | 22 +++++++-------- roles/upgrade/tasks/rollback.yml | 2 +- roles/upgrade/tasks/schema_compatibility.yml | 4 +-- roles/upgrade/tasks/statistics.yml | 2 +- roles/upgrade/tasks/stop_services.yml | 6 ++-- roles/upgrade/tasks/update_extensions.yml | 12 ++++---- roles/upgrade/tasks/upgrade_check.yml | 2 +- update_pgcluster.yml | 24 ++++++++-------- 23 files changed, 115 insertions(+), 68 deletions(-) create mode 100644 roles/update/vars/main.yml diff --git a/config_pgcluster.yml b/config_pgcluster.yml index 52692536e..54e382778 100644 --- a/config_pgcluster.yml +++ b/config_pgcluster.yml @@ -307,6 +307,10 @@ - name: Include system variables ansible.builtin.include_vars: "vars/system.yml" tags: always + + - name: Include OS-specific variables + ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" + tags: always roles: # finish (info) - role: deploy-finish diff --git a/roles/patroni/tasks/custom_wal_dir.yml b/roles/patroni/tasks/custom_wal_dir.yml index d1ae359a5..b39e9c025 100644 --- a/roles/patroni/tasks/custom_wal_dir.yml +++ b/roles/patroni/tasks/custom_wal_dir.yml @@ -39,7 +39,7 @@ become: true become_user: postgres ansible.builtin.command: > - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "CHECKPOINT" + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "CHECKPOINT" - name: Stop patroni service on the Replica (for create symlink) become: true diff --git a/roles/update/README.md b/roles/update/README.md index a3e66c198..3a6b292b1 100644 --- a/roles/update/README.md +++ b/roles/update/README.md @@ -24,19 +24,30 @@ Update all system packages: - `target` - Defines the target for the update. - Available values: 'postgres', 'patroni', 'system' - - Default value: postgres + - Default value: `postgres` - `max_replication_lag_bytes` - Determines the size of the replication lag above which the update will not be performed. - - If the lag is high, you will be prompted to try again later. - - Default value: 10485760 (10 MiB) + - Note: If the lag is high, you will be prompted to try again later. + - Default value: `10485760` (10 MiB) - `max_transaction_sec` - Determines the maximum transaction time, in the presence of which the update will not be performed. - - If long-running transactions are present, you will be prompted to try again later. - - Default value: 15 (seconds) + - Note: If long-running transactions are present, you will be prompted to try again later. + - Default value: `15` (seconds) - `update_extensions` - - If 'true', an attempt will be made to automatically update all extensions for all databases. - - Specify 'false', to avoid updating extensions. - - Default value: true + - Attempt to automatically update all PostgreSQL extensions in all databases. + - Note: Specify 'false', to avoid updating extensions. + - Default value: `true` +- `reboot_host_after_update` + - Restart the server if it is required after the update. + - Default value: `true` +- `reboot_host_timeout` + - Maximum seconds to wait for machine to reboot and respond to a test command. + - Default value: `1800` (30 minutes) +- `reboot_host_post_delay` + - The waiting time (in minutes) for the caches to warm up after restarting the server before updating the next server. + - Note: Applicable when there are multiple replicas. + - Default value: `5` (minutes). + --- ## Plan: @@ -82,6 +93,8 @@ When using load balancing for read-only traffic (the "Type A" and "Type C" schem - Disable `noloadbalance`, `nosync`, `nofailover` parameters in the patroni.yml - Reload patroni service - Make sure replica endpoint is available + - Wait N minutes for caches to warm up after reboot + - Note: variable `reboot_host_post_delay` - Perform the same steps for the next replica server. #### 3. UPDATE: Primary - Switchover Patroni leader role diff --git a/roles/update/tasks/extensions.yml b/roles/update/tasks/extensions.yml index 9339f0d3b..0e631800d 100644 --- a/roles/update/tasks/extensions.yml +++ b/roles/update/tasks/extensions.yml @@ -11,7 +11,7 @@ - name: Get a list of databases ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select datname from pg_catalog.pg_database where datname <> 'template0'" register: databases_list changed_when: false diff --git a/roles/update/tasks/pre_checks.yml b/roles/update/tasks/pre_checks.yml index 44cf0ad08..96281b22e 100644 --- a/roles/update/tasks/pre_checks.yml +++ b/roles/update/tasks/pre_checks.yml @@ -1,12 +1,12 @@ --- - name: '[Pre-Check] (ALL) Test PostgreSQL DB Access' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc 'select 1' + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc 'select 1' changed_when: false - name: '[Pre-Check] Make sure that physical replication is active' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select count(*) from pg_stat_replication where application_name != 'pg_basebackup'" register: pg_replication_state @@ -24,7 +24,7 @@ - name: '[Pre-Check] Make sure there is no high replication lag (more than {{ max_replication_lag_bytes | human_readable }})' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select pg_wal_lsn_diff(pg_current_wal_lsn(),replay_lsn) pg_lag_bytes from pg_stat_replication order by pg_lag_bytes desc limit 1" @@ -53,7 +53,7 @@ - name: '[Pre-Check] Make sure there are no long-running transactions (more than {{ max_transaction_sec }} seconds)' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select pid, usename, client_addr, clock_timestamp() - xact_start as xact_age, state, wait_event_type ||':'|| wait_event as wait_events, left(regexp_replace(query, E'[ \\t\\n\\r]+', ' ', 'g'),100) as query diff --git a/roles/update/tasks/start_traffic.yml b/roles/update/tasks/start_traffic.yml index 4a4cdd372..8fed19beb 100644 --- a/roles/update/tasks/start_traffic.yml +++ b/roles/update/tasks/start_traffic.yml @@ -28,4 +28,14 @@ delay: 2 environment: no_proxy: "{{ inventory_hostname }}" + +# Warming up caches after reboot (is 'reboot_host_post_delay' is defined) +- name: "Wait {{ reboot_host_post_delay }} minutes for caches to warm up after reboot" + ansible.builtin.pause: + minutes: "{{ reboot_host_post_delay }}" + when: + - (reboot_result.rebooted is defined and reboot_result.rebooted) + - (reboot_host_post_delay is defined and reboot_host_post_delay | int > 0) + - (inventory_hostname in groups['secondary'] and groups['secondary'] | length > 1) + ... diff --git a/roles/update/tasks/stop_services.yml b/roles/update/tasks/stop_services.yml index d6f8f4144..534ab0416 100644 --- a/roles/update/tasks/stop_services.yml +++ b/roles/update/tasks/stop_services.yml @@ -16,7 +16,7 @@ become: true become_user: postgres ansible.builtin.command: > - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "CHECKPOINT" + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "CHECKPOINT" - name: "Stop Patroni service on the Cluster Replica ({{ ansible_hostname }})" become: true @@ -32,7 +32,7 @@ become: true become_user: postgres ansible.builtin.command: > - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "CHECKPOINT" + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "CHECKPOINT" - name: "Stop Patroni service on the old Cluster Leader ({{ ansible_hostname }})" become: true diff --git a/roles/update/tasks/stop_traffic.yml b/roles/update/tasks/stop_traffic.yml index aea40853e..d3f74fc4f 100644 --- a/roles/update/tasks/stop_traffic.yml +++ b/roles/update/tasks/stop_traffic.yml @@ -33,7 +33,7 @@ become: true become_user: postgres ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select count(*) from pg_stat_activity where pid <> pg_backend_pid() diff --git a/roles/update/tasks/system.yml b/roles/update/tasks/system.yml index 2e640a2bc..646a1577e 100644 --- a/roles/update/tasks/system.yml +++ b/roles/update/tasks/system.yml @@ -67,6 +67,7 @@ - ansible_os_family == "RedHat" - ansible_distribution_major_version >= '8' +# Reboot (if 'reboot_host_after_update' is 'true') - name: Check if a reboot is required ansible.builtin.stat: path: /var/run/reboot-required @@ -76,6 +77,7 @@ when: - ansible_os_family == "Debian" - ansible_virtualization_type not in ['container', 'docker', 'lxc', 'podman'] # exclude for containers to prevent test failures in CI. + - reboot_host_after_update | bool - name: Check if a reboot is required ansible.builtin.command: needs-restarting -r @@ -85,12 +87,14 @@ when: - ansible_os_family == "RedHat" - ansible_virtualization_type not in ['container', 'docker', 'lxc', 'podman'] # exclude for containers to prevent test failures in CI. + - reboot_host_after_update | bool - name: Rebooting host ansible.builtin.reboot: msg: "Reboot initiated by Ansible due to required system updates" - reboot_timeout: 1800 # 30 minutes + reboot_timeout: "{{ reboot_host_timeout | int }}" test_command: uptime + register: reboot_result when: (reboot_required_debian.stat.exists is defined and reboot_required_debian.stat.exists) or (reboot_required_rhel.rc is defined and reboot_required_rhel.rc != 0) diff --git a/roles/update/tasks/update_extensions.yml b/roles/update/tasks/update_extensions.yml index 019c4ddc4..578a83863 100644 --- a/roles/update/tasks/update_extensions.yml +++ b/roles/update/tasks/update_extensions.yml @@ -1,7 +1,7 @@ --- - name: Get a list of old PostgreSQL extensions ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "select extname from pg_extension e join pg_available_extensions ae on extname = ae.name where installed_version <> default_version" @@ -21,7 +21,7 @@ # excluding: 'pg_repack' (is exists), as it requires re-creation to update - name: Update old PostgreSQL extensions ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "ALTER EXTENSION {{ item }} UPDATE" ignore_errors: true loop: "{{ pg_old_extensions.stdout_lines | reject('match', '^pg_repack$') | list }}" @@ -36,7 +36,7 @@ # excluding: 'pg_stat_statements', because extension pg_stat_kcache depends on it (will be re-created) - name: Update old PostgreSQL extensions ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "ALTER EXTENSION {{ item }} UPDATE" ignore_errors: true loop: "{{ pg_old_extensions.stdout_lines | reject('match', '^(pg_repack|pg_stat_statements|pg_stat_kcache)$') | list }}" @@ -45,7 +45,7 @@ # re-create 'pg_stat_statements' and 'pg_stat_kcache' if an update is required - name: Recreate old pg_stat_statements and pg_stat_kcache extensions to update ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "DROP EXTENSION pg_stat_statements CASCADE; CREATE EXTENSION pg_stat_statements; CREATE EXTENSION pg_stat_kcache" @@ -58,7 +58,7 @@ # re-create the 'pg_repack' if it exists and an update is required - name: Recreate old pg_repack extension to update ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "DROP EXTENSION pg_repack; CREATE EXTENSION pg_repack;" when: diff --git a/roles/update/vars/main.yml b/roles/update/vars/main.yml new file mode 100644 index 000000000..4061f3c8f --- /dev/null +++ b/roles/update/vars/main.yml @@ -0,0 +1,14 @@ +--- + +target: postgres # Defines the target for the update. Available values: 'postgres', 'patroni', 'system' + +update_extensions: true # Attempt will be made to automatically update all PostgreSQL extensions in all databases. + +# if target=system +reboot_host_after_update: true # Restart the server if it is required after the update. +reboot_host_timeout: 1800 # Maximum seconds to wait for machine to reboot and respond to a test command. +reboot_host_post_delay: 5 # The waiting time (in minutes) for the caches to warm up after restarting the server before updating the next server. + +# pre-checks vars +max_replication_lag_bytes: 10485760 # (10 MiB) Determines the size of the replication lag above which the update will not be performed. +max_transaction_sec: 15 # (seconds) Determines the maximum transaction time, in the presence of which the update will not be performed. diff --git a/roles/upgrade/tasks/extensions.yml b/roles/upgrade/tasks/extensions.yml index 555f71b3e..c730e241b 100644 --- a/roles/upgrade/tasks/extensions.yml +++ b/roles/upgrade/tasks/extensions.yml @@ -2,7 +2,7 @@ - name: Get a list of databases ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select datname from pg_catalog.pg_database where datname <> 'template0'" register: databases_list changed_when: false diff --git a/roles/upgrade/tasks/initdb.yml b/roles/upgrade/tasks/initdb.yml index 855c572be..098ee853c 100644 --- a/roles/upgrade/tasks/initdb.yml +++ b/roles/upgrade/tasks/initdb.yml @@ -40,7 +40,7 @@ - name: Get the current install user ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select rolname from pg_roles where oid = 10" changed_when: false register: pg_install_user @@ -49,7 +49,7 @@ - name: Get the current encodig and data_checksums settings ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "show {{ item }}" changed_when: false register: pg_settings diff --git a/roles/upgrade/tasks/post_checks.yml b/roles/upgrade/tasks/post_checks.yml index 602bdb4a6..f6b050f88 100644 --- a/roles/upgrade/tasks/post_checks.yml +++ b/roles/upgrade/tasks/post_checks.yml @@ -2,7 +2,7 @@ - name: Make sure that physical replication is active ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select count(*) from pg_stat_replication where application_name != 'pg_basebackup'" register: pg_replication_state @@ -28,7 +28,7 @@ - name: Create a table "test_replication" with 10000 rows on the Primary ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "drop table IF EXISTS test_replication; create table test_replication as select generate_series(1, 10000)" when: @@ -36,7 +36,7 @@ - name: Wait until the PostgreSQL replica is synchronized ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select count(*) from test_replication" register: count_test until: count_test.stdout | int == 10000 @@ -49,7 +49,7 @@ - name: Drop a table "test_replication" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "drop table IF EXISTS test_replication" when: - inventory_hostname in groups['primary'] diff --git a/roles/upgrade/tasks/post_upgrade.yml b/roles/upgrade/tasks/post_upgrade.yml index 51de2f6a0..591eb9a89 100644 --- a/roles/upgrade/tasks/post_upgrade.yml +++ b/roles/upgrade/tasks/post_upgrade.yml @@ -2,7 +2,7 @@ - name: Ensure the current data directory is the new data directory ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "show data_directory" changed_when: false register: pg_current_datadir @@ -165,7 +165,7 @@ - name: Check the current PostgreSQL version run_once: true ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select current_setting('server_version')" register: postgres_version changed_when: false diff --git a/roles/upgrade/tasks/pre_checks.yml b/roles/upgrade/tasks/pre_checks.yml index adc83ed4e..6ee38ec75 100644 --- a/roles/upgrade/tasks/pre_checks.yml +++ b/roles/upgrade/tasks/pre_checks.yml @@ -38,7 +38,7 @@ - name: '[Pre-Check] Test PostgreSQL database access using a unix socket' ansible.builtin.command: > - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc 'select 1' + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc 'select 1' register: socket_access_result changed_when: false failed_when: "socket_access_result.rc != 0 and 'no pg_hba.conf entry' not in socket_access_result.stderr" @@ -62,7 +62,7 @@ - name: '[Pre-Check] Check the current version of PostgreSQL' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select setting::integer/10000 from pg_settings where name = 'server_version_num'" register: pg_current_version changed_when: false @@ -73,7 +73,7 @@ # for compatibility with Postgres 9.x - name: '[Pre-Check] Check the current version of PostgreSQL' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select substring(setting from '^[0-9]+\.[0-9]+') from pg_settings where name = 'server_version'" register: pg_current_version_9x changed_when: false @@ -112,7 +112,7 @@ # which will later be cleaned up before executing initdb for a new version of PostgreSQL - name: '[Pre-Check] Ensure new data directory is different from the current one' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "show data_directory" changed_when: false register: pg_current_datadir @@ -140,7 +140,7 @@ - name: '[Pre-Check] Make sure that physical replication is active' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select count(*) from pg_stat_replication where application_name != 'pg_basebackup'" register: pg_replication_state @@ -158,7 +158,7 @@ - name: '[Pre-Check] Make sure there is no high replication lag (more than {{ max_replication_lag_bytes | human_readable }})' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select pg_wal_lsn_diff(pg_current_wal_lsn(),replay_lsn) as pg_lag_bytes from pg_stat_replication order by pg_lag_bytes desc limit 1" @@ -185,7 +185,7 @@ # for compatibility with Postgres 9.x - name: '[Pre-Check] Make sure there is no high replication lag (more than {{ max_replication_lag_bytes | human_readable }})' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select pg_xlog_location_diff(pg_current_xlog_location(),replay_location) as pg_lag_bytes from pg_stat_replication order by pg_lag_bytes desc limit 1" @@ -211,7 +211,7 @@ - name: '[Pre-Check] Make sure there are no long-running transactions (more than {{ max_transaction_sec }} seconds)' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select pid, usename, client_addr, clock_timestamp() - xact_start as xact_age, state, wait_event_type ||':'|| wait_event as wait_events, left(regexp_replace(query, E'[ \\t\\n\\r]+', ' ', 'g'),100) as query @@ -244,7 +244,7 @@ # for compatibility with Postgres 9.x - name: '[Pre-Check] Make sure there are no long-running transactions (more than {{ max_transaction_sec }} seconds)' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select pid, usename, client_addr, clock_timestamp() - xact_start as xact_age, state, left(regexp_replace(query, E'[ \\t\\n\\r]+', ' ', 'g'),100) as query from pg_stat_activity @@ -317,7 +317,7 @@ # Tablespaces - name: '[Pre-Check] Check if PostgreSQL tablespaces exist' ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select pg_tablespace_location(oid) as tablespace_location from pg_tablespace where spcname not in ('pg_default','pg_global')" @@ -354,7 +354,7 @@ # Test access via unix socket to be able to perform 'PAUSE' command - name: '[Pre-Check] Test PgBouncer access via unix socket' ansible.builtin.command: >- - psql -h /var/run/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }} + {{ postgresql_bin_dir }}/psql -h /var/run/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }} -p {{ pgbouncer_listen_port }} -U {{ patroni_superuser_username }} -d pgbouncer diff --git a/roles/upgrade/tasks/rollback.yml b/roles/upgrade/tasks/rollback.yml index 0adbc01a5..1fc3bbf3c 100644 --- a/roles/upgrade/tasks/rollback.yml +++ b/roles/upgrade/tasks/rollback.yml @@ -195,7 +195,7 @@ - block: - name: Check the PostgreSQL version ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select current_setting('server_version')" register: postgres_version changed_when: false diff --git a/roles/upgrade/tasks/schema_compatibility.yml b/roles/upgrade/tasks/schema_compatibility.yml index e3a3b460c..a1bb81c2f 100644 --- a/roles/upgrade/tasks/schema_compatibility.yml +++ b/roles/upgrade/tasks/schema_compatibility.yml @@ -2,7 +2,7 @@ - name: Get the current shared_preload_libraries settings ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "show shared_preload_libraries" changed_when: false register: pg_shared_preload_libraries @@ -11,7 +11,7 @@ - name: Get the current cron.database_name settings ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select current_setting('cron.database_name', true)" changed_when: false register: pg_cron_database_name diff --git a/roles/upgrade/tasks/statistics.yml b/roles/upgrade/tasks/statistics.yml index 0320f037e..706171117 100644 --- a/roles/upgrade/tasks/statistics.yml +++ b/roles/upgrade/tasks/statistics.yml @@ -13,7 +13,7 @@ ansible.builtin.shell: | echo $$ > /tmp/pg_terminator.pid for i in {1..{{ vacuumdb_analyze_timeout // 10 }}}; do - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc " + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc " with blocker_pids(pid) as ( select unnest(pg_blocking_pids(pid)) from pg_stat_activity diff --git a/roles/upgrade/tasks/stop_services.yml b/roles/upgrade/tasks/stop_services.yml index 892f2a8ad..dce95798e 100644 --- a/roles/upgrade/tasks/stop_services.yml +++ b/roles/upgrade/tasks/stop_services.yml @@ -2,7 +2,7 @@ - name: "Execute CHECKPOINT before stopping PostgreSQL" ansible.builtin.command: > - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "CHECKPOINT" + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "CHECKPOINT" async: "{{ pg_start_stop_timeout | int }}" # run the command asynchronously poll: 0 register: checkpoint_result @@ -18,7 +18,7 @@ # Wait for the window to appear without high replication lag before stopping PostgreSQL - name: "Wait until replication lag is less than {{ max_replication_lag_bytes | human_readable }}" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select coalesce(max(pg_wal_lsn_diff(pg_current_wal_lsn(),replay_lsn)),1) as pg_lag_bytes from pg_stat_replication" register: pg_lag_bytes @@ -51,7 +51,7 @@ # for compatibility with Postgres 9.x - name: "Wait until replication lag is less than {{ max_replication_lag_bytes | human_readable }}" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select coalesce(max(pg_xlog_location_diff(pg_current_xlog_location(),replay_location)),1) as pg_lag_bytes from pg_stat_replication" register: pg_lag_bytes_9x diff --git a/roles/upgrade/tasks/update_extensions.yml b/roles/upgrade/tasks/update_extensions.yml index 62e5f0bd3..694dd6df5 100644 --- a/roles/upgrade/tasks/update_extensions.yml +++ b/roles/upgrade/tasks/update_extensions.yml @@ -2,7 +2,7 @@ - name: "Get list of installed PostgreSQL extensions (database: {{ pg_target_dbname }})" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "select extname from pg_catalog.pg_extension" register: pg_installed_extensions changed_when: false @@ -11,7 +11,7 @@ - name: "Get list of old PostgreSQL extensions (database: {{ pg_target_dbname }})" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "select extname from pg_catalog.pg_extension e join pg_catalog.pg_available_extensions ae on extname = ae.name where installed_version <> default_version" @@ -34,7 +34,7 @@ # excluding: 'pg_repack' (is exists), as it requires re-creation to update - name: "Update old PostgreSQL extensions (database: {{ pg_target_dbname }})" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "ALTER EXTENSION {{ item }} UPDATE" ignore_errors: true # show the error and continue the playbook execution loop: "{{ pg_old_extensions.stdout_lines | reject('match', '^pg_repack$') | list }}" @@ -48,7 +48,7 @@ # excluding: 'pg_stat_statements','pg_stat_kcache', because extension pg_stat_kcache depends on it (will be re-created) - name: "Update old PostgreSQL extensions (database: {{ pg_target_dbname }})" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "ALTER EXTENSION {{ item }} UPDATE" ignore_errors: true # show the error and continue the playbook execution loop: "{{ pg_old_extensions.stdout_lines | reject('match', '^(pg_repack|pg_stat_statements|pg_stat_kcache)$') | list }}" @@ -56,7 +56,7 @@ # re-create 'pg_stat_statements' and 'pg_stat_kcache' if an update is required - name: "Recreate old pg_stat_statements and pg_stat_kcache extensions to update (database: {{ pg_target_dbname }})" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "DROP EXTENSION pg_stat_statements CASCADE; CREATE EXTENSION pg_stat_statements; CREATE EXTENSION pg_stat_kcache" @@ -70,7 +70,7 @@ # re-create the 'pg_repack' if it exists and an update is required - name: "Recreate old pg_repack extension to update (database: {{ pg_target_dbname }})" ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc + {{ pg_new_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d {{ pg_target_dbname }} -tAXc "DROP EXTENSION pg_repack; CREATE EXTENSION pg_repack;" ignore_errors: true # show the error and continue the playbook execution diff --git a/roles/upgrade/tasks/upgrade_check.yml b/roles/upgrade/tasks/upgrade_check.yml index e9d52329c..6fc42b605 100644 --- a/roles/upgrade/tasks/upgrade_check.yml +++ b/roles/upgrade/tasks/upgrade_check.yml @@ -2,7 +2,7 @@ - name: Get the current shared_preload_libraries settings ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "show shared_preload_libraries" changed_when: false register: pg_shared_preload_libraries_result diff --git a/update_pgcluster.yml b/update_pgcluster.yml index f0b059675..b0e201671 100644 --- a/update_pgcluster.yml +++ b/update_pgcluster.yml @@ -53,12 +53,13 @@ become: true become_user: postgres any_errors_fatal: true - vars: - max_replication_lag_bytes: 10485760 # 10 MiB - max_transaction_sec: 15 # seconds pre_tasks: + - name: Include role variables + ansible.builtin.include_vars: "roles/update/vars/main.yml" - name: Include main variables ansible.builtin.include_vars: "vars/main.yml" + - name: Include OS-specific variables + ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" tasks: - name: Running Pre-Checks ansible.builtin.include_role: @@ -76,14 +77,13 @@ become_method: sudo any_errors_fatal: true environment: "{{ proxy_env | default({}) }}" - vars: - target: postgres # or 'patroni', 'system' pre_tasks: + - name: Include role variables + ansible.builtin.include_vars: "roles/update/vars/main.yml" - name: Include main variables ansible.builtin.include_vars: "vars/main.yml" - name: Include OS-specific variables ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" - tasks: - name: Stop read-only traffic ansible.builtin.include_role: @@ -133,9 +133,9 @@ become_method: sudo any_errors_fatal: true environment: "{{ proxy_env | default({}) }}" - vars: - target: postgres # or 'patroni', 'system' pre_tasks: + - name: Include role variables + ansible.builtin.include_vars: "roles/update/vars/main.yml" - name: Include main variables ansible.builtin.include_vars: "vars/main.yml" - name: Include OS-specific variables @@ -193,11 +193,13 @@ become: true become_user: postgres any_errors_fatal: true - vars: - update_extensions: true # or 'false', to avoid updating extensions pre_tasks: + - name: Include role variables + ansible.builtin.include_vars: "roles/update/vars/main.yml" - name: Include main variables ansible.builtin.include_vars: "vars/main.yml" + - name: Include OS-specific variables + ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" tasks: - name: Update extensions ansible.builtin.include_role: @@ -228,7 +230,7 @@ - name: Check the current PostgreSQL version run_once: true # noqa run-once ansible.builtin.command: >- - psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc + {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc "select current_setting('server_version')" register: postgres_version changed_when: false