Added pgbouncer authentication via auth_user and auth_query (#401)

This commit brings a significant improvement to the database's authentication process through pgbouncer, transitioning from the auth_file method to auth_user and auth_query. Previously, we relied on the auth_file method, which required maintaining and updating a userlist.txt file each time a user was added to or removed from the database. This method was less flexible and efficient, as it required manual management. The new implementation utilizes auth_user and auth_query, automating and simplifying the process. The auth_user in pgbouncer allows a single specified user (pgbouncer_auth_username) to query the user's password for authentication using auth_query. This removes the necessity to manually manage a userlist.txt file, thus streamlining the database users and pgbouncer pool management.
vitabaks · Jul 8, 2023 · f5cbe53 · f5cbe53
1 parent 078d87e
commit f5cbe53
Show file tree

Hide file tree

Showing 7 changed files with 99 additions and 50 deletions.
diff --git a/config_pgcluster.yml b/config_pgcluster.yml
@@ -162,7 +162,7 @@
       when: is_master | bool and postgresql_extensions | length > 0
 
     - role: pgbouncer/config
-      when: pgbouncer_install|bool and pgbouncer_generate_userlist|bool
+      when: pgbouncer_install|bool
 
 - name: config_pgcluster.yml | Check needed restart cluster and prepare for it
   hosts: postgres_cluster

diff --git a/deploy_pgcluster.yml b/deploy_pgcluster.yml
@@ -1,4 +1,5 @@
 ---
+
 - name: Deploy PostgreSQL HA Cluster (based on "Patroni" and "{{ dcs_type }}")
   hosts: all
   become: true
@@ -245,7 +246,7 @@
       when: is_master | bool and postgresql_extensions | length > 0
 
     - role: pgbouncer/config
-      when: pgbouncer_install|bool and pgbouncer_generate_userlist|bool
+      when: pgbouncer_install|bool
 
     - role: netdata
       when: netdata_install is defined and netdata_install|bool

diff --git a/roles/pgbouncer/config/tasks/main.yml b/roles/pgbouncer/config/tasks/main.yml
@@ -19,33 +19,51 @@
   when: existing_pgcluster is not defined or not existing_pgcluster|bool
   tags: pgbouncer, pgbouncer_conf
 
-- name: Get users and password md5 from pg_shadow
-  run_once: true
-  become: true
-  become_user: postgres
-  command: >
-    {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -Atq
-    -c "SELECT concat('\"', usename, '\" \"', passwd, '\"') FROM pg_shadow where usename != '{{ patroni_replication_username }}'"
-  register: pg_shadow_result
-  changed_when: false
-  delegate_to: "{{ groups.master[0] }}"
-  when: pgbouncer_generate_userlist|bool
-  tags: pgbouncer, pgbouncer_generate_userlist
+# if pgbouncer_auth_user is 'false'
+- block:
+    - name: Get users and password md5 from pg_shadow
+      run_once: true
+      become: true
+      become_user: postgres
+      command: >
+        {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -Atq
+        -c "SELECT concat('\"', usename, '\" \"', passwd, '\"') FROM pg_shadow where usename != '{{ patroni_replication_username }}'"
+      register: pg_shadow_result
+      changed_when: false
+      delegate_to: "{{ groups.master[0] }}"
 
-- name: Generate /etc/pgbouncer/userlist.txt
+    - name: "Generate {{ pgbouncer_conf_dir }}/userlist.txt"
+      become: true
+      become_user: postgres
+      copy:
+        content: |
+          {{ pg_shadow_result.stdout }}
+        dest: "{{ pgbouncer_conf_dir }}/userlist.txt"
+      notify: "reload pgbouncer"
+      when:
+        - pg_shadow_result.rc == 0
+        - pg_shadow_result.stdout is defined
+        - pg_shadow_result.stdout | length > 0
+  when: not pgbouncer_auth_user|bool
+  tags: pgbouncer, pgbouncer_conf, pgbouncer_generate_userlist
+
+# if pgbouncer_auth_user is 'true'
+- name: "Create function 'user_search' for pgbouncer 'auth_query' option in all databases"
   become: true
   become_user: postgres
-  copy:
-    content: |
-      {{ pg_shadow_result.stdout }}
-
-    dest: /etc/pgbouncer/userlist.txt
-  notify: "reload pgbouncer"
-  when:
-    - pg_shadow_result.rc == 0
-    - pg_shadow_result.stdout is defined
-    - pg_shadow_result.stdout | length > 0
-    - pgbouncer_generate_userlist|bool
-  tags: pgbouncer, pgbouncer_generate_userlist
+  ansible.builtin.shell: |
+    for db in $({{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -tAXc \
+    "select datname from pg_catalog.pg_database where datname <> 'template0'"); do
+      {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d "$db" -tAXc '
+        CREATE OR REPLACE FUNCTION user_search(uname TEXT) RETURNS TABLE (usename name, passwd text) AS
+        $$
+        SELECT usename, passwd FROM pg_shadow WHERE usename=$1;
+        $$
+        LANGUAGE sql SECURITY DEFINER;
+      '; done
+  args:
+    executable: /bin/bash
+  when: pgbouncer_auth_user|bool and is_master|bool
+  tags: pgbouncer, pgbouncer_conf, pgbouncer_auth_query
 
 ...
diff --git a/roles/pgbouncer/tasks/main.yml b/roles/pgbouncer/tasks/main.yml
@@ -102,46 +102,66 @@
     owner: postgres
     group: postgres
     mode: "0640"
-  when: existing_pgcluster is not defined or not existing_pgcluster|bool
+  when:
+    - (existing_pgcluster is not defined or not existing_pgcluster|bool)
+    - not pgbouncer_auth_user|bool
   tags: pgbouncer
 
 - block:  # for add_pgnode.yml
-    - name: Fetch pgbouncer.ini and userlist.txt conf files from master
+    - name: Fetch pgbouncer.ini file from master
       run_once: true
       fetch:
-        src: "{{ item }}"
+        src: "{{ pgbouncer_conf_dir }}/pgbouncer.ini"
         dest: files/
         validate_checksum: true
         flat: true
-      loop:
-        - /etc/pgbouncer/pgbouncer.ini
-        - /etc/pgbouncer/userlist.txt
       delegate_to: "{{ groups.master[0] }}"
 
-    - name: Copy pgbouncer.ini and userlist.txt conf files to replica
+    - name: Fetch userlist.txt conf file from master
+      run_once: true
+      fetch:
+        src: "{{ pgbouncer_conf_dir }}/userlist.txt"
+        dest: files/
+        validate_checksum: true
+        flat: true
+      delegate_to: "{{ groups.master[0] }}"
+      when: not pgbouncer_auth_user|bool
+
+    - name: Copy pgbouncer.ini file to replica
       copy:
-        src: "files/{{ item }}"
-        dest: /etc/pgbouncer/
+        src: files/pgbouncer.ini
+        dest: "{{ pgbouncer_conf_dir }}"
         owner: postgres
         group: postgres
         mode: "0640"
-      loop:
-        - pgbouncer.ini
-        - userlist.txt
 
-    - name: Remove pgbouncer.ini and userlist.txt conf files from localhost
+    - name: Copy userlist.txt conf file to replica
+      copy:
+        src: files/userlist.txt
+        dest: "{{ pgbouncer_conf_dir }}"
+        owner: postgres
+        group: postgres
+        mode: "0640"
+      when: not pgbouncer_auth_user|bool
+
+    - name: Remove pgbouncer.ini file from localhost
       run_once: true
       file:
-        path: "files/{{ item }}"
+        path: files/pgbouncer.ini
+        state: absent
+      delegate_to: localhost
+
+    - name: Remove userlist.txt conf file from localhost
+      run_once: true
+      file:
+        path: files/userlist.txt
         state: absent
-      loop:
-        - pgbouncer.ini
-        - userlist.txt
       delegate_to: localhost
+      when: not pgbouncer_auth_user|bool
 
     - name: Prepare pgbouncer.ini conf file (replace "listen_addr")
       lineinfile:
-        path: /etc/pgbouncer/pgbouncer.ini
+        path: "{{ pgbouncer_conf_dir }}/pgbouncer.ini"
         regexp: "{{ item.regexp }}"
         line: "{{ item.line }}"
         backrefs: true
@@ -155,7 +175,7 @@
 
     - name: Prepare pgbouncer.ini conf file (replace "listen_addr")
       lineinfile:
-        path: /etc/pgbouncer/pgbouncer.ini
+        path: "{{ pgbouncer_conf_dir }}/pgbouncer.ini"
         regexp: "{{ item.regexp }}"
         line: "{{ item.line }}"
         backrefs: true

diff --git a/roles/pgbouncer/templates/pgbouncer.ini.j2 b/roles/pgbouncer/templates/pgbouncer.ini.j2
@@ -12,7 +12,12 @@ listen_addr = {{ pgbouncer_listen_addr | default('0.0.0.0') }}
 listen_port = {{ pgbouncer_listen_port | default(6432) }}
 unix_socket_dir = /var/run/postgresql
 auth_type = {{ pgbouncer_auth_type }}
-auth_file = /etc/pgbouncer/userlist.txt
+{% if pgbouncer_auth_user | bool %}
+auth_user = {{ pgbouncer_auth_username }}
+auth_query = SELECT usename, passwd FROM user_search($1)
+{% else %} 
+auth_file = {{ pgbouncer_conf_dir }}/userlist.txt
+{% endif %}
 admin_users = {{ pgbouncer_admin_users }}
 stats_users = {{ pgbouncer_stats_users }}
 ignore_startup_parameters = {{ pgbouncer_ignore_startup_parameters }}

diff --git a/tags.md b/tags.md
@@ -54,6 +54,7 @@
 - - pgbouncer_logrotate
 - - pgbouncer_restart
 - - pgbouncer_generate_userlist
+- - pgbouncer_auth_query
 - load_balancing
 - - haproxy
 - - - haproxy_requirements

diff --git a/vars/main.yml b/vars/main.yml
@@ -139,7 +139,8 @@ postgresql_data_checksums: true  # for bootstrap only (initdb)
 postgresql_password_encryption_algorithm: "scram-sha-256"  # or "md5" if your clients do not work with passwords encrypted with SCRAM-SHA-256
 
 # (optional) list of users to be created (if not already exists)
-postgresql_users: []
+postgresql_users:
+  - { name: "{{ pgbouncer_auth_username }}", password: "{{ pgbouncer_auth_password }}", flags: "LOGIN" }
 #  - { name: "mydb-user", password: "mydb-user-pass", flags: "SUPERUSER" }
 #  - { name: "", password: "", flags: "NOSUPERUSER" }
 #  - { name: "", password: "", flags: "NOSUPERUSER" }
@@ -265,6 +266,7 @@ postgresql_pg_hba:
   - { type: "local", database: "all", user: "{{ patroni_superuser_username }}", address: "", method: "trust" }
   - { type: "local", database: "replication", user: "{{ patroni_superuser_username }}", address: "", method: "trust" }
   - { type: "local", database: "all", user: "all", address: "", method: "peer" }
+  - { type: "host", database: "all", user: "{{ pgbouncer_auth_username }}", address: "127.0.0.1/32", method: "trust" } # required for pgbouncer auth_user
   - { type: "host", database: "all", user: "all", address: "127.0.0.1/32", method: "{{ postgresql_password_encryption_algorithm }}" }
   - { type: "host", database: "all", user: "all", address: "::1/128", method: "{{ postgresql_password_encryption_algorithm }}" }
 #  - { type: "host", database: "mydatabase", user: "mydb-user", address: "192.168.0.0/24", method: "{{ postgresql_password_encryption_algorithm }}" }
@@ -295,9 +297,11 @@ pgbouncer_query_wait_timeout: 120
 pgbouncer_default_pool_mode: "session"
 pgbouncer_admin_users: "postgres"  # comma-separated list of users, who are allowed to change settings
 pgbouncer_stats_users: "postgres"  # comma-separated list of users who are just allowed to use SHOW command
-pgbouncer_generate_userlist: true  # generate the authentication file (userlist.txt) from the pg_shadow system table
-pgbouncer_auth_type: "{{ postgresql_password_encryption_algorithm }}"
 pgbouncer_ignore_startup_parameters: "extra_float_digits,geqo,search_path"
+pgbouncer_auth_type: "{{ postgresql_password_encryption_algorithm }}"
+pgbouncer_auth_user: true # or 'false' if you want to manage the list of users for authentication in the database via userlist.txt
+pgbouncer_auth_username: pgbouncer # user who can query the database via the user_search function
+pgbouncer_auth_password: "pgbouncer-pass" # please change password
 
 pgbouncer_pools:
   - { name: "postgres", dbname: "postgres", pool_parameters: "" }