Update postgresql to 9.6.6 and enable it to run without pkg_svc_user=…

…root

Signed-off-by: Irving Popovetsky <[email protected]>

postgresql/README.md

@@ -12,7 +12,7 @@ hab start core/postgresql
 ```
 or
 ```
-docker run starkandwayne/postgresql
+docker run habitat/postgresql
 ```
 if you want to bring up the pre-exported docker image.
 
@@ -22,7 +22,7 @@ This plan supports running clustered PostgreSQL by utilizing Habitat's native le
 
 You can run an example cluster via docker-compose after exporting a docker container from this plan:
 ```
-$ hab pkg export docker core/postgresql
+$ hab pkg export docker $(ls -1t results/*.hart | head -1)
 ```
 
 The docker post-process should create a docker image named `core/postgresql` and it should be available on your local machine
@@ -33,22 +33,35 @@ version: '3'
 
 services:
   pg1:
-    image: core/postgresql
-    command: "start core/postgresql --group cluster --topology leader"
+    image: habitat/postgresql
+    command: --group cluster
+      --topology leader
+    volumes:
+      - pg1-data:/hab/svc/postgresql/data
   pg2:
-    image: core/postgresql
-    command: "start core/postgresql --group cluster --topology leader --peer pg1"
+    image: habitat/postgresql
+    command: --group cluster
+      --topology leader
+      --peer pg1
+    volumes:
+      - pg2-data:/hab/svc/postgresql/data
   pg3:
-    image: core/postgresql
-    command: "start core/postgresql --group cluster --topology leader --peer pg1"
+    image: habitat/postgresql
+    command: --group cluster
+      --topology leader
+      --peer pg1
+    volumes:
+      - pg3-data:/hab/svc/postgresql/data
+
+volumes:
+  pg1-data:
+  pg2-data:
+  pg3-data:
 EOF
 
 docker-compose up
 ```
 
-We intend to support a prod ready clustering solution similar to [patroni](https://github.com/zalando/patroni) and are working on getting there quickly.
-
-
 ## Binding
 
 Consuming services can bind to PostgreSQL via:
@@ -59,13 +72,38 @@ hab start <origin>/<app> --bind database:postgresql.default --peer <pg-host>
 
 Superuser access is exposed to consumers when binding and we advise that required databases, schemas and roles be created and migrations get run in the init hook of the consuming service. The created roles (with restricted access) should then be exposed to the application code that will get run.
 
-To support binding to either standalone or custered PostgreSQL services we suggest using the `.first` field of the binding in the handlebars interpolation:
+In the template of the consuming service, there is currently a difference between the binding syntax for binding to the leader in a clustered topology vs binding to any member.  A [future enhancement](https://github.com/habitat-sh/habitat/issues/4127) may improve upon that, but for now something like this could work if used in an interpreted script file:
+
 ```
-{{#with bind.database.first as |pg| }}
-PGPASSWORD={{pg.cfg.superuser_password}}
-PGUSER={{pg.cfg.superuser_name}}
-PGHOST={{pg.sys.ip}}
-{{/with}}
+# Use the eachAlive and @first helpers to bind to the oldest alive member of the bound service group
+#   - members are processed in the chronological order of when they joined the ring
+# If a later member comes along that is the leader of a cluster topology, override the environment variable definitions with that
+#   - if not, the second section is omitted
+{{~ #if bind.database}}
+  {{~ #eachAlive bind.database.members as |member|}}
+    {{~ #if @first}}
+# First alive member
+PGHOST="{{member.sys.ip}}"
+PGPORT="{{member.cfg.port}}"
+PGUSER="{{member.cfg.superuser_name}}"
+PGPASSWORD="{{member.cfg.superuser_password}}"
+    {{~ /if}}
+
+    {{~ #if member.leader}}
+# Leader node override
+PGHOST="{{member.sys.ip}}"
+PGPORT="{{member.cfg.port}}"
+PGUSER="{{member.cfg.superuser_name}}"
+PGPASSWORD="{{member.cfg.superuser_password}}"
+    {{~ /if}}
+  {{~ /eachAlive}}
+{{~ /if}}
 ```
 
-`.first` will always point to the leader when present.
+However if you're using more of a rigid configuration file syntax then you still need to know and choose in advance whether or not you're expecting to connect to a leader topology or not.
+
+
+# TODO (Potential improvements to this plan):
+- Upgrade logic (detect if the database engine is newer than the data on disk and perform pg_upgrade)
+- Full cluster restart logic (elect the previous leader)
+  - add a `suitability` hook based on the existence of a `recovery.conf` file

postgresql/config/functions.sh

@@ -1,6 +1,10 @@
 init_pgpass() {
-  export PGPASSFILE="{{pkg.svc_config_path}}/.pgpass"
-  chmod 0600 {{pkg.svc_config_path}}/.pgpass
+  cat > {{pkg.svc_var_path}}/.pgpass<<EOF
+*:*:*:{{cfg.superuser.name}}:{{cfg.superuser.password}}
+*:*:*:{{cfg.replication.name}}:{{cfg.replication.password}}
+EOF
+chmod 0600 {{pkg.svc_var_path}}/.pgpass
+  export PGPASSFILE="{{pkg.svc_var_path}}/.pgpass"
 }
 
 write_local_conf() {
@@ -58,15 +62,25 @@ master_ready() {
 
 bootstrap_replica_via_pg_basebackup() {
   echo 'Bootstrapping replica via pg_basebackup from leader '
-
-  rm -rf {{pkg.svc_data_path}}/*
-  pg_basebackup --pgdata={{pkg.svc_data_path}} --xlog-method=stream --dbname='postgres://{{cfg.replication.name}}@{{svc.leader.sys.ip}}:{{cfg.port}}/postgres'
+  rm -rf {{pkg.svc_data_path}}/pgdata/*
+  pg_basebackup --verbose --progress --pgdata={{pkg.svc_data_path}}/pgdata --xlog-method=stream --dbname='postgres://{{cfg.replication.name}}@{{svc.leader.sys.ip}}:{{cfg.port}}/postgres'
 }
 
 ensure_dir_ownership() {
-  echo 'Making sure hab user owns var, config and data paths'
+  echo 'Making sure hab user owns var and data paths'
   chown -RL hab:hab {{pkg.svc_var_path}}
-  chown -RL hab:hab {{pkg.svc_config_path}}
   chown -RL hab:hab {{pkg.svc_data_path}}
-  chmod 0700 {{pkg.svc_data_path}}
+  chmod 0700 {{pkg.svc_data_path}}/pgdata
+}
+
+promote_to_leader() {
+  if [ -f {{pkg.svc_data_path}}/pgdata/recovery.conf ]; then
+    echo "Promoting database"
+    until pg_isready -U {{cfg.superuser.name}} -h localhost -p {{cfg.port}}; do
+      echo "Waiting for database to start"
+      sleep 1
+    done
+
+    pg_ctl promote -D {{pkg.svc_data_path}}/pgdata
+  fi
 }

postgresql/config/postgresql.conf

@@ -17,7 +17,7 @@ datestyle = 'iso, mdy'
 
 default_text_search_config = 'pg_catalog.english'
 
-data_directory = '{{pkg.svc_data_path}}'
+data_directory = '{{pkg.svc_data_path}}/pgdata'
 hba_file = '{{pkg.svc_config_path}}/pg_hba.conf'
 
 wal_level = hot_standby
@@ -44,7 +44,7 @@ dynamic_shared_memory_type = 'none'
 
 {{ #if cfg.replication.enable }}
 archive_mode = 'on'
-archive_command = 'cp %p  {{cfg.replication.archive_path}}/%f'
+archive_command = 'cp %p  {{pkg.svc_data_path}}/archive/%f'
 archive_timeout = '10min'
 max_standby_archive_delay = '30s'
 synchronous_commit = local

postgresql/default.toml

@@ -15,4 +15,3 @@ name = 'replication'
 password = 'replication'
 lag_health_threshold = 1048576
 enable = false
-archive_path = "{{pkg.svc_path}}/archive"

postgresql/hooks/init

@@ -1,24 +1,35 @@
 #!{{pkgPathFor "core/bash"}}/bin/bash
 #
+shopt -s extglob
 
 exec 2>&1
 
 source {{pkg.svc_config_path}}/functions.sh
 
 mkdir -p {{pkg.svc_config_path}}/conf.d
 mkdir -p {{pkg.svc_var_path}}/pg_stat_tmp
-mkdir -p {{pkg.svc_path}}/archive
 
+# Auto-detect pg data in the root of svc_data_path, where older versions of this plan had it
+if [[ -f "{{pkg.svc_data_path}}/PG_VERSION" ]]; then
+  echo "PGDATA detected in the root of the data path ( {{pkg.svc_data_path}} ), relocating it to {{pkg.svc_data_path}}/pgdata"
+  mkdir -p {{pkg.svc_data_path}}/pgdata
+  # bash extended globbing can cleanly move everything under a subfolder http://www.linuxjournal.com/content/bash-extended-globbing
+  mv {{pkg.svc_data_path}}/!(pgdata) {{pkg.svc_data_path}}/pgdata/
+  chmod 0700 {{pkg.svc_data_path}}/pgdata
+else
+  mkdir -p {{pkg.svc_data_path}}/pgdata
+fi
+
+mkdir -p {{pkg.svc_data_path}}/archive
 init_pgpass
 ensure_dir_ownership
 
 {{#unless svc.me.follower ~}}
-if [[ ! -f "{{pkg.svc_data_path}}/PG_VERSION" ]]; then
+if [[ ! -f "{{pkg.svc_data_path}}/pgdata/PG_VERSION" ]]; then
   echo " Database does not exist, creating with 'initdb'"
-  chpst -U hab:hab -u hab:hab \
     initdb -U {{cfg.superuser.name}} \
     -E utf8 \
-    -D {{pkg.svc_data_path}} \
+    -D {{pkg.svc_data_path}}/pgdata \
     --pwfile {{pkg.svc_config_path}}/pwfile \
     --locale POSIX \
     --data-checksums

postgresql/hooks/reconfigure

@@ -4,15 +4,6 @@ source {{pkg.svc_config_path}}/functions.sh
 
 init_pgpass
 
-{{ #unless svc.me.follower }}
-if [ -f {{pkg.svc_data_path}}/recovery.conf ]; then
-  echo "Promoting database"
-  until pg_isready -U {{cfg.superuser.name}} -h localhost -p {{cfg.port}}; do
-    echo "Waiting for database to start"
-    sleep 1
-  done
-
-  chpst -U hab:hab -u hab:hab pg_ctl promote -D {{pkg.svc_data_path}}
-fi
-
+{{~ #unless svc.me.follower }}
+promote_to_leader
 {{~/unless}}

postgresql/hooks/run

@@ -12,16 +12,16 @@ init_pgpass
 write_local_conf
 
 {{#if svc.me.follower }}
-if [[ ! -f "{{pkg.svc_data_path}}/PG_VERSION" ]]; then
+if [[ ! -f "{{pkg.svc_data_path}}/pgdata/PG_VERSION" ]]; then
   bootstrap_replica_via_pg_basebackup
 fi
 
-cp {{pkg.svc_config_path}}/recovery.conf {{pkg.svc_data_path}}/
+cp {{pkg.svc_config_path}}/recovery.conf {{pkg.svc_data_path}}/pgdata/recovery.conf
 {{/if}}
 
 ensure_dir_ownership
 
 echo "Starting PostgreSQL"
-export PGDATA={{pkg.svc_data_path}}
-exec chpst -U hab:hab -u hab:hab postgres \
+export PGDATA={{pkg.svc_data_path}}/pgdata
+exec postgres \
   -c config_file={{pkg.svc_config_path}}/postgresql.conf

postgresql/plan.sh

@@ -1,12 +1,12 @@
 pkg_name=postgresql
-pkg_version=9.6.3
+pkg_version=9.6.6
 pkg_origin=core
 pkg_maintainer="The Habitat Maintainers <[email protected]>"
 pkg_description="PostgreSQL is a powerful, open source object-relational database system."
 pkg_upstream_url="https://www.postgresql.org/"
 pkg_license=('PostgreSQL')
 pkg_source=https://ftp.postgresql.org/pub/source/v${pkg_version}/${pkg_name}-${pkg_version}.tar.bz2
-pkg_shasum=1645b3736901f6d854e695a937389e68ff2066ce0cde9d73919d6ab7c995b9c6
+pkg_shasum=399cdffcb872f785ba67e25d275463d74521566318cfef8fe219050d063c8154
 
 pkg_deps=(
   core/bash
@@ -44,13 +44,10 @@ pkg_exports=(
 )
 pkg_exposes=(port)
 
-pkg_svc_user=root
-pkg_svc_group=$pkg_svc_user
-
-ext_postgis_version=2.3.2
+ext_postgis_version=2.4.2
 ext_postgis_source=http://download.osgeo.org/postgis/source/postgis-${ext_postgis_version}.tar.gz
 ext_postgis_filename=postgis-${ext_postgis_version}.tar.gz
-ext_postgis_shasum=e92e34c18f078a3d1a2503cd870efdc4fa9e134f0bcedbbbdb8b46b0e6af09e4
+ext_postgis_shasum=23625bc99ed440d53a20225721095a3f5c653b62421c4d597c8038f0d7a321d9
 
 do_before() {
   ext_postgis_dirname="postgis-${ext_postgis_version}"