Add a test for master restart after being killed

hack/run_test

@@ -32,6 +32,17 @@ function cleanup() {
 }
 trap cleanup EXIT
 
+function cleanup_volume() {
+  local volume_dir="$1"; shift
+  local cid="$1"; shift
+  local volume_args="-v ${volume_dir}:/var/lib/pgsql/data:Z"
+
+  docker stop $cid
+  local rm_cmd='/bin/rm -rf /var/lib/pgsql/data/*'
+  docker run $volume_args --rm $IMAGE_NAME /bin/sh -c "$rm_cmd"
+  rmdir ${volume_dir}
+}
+
 function get_cid() {
   local id="$1" ; shift || return 1
   echo $(cat "$CIDFILE_DIR/$id")
@@ -99,6 +110,13 @@ function create_container() {
   echo "Created container $(cat $cidfile)"
 }
 
+function create_volume() {
+  local volume_dir
+  volume_dir=`mktemp -d --tmpdir pg-testdata.XXXXX`
+  chmod a+rwx ${volume_dir}
+  echo $volume_dir
+}
+
 function assert_login_access() {
   local PGUSER=$1 ; shift
   local PASS=$1 ; shift
@@ -269,7 +287,8 @@ function run_slave() {
 
 function run_master() {
   local suffix="$1"; shift
-  docker run $cluster_args -p 5432  \
+  master_args=${master_args-}
+  docker run $cluster_args $master_args \
     -d --cidfile ${CIDFILE_DIR}/master-${suffix}.cid $IMAGE_NAME run-postgresql-master >/dev/null
 }
 
@@ -309,7 +328,7 @@ function test_value_replication() {
     CONTAINER_IP=$slave_ip
     for i in $(seq $max_attempts); do
       result="$(postgresql_cmd -At -c "select * from $table_name")"
-      if [[ ! -z "${result}" ]]; then
+      if [[ "$result" == "$value" ]]; then
         echo "${slave_ip} successfully got value from MASTER ${master_ip}"
         break
       fi
@@ -331,9 +350,47 @@ function setup_replication_cluster() {
 
   # Run the PostgreSQL slaves
   local i
-  for i in $(seq ${SLAVE_NUM:-1}); do
+  for i in $(seq ${slave_num:-1}); do
     slave_cids="$slave_cids $(run_slave $cid_suffix-$i)"
   done
+
+}
+
+function run_master_restart_test() {
+  echo "Testing failed master restart"
+  local cluster_args="-e POSTGRESQL_ADMIN_PASSWORD=pass -e POSTGRESQL_MASTER_USER=$PGUSER -e POSTGRESQL_MASTER_PASSWORD=$PASS"
+  local cid_suffix="mrestart"
+  local table_name="t1"
+  local master_ip=
+  local slave_cids=
+
+  local volume_dir=$(create_volume)
+  master_args="-v ${volume_dir}:/var/lib/pgsql/data:Z"
+
+  # Setup the cluster
+  slave_num=2 setup_replication_cluster
+
+  # Check if the master knows about the slaves
+  CONTAINER_IP=$master_ip
+  test_slave_visibility
+
+  echo "Kill the master and create a new one"
+  local cidfile=$CIDFILE_DIR/master-$cid_suffix.cid
+  docker kill $(cat $cidfile)
+  # Don't forget to remove its .cid file
+  rm $cidfile
+
+  run_master $cid_suffix
+  CONTAINER_IP=$(get_container_ip master-$cid_suffix.cid)
+
+  # Check if the new master sees existing slaves
+  test_slave_visibility
+
+  # Check if the replication works
+  table_name="t1" test_value_replication
+
+  # Cleanup the volume
+  cleanup_volume $volume_dir $(cat $cidfile)
 }
 
 function run_replication_test() {
@@ -363,10 +420,7 @@ function run_change_password_test() {
   local password='password'
   local admin_password='adminPassword'
 
-  local volume_dir
-  volume_dir=`mktemp -d --tmpdir pg-testdata.XXXXX`
-  chmod a+rwx ${volume_dir}
-
+  local volume_dir=$(create_volume)
   local volume_options="-v ${volume_dir}:/var/lib/pgsql/data:Z"
 
   DOCKER_ARGS="
@@ -421,9 +475,7 @@ $volume_options
   # into account 0077 umask of PostgreSQL server, we are unable to remove files
   # created by server.  That's why we need to let docker escalate the privileges
   # again.
-  local rm_cmd='/bin/rm -rf /var/lib/pgsql/data/*'
-  docker run $volume_options --rm $IMAGE_NAME /bin/sh -c "$rm_cmd"
-  rmdir ${volume_dir}
+  cleanup_volume $volume_dir $(get_cid ${name})
 
   echo "  Success!"
 }
@@ -445,3 +497,4 @@ DOCKER_ARGS="-u 12345" PGUSER=user3 PASS=pass1 ADMIN_PASS=r00t run_tests admin_a
 DB=postgres DOCKER_ARGS="-u 12345" ADMIN_PASS=rOOt run_tests only_admin_altuid
 run_change_password_test
 DB=postgres PGUSER=master PASS=master run_replication_test
+DB=postgres PGUSER=master PASS=master run_master_restart_test