(PCP-293 & PCP-294)

acceptance/lib/pxp-agent/test_helper.rb

@@ -1,5 +1,8 @@
 require 'pxp-agent/config_helper.rb'
 require 'pcp/client'
+require 'net/http'
+require 'openssl'
+require 'json'
 
 # This file contains general test helper methods for pxp-agent acceptance tests
 
@@ -37,9 +40,19 @@ def expect_file_on_host_to_contain(host, file, expected, seconds=30)
 
 # Some helpers for working with a pcp-broker 'lein tk' instance
 def run_pcp_broker(host)
-  on(host, 'cd ~/pcp-broker; export LEIN_ROOT=ok; lein tk </dev/null >/var/log/pcp-broker.log 2>&1 &')
+  on(host, 'cd /opt/puppet-git-repos/pcp-broker; export LEIN_ROOT=ok; lein tk </dev/null >/var/log/pcp-broker.log 2>&1 &')
   assert(port_open_within?(host, PCP_BROKER_PORT, 60),
          "pcp-broker port #{PCP_BROKER_PORT.to_s} not open within 1 minutes of starting the broker")
+  broker_state = nil
+  attempts = 0
+  until broker_state == "running" or attempts == 100 do
+    broker_state = get_pcp_broker_status(host)
+    if broker_state != "running"
+      attempts += 1
+      sleep 0.5
+    end
+  end
+  assert_equal("running", broker_state, "Shortly after startup, pcp-broker should report its state as being 'running'")
 end
 
 def kill_pcp_broker(host)
@@ -49,6 +62,21 @@ def kill_pcp_broker(host)
   end
 end
 
+def get_pcp_broker_status(host)
+  uri = URI.parse("https://#{host}:#{PCP_BROKER_PORT}/status/v1/services/broker-service")
+  begin
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    res = http.get(uri.request_uri)
+    document = JSON.load(res.body)
+    return document["state"]
+  rescue => e
+    puts e.inspect
+    return nil
+  end
+end
+
 # Query pcp-broker's inventory of associated clients
 # Reference: https://github.com/puppetlabs/pcp-specifications/blob/master/pcp/inventory.md
 # @param broker hostname or beaker host object of the pcp-broker host

acceptance/setup/common/010_Setup_Broker.rb

@@ -17,7 +17,7 @@
 
 step 'Clone pcp-broker to master' do
   clone_git_repo_on(master, GIT_CLONE_FOLDER,
-    extract_repo_info_from(build_git_url('pcp-broker', nil, nil, 'https')).merge(:rev => '602c003'))
+    extract_repo_info_from(build_git_url('pcp-broker', nil, nil, 'https')))
 end
 
 step 'Download lein bootstrap' do

acceptance/tests/reconnect_after_broker_unavailable.rb

@@ -0,0 +1,28 @@
+require 'pxp-agent/config_helper.rb'
+require 'pxp-agent/test_helper.rb'
+
+test_name 'C94789 - An associated agent should automatically reconnect when the broker was temporarily unavailable'
+
+step 'Ensure each agent host has pxp-agent running and associated' do
+  agents.each_with_index do |agent, i|
+    on agent, puppet('resource service pxp-agent ensure=stopped')
+    cert_dir = configure_std_certs_on_host(agent)
+    create_remote_file(agent, pxp_agent_config_file(agent), pxp_config_json_using_test_certs(master, agent, i + 1, cert_dir).to_s)
+    on(agent, "rm -rf #{logfile(agent)}")
+    on agent, puppet('resource service pxp-agent ensure=running')
+    assert(is_associated?(master, "pcp://client0#{i+1}.example.com/agent"),
+           "Agent identity pcp://client0#{i+1}.example.com/agent for agent host #{agent} does not appear in pcp-broker's client inventory")
+  end
+end
+
+step 'On master, stop then restart the broker' do
+  kill_pcp_broker(master)
+  run_pcp_broker(master)
+end
+
+step 'On each agent, test that a 2nd association has occurred' do
+  agents.each_with_index do |agent, i|
+    assert(is_associated?(master, "pcp://client0#{i+1}.example.com/agent"),
+           "Agent identity pcp://client0#{i+1}.example.com/agent for agent host #{agent} does not appear in pcp-broker's client inventory")
+  end
+end