diff --git a/acceptance/lib/pxp-agent/test_helper.rb b/acceptance/lib/pxp-agent/test_helper.rb index deed2f5d..94d1785c 100644 --- a/acceptance/lib/pxp-agent/test_helper.rb +++ b/acceptance/lib/pxp-agent/test_helper.rb @@ -1,5 +1,8 @@ require 'pxp-agent/config_helper.rb' require 'pcp/client' +require 'net/http' +require 'openssl' +require 'json' # This file contains general test helper methods for pxp-agent acceptance tests @@ -37,9 +40,19 @@ def expect_file_on_host_to_contain(host, file, expected, seconds=30) # Some helpers for working with a pcp-broker 'lein tk' instance def run_pcp_broker(host) - on(host, 'cd ~/pcp-broker; export LEIN_ROOT=ok; lein tk /var/log/pcp-broker.log 2>&1 &') + on(host, 'cd /opt/puppet-git-repos/pcp-broker; export LEIN_ROOT=ok; lein tk /var/log/pcp-broker.log 2>&1 &') assert(port_open_within?(host, PCP_BROKER_PORT, 60), "pcp-broker port #{PCP_BROKER_PORT.to_s} not open within 1 minutes of starting the broker") + broker_state = nil + attempts = 0 + until broker_state == "running" or attempts == 100 do + broker_state = get_pcp_broker_status(host) + if broker_state != "running" + attempts += 1 + sleep 0.5 + end + end + assert_equal("running", broker_state, "Shortly after startup, pcp-broker should report its state as being 'running'") end def kill_pcp_broker(host) @@ -49,6 +62,21 @@ def kill_pcp_broker(host) end end +def get_pcp_broker_status(host) + uri = URI.parse("https://#{host}:#{PCP_BROKER_PORT}/status/v1/services/broker-service") + begin + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + res = http.get(uri.request_uri) + document = JSON.load(res.body) + return document["state"] + rescue => e + puts e.inspect + return nil + end +end + # Query pcp-broker's inventory of associated clients # Reference: https://github.com/puppetlabs/pcp-specifications/blob/master/pcp/inventory.md # @param broker hostname or beaker host object of the pcp-broker host diff --git a/acceptance/setup/common/010_Setup_Broker.rb b/acceptance/setup/common/010_Setup_Broker.rb index 441406bf..18dab9b5 100644 --- a/acceptance/setup/common/010_Setup_Broker.rb +++ b/acceptance/setup/common/010_Setup_Broker.rb @@ -17,7 +17,7 @@ step 'Clone pcp-broker to master' do clone_git_repo_on(master, GIT_CLONE_FOLDER, - extract_repo_info_from(build_git_url('pcp-broker', nil, nil, 'https')).merge(:rev => '602c003')) + extract_repo_info_from(build_git_url('pcp-broker', nil, nil, 'https'))) end step 'Download lein bootstrap' do diff --git a/acceptance/tests/reconnect_after_broker_unavailable.rb b/acceptance/tests/reconnect_after_broker_unavailable.rb new file mode 100644 index 00000000..21fdc0b0 --- /dev/null +++ b/acceptance/tests/reconnect_after_broker_unavailable.rb @@ -0,0 +1,28 @@ +require 'pxp-agent/config_helper.rb' +require 'pxp-agent/test_helper.rb' + +test_name 'C94789 - An associated agent should automatically reconnect when the broker was temporarily unavailable' + +step 'Ensure each agent host has pxp-agent running and associated' do + agents.each_with_index do |agent, i| + on agent, puppet('resource service pxp-agent ensure=stopped') + cert_dir = configure_std_certs_on_host(agent) + create_remote_file(agent, pxp_agent_config_file(agent), pxp_config_json_using_test_certs(master, agent, i + 1, cert_dir).to_s) + on(agent, "rm -rf #{logfile(agent)}") + on agent, puppet('resource service pxp-agent ensure=running') + assert(is_associated?(master, "pcp://client0#{i+1}.example.com/agent"), + "Agent identity pcp://client0#{i+1}.example.com/agent for agent host #{agent} does not appear in pcp-broker's client inventory") + end +end + +step 'On master, stop then restart the broker' do + kill_pcp_broker(master) + run_pcp_broker(master) +end + +step 'On each agent, test that a 2nd association has occurred' do + agents.each_with_index do |agent, i| + assert(is_associated?(master, "pcp://client0#{i+1}.example.com/agent"), + "Agent identity pcp://client0#{i+1}.example.com/agent for agent host #{agent} does not appear in pcp-broker's client inventory") + end +end