Attempt marathon connection during watch loop

This increases the resiliency in the case that marathon cannot be
immediately found. A connection attempt will be made at the start of
each watch interval.

lib/synapse/service_watcher/marathon.rb

@@ -6,12 +6,7 @@ module Synapse
   class MarathonWatcher < BaseWatcher
     def start
       @check_interval = @discovery['check_interval'] || 10.0
-
-      @marathon_api = URI.join(@discovery['marathon_api_url'], "/v2/apps/#{@discovery['application_name']}/tasks")
-      @connection = Net::HTTP.new(@marathon_api.host, @marathon_api.port)
-      @connection.open_timeout = 5
-      @connection.start
-
+      @connection = nil
       @watcher = Thread.new { watch }
     end
 
@@ -34,12 +29,31 @@ def validate_discovery_opts
       end
     end
 
+    def attempt_marathon_connection
+      @marathon_api = URI.join(@discovery['marathon_api_url'], "/v2/apps/#{@discovery['application_name']}/tasks")
+
+      begin
+        @connection = Net::HTTP.new(@marathon_api.host, @marathon_api.port)
+        @connection.open_timeout = 5
+        @connection.start
+      rescue => ex
+        @connection = nil
+        log.error "synapse: could not connect to marathon at #{@marathon_api}: #{ex}"
+
+        raise ex
+      end
+    end
+
     def watch
       until @should_exit
         retry_count = 0
         start = Time.now
 
         begin
+          if @connection.nil?
+            attempt_marathon_connection
+          end
+
           req = Net::HTTP::Get.new(@marathon_api.request_uri)
           req['Accept'] = 'application/json'
           response = @connection.request(req)

spec/lib/synapse/service_watcher_marathon_spec.rb

@@ -43,6 +43,18 @@
   end
 
   describe '#watch' do
+    context 'when synapse cannot connect to marathon' do
+      before do
+        allow(Net::HTTP).to receive(:new).
+          with(marathon_host, marathon_port.to_i).
+          and_raise(Errno::ECONNREFUSED)
+      end
+
+      it 'does not crash' do
+        expect { subject.start }.not_to raise_error
+      end
+    end
+
     it 'requests the proper API endpoint one time' do
       subject.start
       expect(a_request(:get, marathon_request_uri)).to have_been_made.times(1)
@@ -114,7 +126,7 @@
       context 'when marathon returns invalid response' do
         let(:marathon_response) { [] }
         it 'does not blow up' do
-          expect(subject.start).to_not raise_error
+          expect { subject.start }.to_not raise_error
         end
       end