feat: add configurable retries with exponential backoff (#56)

On windows blackbox was preventing log rotation when configured with invalid
syslog credentials as it was locking the file while trying to endlessly reconnect.
This change makes it so that blackbox exits which unlocks the file and indicates
that the config is invalid.

Signed-off-by: Andrew Crump <[email protected]>

Author: rroberts2222

jobs/syslog_forwarder_windows/spec

@@ -63,6 +63,10 @@ properties:
     description: limit goprocess to a single cpu via gomaxprocs
     default: true
 
+  syslog.blackbox.max_retries:
+    description: maximum number of times to retry the connection with an exponential backoff between attempts
+    default: 10
+
   syslog.migration.disabled:
     default: false
     description: |

jobs/syslog_forwarder_windows/templates/blackbox_config.yml.erb

@@ -48,6 +48,7 @@ syslog:
     transport: <%= syslog_transport %>
     address:  <%= syslog_address %>:<%= syslog_port %>
     ca: <%= syslog_ca %>
+    max_retries: <%= p("syslog.blackbox.max_retries") %>
   exclude_file_pattern: '*.[0-9].*.log'
 
 <% end %>