Merge pull request brooklyncentral#1318 from sjcorbett/ssh-connection…

…-refactor

Avoid thundering herd when SoftwareProcess rebinds its sensors

software/base/src/main/java/brooklyn/entity/basic/SoftwareProcess.java

@@ -13,6 +13,7 @@
 import brooklyn.location.MachineProvisioningLocation;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.flags.SetFromFlag;
+import brooklyn.util.time.Duration;
 
 public interface SoftwareProcess extends Entity, Startable {
 
@@ -69,6 +70,15 @@ public interface SoftwareProcess extends Entity, Startable {
     public static final MapConfigKey<Object> PROVISIONING_PROPERTIES = new MapConfigKey<Object>(Object.class,
             "provisioning.properties", "Custom properties to be passed in when provisioning a new machine", MutableMap.<String,Object>of());
 
+    @SetFromFlag("maxRebindSensorsDelay")
+    ConfigKey<Duration> MAXIMUM_REBIND_SENSOR_CONNECT_DELAY = ConfigKeys.newConfigKey(Duration.class,
+            "softwareProcess.maxSensorRebindDelay",
+            "The maximum delay to apply when reconnecting sensors when rebinding to this entity. " +
+                    "Brooklyn will wait a random amount of time, up to the value of this config key, to " +
+                    "avoid a thundering herd problem when the entity shares its machine with " +
+                    "several others. Set to null or to 0 to disable any delay.",
+            Duration.TEN_SECONDS);
+
     /** controls the behavior when starting (stop, restart) {@link Startable} children as part of the start (stop, restart) effector on this entity
      * <p>
      * (NB: restarts are currently not propagated to children in the default {@link SoftwareProcess}

software/base/src/main/java/brooklyn/entity/basic/SoftwareProcessImpl.java

@@ -5,6 +5,8 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 
@@ -54,7 +56,7 @@ public abstract class SoftwareProcessImpl extends AbstractEntity implements Soft
 
     private static final SoftwareProcessDriverLifecycleEffectorTasks LIFECYCLE_TASKS =
             new SoftwareProcessDriverLifecycleEffectorTasks();
-    
+
     protected boolean connectedSensors = false;
 
     public SoftwareProcessImpl() {
@@ -179,7 +181,19 @@ protected void postRebind() {
     }
 
     protected void callRebindHooks() {
-        connectSensors();
+        Duration configuredMaxDelay = getConfig(MAXIMUM_REBIND_SENSOR_CONNECT_DELAY);
+        if (configuredMaxDelay == null || Duration.ZERO.equals(configuredMaxDelay)) {
+            connectSensors();
+        } else {
+            long delay = (long) (Math.random() * configuredMaxDelay.toMilliseconds());
+            log.debug("Scheduled reconnection of sensors on {} in {}ms", this, delay);
+            Timer timer = new Timer();
+            timer.schedule(new TimerTask() {
+                @Override public void run() {
+                    connectSensors();
+                }
+            }, delay);
+        }
         // don't wait here - it may be long-running, e.g. if remote entity has died, and we don't want to block rebind waiting or cause it to fail
         // the service will subsequently show service not up and thus failure
 //        waitForServiceUp();