Merge pull request #2228 from HubSpot/live-configuration

Support global overrides of request placement strategy/rack sensitivity

SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java

@@ -26,6 +26,7 @@
 import com.hubspot.singularity.config.CustomExecutorConfiguration;
 import com.hubspot.singularity.config.HistoryPurgingConfiguration;
 import com.hubspot.singularity.config.MesosConfiguration;
+import com.hubspot.singularity.config.OverrideConfiguration;
 import com.hubspot.singularity.config.S3Configuration;
 import com.hubspot.singularity.config.S3GroupConfiguration;
 import com.hubspot.singularity.config.SMTPConfiguration;
@@ -408,6 +409,12 @@ public UIConfiguration uiConfiguration(final SingularityConfiguration config) {
     return config.getUiConfiguration();
   }
 
+  @Provides
+  @Singleton
+  public OverrideConfiguration overrideConfiguration() {
+    return new OverrideConfiguration();
+  }
+
   @Provides
   @Singleton
   public CustomExecutorConfiguration customExecutorConfiguration(

SingularityService/src/main/java/com/hubspot/singularity/config/OverrideConfiguration.java

@@ -0,0 +1,32 @@
+package com.hubspot.singularity.config;
+
+import com.hubspot.singularity.AgentPlacement;
+import java.util.Optional;
+
+/**
+ * Stores temporary overrides for certain Singularity behavior,
+ * because calling configuration setters directly doesn't work.
+ */
+public class OverrideConfiguration {
+  /** If false, ignore rack sensitive requests. If true, work normally. */
+  private boolean allowRackSensitivity = true;
+
+  /** If set, overrides agent placement for all requests to the specified value. */
+  private Optional<AgentPlacement> agentPlacementOverride = Optional.empty();
+
+  public boolean isAllowRackSensitivity() {
+    return allowRackSensitivity;
+  }
+
+  public void setAllowRackSensitivity(boolean allowRackSensitivity) {
+    this.allowRackSensitivity = allowRackSensitivity;
+  }
+
+  public Optional<AgentPlacement> getAgentPlacementOverride() {
+    return agentPlacementOverride;
+  }
+
+  public void setAgentPlacementOverride(Optional<AgentPlacement> agentPlacementOverride) {
+    this.agentPlacementOverride = agentPlacementOverride;
+  }
+}

SingularityService/src/main/java/com/hubspot/singularity/data/AbstractMachineManager.java

@@ -32,6 +32,7 @@ public abstract class AbstractMachineManager<T extends SingularityMachineAbstrac
   private final Transcoder<SingularityMachineStateHistoryUpdate> historyTranscoder;
   private final Transcoder<SingularityExpiringMachineState> expiringMachineStateTranscoder;
   private final int maxHistoryEntries;
+  private final SingularityConfiguration configuration;
 
   public AbstractMachineManager(
     CuratorFramework curator,
@@ -46,6 +47,7 @@ public AbstractMachineManager(
     this.historyTranscoder = historyTranscoder;
     this.expiringMachineStateTranscoder = expiringMachineStateTranscoder;
     this.maxHistoryEntries = configuration.getMaxMachineHistoryEntries();
+    this.configuration = configuration;
   }
 
   protected abstract String getRoot();
@@ -154,6 +156,10 @@ protected List<T> getObjectsNoCache(String root) {
 
   protected abstract void deleteFromLeaderCache(String objectId);
 
+  public SingularityConfiguration getConfiguration() {
+    return configuration;
+  }
+
   public enum StateChangeResult {
     FAILURE_NOT_FOUND,
     FAILURE_ALREADY_AT_STATE,

SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityAgentAndRackManager.java

@@ -22,6 +22,7 @@
 import com.hubspot.singularity.SingularityTask;
 import com.hubspot.singularity.SingularityTaskId;
 import com.hubspot.singularity.SingularityTaskRequest;
+import com.hubspot.singularity.config.OverrideConfiguration;
 import com.hubspot.singularity.config.SingularityConfiguration;
 import com.hubspot.singularity.data.AbstractMachineManager;
 import com.hubspot.singularity.data.AgentManager;
@@ -64,11 +65,13 @@ public class SingularityAgentAndRackManager {
   private final SingularityAgentAndRackHelper agentAndRackHelper;
   private final AtomicInteger activeAgentsLost;
   private final SingularityLeaderCache leaderCache;
+  private final OverrideConfiguration overrides;
 
   @Inject
   SingularityAgentAndRackManager(
     SingularityAgentAndRackHelper agentAndRackHelper,
     SingularityConfiguration configuration,
+    OverrideConfiguration overrides,
     SingularityExceptionNotifier exceptionNotifier,
     RackManager rackManager,
     AgentManager agentManager,
@@ -80,6 +83,7 @@ public class SingularityAgentAndRackManager {
     SingularityLeaderCache leaderCache
   ) {
     this.configuration = configuration;
+    this.overrides = overrides;
 
     this.exceptionNotifier = exceptionNotifier;
     this.agentAndRackHelper = agentAndRackHelper;
@@ -159,10 +163,12 @@ AgentMatchState doesOfferMatch(
       return AgentMatchState.AGENT_ATTRIBUTES_DO_NOT_MATCH;
     }
 
-    final AgentPlacement agentPlacement = taskRequest
-      .getRequest()
-      .getAgentPlacement()
-      .orElse(configuration.getDefaultAgentPlacement());
+    final AgentPlacement agentPlacement = maybeOverrideAgentPlacement(
+      taskRequest
+        .getRequest()
+        .getAgentPlacement()
+        .orElse(configuration.getDefaultAgentPlacement())
+    );
 
     if (
       !taskRequest.getRequest().isRackSensitive() &&
@@ -244,7 +250,9 @@ AgentMatchState doesOfferMatch(
       }
     }
 
-    if (taskRequest.getRequest().isRackSensitive()) {
+    if (
+      overrides.isAllowRackSensitivity() && taskRequest.getRequest().isRackSensitive()
+    ) {
       final boolean isRackOk = isRackOk(
         countPerRack,
         sanitizedRackId,
@@ -363,6 +371,10 @@ private boolean isPreferred(
     );
   }
 
+  private AgentPlacement maybeOverrideAgentPlacement(AgentPlacement placement) {
+    return overrides.getAgentPlacementOverride().orElse(placement);
+  }
+
   private boolean isPreferredByAllowedAttributes(
     SingularityOfferHolder offerHolder,
     SingularityTaskRequest taskRequest

SingularityService/src/main/java/com/hubspot/singularity/resources/AbstractLeaderAwareResource.java

@@ -71,6 +71,8 @@ protected <T, Q> T maybeProxyToLeader(
     switch (request.getMethod().toUpperCase()) {
       case "POST":
         requestBuilder = httpClient.preparePost(url);
+        // necessary to ensure POST requests without bodies work (will hit 30 second timeout otherwise)
+        requestBuilder.setBody("");
         break;
       case "PUT":
         requestBuilder = httpClient.preparePut(url);

SingularityService/src/main/java/com/hubspot/singularity/resources/AbstractMachineResource.java

@@ -10,6 +10,7 @@
 import com.hubspot.singularity.SingularityUser;
 import com.hubspot.singularity.api.SingularityMachineChangeRequest;
 import com.hubspot.singularity.auth.SingularityAuthorizer;
+import com.hubspot.singularity.config.SingularityConfiguration;
 import com.hubspot.singularity.data.AbstractMachineManager;
 import com.hubspot.singularity.data.AbstractMachineManager.StateChangeResult;
 import com.hubspot.singularity.data.SingularityValidator;
@@ -18,6 +19,7 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.function.Consumer;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Response.Status;
 import org.apache.curator.framework.recipes.leader.LeaderLatch;
@@ -132,6 +134,16 @@ protected void decommission(
     saveExpiring(decommissionRequest, user, objectId);
   }
 
+  protected void configure(
+    Consumer<SingularityConfiguration> configurer,
+    SingularityUser user,
+    SingularityAction action
+  ) {
+    authorizationHelper.checkAdminAuthorization(user);
+    validator.checkActionEnabled(action);
+    configurer.accept(manager.getConfiguration());
+  }
+
   public void updateDecommissionAgent(
     SingularityUser user,
     String agentId,

SingularityService/src/main/java/com/hubspot/singularity/resources/SingularityConfigurationResource.java

@@ -1,28 +1,61 @@
 package com.hubspot.singularity.resources;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.inject.Inject;
+import com.hubspot.singularity.AgentPlacement;
+import com.hubspot.singularity.Singularity;
 import com.hubspot.singularity.SingularityLimits;
+import com.hubspot.singularity.SingularityUser;
+import com.hubspot.singularity.auth.SingularityAuthorizer;
 import com.hubspot.singularity.config.ApiPaths;
+import com.hubspot.singularity.config.OverrideConfiguration;
 import com.hubspot.singularity.config.SingularityConfiguration;
+import com.ning.http.client.AsyncHttpClient;
+import io.dropwizard.auth.Auth;
 import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import io.swagger.v3.oas.annotations.tags.Tags;
+import java.util.Optional;
+import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.curator.framework.recipes.leader.LeaderLatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Path(ApiPaths.CONFIGURATION_RESOURCE_PATH)
 @Produces({ MediaType.APPLICATION_JSON })
-@Schema(title = "Exposes some Singularity configuration values")
+@Schema(title = "Exposes some live Singularity configuration")
 @Tags({ @Tag(name = "Singularity configuration") })
-public class SingularityConfigurationResource {
+public class SingularityConfigurationResource extends AbstractLeaderAwareResource {
+  private static final Logger LOG = LoggerFactory.getLogger(
+    SingularityConfigurationResource.class
+  );
   private final SingularityConfiguration config;
+  private final SingularityAuthorizer auth;
+  private final OverrideConfiguration overrides;
 
   @Inject
-  public SingularityConfigurationResource(SingularityConfiguration config) {
+  public SingularityConfigurationResource(
+    SingularityConfiguration config,
+    OverrideConfiguration overrides,
+    SingularityAuthorizer authorizationHelper,
+    LeaderLatch leaderLatch,
+    AsyncHttpClient httpClient,
+    @Singularity ObjectMapper objectMapper
+  ) {
+    super(httpClient, leaderLatch, objectMapper);
     this.config = config;
+    this.overrides = overrides;
+    this.auth = authorizationHelper;
   }
 
   @GET
@@ -31,4 +64,91 @@ public SingularityConfigurationResource(SingularityConfiguration config) {
   public SingularityLimits getSingularityLimits() {
     return new SingularityLimits(config.getMaxDecommissioningAgents());
   }
+
+  @POST
+  @Path("/rack-sensitive/enable")
+  @Operation(summary = "Enable global rack sensitivity, respecting request settings")
+  public Response enableGlobalRackSensitivity(
+    @Context HttpServletRequest requestContext,
+    @Parameter(hidden = true) @Auth SingularityUser user
+  ) {
+    return maybeProxyToLeader(
+      requestContext,
+      Response.class,
+      null,
+      () -> {
+        auth.checkAdminAuthorization(user);
+        LOG.info("Config override - allowRackSensitivity=true");
+        overrides.setAllowRackSensitivity(true);
+        return Response.ok().build();
+      }
+    );
+  }
+
+  @POST
+  @Path("/rack-sensitive/disable")
+  @Operation(summary = "Disable global rack sensitivity, overriding request settings")
+  public Response disableGlobalRackSensitivity(
+    @Context HttpServletRequest requestContext,
+    @Parameter(hidden = true) @Auth SingularityUser user
+  ) {
+    auth.checkAdminAuthorization(user);
+    return maybeProxyToLeader(
+      requestContext,
+      Response.class,
+      null,
+      () -> {
+        LOG.info("Config override - allowRackSensitivity=false");
+        overrides.setAllowRackSensitivity(false);
+        return Response.ok().build();
+      }
+    );
+  }
+
+  @POST
+  @Path("/placement-strategy/override/set/{strategy}")
+  @Operation(
+    summary = "Set global placement strategy override, causing scheduling to ignore the default and request settings."
+  )
+  public Response setPlacementStrategyOverride(
+    @Context HttpServletRequest requestContext,
+    @Parameter(required = false, description = "Placement strategy name") @PathParam(
+      "strategy"
+    ) AgentPlacement strategy,
+    @Parameter(hidden = true) @Auth SingularityUser user
+  ) {
+    auth.checkAdminAuthorization(user);
+    return maybeProxyToLeader(
+      requestContext,
+      Response.class,
+      null,
+      () -> {
+        LOG.info("Config override - agentPlacementOverride={}", strategy);
+        overrides.setAgentPlacementOverride(Optional.ofNullable(strategy));
+        return Response.ok().build();
+      }
+    );
+  }
+
+  @POST
+  @Path("/placement-strategy/override/clear")
+  @Operation(
+    summary = "Clear global placement strategy override, causing scheduling to respect the default and request settings."
+  )
+  public Response disableSeparatePlacement(
+    @Context HttpServletRequest requestContext,
+    @Parameter(hidden = true) @Auth SingularityUser user
+  ) {
+    auth.checkAdminAuthorization(user);
+    return maybeProxyToLeader(
+      requestContext,
+      Response.class,
+      null,
+      () -> {
+        LOG.info("Config override - agentPlacementOverride=");
+        overrides.setAgentPlacementOverride(Optional.empty());
+        return Response.ok().build();
+      }
+    );
+  }
 }

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityAutoScaleSpreadAllPoller.java

@@ -56,6 +56,7 @@ public void runActionOnPoll() {
       lock.runWithRequestLock(
         () -> {
           SingularityRequest request = requestWithState.getRequest();
+          // global override not supported here
           AgentPlacement placement = request
             .getAgentPlacement()
             .orElse(defaultAgentPlacement);