From 9fffc596b661b7bea221db7a406aa3559a9deeae Mon Sep 17 00:00:00 2001 From: Lars Wander Date: Fri, 7 Jul 2017 12:41:53 -0400 Subject: [PATCH] feat(deploy): Scale down orca & rosco during deploys When possible, orca & rosco will be scaled down during deployments (if they aren't handling any work). --- .../deployment/v1/DistributedDeployer.java | 57 ++++++++++++------- .../distributed/DistributedService.java | 1 + .../google/GoogleDistributedService.java | 7 +++ .../google/GoogleProviderUtils.java | 9 +++ .../KubernetesDistributedService.java | 6 ++ .../kubernetes/KubernetesProviderUtils.java | 5 ++ 6 files changed, 65 insertions(+), 20 deletions(-) diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/deployment/v1/DistributedDeployer.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/deployment/v1/DistributedDeployer.java index 87346fb3e9..6c4e8896ee 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/deployment/v1/DistributedDeployer.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/deployment/v1/DistributedDeployer.java @@ -276,16 +276,45 @@ private void reapRoscoServerGroups(AccountDeploymentDetails< List allRoscos = new ArrayList<>(executionsByServerGroupVersion.keySet()); allRoscos.sort(Integer::compareTo); - int roscoCount = allRoscos.size(); - if (roscoCount <= MAX_REMAINING_SERVER_GROUPS) { + cleanupServerGroups(details, roscoService, roscoSettings, executionsByServerGroupVersion, allRoscos); + } + + private void cleanupServerGroups(AccountDeploymentDetails details, + DistributedService service, + ServiceSettings settings, + Map workloadByVersion, + List runningVersions) { + int runningVersionCount = runningVersions.size(); + + if (runningVersionCount <= 1) { + log.info("There are no extra services to cleanup. Running count = " + runningVersionCount); return; } - allRoscos = allRoscos.subList(0, roscoCount - MAX_REMAINING_SERVER_GROUPS); - for (Integer roscoVersion : allRoscos) { - if (executionsByServerGroupVersion.get(roscoVersion) == 0) { - DaemonTaskHandler.message("Reaping old rosco server group sequence " + roscoVersion); - roscoService.deleteVersion(details, roscoSettings, roscoVersion); + /* + * To visualize the array slicing below, say we have 5 running + * server groups, 3 of which we can destroy, one we can scale down, + * and one that's actively serving traffic, as illustrated here: + * + * + * | running - MAX_REMAINING | MAX_REMAINING | + * | No longer in use | backup | active | + * server groups: | v001 | v002 | v003 | v004 | v005 | + */ + List killableVersions = runningVersions.subList(0, runningVersionCount - MAX_REMAINING_SERVER_GROUPS); + List shrinkableVersions = runningVersions.subList(runningVersionCount - MAX_REMAINING_SERVER_GROUPS, runningVersionCount - 1); + + for (Integer version : killableVersions) { + if (workloadByVersion.get(version) == 0) { + DaemonTaskHandler.message("Reaping old server group sequence " + version); + service.deleteVersion(details, settings, version); + } + } + + for (Integer version : shrinkableVersions) { + if (workloadByVersion.get(version) == 0) { + DaemonTaskHandler.message("Shrinking old server group sequence " + version); + service.resizeVersion(details, settings, version, 0); } } } @@ -318,18 +347,6 @@ private void reapOrcaServerGroups(AccountDeploymentDetails allOrcas = new ArrayList<>(executionsByServerGroupVersion.keySet()); allOrcas.sort(Integer::compareTo); - int orcaCount = allOrcas.size(); - if (orcaCount <= MAX_REMAINING_SERVER_GROUPS) { - return; - } - - allOrcas = allOrcas.subList(0, orcaCount - MAX_REMAINING_SERVER_GROUPS); - for (Integer orcaVersion : allOrcas) { - // TODO(lwander) consult clouddriver to ensure this orca isn't enabled - if (executionsByServerGroupVersion.get(orcaVersion) == 0) { - DaemonTaskHandler.message("Reaping old orca instance " + orcaVersion); - orcaService.deleteVersion(details, orcaSettings, orcaVersion); - } - } + cleanupServerGroups(details, orcaService, orcaSettings, executionsByServerGroupVersion, allOrcas); } } diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/DistributedService.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/DistributedService.java index 8d573b6b7f..0e39317244 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/DistributedService.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/DistributedService.java @@ -59,6 +59,7 @@ void ensureRunning(AccountDeploymentDetails details, S connectToService(AccountDeploymentDetails details, SpinnakerRuntimeSettings runtimeSettings, SpinnakerService sidecar); String connectCommand(AccountDeploymentDetails details, SpinnakerRuntimeSettings runtimeSettings); void deleteVersion(AccountDeploymentDetails details, ServiceSettings settings, Integer version); + void resizeVersion(AccountDeploymentDetails details, ServiceSettings settings, int version, int targetSize); boolean isRequiredToBootstrap(); DeployPriority getDeployPriority(); SpinnakerService getService(); diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleDistributedService.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleDistributedService.java index 5261a6a641..e88e7f6a56 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleDistributedService.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleDistributedService.java @@ -91,6 +91,13 @@ default Provider.ProviderType getProviderType() { return Provider.ProviderType.GOOGLE; } + @Override + default void resizeVersion(AccountDeploymentDetails details, ServiceSettings serviceSettings, int version, int targetSize) { + String name = getVersionedName(version); + String zone = serviceSettings.getLocation(); + GoogleProviderUtils.resize(details, zone, name, targetSize); + } + default String getStartupScript() { return String.join("\n", "#!/usr/bin/env bash", "", diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleProviderUtils.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleProviderUtils.java index 5b4c7b3206..1bd1954ec7 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleProviderUtils.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleProviderUtils.java @@ -396,6 +396,15 @@ static String ensureSpinnakerNetworkExists(AccountDeploymentDetails details, String zone, String managedInstaceGroupName, int targetSize) { + Compute compute = getCompute(details); + try { + compute.instanceGroupManagers().resize(details.getAccount().getProject(), zone, managedInstaceGroupName, targetSize); + } catch (IOException e) { + throw new HalException(FATAL, "Unable to resize instance group manager " + managedInstaceGroupName + ": " + e.getMessage(), e); + } + } + static Compute getCompute(AccountDeploymentDetails details) { ConfigProblemSetBuilder problemSetBuilder = new ConfigProblemSetBuilder(null); GoogleNamedAccountCredentials credentials = details.getAccount().getNamedAccountCredentials("", problemSetBuilder); diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/KubernetesDistributedService.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/KubernetesDistributedService.java index bc0b95a7fe..fb97e92a69 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/KubernetesDistributedService.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/KubernetesDistributedService.java @@ -105,6 +105,12 @@ default Map> getAvailabilityZones(ServiceSettings settings) return availabilityZones; } + default void resizeVersion(AccountDeploymentDetails details, ServiceSettings settings, int version, int targetSize) { + String name = getVersionedName(version); + String namespace = getNamespace(settings); + KubernetesProviderUtils.resize(details, namespace, name, targetSize); + } + @Override default Map buildRollbackPipeline(AccountDeploymentDetails details, SpinnakerRuntimeSettings runtimeSettings) { ServiceSettings settings = runtimeSettings.getServiceSettings(getService()); diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/KubernetesProviderUtils.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/KubernetesProviderUtils.java index ef5b5192d5..4c0cee2e18 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/KubernetesProviderUtils.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/KubernetesProviderUtils.java @@ -134,6 +134,11 @@ static KubernetesClient getClient(AccountDeploymentDetails de return new DefaultKubernetesClient(config); } + static void resize(AccountDeploymentDetails details, String namespace, String replicaSetName, int targetSize) { + KubernetesClient client = getClient(details); + client.extensions().replicaSets().inNamespace(namespace).withName(replicaSetName).scale(targetSize); + } + static void upsertSecret(AccountDeploymentDetails details, Set> files, String secretName, String namespace) { KubernetesClient client = getClient(details);