Skip to content

Commit

Permalink
feat(deployments): allows for custom sizing for kubernetes distribute…
Browse files Browse the repository at this point in the history
…d deployments in halconfig (#724)
  • Loading branch information
omawhite authored and edwinavalos committed Oct 11, 2017
1 parent ce595a1 commit 5795c82
Show file tree
Hide file tree
Showing 10 changed files with 492 additions and 23 deletions.
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,16 @@ The __daemon__ validates and generates Spinnaker config using your
__halconfig__. It must run on a machine that has any credentials needed by
Spinnaker in order to validate your configuration.

### debugging

To run a daemon locally for JVM debugging, set the java system property ```DEBUG=true```. For example:
```
./gradlew halyard-web:run -DDEBUG=true
```

It listens for the debugger on port 9099, and does _not_ wait for the debugger before running
halyard. To change these, check out the relevant bits in halyard-web/halyard-web.gradle

## hal

__hal__ is a CLI for making changes to your __halconfig__ via the __daemon__.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
* Copyright 2017 Target, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.halyard.config.model.v1.node;

import lombok.Data;
import lombok.EqualsAndHashCode;

import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;

/**
* A CustomSizing is a map of maps where you can hack in provider-specific settings related to instance/container/pod sizes.
*/
@Data
@EqualsAndHashCode(callSuper = false)
public class CustomSizing implements Map<String, Map> {
Map<String, Map> componentSizings = new HashMap<>();

public static String stringOrNull(Object value) {
return value != null ? String.valueOf(value) : null;
}

@Override
public int size() {
return componentSizings.size();
}

@Override
public boolean isEmpty() {
return componentSizings.isEmpty();
}

@Override
public boolean containsKey(Object key) {
return componentSizings.containsKey(key);
}

@Override
public boolean containsValue(Object value) {
return componentSizings.containsValue(value);
}

@Override
public Map get(Object key) {
return componentSizings.get(key);
}

@Override
public Map put(String key, Map value) {
return componentSizings.put(key, value);
}

@Override
public Map remove(Object key) {
return componentSizings.remove(key);
}

@Override
public void putAll(Map m) {
componentSizings.putAll(m);
}

@Override
public void clear() {
componentSizings.clear();
}

@Override
public Set<String> keySet() {
return componentSizings.keySet();
}

@Override
public Collection<Map> values() {
return componentSizings.values();
}

@Override
public Set<Entry<String, Map>> entrySet() {
return componentSizings.entrySet();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ public static Size fromString(String name) {
private Consul consul = new Consul();
private Vault vault = new Vault();
private String location;
private CustomSizing customSizing = new CustomSizing();

@Data
public static class Consul {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,44 @@ deploymentConfigurations:
out.deploymentConfigurations[0].deploymentEnvironment.location == 'myLocation'
}

void "parses deployment custom sizings"() {
setup:
String config = """
deploymentConfigurations:
- deploymentEnvironment:
customSizing:
clouddriver:
requests:
memory: 64Mi
cpu: 250m
limits:
memory: 128Mi
cpu: 500m
echo:
requests:
memory: 128Mi
cpu: 500m
limits:
memory: 64Mi
cpu: 250m
"""
InputStream stream = new ByteArrayInputStream(config.getBytes(StandardCharsets.UTF_8))
Halconfig out = null

when:
out = parser.parseHalconfig(stream)

then:
out.deploymentConfigurations[0].deploymentEnvironment.customSizing['clouddriver'].requests.memory == '64Mi'
out.deploymentConfigurations[0].deploymentEnvironment.customSizing['clouddriver'].requests.cpu == '250m'
out.deploymentConfigurations[0].deploymentEnvironment.customSizing['clouddriver'].limits.memory == '128Mi'
out.deploymentConfigurations[0].deploymentEnvironment.customSizing['clouddriver'].limits.cpu == '500m'
out.deploymentConfigurations[0].deploymentEnvironment.customSizing['echo'].limits.memory == '64Mi'
out.deploymentConfigurations[0].deploymentEnvironment.customSizing['echo'].limits.cpu == '250m'
out.deploymentConfigurations[0].deploymentEnvironment.customSizing['echo'].requests.memory == '128Mi'
out.deploymentConfigurations[0].deploymentEnvironment.customSizing['echo'].requests.cpu == '500m'
}

@Unroll("parses authn: #authnProvider:#propertyName value should be #propertyValue")
void "parses all authn properties"() {
setup:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,17 @@
import com.netflix.spinnaker.halyard.deploy.services.v1.GenerateService;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.RunningServiceDetails;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.SpinnakerRuntimeSettings;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.*;

import java.util.*;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.ConfigSource;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.HasServiceSettings;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.ServiceSettings;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.SpinnakerMonitoringDaemonService;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.SpinnakerService;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* This interface represents the cloud-environments specific information/operations required to install a service.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesHttpGetAction;
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesImageDescription;
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesProbe;
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesResourceDescription;
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesSecretVolumeSource;
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesTcpSocketAction;
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesVolumeMount;
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesVolumeSource;
import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.KubernetesVolumeSourceType;
import com.netflix.spinnaker.halyard.config.model.v1.node.CustomSizing;
import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentEnvironment;
import com.netflix.spinnaker.halyard.config.model.v1.node.Provider;
import com.netflix.spinnaker.halyard.config.model.v1.providers.kubernetes.KubernetesAccount;
Expand Down Expand Up @@ -299,10 +301,9 @@ default Map<String, Object> getServerGroupDescription(
DeployKubernetesAtomicOperationDescription description = new DeployKubernetesAtomicOperationDescription();
SpinnakerMonitoringDaemonService monitoringService = getMonitoringDaemonService();
ServiceSettings settings = runtimeSettings.getServiceSettings(getService());
DeploymentEnvironment.Size size = details
DeploymentEnvironment deploymentEnvironment = details
.getDeploymentConfiguration()
.getDeploymentEnvironment()
.getSize();
.getDeploymentEnvironment();

String accountName = details.getAccount().getName();
String namespace = getNamespace(settings);
Expand Down Expand Up @@ -335,13 +336,13 @@ default Map<String, Object> getServerGroupDescription(

List<KubernetesContainerDescription> containers = new ArrayList<>();
ServiceSettings serviceSettings = runtimeSettings.getServiceSettings(getService());
KubernetesContainerDescription container = buildContainer(name, serviceSettings, configSources, size);
KubernetesContainerDescription container = buildContainer(name, serviceSettings, configSources, deploymentEnvironment);
containers.add(container);

ServiceSettings monitoringSettings = runtimeSettings.getServiceSettings(monitoringService);
if (monitoringSettings.getEnabled() && serviceSettings.getMonitored()) {
serviceSettings = runtimeSettings.getServiceSettings(monitoringService);
container = buildContainer(monitoringService.getServiceName(), serviceSettings, configSources, size);
container = buildContainer(monitoringService.getServiceName(), serviceSettings, configSources, deploymentEnvironment);
containers.add(container);
}

Expand All @@ -350,7 +351,7 @@ default Map<String, Object> getServerGroupDescription(
return getObjectMapper().convertValue(description, new TypeReference<Map<String, Object>>() { });
}

default KubernetesContainerDescription buildContainer(String name, ServiceSettings settings, List<ConfigSource> configSources, DeploymentEnvironment.Size size) {
default KubernetesContainerDescription buildContainer(String name, ServiceSettings settings, List<ConfigSource> configSources, DeploymentEnvironment deploymentEnvironment) {
KubernetesContainerDescription container = new KubernetesContainerDescription();
KubernetesProbe readinessProbe = new KubernetesProbe();
KubernetesHandler handler = new KubernetesHandler();
Expand All @@ -373,14 +374,7 @@ default KubernetesContainerDescription buildContainer(String name, ServiceSettin
readinessProbe.setHandler(handler);
container.setReadinessProbe(readinessProbe);

/* TODO(lwander) this needs work
SizingTranslation.ServiceSize serviceSize = sizingTranslation.getServiceSize(size, service);
KubernetesResourceDescription resources = new KubernetesResourceDescription();
resources.setCpu(serviceSize.getCpu());
resources.setMemory(serviceSize.getRam());
container.setRequests(resources);
container.setLimits(resources);
*/
applyCustomSize(container, deploymentEnvironment, name);

KubernetesImageDescription imageDescription = KubernetesUtil.buildImageDescription(settings.getArtifactId());
container.setImageDescription(imageDescription);
Expand Down Expand Up @@ -426,6 +420,31 @@ default KubernetesContainerDescription buildContainer(String name, ServiceSettin
return container;
}

default void applyCustomSize(KubernetesContainerDescription container, DeploymentEnvironment deploymentEnvironment, String componentName) {
Map<String, Map> componentSizing = deploymentEnvironment.getCustomSizing().get(componentName);

if (componentSizing != null) {

if (componentSizing.get("requests") != null) {
container.setRequests(retrieveKubernetesResourceDescription(componentSizing, "requests"));
}

if (componentSizing.get("limits") != null) {
container.setLimits(retrieveKubernetesResourceDescription(componentSizing, "limits"));
}
}

/* TODO(lwander) this needs work
SizingTranslation.ServiceSize serviceSize = sizingTranslation.getServiceSize(deploymentEnvironment.getSize(), service);
*/
}

default KubernetesResourceDescription retrieveKubernetesResourceDescription(Map<String, Map> componentSizing, String resourceType) {
KubernetesResourceDescription requests = new KubernetesResourceDescription();
requests.setCpu(CustomSizing.stringOrNull(componentSizing.get(resourceType).get("cpu")));
requests.setMemory(CustomSizing.stringOrNull(componentSizing.get(resourceType).get("memory")));
return requests;
}

default void ensureRunning(
AccountDeploymentDetails<KubernetesAccount> details,
Expand Down Expand Up @@ -477,12 +496,13 @@ default void ensureRunning(
}

List<Container> containers = new ArrayList<>();
containers.add(ResourceBuilder.buildContainer(serviceName, settings, configSources));
DeploymentEnvironment deploymentEnvironment = details.getDeploymentConfiguration().getDeploymentEnvironment();
containers.add(ResourceBuilder.buildContainer(serviceName, settings, configSources, deploymentEnvironment));

for (SidecarService sidecarService : getSidecars(runtimeSettings)) {
String sidecarName = sidecarService.getService().getServiceName();
ServiceSettings sidecarSettings = resolvedConfiguration.getServiceSettings(sidecarService.getService());
containers.add(ResourceBuilder.buildContainer(sidecarName, sidecarSettings, configSources));
containers.add(ResourceBuilder.buildContainer(sidecarName, sidecarSettings, configSources, deploymentEnvironment));
}

List<Volume> volumes = configSources.stream().map(c -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,28 @@

package com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.distributed.kubernetes;

import com.netflix.spinnaker.halyard.config.model.v1.node.CustomSizing;
import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentEnvironment;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.ConfigSource;
import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.ServiceSettings;
import io.fabric8.kubernetes.api.model.*;
import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.ContainerBuilder;
import io.fabric8.kubernetes.api.model.ContainerPortBuilder;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.ProbeBuilder;
import io.fabric8.kubernetes.api.model.QuantityBuilder;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder;
import io.fabric8.kubernetes.api.model.VolumeMount;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

class ResourceBuilder {
static Container buildContainer(String name, ServiceSettings settings, List<ConfigSource> configSources) {
static Container buildContainer(String name, ServiceSettings settings, List<ConfigSource> configSources, DeploymentEnvironment deploymentEnvironment) {
int port = settings.getPort();
List<EnvVar> envVars = settings.getEnv().entrySet().stream().map(e -> {
EnvVarBuilder envVarBuilder = new EnvVarBuilder();
Expand Down Expand Up @@ -62,16 +75,37 @@ static Container buildContainer(String name, ServiceSettings settings, List<Conf
List<VolumeMount> volumeMounts = configSources.stream().map(c -> {
return new VolumeMountBuilder().withMountPath(c.getMountPath()).withName(c.getId()).build();
}).collect(Collectors.toList());
ContainerBuilder containerBuilder = new ContainerBuilder();

ContainerBuilder containerBuilder = new ContainerBuilder();
containerBuilder = containerBuilder
.withName(name)
.withImage(settings.getArtifactId())
.withPorts(new ContainerPortBuilder().withContainerPort(port).build())
.withVolumeMounts(volumeMounts)
.withEnv(envVars)
.withReadinessProbe(probeBuilder.build());
.withReadinessProbe(probeBuilder.build())
.withResources(buildResourceRequirements(name, deploymentEnvironment));

return containerBuilder.build();
}

static ResourceRequirements buildResourceRequirements(String serviceName, DeploymentEnvironment deploymentEnvironment) {
Map<String, Map> customSizing = deploymentEnvironment.getCustomSizing().get(serviceName);

if (customSizing == null) {
return null;
}

ResourceRequirementsBuilder resourceRequirementsBuilder = new ResourceRequirementsBuilder();
if (customSizing.get("requests") != null) {
resourceRequirementsBuilder.addToRequests("memory", new QuantityBuilder().withAmount(CustomSizing.stringOrNull(customSizing.get("requests").get("memory"))).build());
resourceRequirementsBuilder.addToRequests("cpu", new QuantityBuilder().withAmount(CustomSizing.stringOrNull(customSizing.get("requests").get("cpu"))).build());
}
if (customSizing.get("limits") != null) {
resourceRequirementsBuilder.addToLimits("memory", new QuantityBuilder().withAmount(CustomSizing.stringOrNull(customSizing.get("limits").get("memory"))).build());
resourceRequirementsBuilder.addToLimits("cpu", new QuantityBuilder().withAmount(CustomSizing.stringOrNull(customSizing.get("limits").get("cpu"))).build());
}

return resourceRequirementsBuilder.build();
}
}
Loading

0 comments on commit 5795c82

Please sign in to comment.