From 8805f0a5c76498d0e60a8375182a8b4891a11db0 Mon Sep 17 00:00:00 2001 From: seder Date: Fri, 24 Apr 2015 14:07:58 +0200 Subject: [PATCH] Add configurable network mode for docker containers This commit adds the possibility to configure the network mode of the containerized Jenkins slaves provisioned by Mesos. For now it is possible to pick "Bridge" or "Host", since these are the two supported modes by Mesos (version 0.21.x). Additionally, in "Bridge" mode it is also possible to define port mappings. Issue: JENKINS-26324 --- pom.xml | 1 + .../plugins/mesos/JenkinsScheduler.java | 129 ++++++++++++++++-- .../jenkinsci/plugins/mesos/MesosCloud.java | 28 +++- .../plugins/mesos/MesosSlaveInfo.java | 77 ++++++++++- .../plugins/mesos/MesosCloud/config.jelly | 31 +++++ 5 files changed, 239 insertions(+), 27 deletions(-) diff --git a/pom.xml b/pom.xml index d40418b0a..680a1dbe8 100644 --- a/pom.xml +++ b/pom.xml @@ -119,6 +119,7 @@ maven-compiler-plugin 1.6 + 1.6 diff --git a/src/main/java/org/jenkinsci/plugins/mesos/JenkinsScheduler.java b/src/main/java/org/jenkinsci/plugins/mesos/JenkinsScheduler.java index ec2e41b1b..a806f08b4 100644 --- a/src/main/java/org/jenkinsci/plugins/mesos/JenkinsScheduler.java +++ b/src/main/java/org/jenkinsci/plugins/mesos/JenkinsScheduler.java @@ -28,19 +28,20 @@ import java.util.Queue; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Level; import java.util.logging.Logger; import jenkins.model.Jenkins; import net.sf.json.JSONObject; -import com.google.protobuf.ByteString; - import org.apache.commons.lang.StringUtils; import org.apache.mesos.MesosSchedulerDriver; import org.apache.mesos.Protos.Attribute; import org.apache.mesos.Protos.CommandInfo; import org.apache.mesos.Protos.ContainerInfo; import org.apache.mesos.Protos.ContainerInfo.DockerInfo; +import org.apache.mesos.Protos.ContainerInfo.DockerInfo.Network; +import org.apache.mesos.Protos.ContainerInfo.DockerInfo.PortMapping; import org.apache.mesos.Protos.Credential; import org.apache.mesos.Protos.ExecutorID; import org.apache.mesos.Protos.Filters; @@ -57,12 +58,13 @@ import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.Protos.TaskStatus; import org.apache.mesos.Protos.Value; -import org.apache.mesos.Protos.Value.Type; +import org.apache.mesos.Protos.Value.Range; import org.apache.mesos.Protos.Volume; import org.apache.mesos.Protos.Volume.Mode; import org.apache.mesos.Scheduler; import org.apache.mesos.SchedulerDriver; +import com.google.protobuf.ByteString; public class JenkinsScheduler implements Scheduler { private static final String SLAVE_JAR_URI_SUFFIX = "jnlpJars/slave.jar"; @@ -147,7 +149,7 @@ public synchronized void stop() { if (driver != null) { driver.stop(); } else { - LOGGER.warning("Unable to stop Mesos driver: driver is null."); + LOGGER.warning("Unable to stop Mesos driver: driver is null."); } running = false; } @@ -247,7 +249,12 @@ public synchronized void resourceOffers(SchedulerDriver driver, List offe if (matches(offer, request)) { matched = true; LOGGER.info("Offer matched! Creating mesos task"); - createMesosTask(offer, request); + + try { + createMesosTask(offer, request); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } requests.remove(request); break; } @@ -262,6 +269,7 @@ public synchronized void resourceOffers(SchedulerDriver driver, List offe private boolean matches(Offer offer, Request request) { double cpus = -1; double mem = -1; + List ports = null; for (Resource resource : offer.getResourcesList()) { if (resource.getName().equals("cpus")) { @@ -279,7 +287,11 @@ private boolean matches(Offer offer, Request request) { } else if (resource.getName().equals("disk")) { LOGGER.fine("Ignoring disk resources from offer"); } else if (resource.getName().equals("ports")) { - LOGGER.fine("Ignoring ports resources from offer"); + if (resource.getType().equals(Value.Type.RANGES)) { + ports = resource.getRanges().getRangeList(); + } else { + LOGGER.severe("Ports resource was not a range: " + resource.getType().toString()); + } } else { LOGGER.warning("Ignoring unknown resource type: " + resource.getName()); } @@ -288,22 +300,35 @@ private boolean matches(Offer offer, Request request) { if (cpus < 0) LOGGER.fine("No cpus resource present"); if (mem < 0) LOGGER.fine("No mem resource present"); + boolean hasPortMappings = request.request.slaveInfo.getContainerInfo().hasPortMappings(); + boolean hasPortResources = ports != null && !ports.isEmpty(); + + if (hasPortMappings && !hasPortResources) { + LOGGER.severe("No ports resource present"); + } + // Check for sufficient cpu and memory resources in the offer. double requestedCpus = request.request.cpus; double requestedMem = (1 + JVM_MEM_OVERHEAD_FACTOR) * request.request.mem; // Get matching slave attribute for this label. JSONObject slaveAttributes = getMesosCloud().getSlaveAttributeForLabel(request.request.slaveInfo.getLabelString()); - if (requestedCpus <= cpus && requestedMem <= mem && slaveAttributesMatch(offer, slaveAttributes)) { + if (requestedCpus <= cpus + && requestedMem <= mem + && !(hasPortMappings && !hasPortResources) + && slaveAttributesMatch(offer, slaveAttributes)) { return true; } else { + String requestedPorts = StringUtils.join(request.request.slaveInfo.getContainerInfo().getPortMappings().toArray(), "/"); + LOGGER.info( "Offer not sufficient for slave request:\n" + offer.getResourcesList().toString() + "\n" + offer.getAttributesList().toString() + "\nRequested for Jenkins slave:\n" + - " cpus: " + requestedCpus + "\n" + - " mem: " + requestedMem + "\n" + + " cpus: " + requestedCpus + "\n" + + " mem: " + requestedMem + "\n" + + " ports: " + requestedPorts + "\n" + " attributes: " + (slaveAttributes == null ? "" : slaveAttributes.toString())); return false; } @@ -345,8 +370,36 @@ private boolean slaveAttributesMatch(Offer offer, JSONObject slaveAttributes) { return slaveTypeMatch; } + private List findPortsToUse(Offer offer, Request request, int maxCount) { + List portsToUse = new ArrayList(); + List portRangesList = null; + + for (Resource resource : offer.getResourcesList()) { + if (resource.getName().equals("ports")) { + portRangesList = resource.getRanges().getRangeList(); + break; + } + } + + LOGGER.fine("portRangesList=" + portRangesList); + + int portRangeIndex = 0; + while (portsToUse.size() < maxCount && portRangeIndex < portRangesList.size()) { + Value.Range currentPortRange = portRangesList.get(portRangeIndex); + long nextPort = currentPortRange.getBegin(); + + while (portsToUse.size() < maxCount && nextPort < currentPortRange.getEnd()) { + portsToUse.add((int)nextPort); + nextPort++; + } + } + + return portsToUse; + } + private void createMesosTask(Offer offer, Request request) { - TaskID taskId = TaskID.newBuilder().setValue(request.request.slave.name).build(); + final String slaveName = request.request.slave.name; + TaskID taskId = TaskID.newBuilder().setValue(slaveName).build(); LOGGER.info("Launching task " + taskId.getValue() + " with URI " + joinPaths(jenkinsMaster, SLAVE_JAR_URI_SUFFIX)); @@ -412,18 +465,66 @@ private void createMesosTask(Offer offer, Request request) { MesosSlaveInfo.ContainerInfo containerInfo = request.request.slaveInfo.getContainerInfo(); if (containerInfo != null) { ContainerInfo.Type containerType = ContainerInfo.Type.valueOf(containerInfo.getType()); - ContainerInfo.Builder containerInfoBuilder = ContainerInfo.newBuilder().setType(containerType); + + ContainerInfo.Builder containerInfoBuilder = ContainerInfo.newBuilder() // + .setType(containerType) // + .setHostname(slaveName); + switch(containerType) { case DOCKER: LOGGER.info("Launching in Docker Mode:" + containerInfo.getDockerImage()); - DockerInfo.Builder dockerInfoBuilder = DockerInfo.newBuilder(); + DockerInfo.Builder dockerInfoBuilder = DockerInfo.newBuilder() // + .setImage(containerInfo.getDockerImage()); + if (containerInfo.getParameters() != null) { for (MesosSlaveInfo.Parameter parameter : containerInfo.getParameters()) { LOGGER.info("Adding Docker parameter '" + parameter.getKey() + ":" + parameter.getValue() + "'"); dockerInfoBuilder.addParameters(Parameter.newBuilder().setKey(parameter.getKey()).setValue(parameter.getValue()).build()); } } - containerInfoBuilder.setDocker(dockerInfoBuilder.setImage(containerInfo.getDockerImage())); + + String networking = request.request.slaveInfo.getContainerInfo().getNetworking(); + dockerInfoBuilder.setNetwork(Network.valueOf(networking)); + + if (request.request.slaveInfo.getContainerInfo().hasPortMappings()) { + List portMappings = request.request.slaveInfo.getContainerInfo().getPortMappings(); + int portToUseIndex = 0; + List portsToUse = findPortsToUse(offer, request, portMappings.size()); + + Value.Ranges.Builder portRangesBuilder = Value.Ranges.newBuilder(); + + for (MesosSlaveInfo.PortMapping portMapping : portMappings) { + PortMapping.Builder portMappingBuilder = PortMapping.newBuilder() // + .setContainerPort(portMapping.getContainerPort()) // + .setProtocol(portMapping.getProtocol()); + + int portToUse = portMapping.getHostPort() == null ? portsToUse.get(portToUseIndex++) : portMapping.getHostPort(); + + portMappingBuilder.setHostPort(portToUse); + + portRangesBuilder.addRange( + Value.Range + .newBuilder() + .setBegin(portToUse) + .setEnd(portToUse) + ); + + LOGGER.finest("Adding portMapping: " + portMapping); + dockerInfoBuilder.addPortMappings(portMappingBuilder); + } + + taskBuilder.addResources( + Resource + .newBuilder() + .setName("ports") + .setType(Value.Type.RANGES) + .setRanges(portRangesBuilder) + ); + } else { + LOGGER.fine("No portMappings found"); + } + + containerInfoBuilder.setDocker(dockerInfoBuilder); break; default: LOGGER.warning("Unknown container type:" + containerInfo.getType()); @@ -607,7 +708,7 @@ public static void supervise() { cloud.stopScheduler(); } } else { - LOGGER.info("Schedular already stopped. NOOP."); + LOGGER.info("Scheduler already stopped. NOOP."); } } catch (Exception e) { LOGGER.info("Exception: " + e); diff --git a/src/main/java/org/jenkinsci/plugins/mesos/MesosCloud.java b/src/main/java/org/jenkinsci/plugins/mesos/MesosCloud.java index cfc42dd6d..983fcb6fb 100644 --- a/src/main/java/org/jenkinsci/plugins/mesos/MesosCloud.java +++ b/src/main/java/org/jenkinsci/plugins/mesos/MesosCloud.java @@ -46,6 +46,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.mesos.MesosNativeLibrary; +import org.apache.mesos.Protos.ContainerInfo.DockerInfo.Network; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.QueryParameter; import org.kohsuke.stapler.StaplerRequest; @@ -101,10 +102,10 @@ public static void init() { for (Cloud c : jenkins.clouds) { if( c instanceof MesosCloud) { - // Register mesos framework on init, if on demand registration is not enabled. - if (!((MesosCloud) c).isOnDemandRegistration()) { + // Register mesos framework on init, if on demand registration is not enabled. + if (!((MesosCloud) c).isOnDemandRegistration()) { ((MesosCloud)c).restartMesos(); - } + } } } } @@ -441,9 +442,9 @@ public boolean configure(StaplerRequest request, JSONObject object) .getBoolean("readOnly"))); } } - + List parameters = new ArrayList(); - + if (containerInfoJson.has("parameters")) { JSONArray parametersJson = containerInfoJson.getJSONArray("parameters"); for (Object obj : parametersJson) { @@ -452,9 +453,24 @@ public boolean configure(StaplerRequest request, JSONObject object) } } + List portMappings = new ArrayList(); + + final String networking = containerInfoJson.getString("networking"); + if (Network.BRIDGE.equals(Network.valueOf(networking)) && containerInfoJson.has("portMappings")) { + JSONArray portMappingsJson = containerInfoJson + .getJSONArray("portMappings"); + for (Object obj : portMappingsJson) { + JSONObject portMappingJson = (JSONObject) obj; + portMappings.add(new MesosSlaveInfo.PortMapping( + portMappingJson.getInt("containerPort"), + portMappingJson.getInt("hostPort"), + portMappingJson.getString("protocol"))); + } + } + containerInfo = new MesosSlaveInfo.ContainerInfo( containerInfoJson.getString("type"), - containerInfoJson.getString("dockerImage"), volumes, parameters); + containerInfoJson.getString("dockerImage"), volumes, parameters, networking, portMappings); } List additionalURIs = new ArrayList(); diff --git a/src/main/java/org/jenkinsci/plugins/mesos/MesosSlaveInfo.java b/src/main/java/org/jenkinsci/plugins/mesos/MesosSlaveInfo.java index 348f175d1..89da2af94 100644 --- a/src/main/java/org/jenkinsci/plugins/mesos/MesosSlaveInfo.java +++ b/src/main/java/org/jenkinsci/plugins/mesos/MesosSlaveInfo.java @@ -2,6 +2,7 @@ import hudson.model.Descriptor.FormException; +import java.util.Collections; import java.util.List; import java.util.logging.Logger; @@ -10,6 +11,7 @@ import net.sf.json.JSONSerializer; import org.apache.commons.lang.StringUtils; +import org.apache.mesos.Protos.ContainerInfo.DockerInfo.Network; import org.kohsuke.stapler.DataBoundConstructor; public class MesosSlaveInfo { @@ -123,7 +125,9 @@ public String getJvmArgs() { return jvmArgs; } - public String getJnlpArgs() {return jnlpArgs; } + public String getJnlpArgs() { + return jnlpArgs; + } public ExternalContainerInfo getExternalContainerInfo() { return externalContainerInfo; @@ -176,14 +180,28 @@ public static class ContainerInfo { private final String dockerImage; private final List volumes; private final List parameters; + private final String networking; + private final List portMappings; @DataBoundConstructor - public ContainerInfo(String type, String dockerImage, List volumes, List parameters) + public ContainerInfo(String type, String dockerImage, List volumes, List parameters, String networking, List portMappings) throws FormException { this.type = type; this.dockerImage = dockerImage; this.volumes = volumes; this.parameters = parameters; + + if (networking == null) { + this.networking = Network.BRIDGE.toString(); + } else { + this.networking = networking; + } + + if (Network.HOST.equals(Network.valueOf(networking))) { + this.portMappings = Collections.emptyList(); + } else { + this.portMappings = portMappings; + } } public String getType() { @@ -197,31 +215,76 @@ public String getDockerImage() { public List getVolumes() { return volumes; } - + public List getParameters() { return parameters; } + + public String getNetworking() { + return networking; + } + + public List getPortMappings() { + return portMappings; + } + + public boolean hasPortMappings() { + return portMappings != null && !portMappings.isEmpty(); + } } - + public static class Parameter { private final String key; private final String value; - + @DataBoundConstructor public Parameter(String key, String value) { this.key = key; this.value = value; } - + public String getKey() { return key; } - + public String getValue() { return value; } } + public static class PortMapping { + + // TODO validate 1 to 65535 + private final Integer containerPort; + private final Integer hostPort; + private final String protocol; + + @DataBoundConstructor + public PortMapping(Integer containerPort, Integer hostPort, String protocol) { + this.containerPort = containerPort; + this.hostPort = hostPort; + this.protocol = protocol; + } + + public Integer getContainerPort() { + return containerPort; + } + + public Integer getHostPort() { + return hostPort; + } + + public String getProtocol() { + return protocol; + } + + @Override + public String toString() { + return (hostPort == null ? 0 : hostPort) + ":" + containerPort; + } + + } + public static class Volume { private final String containerPath; private final String hostPath; diff --git a/src/main/resources/org/jenkinsci/plugins/mesos/MesosCloud/config.jelly b/src/main/resources/org/jenkinsci/plugins/mesos/MesosCloud/config.jelly index 171f16d48..d9dd3133d 100644 --- a/src/main/resources/org/jenkinsci/plugins/mesos/MesosCloud/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/mesos/MesosCloud/config.jelly @@ -103,6 +103,37 @@ + + + + + +
+ + + + + + + + + + + +
+
+
+ +
+
+ + + + +