diff --git a/docs/_posts/2000-01-06-docker-swarm.md b/docs/_posts/2000-01-06-docker-swarm.md new file mode 100644 index 0000000000..35d1a678ff --- /dev/null +++ b/docs/_posts/2000-01-06-docker-swarm.md @@ -0,0 +1,115 @@ +--- +title: "Docker Swarm" +bg: edge_water +color: black +fa-icon: img/docker_swarm_icon.png +--- + +### Before starting + +Setup a [docker swarm](https://docs.docker.com/get-started/part4/) with at least two nodes. + +#### Constellations + +* One manager and multiple workers. The hub will run on the manager and the +created browser containers will run on the workers. +* A high available docker swarm with multiple managers. The manager that runs the hub will +not run any created browser container. + +_Info:_ Currently we do not support running created browser containers on the same node as +the hub. Running the browser containers and the hub on the same node leads to communication +problems, which we hope to fix soon. + +#### Images + +Pull images: +{% highlight shell %} +docker pull dosel/zalenium +docker pull elgalu/selenium +{% endhighlight %} + + +### Run Zalenium + +Provide a docker-compose.yml file and deploy it: +{% highlight shell %} +docker stack deploy -c ./docker-compose.yml STACK +{% endhighlight %} + +_Info:_ Use an appropriate name for `STACK`. + +docker-compose.yml example: +{% highlight shell %} +version: "3.7" + +services: + zalenium: + image: dosel/zalenium + hostname: zalenium + deploy: + placement: + constraints: + - node.role == manager + labels: + - "de.zalando.gridRole=hub" # important for us to identify the node which runs zalenium hub + ports: + - "4444:4444" + - "8000:8000" # port for remote debugging zalenium code + networks: + - zalenium # attachable overlay network to use + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/videos:/home/seluser/videos + environment: + - PULL_SELENIUM_IMAGE=true + - ZALENIUM_PROXY_CLEANUP_TIMEOUT=1800 + command: ["start", "--swarmOverlayNetwork", "STACK_zalenium", "--videoRecordingEnabled", "false"] + +networks: + zalenium: + driver: overlay + attachable: true + +{% endhighlight %} + +_Info:_ It is important to give the service that runs the zalenium hub the label +`"de.zalando.gridRole=hub"`. This helps to identify the node which runs the hub +and created browser containers will not be deployed on this node, which would cause +communication problems between the hub and the created browser containers. + +Video recording and logs are currently not supported, but we hope to re-enable this +feature with docker swarm. + +#### Network + +The created overlay network must be passed as argument `--swarmOverlayNetwork` to zalenium, +which will actually switch Zalenium to Docker Swarm mode. + +Make sure passing the network name with its stack name as prefix. + +In our example we named our network "zalenium" and the stack was named "STACK" so the network +will have the name `"STACK_zalenium"`, which we passed to `"--swarmOverlayNetwork"`. + + +### Technical Information + +__Synchronized docker operations__ + +Docker operations run in synchronized blocks to prevent stale browser containers remain forever. + +see also: +- [eclipse-ee4j/jersey#3772](https://github.com/eclipse-ee4j/jersey/issues/3772) +- [zalando/zalenium#808](https://github.com/zalando/zalenium/issues/808) + +__One service per test session__ + +For each test session we deploy a new service that will create a browser container to run tests. + +Working with one service and adapt the number of replicas does not work because we can't +control which browser containers will be removed when decreasing replicas. It can and +will happen that docker will remove a browser container with a running test to fulfill +number of replicas. + + + + diff --git a/docs/_posts/2000-01-06-kubernetes.md b/docs/_posts/2000-01-07-kubernetes.md similarity index 100% rename from docs/_posts/2000-01-06-kubernetes.md rename to docs/_posts/2000-01-07-kubernetes.md diff --git a/docs/_posts/2000-01-07-usage.md b/docs/_posts/2000-01-08-usage.md similarity index 100% rename from docs/_posts/2000-01-07-usage.md rename to docs/_posts/2000-01-08-usage.md diff --git a/docs/_posts/2000-01-08-how.md b/docs/_posts/2000-01-09-how.md similarity index 100% rename from docs/_posts/2000-01-08-how.md rename to docs/_posts/2000-01-09-how.md diff --git a/docs/_posts/2000-01-09-contributing.md b/docs/_posts/2000-01-10-contributing.md similarity index 100% rename from docs/_posts/2000-01-09-contributing.md rename to docs/_posts/2000-01-10-contributing.md diff --git a/docs/_posts/2000-01-10-faq.md b/docs/_posts/2000-01-11-faq.md similarity index 100% rename from docs/_posts/2000-01-10-faq.md rename to docs/_posts/2000-01-11-faq.md diff --git a/docs/_posts/2000-01-11-links.md b/docs/_posts/2000-01-12-links.md similarity index 100% rename from docs/_posts/2000-01-11-links.md rename to docs/_posts/2000-01-12-links.md diff --git a/docs/docker/docker-compose-swarm.yml b/docs/docker/docker-compose-swarm.yml index d60919277e..7733ef6b01 100644 --- a/docs/docker/docker-compose-swarm.yml +++ b/docs/docker/docker-compose-swarm.yml @@ -1,11 +1,15 @@ version: "3.7" + services: zalenium: image: dosel/zalenium + hostname: zalenium deploy: placement: constraints: - node.role == manager + labels: + - "de.zalando.gridRole=hub" # important for us to identify the node which runs zalenium hub ports: - "4444:4444" networks: @@ -15,8 +19,9 @@ services: - /tmp/videos:/home/seluser/videos environment: - PULL_SELENIUM_IMAGE=true - command: ["start", "--swarmOverlayNetwork", "_zalenium", "--videoRecordingEnabled", "false"] + command: ["start", "--swarmOverlayNetwork", "STACK_zalenium", "--videoRecordingEnabled", "false"] + networks: zalenium: driver: overlay - attachable: true \ No newline at end of file + attachable: true diff --git a/docs/img/docker_swarm_icon.png b/docs/img/docker_swarm_icon.png new file mode 100644 index 0000000000..5a50a9dedf Binary files /dev/null and b/docs/img/docker_swarm_icon.png differ diff --git a/src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmContainerClient.java b/src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmContainerClient.java index d1a664d14c..10bff3f661 100644 --- a/src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmContainerClient.java +++ b/src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmContainerClient.java @@ -245,39 +245,46 @@ private ContainerSpec buildContainerSpec(List flattenedEnvVars, String i } private TaskSpec buildTaskSpec(ContainerSpec containerSpec) { - final RestartPolicy restartPolicy = RestartPolicy.builder() - .condition("on-failure") - .build(); + try { + final RestartPolicy restartPolicy = RestartPolicy.builder() + .condition("on-failure") + .build(); + String hostname = SwarmUtilities.getHubHostname(); + final List placementList = new ArrayList<>(); - final List placementList = new ArrayList<>(); + placementList.add("node.hostname != " + hostname); - placementList.add("node.role==worker"); + final Placement placement = Placement.create(placementList); - final Placement placement = Placement.create(placementList); - Resources.Builder resourceBuilder = Resources.builder(); - String cpuLimit = getSeleniumContainerCpuLimit(); - String memLimit = getSeleniumContainerMemoryLimit(); + Resources.Builder resourceBuilder = Resources.builder(); + String cpuLimit = getSeleniumContainerCpuLimit(); + String memLimit = getSeleniumContainerMemoryLimit(); - if (!Strings.isNullOrEmpty(cpuLimit)) { - resourceBuilder.nanoCpus(Long.valueOf(cpuLimit)); - } + if (!Strings.isNullOrEmpty(cpuLimit)) { + resourceBuilder.nanoCpus(Long.valueOf(cpuLimit)); + } - if (!Strings.isNullOrEmpty(memLimit)) { - resourceBuilder.memoryBytes(Long.valueOf(memLimit)); - } + if (!Strings.isNullOrEmpty(memLimit)) { + resourceBuilder.memoryBytes(Long.valueOf(memLimit)); + } - ResourceRequirements resourceRequirements = ResourceRequirements.builder() - .limits(resourceBuilder.build()) - .build(); + ResourceRequirements resourceRequirements = ResourceRequirements.builder() + .limits(resourceBuilder.build()) + .build(); + + final TaskSpec.Builder taskSpecBuilder = TaskSpec.builder() + .resources(resourceRequirements) + .restartPolicy(restartPolicy) + .placement(placement) + .containerSpec(containerSpec); - final TaskSpec.Builder taskSpecBuilder = TaskSpec.builder() - .resources(resourceRequirements) - .restartPolicy(restartPolicy) - .placement(placement) - .containerSpec(containerSpec); + return taskSpecBuilder.build(); + } catch (DockerException | InterruptedException e) { + e.printStackTrace(); + } - return taskSpecBuilder.build(); + return null; } private ServiceSpec buildServiceSpec(TaskSpec taskSpec, String nodePort, String noVncPort) { diff --git a/src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmUtilities.java b/src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmUtilities.java index e3e53b0721..a8f62926ee 100644 --- a/src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmUtilities.java +++ b/src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmUtilities.java @@ -108,6 +108,27 @@ public static synchronized Task getTaskByContainerId(String containerId) throws return null; } + public static synchronized Task getTaskByContainerLabel(String labelKey, String labelValue) throws DockerException, InterruptedException { + List tasks = dockerClient.listTasks(); + + for (Task task : CollectionUtils.emptyIfNull(tasks)) { + ContainerStatus containerStatus = task.status().containerStatus(); + + if (containerStatus != null) { + ContainerInfo containerInfo = dockerClient.inspectContainer(containerStatus.containerId()); + ImmutableMap labels = containerInfo.config().labels(); + boolean hasLabel = labels != null && labels.containsKey(labelKey); + boolean labelHasValue = labels != null && labels.get(labelKey).equals(labelValue); + + if (hasLabel && labelHasValue) { + return task; + } + } + } + + return null; + } + public static synchronized Task getTaskByServiceId(String serviceId) throws DockerException, InterruptedException { String serviceName = dockerClient.inspectService(serviceId).spec().name(); Task.Criteria criteria = Task.Criteria.builder().serviceName(serviceName).build(); @@ -149,6 +170,22 @@ public static String getSwarmIp(ContainerInfo containerInfo) { return ipAddress; } + + public static synchronized String getHubHostname () throws DockerException, InterruptedException { + String hubLabelKey = "de.zalando.gridRole"; + String hubLabelValue = "hub"; + Task task = getTaskByContainerLabel(hubLabelKey, hubLabelValue); + + if (task != null) { + String nodeId = task.nodeId(); + NodeInfo nodeInfo = dockerClient.inspectNode(nodeId); + + return nodeInfo.description().hostname(); + } + + return null; + } + public static boolean isSwarmActive() { return !overlayNetwork.isEmpty(); }