Improving Docker Swarm Implementation (#984)

* Add functions to return the hub hostname

* Add placement constraint based on hostname

* Add docker-swarm md file

* Add documentation for using docker swarm

* Update docker/docker-compose-swarm.yml file

* Improve docker swarm documentation after review

docs/_posts/2000-01-06-docker-swarm.md

@@ -0,0 +1,115 @@
+---
+title: "Docker Swarm"
+bg: edge_water
+color: black
+fa-icon: img/docker_swarm_icon.png
+---
+
+### Before starting
+
+Setup a [docker swarm](https://docs.docker.com/get-started/part4/) with at least two nodes.
+
+#### Constellations
+
+* One manager and multiple workers. The hub will run on the manager and the 
+created browser containers will run on the workers.
+* A high available docker swarm with multiple managers. The manager that runs the hub will 
+not run any created browser container.
+
+_Info:_ Currently we do not support running created browser containers on the same node as
+the hub. Running the browser containers and the hub on the same node leads to communication
+problems, which we hope to fix soon.
+
+#### Images
+
+Pull images:
+{% highlight shell %}
+docker pull dosel/zalenium
+docker pull elgalu/selenium
+{% endhighlight %}
+
+
+### Run Zalenium
+
+Provide a docker-compose.yml file and deploy it:
+{% highlight shell %}
+docker stack deploy -c ./docker-compose.yml STACK
+{% endhighlight %}
+
+_Info:_ Use an appropriate name for `STACK`.
+
+docker-compose.yml example:
+{% highlight shell %}
+version: "3.7"
+
+services:
+  zalenium:
+    image: dosel/zalenium
+    hostname: zalenium
+    deploy:
+      placement:
+        constraints:
+            - node.role == manager
+    labels:
+        - "de.zalando.gridRole=hub" # important for us to identify the node which runs zalenium hub
+    ports:
+        - "4444:4444"
+        - "8000:8000" # port for remote debugging zalenium code
+    networks:
+        - zalenium # attachable overlay network to use
+    volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
+        - /tmp/videos:/home/seluser/videos
+    environment:
+        - PULL_SELENIUM_IMAGE=true
+        - ZALENIUM_PROXY_CLEANUP_TIMEOUT=1800
+    command: ["start", "--swarmOverlayNetwork", "STACK_zalenium", "--videoRecordingEnabled", "false"]
+
+networks:
+    zalenium:
+        driver: overlay
+        attachable: true
+
+{% endhighlight %}
+
+_Info:_ It is important to give the service that runs the zalenium hub the label
+`"de.zalando.gridRole=hub"`. This helps to identify the node which runs the hub
+and created browser containers will not be deployed on this node, which would cause
+communication problems between the hub and the created browser containers.
+
+Video recording and logs are currently not supported, but we hope to re-enable this
+feature with docker swarm.
+
+#### Network
+
+The created overlay network must be passed as argument `--swarmOverlayNetwork` to zalenium,
+which will actually switch Zalenium to Docker Swarm mode.
+
+Make sure passing the network name with its stack name as prefix.
+
+In our example we named our network "zalenium" and the stack was named "STACK" so the network
+will have the name `"STACK_zalenium"`, which we passed to `"--swarmOverlayNetwork"`.
+
+
+### Technical Information
+
+__Synchronized docker operations__
+
+Docker operations run in synchronized blocks to prevent stale browser containers remain forever.
+
+see also:
+- [eclipse-ee4j/jersey#3772](https://github.com/eclipse-ee4j/jersey/issues/3772)
+- [zalando/zalenium#808](https://github.com/zalando/zalenium/issues/808)
+
+__One service per test session__
+
+For each test session we deploy a new service that will create a browser container to run tests.
+
+Working with one service and adapt the number of replicas does not work because we can't
+control which browser containers will be removed when decreasing replicas. It can and
+will happen that docker will remove a browser container with a running test to fulfill
+number of replicas.
+
+
+
+

docs/docker/docker-compose-swarm.yml

@@ -1,11 +1,15 @@
 version: "3.7"
+
 services:
   zalenium:
     image: dosel/zalenium
+    hostname: zalenium
     deploy:
       placement:
         constraints:
           - node.role == manager
+    labels:
+      - "de.zalando.gridRole=hub" # important for us to identify the node which runs zalenium hub
     ports:
       - "4444:4444"
     networks:
@@ -15,8 +19,9 @@ services:
       - /tmp/videos:/home/seluser/videos
     environment:
       - PULL_SELENIUM_IMAGE=true
-    command: ["start", "--swarmOverlayNetwork", "<stack-name>_zalenium", "--videoRecordingEnabled", "false"]
+    command: ["start", "--swarmOverlayNetwork", "STACK_zalenium", "--videoRecordingEnabled", "false"]
+
 networks:
   zalenium:
     driver: overlay
-    attachable: true
+    attachable: true

src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmContainerClient.java

@@ -245,39 +245,46 @@ private ContainerSpec buildContainerSpec(List<String> flattenedEnvVars, String i
     }
 
     private TaskSpec buildTaskSpec(ContainerSpec containerSpec) {
-        final RestartPolicy restartPolicy = RestartPolicy.builder()
-                .condition("on-failure")
-                .build();
+        try {
+            final RestartPolicy restartPolicy = RestartPolicy.builder()
+                    .condition("on-failure")
+                    .build();
+            String hostname = SwarmUtilities.getHubHostname();
+            final List<String> placementList = new ArrayList<>();
 
-        final List<String> placementList = new ArrayList<>();
+            placementList.add("node.hostname != " + hostname);
 
-        placementList.add("node.role==worker");
+            final Placement placement = Placement.create(placementList);
 
-        final Placement placement = Placement.create(placementList);
 
-        Resources.Builder resourceBuilder = Resources.builder();
-        String cpuLimit = getSeleniumContainerCpuLimit();
-        String memLimit = getSeleniumContainerMemoryLimit();
+            Resources.Builder resourceBuilder = Resources.builder();
+            String cpuLimit = getSeleniumContainerCpuLimit();
+            String memLimit = getSeleniumContainerMemoryLimit();
 
-        if (!Strings.isNullOrEmpty(cpuLimit)) {
-            resourceBuilder.nanoCpus(Long.valueOf(cpuLimit));
-        }
+            if (!Strings.isNullOrEmpty(cpuLimit)) {
+                resourceBuilder.nanoCpus(Long.valueOf(cpuLimit));
+            }
 
-        if (!Strings.isNullOrEmpty(memLimit)) {
-            resourceBuilder.memoryBytes(Long.valueOf(memLimit));
-        }
+            if (!Strings.isNullOrEmpty(memLimit)) {
+                resourceBuilder.memoryBytes(Long.valueOf(memLimit));
+            }
 
-        ResourceRequirements resourceRequirements = ResourceRequirements.builder()
-                .limits(resourceBuilder.build())
-                .build();
+            ResourceRequirements resourceRequirements = ResourceRequirements.builder()
+                    .limits(resourceBuilder.build())
+                    .build();
+
+            final TaskSpec.Builder taskSpecBuilder = TaskSpec.builder()
+                    .resources(resourceRequirements)
+                    .restartPolicy(restartPolicy)
+                    .placement(placement)
+                    .containerSpec(containerSpec);
 
-        final TaskSpec.Builder taskSpecBuilder = TaskSpec.builder()
-                .resources(resourceRequirements)
-                .restartPolicy(restartPolicy)
-                .placement(placement)
-                .containerSpec(containerSpec);
+            return taskSpecBuilder.build();
+        } catch (DockerException | InterruptedException e) {
+            e.printStackTrace();
+        }
 
-        return taskSpecBuilder.build();
+        return null;
     }
 
     private ServiceSpec buildServiceSpec(TaskSpec taskSpec, String nodePort, String noVncPort) {

src/main/java/de/zalando/ep/zalenium/container/swarm/SwarmUtilities.java

@@ -108,6 +108,27 @@ public static synchronized Task getTaskByContainerId(String containerId) throws
         return null;
     }
 
+    public static synchronized Task getTaskByContainerLabel(String labelKey, String labelValue) throws DockerException, InterruptedException {
+        List<Task> tasks = dockerClient.listTasks();
+
+        for (Task task : CollectionUtils.emptyIfNull(tasks)) {
+            ContainerStatus containerStatus = task.status().containerStatus();
+
+            if (containerStatus != null) {
+                ContainerInfo containerInfo = dockerClient.inspectContainer(containerStatus.containerId());
+                ImmutableMap<String, String> labels = containerInfo.config().labels();
+                boolean hasLabel = labels != null && labels.containsKey(labelKey);
+                boolean labelHasValue = labels != null && labels.get(labelKey).equals(labelValue);
+
+                if (hasLabel && labelHasValue) {
+                    return task;
+                }
+            }
+        }
+
+        return null;
+    }
+
     public static synchronized Task getTaskByServiceId(String serviceId) throws DockerException, InterruptedException {
         String serviceName = dockerClient.inspectService(serviceId).spec().name();
         Task.Criteria criteria = Task.Criteria.builder().serviceName(serviceName).build();
@@ -149,6 +170,22 @@ public static String getSwarmIp(ContainerInfo containerInfo) {
         return ipAddress;
     }
 
+
+    public static synchronized String getHubHostname () throws DockerException, InterruptedException {
+        String hubLabelKey = "de.zalando.gridRole";
+        String hubLabelValue = "hub";
+        Task task = getTaskByContainerLabel(hubLabelKey, hubLabelValue);
+
+        if (task != null) {
+            String nodeId = task.nodeId();
+            NodeInfo nodeInfo = dockerClient.inspectNode(nodeId);
+
+            return nodeInfo.description().hostname();
+        }
+
+        return null;
+    }
+
     public static boolean isSwarmActive() {
         return !overlayNetwork.isEmpty();
     }