Update Datastore+Tracingstore Standalone Deployment Templates

webknossos-datastore/deployment/README.md

@@ -10,17 +10,14 @@ webKnossos expects the following file structure:
 ```
 docker-compose.yml
 binaryData/ # path can be specified in config/environment
-    <Team name>/ 
+    <Team name>/
         <Dataset 1>/ # Can be converted from image stack with webknossos-cuber
             color/
                 1 # mag1
                 2 # mag2
                 4 # mag4
                 ...
         ...
-fossildb/
-    data/
-    backup/
 config/
 tmp/
 ```

webknossos-datastore/deployment/config/docker.conf → webknossos-datastore/deployment/config/datastore-docker.conf

@@ -18,9 +18,4 @@ datastore {
     uri = "localhost:9000" # TODO
     secured = false # TODO
   }
-
-  fossildb {
-    address = "fossildb"
-    port = 7155
-  }
 }

webknossos-datastore/deployment/config/environment

@@ -2,9 +2,6 @@ DATASTORE_TAG=master
 DATASTORE_PORT=9090
 DATASTORE_DEBUG_PORT=5011
 
-FOSSILDB_TAG=master
-FOSSILDB_PORT=7155
-
 USER_UID=1000
 USER_GID=1000
 USER_GROUP_1=nogroup

webknossos-datastore/deployment/docker-compose.yml

@@ -3,45 +3,27 @@ version: '2.1'
 services:
   webknossos-datastore:
     image: scalableminds/webknossos-datastore:${DATASTORE_TAG}
-    entrypoint: bin/standalone-datastore
+    entrypoint: bin/webknossos-datastore
     command:
       - -J-Xmx20G
       - -J-Xms1G
       - -J-XX:+UseG1GC
       - -jvm-debug
       - "5011"
-      - -Dlogback.configurationFile=conf/logback-docker.xml
-      - -Dlogger.file=conf/logback-docker.xml
       - -Dconfig.file=conf/docker.conf
       - $DATASTORE_ARGS_1
       - $DATASTORE_ARGS_2
-    links:
-      - fossildb
     ports:
       - $DATASTORE_PORT:9090
       - $DATASTORE_DEBUG_PORT:5011
     user: $USER_UID:$USER_GID
     group_add:
       - $USER_GROUP_1
+      - $USER_GROUP_2
     volumes:
       - $BINARY_DATA_PATH:/srv/webknossos-datastore/binaryData
       - ./tmp:/tmp
-      - ./config/docker.conf:/srv/webknossos-datastore/conf/docker.conf
+      - ./config/datastore-docker.conf:/srv/webknossos-datastore/conf/docker.conf
     environment:
       - NEW_RELIC_LICENSE_KEY
       - NEW_RELIC_APP_NAME
-
-  fossildb:
-    image: scalableminds/fossildb:${FOSSILDB_TAG}
-    command:
-      - fossildb
-      - -c
-      - skeletons,skeletonUpdates,volumes,volumeData
-    ports:
-      - $FOSSILDB_PORT:7155
-    user: $USER_UID:$USER_GID
-    group_add:
-      - $USER_GROUP_1
-    volumes:
-      - "./fossildb/data:/fossildb/data"
-      - "./fossildb/backup:/fossildb/backup"

webknossos-datastore/deployment/webknossos-datastore.service

@@ -8,7 +8,7 @@ Restart=always
 RestartSec=3
 WorkingDirectory=/usr/lib/webknossos-datastore
 EnvironmentFile=/usr/lib/webknossos-datastore/config/environment
-ExecStart=/usr/local/bin/docker-compose up --force-recreate --abort-on-container-exit -t 2 fossildb webknossos-datastore
+ExecStart=/usr/local/bin/docker-compose up --force-recreate --abort-on-container-exit -t 2 webknossos-datastore
 
 [Install]
 WantedBy=default.target

webknossos-tracingstore/deployment/README.md

@@ -0,0 +1,97 @@
+# Deployment
+
+## Requirements
+* Linux (we have good experiences with Ubuntu and Debian, but others should work as well)
+* a recent versions of docker & docker-compose
+* Public domain name with SSL certificate
+
+## Prepare file system
+webKnossos expects the following file structure:
+```
+docker-compose.yml
+fossildb/
+    data/
+    backup/
+config/
+tmp/
+```
+Ensure that all those directories have the correct access rights.
+
+## Add the `deployment` files
+Copy the files from the `deployment` folder to `/usr/lib/webknossos-tracingstore` and customize the files in the `config` subfolder.
+
+## Manage the webknossos-tracingstore service
+Depending on you Linux distribution, the service unit file should be linked to `/lib/systemd/system/webknossos-tracingstore.service` or `/usr/lib/systemd/system/webknossos-tracingstore.service`.
+Usage:
+
+```
+# Reload configuration
+systemctl daemon-reload
+
+# Enable for auto-start
+systemctl enable webknossos-tracingstore
+
+systemctl start webknossos-tracingstore
+
+systemctl stop webknossos-tracingstore
+```
+
+## Using a cluster proxy/firewall for HTTP(S) routing
+If your cluster enviroment has a firewall that supports HTTP(S) routing, you can expose the tracingstore directly on Port 80.
+
+## Using nginx for HTTP(S) routing
+Nginx is a high performance HTTP server that allows for proxing HTTP(S) request. This is useful, because the tracingstore doesn't support HTTPS by itself. So, you can put the nginx in front of the tracingstore to accept HTTPS requests from the outside and route them as regular HTTP requests to the tracingstore.
+
+[DigitalOcean has a great tutorial for setting up nginx](https://www.digitalocean.com/community/tutorials/understanding-nginx-http-proxying-load-balancing-buffering-and-caching).
+
+We use the free and automatad [Letsencrypt](https://letsencrypt.org/) service for our SSL certificates. [Tutorial: Setting up Letsencrypt with nginx](https://www.digitalocean.com/community/tutorials/how-to-secure-nginx-with-let-s-encrypt-on-ubuntu-16-04).
+
+Example configuration:
+```
+upstream webknossos-tracingstore-0 {
+  server localhost:9090;
+}
+
+server {
+  listen 80;
+  listen [::]:80;
+  server_name tracingstore-0.mylab.com;
+
+  # For Letsencrypt
+  location ~ /.well-known/ {
+    root /usr/share/nginx/html;
+    allow all;
+    break;
+  }
+
+  return 301 https://$server_name$request_uri;
+}
+
+server {
+  listen 443 ssl;
+  listen [::]:443 ssl;
+  server_name  tracingstore-0.mylab.com;
+
+  access_log  /var/log/nginx/tracingstore-0.mylab.com.access.log;
+  error_log   /var/log/nginx/tracingstore-0.mylab.com.error.log debug;
+  rewrite_log on;
+
+  # For Letsencrypt
+  location ~ /.well-known/ {
+    root /usr/share/nginx/html;
+    allow all;
+  }
+
+  ssl                 on;
+  ssl_certificate     /etc/letsencrypt/live/tracingstore-0.mylab.com/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/tracingstore-0.mylab.com/privkey.pem;
+  ssl_protocols       SSLv3 TLSv1 TLSv1.1 TLSv1.2;
+  ssl_ciphers         HIGH:!aNULL:!MD5;
+
+  client_max_body_size 0;
+
+  location / {
+    proxy_pass http://webknossos-tracingstore-0;
+  }
+}
+```

webknossos-tracingstore/deployment/config/environment

@@ -0,0 +1,10 @@
+DATASTORE_TAG=master
+DATASTORE_PORT=9090
+DATASTORE_DEBUG_PORT=5011
+
+FOSSILDB_TAG=master
+FOSSILDB_PORT=7155
+
+USER_UID=1000
+USER_GID=1000
+USER_GROUP_1=nogroup

webknossos-tracingstore/deployment/config/tracingstore-docker.conf

@@ -0,0 +1,26 @@
+include "standalone-tracingstore.conf"
+
+#########################################################################
+# Do not change the ports, they are only used inside the containers.    #
+# To change the ports on the host, change them in the environment file. #
+#########################################################################
+
+http {
+  uri = "localhost" # TODO
+  port = 9090
+}
+
+tracingstore {
+  key = "70d1" # TODO
+  name = "mytracingstore" # TODO
+
+  oxalis {
+    uri = "localhost:9000" # TODO
+    secured = false # TODO
+  }
+
+  fossildb {
+    address = "fossildb"
+    port = 7155
+  }
+}

webknossos-tracingstore/deployment/docker-compose.yml

@@ -0,0 +1,53 @@
+version: '2.1'
+
+services:
+  webknossos-tracingstore:
+    image: scalableminds/webknossos-tracingstore:${DATASTORE_TAG}
+    entrypoint: bin/webknossos-tracingstore
+    command:
+      - -J-Xmx20G
+      - -J-Xms1G
+      - -J-XX:+UseG1GC
+      - -jvm-debug
+      - "5011"
+      - -Dconfig.file=conf/docker.conf
+      - $DATASTORE_ARGS_1
+      - $DATASTORE_ARGS_2
+    links:
+      - fossildb
+    ports:
+      - $TRACINGSTORE_PORT:9090
+      - $TRACINGSTORE_DEBUG_PORT:5011
+    user: $USER_UID:$USER_GID
+    group_add:
+      - $USER_GROUP_1
+      - $USER_GROUP_2
+    volumes:
+      - ./tmp:/tmp
+      - ./config/tracingstore-docker.conf:/srv/webknossos-tracingstore/conf/docker.conf
+    environment:
+      - NEW_RELIC_LICENSE_KEY
+      - NEW_RELIC_APP_NAME
+
+  fossildb:
+    image: scalableminds/fossildb:${FOSSILDB_TAG}
+    command:
+      - fossildb
+      - -c
+      - skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates
+    ports:
+      - $FOSSILDB_PORT:7155
+    user: $USER_UID:$USER_GID
+    group_add:
+      - $USER_GROUP_1
+    volumes:
+      - "./fossildb/data:/fossildb/data"
+      - "./fossildb/backup:/fossildb/backup"
+
+  fossildb-backup:
+    image: scalableminds/fossildb-client:${FOSSILDB_TAG}
+    command:
+      - fossildb
+      - backup
+    links:
+      - fossildb

webknossos-tracingstore/deployment/webknossos-tracingstore.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=webKnossos tracingstore
+Requires=docker.service
+After=docker.service
+
+[Service]
+Restart=always
+RestartSec=3
+WorkingDirectory=/usr/lib/webknossos-tracingstore
+EnvironmentFile=/usr/lib/webknossos-tracingstore/config/environment
+ExecStart=/usr/local/bin/docker-compose up --force-recreate --abort-on-container-exit -t 2 fossildb webknossos-tracingstore
+
+[Install]
+WantedBy=default.target