From 973b6400f93199dd3d6c39e9226a874cdf3fbf05 Mon Sep 17 00:00:00 2001 From: Manuel Holtgrewe Date: Mon, 10 Jul 2023 09:15:05 +0200 Subject: [PATCH] feat: adding nginx for serving track data (#15) (#20) --- .ci/config/nginx/nginx.conf | 35 +++++++++++++++++++++++++++++++++ .editorconfig | 4 ++++ .env.ci | 5 +++++ README.md | 32 ++++++++++++++++++++++++++++++ docker-compose.override.yml-dev | 6 ++++++ docker-compose.yml | 17 ++++++++++++++++ env.tpl | 9 +++++++++ utils/nginx/nginx.conf | 35 +++++++++++++++++++++++++++++++++ 8 files changed, 143 insertions(+) create mode 100644 .ci/config/nginx/nginx.conf create mode 100644 utils/nginx/nginx.conf diff --git a/.ci/config/nginx/nginx.conf b/.ci/config/nginx/nginx.conf new file mode 100644 index 0000000..5c13ff0 --- /dev/null +++ b/.ci/config/nginx/nginx.conf @@ -0,0 +1,35 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + + gzip on; + + # The full static data directory is mounted into the container but we only + # serve the "nginx" sub directory (with indices in the default + # configuration). + server { + location / { + root /data/nginx; + autoindex on; + } + } +} diff --git a/.editorconfig b/.editorconfig index 7ca29f0..519c43a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,6 +12,10 @@ trim_trailing_whitespace = true indent_style = space indent_size = 4 +[*.conf] +indent_style = space +indent_size = 2 + [*.py] line_length=120 known_first_party=varfish diff --git a/.env.ci b/.env.ci index 7c56736..c93cdb6 100644 --- a/.env.ci +++ b/.env.ci @@ -61,6 +61,11 @@ # -- General Container Configuration ----------------------------------------- +# Base directory for configuration. +# config_basedir: ./.dev/config +## In CI: set to directory with appropriate config. +config_basedir=./.ci/config + # Base directory for volumes. # volumes_basedir: ./volumes ## In CI: set to a directory with minimal data for spinning up the containers. diff --git a/README.md b/README.md index 12f5428..baf9275 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,30 @@ ln -Tsr .dev/volumes/varfish-static/data/download/worker/masked-repeat-grch38-*/ .dev/volumes/varfish-static/data/worker/grch38/features/masked_repeat.bin ln -Tsr .dev/volumes/varfish-static/data/download/worker/masked-segdup-grch38-*/masked-segdup.bin \ .dev/volumes/varfish-static/data/worker/grch38/features/masked_seqdup.bin + +## +## tracks +## + +mkdir -p .dev/volumes/varfish-static/data/nginx/grch3{7,8} + +paths_37=$(find .dev/volumes/varfish-static/data/download/tracks/ -type f -name '*.bed' -or -name '*.bed.gz' | sort | grep grch37) +for path in $paths_37; do + if [[ -e ${path}.tbi ]]; then + ln -sr $path ${path}.tbi .dev/volumes/varfish-static/data/nginx/grch37 + else + ln -sr $path .dev/volumes/varfish-static/data/nginx/grch37 + fi +done + +paths_38=$(find .dev/volumes/varfish-static/data/download/tracks/ -type f -name '*.bed' -or -name '*.bed.gz' | sort | grep grch38) +for path in $paths_38; do + if [[ -e ${path}.tbi ]]; then + ln -sr $path ${path}.tbi .dev/volumes/varfish-static/data/nginx/grch38 + else + ln -sr $path .dev/volumes/varfish-static/data/nginx/grch38 + fi +done ``` To create an in-house database: @@ -238,6 +262,14 @@ varfish-server-worker db to-bin \ --path-output-bin .dev/volumes/varfish-static/data/worker/grch37/strucvars/inhouse.bin ``` +### Setup Configuration + +The next step step is to create the configuration files in `.dev/config`. + +```bash session +mkdir -p .dev/config/nginx +cp utils/nginx/nginx.conf .dev/config/nginx +``` ### Startup and Check diff --git a/docker-compose.override.yml-dev b/docker-compose.override.yml-dev index 14f24d0..a85f22a 100644 --- a/docker-compose.override.yml-dev +++ b/docker-compose.override.yml-dev @@ -10,6 +10,7 @@ # - `3001` -- annonars # - `3002` -- mehari # - `3003` -- viguno +# - `3004` -- nginx # - `3010` -- minio # - `3011` -- minio console @@ -29,6 +30,11 @@ services: ports: - "3003:8080" + # map nginx to port 3004 + nginx: + ports: + - "3004:80" + # map Minio S3 to port 9000 and console to 9001 minio: ports: diff --git a/docker-compose.yml b/docker-compose.yml index c7f53a2..69658fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,23 @@ services: # Mount Docker socket into container so traefik can react to events. - "/var/run/docker.sock:/var/run/docker.sock:ro" + # -- nginx ----------------------------------------------------------------- + # + # We serve static files such as browser tracks with nginx. + + nginx: + container_name: nginx + hostname: nginx + image: ${image_nginx_name:-nginx}:${image_nginx_version:-1} + volumes: + - type: bind + source: ${volumes_basedir:-./.dev/volumes}/varfish-static/data + target: /data + read_only: true + - type: bind + source: ${config_basedir:-./.dev/config}/nginx/nginx.conf + target: /etc/nginx/nginx.conf + # -- Mehari ---------------------------------------------------------------- # # Mehari provides the transcript-related information. diff --git a/env.tpl b/env.tpl index 49cf348..7f78d10 100644 --- a/env.tpl +++ b/env.tpl @@ -59,8 +59,17 @@ # Version of the "mc" (Minio client) image to use. # image_mc_version=latest +# Name of the nginx image to use. +# image_nginx_name=nginx + +# Version of the nginx image to use. +# image_nginx_version=1 + # -- General Container Configuration ----------------------------------------- +# Base directory for configuration. +# config_basedir: ./.dev/config + # Base directory for volumes. # volumes_basedir: ./.dev/volumes diff --git a/utils/nginx/nginx.conf b/utils/nginx/nginx.conf new file mode 100644 index 0000000..5c13ff0 --- /dev/null +++ b/utils/nginx/nginx.conf @@ -0,0 +1,35 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + + gzip on; + + # The full static data directory is mounted into the container but we only + # serve the "nginx" sub directory (with indices in the default + # configuration). + server { + location / { + root /data/nginx; + autoindex on; + } + } +}