diff --git a/Makefile b/Makefile index 6ddcb75e7c..88e80bfe4e 100644 --- a/Makefile +++ b/Makefile @@ -249,11 +249,50 @@ ifeq ($(package), all) ./dist/galley-schema --keyspace galley_test --replication-factor 1 --reset ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 --reset ./dist/spar-schema --keyspace spar_test --replication-factor 1 --reset +ifeq ($(INTEGRATION_FEDERATION_TESTS), 1) + ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 --reset + ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 --reset + ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 --reset + ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 --reset +endif else $(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 --reset +ifeq ($(INTEGRATION_FEDERATION_TESTS), 1) + $(EXE_SCHEMA) --keyspace $(package)_test2 --replication-factor 1 --reset +endif endif ./dist/brig-index reset --elasticsearch-server http://localhost:9200 > /dev/null +# Usage: +# +# Migrate all keyspaces and reset the ES index +# make db-migrate +# +# Migrate keyspace for only one service, say galley: +# make db-migrate package=galley +.PHONY: db-reset +db-reset: c + @echo "Make sure you have ./deploy/dockerephemeral/run.sh running in another window!" +ifeq ($(package), all) + ./dist/brig-schema --keyspace brig_test --replication-factor 1 --reset + ./dist/galley-schema --keyspace galley_test --replication-factor 1 --reset + ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 --reset + ./dist/spar-schema --keyspace spar_test --replication-factor 1 --reset +ifeq ($(INTEGRATION_FEDERATION_TESTS), 1) + ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 --reset + ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 --reset + ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 --reset + ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 --reset +endif +else + $(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 --reset +ifeq ($(INTEGRATION_FEDERATION_TESTS), 1) + $(EXE_SCHEMA) --keyspace $(package)_test2 --replication-factor 1 --reset +endif +endif + ./dist/brig-index reset --elasticsearch-index directory_test --elasticsearch-server http://localhost:9200 > /dev/null + ./dist/brig-index reset --elasticsearch-index directory_test2 --elasticsearch-server http://localhost:9200 > /dev/null + # Usage: # # Migrate all keyspaces and reset the ES index @@ -267,7 +306,14 @@ db-migrate: c ./dist/galley-schema --keyspace galley_test --replication-factor 1 > /dev/null ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 > /dev/null ./dist/spar-schema --keyspace spar_test --replication-factor 1 > /dev/null - ./dist/brig-index reset --elasticsearch-server http://localhost:9200 > /dev/null +ifeq ($(INTEGRATION_FEDERATION_TESTS), 1) + ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 > /dev/null + ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 > /dev/null + ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 > /dev/null + ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 > /dev/null +endif + ./dist/brig-index reset --elasticsearch-index-prefix directory --elasticsearch-server http://localhost:9200 > /dev/null + ./dist/brig-index reset --elasticsearch-index-prefix directory2 --elasticsearch-server http://localhost:9200 > /dev/null ################################# ## dependencies diff --git a/changelog.d/5-internal/local-end2end-setup b/changelog.d/5-internal/local-end2end-setup new file mode 100644 index 0000000000..ac38cd1546 --- /dev/null +++ b/changelog.d/5-internal/local-end2end-setup @@ -0,0 +1 @@ +New integration test script with support for running end2end tests locally diff --git a/charts/gundeck/templates/tests/configmap.yaml b/charts/gundeck/templates/tests/configmap.yaml index 6829860247..c5df36067c 100644 --- a/charts/gundeck/templates/tests/configmap.yaml +++ b/charts/gundeck/templates/tests/configmap.yaml @@ -19,7 +19,7 @@ data: # some gundeck integration tests make use of two different # cannon instances to test the distributed case. when running # the integration tests locally, the two instances will be spun - # up separately (see `wire-server/services/integration.sh`). + # up separately (see `wire-server/services/run-services`). # # here, we spin up two replicas, provide the integration tests # with the same service coordinates, and rely on the k8s load diff --git a/deploy/dockerephemeral/coredns-config/db.example.com b/deploy/dockerephemeral/coredns-config/db.example.com index 941502a432..39beb50578 100644 --- a/deploy/dockerephemeral/coredns-config/db.example.com +++ b/deploy/dockerephemeral/coredns-config/db.example.com @@ -12,4 +12,6 @@ $ORIGIN example.com. www IN A 127.0.0.1 IN AAAA ::1 -_wire-server-federator._tcp IN SRV 0 0 443 federator.integration.example.com. +_wire-server-federator._tcp IN SRV 0 0 8443 localhost. +_wire-server-federator._tcp.b IN SRV 0 0 9443 localhost. + diff --git a/hack/bin/cabal-run-integration.sh b/hack/bin/cabal-run-integration.sh index 285744f0e1..57cf9a2874 100755 --- a/hack/bin/cabal-run-integration.sh +++ b/hack/bin/cabal-run-integration.sh @@ -46,7 +46,7 @@ run_integration_tests() { service_dir="$TOP_LEVEL/services/$package" cd "$service_dir" - "$TOP_LEVEL/services/integration.sh" \ + "$TOP_LEVEL/services/run-services" \ "$TOP_LEVEL/dist/$package-integration" \ -s "$service_dir/$package.integration.yaml" \ -i "$TOP_LEVEL/services/integration.yaml" \ diff --git a/nix/overlay.nix b/nix/overlay.nix index 1f20335dbb..7657680fd7 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -87,7 +87,7 @@ self: super: { inherit (super) stdenv fetchurl; }; - helm = super.callPackage ./pkgs/helm {}; + helm = super.callPackage ./pkgs/helm { }; helmfile = staticBinary { pname = "helmfile"; diff --git a/nix/wire-server.nix b/nix/wire-server.nix index c215298f8b..8fc1b22edd 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -372,7 +372,8 @@ in pkgs.kind pkgs.netcat pkgs.niv - pkgs.python3 + (pkgs.python3.withPackages + (ps: with ps; [ pyyaml ])) pkgs.rsync pkgs.wget pkgs.yq diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml index 41b4287f9f..f38e8507c1 100644 --- a/services/brig/brig.integration.yaml +++ b/services/brig/brig.integration.yaml @@ -195,6 +195,4 @@ optSettings: setEnableMLS: true logLevel: Warn -# ^ NOTE: We log too much in brig, if we set this to Info like other services, running tests -# produces too many logs, hence this is set to Warn. logNetStrings: false diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index c614172794..af164ff20f 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -146,7 +146,18 @@ elasticServerParser = restrictedElasticSettingsParser :: Parser ElasticSettings restrictedElasticSettingsParser = do server <- elasticServerParser - pure $ localElasticSettings & esServer .~ server + prefix <- + strOption + ( long "elasticsearch-index-prefix" + <> metavar "PREFIX" + <> help "Elasticsearch Index Prefix. The actual index name will be PREFIX_test." + <> value "directory" + <> showDefault + ) + pure $ + localElasticSettings + & esServer .~ server + & esIndex .~ ES.IndexName (prefix <> "_test") indexNameParser :: Parser ES.IndexName indexNameParser = diff --git a/services/cannon/cannon.integration.yaml b/services/cannon/cannon.integration.yaml index 5bd5d2a706..f64f3c104f 100644 --- a/services/cannon/cannon.integration.yaml +++ b/services/cannon/cannon.integration.yaml @@ -21,5 +21,5 @@ drainOpts: millisecondsBetweenBatches: 500 minBatchSize: 5 -logLevel: Info +logLevel: Warn logNetStrings: false diff --git a/services/cargohold/cargohold.integration.yaml b/services/cargohold/cargohold.integration.yaml index d43b06f019..0f85c2b42c 100644 --- a/services/cargohold/cargohold.integration.yaml +++ b/services/cargohold/cargohold.integration.yaml @@ -24,5 +24,5 @@ settings: downloadLinkTTL: 300 # Seconds federationDomain: example.com -logLevel: Info +logLevel: Warn logNetStrings: false diff --git a/services/federator/federator.integration.yaml b/services/federator/federator.integration.yaml index 42e08d35c5..9562b697e1 100644 --- a/services/federator/federator.integration.yaml +++ b/services/federator/federator.integration.yaml @@ -14,7 +14,7 @@ galley: host: 0.0.0.0 port: 8085 -logLevel: Debug +logLevel: Warn logNetStrings: false optSettings: diff --git a/services/federator/src/Federator/Remote.hs b/services/federator/src/Federator/Remote.hs index 1ea810df18..a978c4a14d 100644 --- a/services/federator/src/Federator/Remote.hs +++ b/services/federator/src/Federator/Remote.hs @@ -123,7 +123,9 @@ interpretRemote = interpret $ \case let path = LBS.toStrict . toLazyByteString $ HTTP.encodePathSegments ["federation", componentName component, rpc] - req' = HTTP2.requestBuilder HTTP.methodPost path headers body + -- filter out Host header, because the HTTP2 client adds it back + headers' = filter ((/= "Host") . fst) headers + req' = HTTP2.requestBuilder HTTP.methodPost path headers' body tlsConfig = mkTLSConfig settings hostname port resp <- mapError (RemoteError target) . (fromEither @FederatorClientHTTP2Error =<<) . embed $ diff --git a/services/galley/galley.integration.yaml b/services/galley/galley.integration.yaml index 6a90f1b58e..87f02e7e34 100644 --- a/services/galley/galley.integration.yaml +++ b/services/galley/galley.integration.yaml @@ -73,7 +73,7 @@ settings: status: disabled lockStatus: locked -logLevel: Info +logLevel: Warn logNetStrings: false journal: # if set, journals; if not set, disables journaling diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml index 5457469af0..1d56cfdf88 100644 --- a/services/gundeck/gundeck.integration.yaml +++ b/services/gundeck/gundeck.integration.yaml @@ -37,5 +37,5 @@ settings: hard: 30 # more than this number of threads will not be allowed soft: 10 # more than this number of threads will be warned about -logLevel: Info +logLevel: Warn logNetStrings: false diff --git a/services/integration.sh b/services/integration.sh deleted file mode 100755 index 648a60aaba..0000000000 --- a/services/integration.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -USAGE="$0 [args...]" -EXE=${1:?$USAGE} -TOP_LEVEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -DIR="${TOP_LEVEL}/services" -PARENT_PID=$$ -rm -f /tmp/integration.* # remove previous temp files, if any -EXIT_STATUS_LOCATION=$(mktemp "/tmp/integration.XXXXXXXXXXX") -echo 1 >"${EXIT_STATUS_LOCATION}" - -function kill_all() { - # kill the process tree of the PARENT_PID - kill -9 -${PARENT_PID} &> /dev/null -} - -function list_descendants () { - local children - children="$(pgrep -P "$1")" - for pid in $children - do - list_descendants "$pid" - done - echo "$children" -} - -function kill_gracefully() { - pkill "gundeck|brig|galley|cargohold|cannon|spar|nginz|stern" - sleep 1 - kill $(list_descendants "$PARENT_PID") &> /dev/null -} - -trap "kill_gracefully; kill_all" INT TERM ERR - -function check_prerequisites() { - if ! ( nc -z 127.0.0.1 9042 \ - && nc -z 127.0.0.1 9200 \ - && nc -z 127.0.0.1 6379 ); then - echo "Databases not up. Maybe run 'deploy/dockerephemeral/run.sh' in a separate terminal first?"; exit 1; - fi - if [ ! -f "${TOP_LEVEL}/dist/brig" ] \ - && [ ! -f "${TOP_LEVEL}/dist/galley" ] \ - && [ ! -f "${TOP_LEVEL}/dist/cannon" ] \ - && [ ! -f "${TOP_LEVEL}/dist/gundeck" ] \ - && [ ! -f "${TOP_LEVEL}/dist/cargohold" ] \ - && [ ! -f "${TOP_LEVEL}/dist/stern" ] \ - && [ ! -f "${TOP_LEVEL}/dist/spar" ]; then - echo "Not all services are compiled. How about you run 'cd ${TOP_LEVEL} && make' first?"; exit 1; - fi -} - -blue=6 -green=10 -orange=3 -yellow=11 -purpleish=13 - -if [[ $INTEGRATION_USE_REAL_AWS -eq 1 ]]; then - echo 'Attempting to run integration tests using real AWS services!' - [ -z "$AWS_REGION" ] && echo "Need to set AWS_REGION in your environment" && exit 1; - [ -z "$AWS_ACCESS_KEY_ID" ] && echo "Need to set AWS_ACCESS_KEY_ID in your environment" && exit 1; - [ -z "$AWS_SECRET_ACCESS_KEY" ] && echo "Need to set AWS_SECRET_ACCESS_KEY in your environment" && exit 1; - "${TOP_LEVEL}"/services/gen-aws-conf.sh - integration_file_extension='-aws.yaml' -elif [[ $INTEGRATION_CARGOHOLD_ONLY_COMPAT -eq 1 ]]; then - echo "Running tests using specific S3 buckets for cargohold using folder $CARGOHOLD_COMPAT_CONFIG_FOLDER" - if [ ! -f "${CARGOHOLD_COMPAT_CONFIG_FOLDER}/env.sh" ] \ - && [ ! -f "${CARGOHOLD_COMPAT_CONFIG_FOLDER}/cargohold.integration.yaml" ]; then - echo 'expecting a CARGOHOLD_COMPAT_CONFIG_FOLDER/cargohold.integration.yaml and' - echo 'expecting a CARGOHOLD_COMPAT_CONFIG_FOLDER/env.sh' - exit 1; - fi -else - # brig,gundeck,galley use the amazonka library's 'Discover', which expects AWS credentials - # even if those are not used/can be dummy values with the fake sqs/ses/etc containers used - # (see deploy/dockerephemeral/docker-compose.yaml ) - echo 'Running tests using mocked AWS services' - export AWS_REGION=eu-west-1 - export AWS_ACCESS_KEY_ID=dummykey - export AWS_SECRET_ACCESS_KEY=dummysecret - integration_file_extension='.yaml' -fi - -function run() { - service=$1 - instance=$2 - colour=$3 - configfile=${4:-"${service}${instance}.integration${integration_file_extension}"} - # Check if we're on a Mac - if [[ "$OSTYPE" == "darwin"* ]]; then - # Mac sed uses '-l' to set line-by-line buffering - UNBUFFERED=-l - # Test if sed supports buffer settings. GNU sed does, busybox does not. - elif sed -u '' /dev/null 2>&1; then - UNBUFFERED=-u - else - echo -e "\n\nWARNING: log output is buffered and may not show on your screen!\n\n" - UNBUFFERED='' - fi - ( ( cd "${DIR}/${service}" && "${TOP_LEVEL}/dist/${service}" -c "${configfile}" ) || kill_all) \ - | sed ${UNBUFFERED} -e "s/^/$(tput setaf ${colour})[${service}] /" -e "s/$/$(tput sgr0)/" & -} - - -if [[ $INTEGRATION_CARGOHOLD_ONLY_COMPAT -eq 1 ]]; then - source "${CARGOHOLD_COMPAT_CONFIG_FOLDER}/env.sh" - echo run cargohold "" ${purpleish} "${CARGOHOLD_COMPAT_CONFIG_FOLDER}/cargohold.integration.yaml" - run cargohold "" ${purpleish} "${CARGOHOLD_COMPAT_CONFIG_FOLDER}/cargohold.integration.yaml" -else - check_prerequisites - run brig "" ${green} - run galley "" ${yellow} - run gundeck "" ${blue} - run cannon "" ${orange} - run cannon "2" ${orange} - run cargohold "" ${purpleish} - run spar "" ${orange} - run federator "" ${blue} - run stern "" ${yellow} -fi - -function run_nginz() { - colour=$1 - - if [[ ! ${COMPILE_NGINX_USING_NIX:-1} -eq 0 ]]; then - # For nix we don't need LD_LIBRARY_PATH; we link against libzauth directly. - nginz=$(nix-build "${TOP_LEVEL}/nix" -A pkgs.nginz --no-out-link ) - (cd ${NGINZ_WORK_DIR} && ${nginz}/bin/nginx -p ${NGINZ_WORK_DIR} -c ${NGINZ_WORK_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \ - | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" & - else - prefix=$([ -w /usr/local ] && echo /usr/local || echo "${HOME}/.wire-dev") - - (cd ${NGINZ_WORK_DIR} && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${prefix}/lib/ ${TOP_LEVEL}/dist/nginx -p ${NGINZ_WORK_DIR} -c ${NGINZ_WORK_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \ - | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" & - fi -} - -NGINZ_PORT="" - -if [[ ! ${INTEGRATION_USE_NGINZ:-1} -eq 0 ]]; then - NGINZ_PORT=8080 - # Note: for integration tests involving nginz, - # nginz and brig must share the same zauth public/private keys - export NGINZ_WORK_DIR="$TOP_LEVEL/services/nginz/integration-test" - - run_nginz ${purpleish} -fi - -# the ports are copied from ./integration.yaml -if [[ $INTEGRATION_CARGOHOLD_ONLY_COMPAT -eq 1 ]]; then - PORT_LIST="8084" -else - PORT_LIST="8082 8083 8084 8085 8086 8088 $NGINZ_PORT" -fi - -while [ "$all_services_are_up" == "" ]; do - export all_services_are_up="1" - for port in $PORT_LIST; do - ( curl --write-out '%{http_code}' --silent --output /dev/null http://localhost:"$port"/i/status \ - | grep -q '^20[04]' ) \ - || export all_services_are_up="" - done - sleep 1 -done -echo "all services are up!" - -( ${EXE} "${@:2}" && echo 0 > "${EXIT_STATUS_LOCATION}" && kill_gracefully ) || kill_gracefully & - -wait -exit $(<"${EXIT_STATUS_LOCATION}") diff --git a/services/integration.yaml b/services/integration.yaml index 97adbc1e48..20e002ec88 100644 --- a/services/integration.yaml +++ b/services/integration.yaml @@ -88,7 +88,7 @@ backendTwo: port: 9084 cannon: host: 127.0.0.1 - port: 9086 + port: 9083 redis2: host: 127.0.0.1 diff --git a/services/nginz/integration-test/conf/nginz/integration.conf b/services/nginz/integration-test/conf/nginz/integration.conf new file mode 100644 index 0000000000..baae352c92 --- /dev/null +++ b/services/nginz/integration-test/conf/nginz/integration.conf @@ -0,0 +1,19 @@ +# plain TCP/http listening for integration tests only. +listen 8080; +listen 8081; + +# for nginx-without-tls, we need to use a separate port for http2 traffic, +# as nginx cannot handle unencrypted http1 and http2 trafic on the same +# port. +# This port is only used for trying out nginx http2 forwarding without TLS locally and should not +# be ported to any production nginz config. +listen 8090 http2; + +######## TLS/SSL block start ############## +# +# Most integration tests simply use the http ports 8080 and 8081 +# But to also test tls forwarding, this port can be used. +# This applies only locally, as for kubernetes (helm chart) based deployments, +# TLS is terminated at the ingress level, not at nginz level +listen 8443 ssl http2; +listen [::]:8443 ssl http2; diff --git a/services/nginz/integration-test/conf/nginz/nginx.conf b/services/nginz/integration-test/conf/nginz/nginx.conf index e1d5f3fb6c..564923b263 100644 --- a/services/nginz/integration-test/conf/nginz/nginx.conf +++ b/services/nginz/integration-test/conf/nginz/nginx.conf @@ -1,6 +1,6 @@ worker_processes 4; worker_rlimit_nofile 1024; -pid /tmp/nginz.pid; +include pid.conf; # for easy overriding # nb. start up errors (eg. misconfiguration) may still end up in /$(LOG_PATH)/error.log error_log stderr warn; @@ -106,25 +106,7 @@ http { # server { - # plain TCP/http listening for integration tests only. - listen 8080; - listen 8081; - - # for nginx-without-tls, we need to use a separate port for http2 traffic, - # as nginx cannot handle unencrypted http1 and http2 trafic on the same - # port. - # This port is only used for trying out nginx http2 forwarding without TLS locally and should not - # be ported to any production nginz config. - listen 8090 http2; - - ######## TLS/SSL block start ############## - # - # Most integration tests simply use the http ports 8080 and 8081 - # But to also test tls forwarding, this port can be used. - # This applies only locally, as for kubernetes (helm chart) based deployments, - # TLS is terminated at the ingress level, not at nginz level - listen 8443 ssl http2; - listen [::]:8443 ssl http2; + include integration.conf; # self-signed certificates generated using wire-server/hack/bin/selfsigned.sh ssl_certificate integration-leaf.pem; diff --git a/services/nginz/integration-test/conf/nginz/pid.conf b/services/nginz/integration-test/conf/nginz/pid.conf new file mode 100644 index 0000000000..e722aa5ae2 --- /dev/null +++ b/services/nginz/integration-test/conf/nginz/pid.conf @@ -0,0 +1 @@ +pid /tmp/nginz.pid; diff --git a/services/run-services b/services/run-services new file mode 100755 index 0000000000..9fdbf894e6 --- /dev/null +++ b/services/run-services @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 + +from dataclasses import dataclass, replace +import logging +import os +import re +import select +import signal +import shutil +import socket +import subprocess +import yaml +import urllib.request +import urllib.error +import sys +import tempfile +import time +import traceback +import threading + +@dataclass +class SpawnFailException(Exception): + failed_instances: object + +class Colors: + GREEN = "\x1b[38;5;10m" + YELLOW = "\x1b[38;5;11m" + BLUE = "\x1b[38;5;6m" + PURPLEISH = "\x1b[38;5;13m" + ORANGE = "\x1b[38;5;3m" + RED = "\x1b[38;5;1m" + RESET = "\x1b[0m" + +@dataclass(frozen=True) +class Service: + name: str + color: str + _internal_name: str = None + check_status: bool = True + level: str = None + config: str = None + + def with_level(self, level=None): + if level is None: + level = os.environ.get("INTEGRATION_{self.name.capitalize()}_LEVEL") + return replace(self, level=level) + + @property + def internal_name(self): + if self._internal_name is None: + return self.name + else: + return self._internal_name + + def path(self): + return os.path.join(ROOT, "dist", self.name) + + def config_file(self): + if self.config is None: + base = self.name + else: + base = self.config + return os.path.join(ROOT, "services", self.name, + base + ".integration.yaml") + + + def spawn(self, config_file, environment): + return subprocess.Popen([self.path(), "-c", config_file], + encoding='utf-8', + cwd=os.path.join(ROOT, "services", self.name), + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + def check_exists(self): + if not os.path.exists(self.path()): + raise Exception(f"{self.name} not found") + +@dataclass(frozen=True) +class Nginz: + color: str + level: str = None + + @property + def name(self): return "nginz" + + @property + def internal_name(self): return self.name + + @property + def check_status(self): return True + + def config_file(self): + return os.path.join(ROOT, "services", "nginz", "integration-test", + "conf", "nginz","nginx.conf") + + def spawn(self, config_file, environment): + cwd = os.path.join(ROOT, "services", "nginz", "integration-test") + return subprocess.Popen([shutil.which("nginx"), "-p", cwd, "-c", + config_file, + "-g", "daemon off;"], + encoding='utf-8', cwd=cwd, env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + def check_exists(self): + if shutil.which("nginx") is None: + raise Exception("nginx not found") + +@dataclass(frozen=True) +class Instance: + service: Service + port: int + thread: threading.Thread = None + process: subprocess.Popen = None + exception: Exception = None + + def check_status(self): + self.process.poll() + if self.process.returncode is not None: + raise Exception(f"{self.service.name} has terminated") + if not self.service.check_status: + return True + try: + with urllib.request.urlopen(f"http://localhost:{self.port}/i/status") as resp: + return resp.status in [200, 204] + except urllib.error.URLError: + return False + + def spawn(self, service_map, environment, suffix, domain, backend_name): + try: + config_file = self.modified_config_file(service_map, suffix, domain) + sub = self.service.spawn(config_file, environment) + t = threading.Thread(target=lambda: color_output(sub, self.service, backend_name)) + t.start() + return Instance(self.service, self.port, t, sub) + except Exception as e: + return Instance(self.service, self.port, exception=e) + + def modified_config_file(self, service_map, suffix, domain): + """Overwrite port configuration on this service using the provided + service_map. + + This works by creating an unnamed pipe, writing the modified config + file to it, and returning a path to the read end of the pipe (in + /proc).""" + + with open(self.service.config_file()) as f: + data = yaml.safe_load(f) + + # set ports of other services + for service in service_map: + if service.internal_name in data: + data[service.internal_name]['port'] = service_map[service] + + # set cassandra keyspace + if 'cassandra' in data: + data['cassandra']['keyspace'] = f"{self.service.name}_test{suffix}" + + # set elasticseach index + if 'elasticsearch' in data: + data['elasticsearch']['index'] = f"directory{suffix}_test" + + # set federation domain + if 'optSettings' in data: + data['optSettings']['setFederationDomain'] = domain + elif 'settings' in data: + data['settings']['federationDomain'] = domain + + # set log level + if self.service.level is not None: + if 'logLevel' in data: + data['logLevel'] = self.service.level + elif 'saml' in data: + data['saml']['logLevel'] = self.service.level + + self.set_own_port(data) + + # write modified config file to pipe + return make_pipe(yaml.dump(data).encode('utf-8')) + + def set_own_port(self, data): + # spar's own port is in a different place + if self.service.name == 'spar': + data['saml']['spPort'] = self.port + elif self.service.name in data: + data[self.service.name]['port'] = self.port + +class DummyInstance(Instance): + def spawn(self, service_map, environment, suffix, domain, backend_name): + return self + + def modified_config_file(self, service_map, suffix, domain): + return "" + + def check_status(self): + return True + +class FederatorInstance(Instance): + def __init__(self, internal_port, external_port): + self.external_port = external_port + super().__init__(FEDERATOR, internal_port) + + def set_own_port(self, data): + # set external port only, as the internal one is part of the service + # map and is set by the general config logic + data['federatorExternal']['port'] = self.external_port + +class NginzInstance(Instance): + def __init__(self, local_port, http2_port, ssl_port, fed_port): + self.http2_port = http2_port + self.ssl_port = ssl_port + self.fed_port = fed_port + super().__init__(NGINZ, local_port) + + def modified_config_file(self, service_map, suffix, domain): + # Create a whole temporary directory and copy all nginx's config files. + # This is necessary because nginx assumes local imports are relative to + # the location of the main configuration file. + self.tmpdir = tempfile.TemporaryDirectory() + shutil.copytree(os.path.dirname(self.service.config_file()), + self.tmpdir.name, + dirs_exist_ok=True) + + # override port configuration + with open(os.path.join(self.tmpdir.name, "integration.conf"), 'w') as f: + override = f""" + listen {self.port}; + listen {self.http2_port} http2; + listen {self.ssl_port} ssl http2; + listen [::]:{self.ssl_port} ssl http2;""" + print(override, file=f) + + # override upstreams + with open(os.path.join(self.tmpdir.name, "upstreams"), 'w') as f: + for service, port in service_map.items(): + print(f"upstream {service.internal_name} {{", file=f) + print(f" least_conn;", file=f) + print(f" keepalive 32;", file=f) + print(f" server 127.0.0.1:{port} max_fails=3 weight=1;", file=f) + print("}", file=f) + print("upstream federator_external {", file=f) + print(f" server 127.0.0.1:{self.fed_port} max_fails=3 weight=1;", file=f) + print("}", file=f) + + # override pid configuration + with open(os.path.join(self.tmpdir.name, "pid.conf"), 'w') as f: + pid = os.path.join(self.tmpdir.name, "nginz.pid") + print(f"pid {pid};", file=f) + + return os.path.join(self.tmpdir.name, "nginx.conf") + +def check_prerequisites(services): + try: + for port in (9042, 9200, 6379): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect(("127.0.0.1", port)) + except Exception as e: + logging.error(f"{Colors.RED}Databases not up. Try running 'deploy/dockerephemeral/run.sh'. {Colors.RESET}") + sys.exit(1) + + try: + for service in services: + service.check_exists() + except Exception as e: + logging.error(Colors.RED + str(e) + Colors.RESET) + sys.exit(1) + +def color_output(sub, service, backend_name): + if backend_name is not None: + backend_name = "@" + backend_name + try: + for line in sub.stdout: + logging.info(f"{service.color}[{service.name}{backend_name}] {line.rstrip()}{Colors.RESET}") + finally: + sub.terminate() + sub.wait() + +def find_root(base): + # find git repository + root = os.path.realpath(base) + while not os.path.exists(os.path.join(root, ".git")): + p = os.path.dirname(root) + if p == root: raise Exception("Could not find wire-server root") + root = p + return root + +def make_pipe(data): + (r, w) = os.pipe() + os.write(w, data) + os.close(w) + return f"/proc/{os.getpid()}/fd/{r}" + +def cleanup_instances(instances): + for instance in instances: + if instance.process is None: continue + instance.process.terminate() + + for instance in instances: + if instance.thread is None: continue + instance.thread.join(timeout=0.1) + # some services don't react promptly to SIGTERM, so we give them a + # nudge if they don't terminate within a few milliseconds + if instance.thread.is_alive(): + instance.process.terminate() + instance.thread.join(timeout=0.1) + if instance.thread.is_alive(): + print("force-killing", instance.service.name) + instance.process.send_signal(signal.SIGKILl) + instance.thread.join() + +def start_backend(services, suffix, domain, backend_name): + # build a service map by choosing an arbitrary instance of each service + service_map = dict((s.service, s.port) for s in services) + + instances = set() + for blueprint in services: + instances.add(blueprint.spawn(service_map, environment, suffix, domain, backend_name)) + + failed_instances = [instance for instance in instances + if instance.exception is not None] + + # check instances + to_be_checked = [instance for instance in instances + if instance.exception is None] + start_time = time.time() + while to_be_checked: + if time.time() - start_time >= 5: + print(f"{Colors.RED}Timeout while spawing services{Colors.RESET}") + failed_instances.extend(to_be_checked) + break + + to_be_checked_again = set() + for instance in to_be_checked: + try: + if not instance.check_status(): + to_be_checked_again.add(instance) + except Exception as e: + failed_instances.append(replace(instance, exception=e)) + + to_be_checked = to_be_checked_again + time.sleep(0.05) + + # TODO: elapse timeout so that the timeout thread doesn't hold up the + # process + + if failed_instances: + cleanup_instances(instances) + raise SpawnFailException(failed_instances) + + return instances + +ENABLE_FEDERATION = os.environ.get("INTEGRATION_FEDERATION_TESTS") == "1" +LEVEL = os.environ.get("INTEGRATION_LEVEL") +BRIG = Service("brig", Colors.GREEN).with_level(LEVEL) +GALLEY = Service("galley", Colors.YELLOW).with_level(LEVEL) +GUNDECK = Service("gundeck", Colors.BLUE).with_level(LEVEL) +CANNON = Service("cannon", Colors.ORANGE).with_level(LEVEL) +CANNON2 = Service("cannon", Colors.ORANGE, + "cannon2", config="cannon2").with_level(LEVEL) +CARGOHOLD = Service("cargohold", Colors.PURPLEISH).with_level(LEVEL) +SPAR = Service("spar", Colors.ORANGE).with_level(LEVEL) +FEDERATOR = Service("federator", Colors.BLUE, + "federatorInternal", + check_status=False).with_level(LEVEL) +STERN = Service("stern", Colors.YELLOW).with_level(LEVEL) +PROXY = Service("proxy", Colors.RED).with_level(LEVEL) +NGINZ = Nginz(Colors.PURPLEISH) + +if __name__ == '__main__': + logging.basicConfig(encoding='utf-8', level=logging.INFO, + format='%(message)s') + ROOT = find_root(os.getcwd()) + if ROOT is None: + error("This script needs to be run within the wire-server direnv") + + environment = { + 'AWS_REGION': "eu-west-1", + 'AWS_ACCESS_KEY_ID': "dummykey", + 'AWS_SECRET_ACCESS_KEY': "dummysecret" + } + + backend_a = [ + Instance(BRIG, 8082), + Instance(GALLEY, 8085), + Instance(GUNDECK, 8086), + Instance(CANNON, 8083), + Instance(CANNON2, 8183), + Instance(CARGOHOLD, 8084), + Instance(SPAR, 8088), + DummyInstance(PROXY, 8087), + FederatorInstance(8097, 8098), + NginzInstance( + local_port=8080, + http2_port=8090, + ssl_port=8443, + fed_port=8098) + ] + + backend_b = [ + Instance(BRIG, 9082), + Instance(GALLEY, 9085), + Instance(GUNDECK, 9086), + Instance(CANNON, 9083), + Instance(CANNON2, 9183), + Instance(CARGOHOLD, 9084), + Instance(SPAR, 9088), + DummyInstance(PROXY, 9087), + FederatorInstance(9097, 9098), + NginzInstance( + local_port=9080, + http2_port=9090, + ssl_port=9443, + fed_port=9098) + ] + + check_prerequisites(set(s.service for s in backend_a)) + + try: + instances = set() + instances |= start_backend(backend_a, "", "example.com", "A") + if ENABLE_FEDERATION: + instances |= start_backend(backend_b, "2", "b.example.com", "B") + + # run main script or just wait forever + if len(sys.argv) == 1: + print("(This will hang, Control+C to close.)") + print("Now you can manually curl them or start an integration test executable manually with e.g. \n(first cd to a service dir for correct working directory)\n cd services/brig && ../../dist/brig-integration -s brig.integration.yaml -i ../integration.yaml") + signal.pause() + else: + ret = subprocess.run(sys.argv[1:], + env=dict(list(os.environ.items()) + + list(environment.items()))) + sys.exit(ret.returncode) + except KeyboardInterrupt: + pass + except SpawnFailException as e: + print(f"{Colors.RED}The following services failed to start:{Colors.RESET}") + for instance in e.failed_instances: + print(f"{instance.service.name} at port {instance.port}" + + (f" ({instance.exception})" if instance.exception else "")) + finally: + cleanup_instances(instances) diff --git a/services/spar/spar.integration.yaml b/services/spar/spar.integration.yaml index 77792b4ee1..6a1eb2f398 100644 --- a/services/spar/spar.integration.yaml +++ b/services/spar/spar.integration.yaml @@ -1,6 +1,6 @@ saml: version: SAML2.0 - logLevel: Info + logLevel: Warn spHost: 0.0.0.0 spPort: 8088 diff --git a/services/start-services-only.sh b/services/start-services-only.sh index 9d7ea41c09..374e12f285 100755 --- a/services/start-services-only.sh +++ b/services/start-services-only.sh @@ -2,10 +2,9 @@ # Run all haskell services without immediately starting a test executable. # Can be useful for manually poking at the API. - set -eo pipefail SERVICES_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# call integration.sh, show a message, then sleep (instead of executing a test executable) -"$SERVICES_DIR/integration.sh" bash -c 'printf "(This will hang, Control+C to close.)\nNow you can manually curl them or start an integration test executable manually with e.g. \n(first cd to a service dir for correct working directory)\n cd services/brig && ../../dist/brig-integration -s brig.integration.yaml -i ../integration.yaml\n" && sleep 1000000' +# call run-services, show a message, then sleep (instead of executing a test executable) +exec "$SERVICES_DIR/run-services" diff --git a/tools/stern/src/Stern/Intra.hs b/tools/stern/src/Stern/Intra.hs index c27ef36829..0ca2c9dd38 100644 --- a/tools/stern/src/Stern/Intra.hs +++ b/tools/stern/src/Stern/Intra.hs @@ -122,7 +122,7 @@ backendApiVersion :: Version backendApiVersion = V2 -- | Make sure the backend supports `backendApiVersion`. Crash if it doesn't. (This is called --- in `Stern.API` so problems make `./services/integration.sh` crash.) +-- in `Stern.API` so problems make `./services/run-service` crash.) assertBackendApiVersion :: App () assertBackendApiVersion = recoverAll (constantDelay 1000000 <> limitRetries 5) $ \_retryStatus -> do b <- view brig diff --git a/treefmt.toml b/treefmt.toml index 2984c8f027..6dd52aa914 100644 --- a/treefmt.toml +++ b/treefmt.toml @@ -33,7 +33,7 @@ excludes = [ "services/spar/test-scim-suite/runsuite.sh", "services/spar/test-scim-suite/run.sh", "services/brig/federation-tests.sh", - "services/integration.sh", + "services/run-services", "hack/bin/create_test_team_members.sh", "hack/bin/create_test_team_scim.sh", "hack/bin/create_test_user.sh",