From e4137c13618f57f1f69c442644d287e6acc8b6dc Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 24 May 2022 14:35:36 +0100
Subject: [PATCH 01/27] Mutiny: oust the monolith image and replace with the
 workers image

---
 docker/complement/Dockerfile                  |  46 +++++--
 docker/complement/SynapseWorkers.Dockerfile   |  40 ------
 docker/complement/conf/homeserver.yaml        | 129 ------------------
 docker/complement/conf/log_config.yaml        |  24 ----
 .../postgres.supervisord.conf                 |   0
 .../start-complement-synapse-workers.sh       |   0
 docker/complement/conf/start.sh               |  30 ----
 .../workers-shared.yaml                       |   0
 8 files changed, 32 insertions(+), 237 deletions(-)
 delete mode 100644 docker/complement/SynapseWorkers.Dockerfile
 delete mode 100644 docker/complement/conf/homeserver.yaml
 delete mode 100644 docker/complement/conf/log_config.yaml
 rename docker/complement/{conf-workers => conf}/postgres.supervisord.conf (100%)
 rename docker/complement/{conf-workers => conf}/start-complement-synapse-workers.sh (100%)
 delete mode 100755 docker/complement/conf/start.sh
 rename docker/complement/{conf-workers => conf}/workers-shared.yaml (100%)

diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 4823ce7364a9..99a09cbc2bab 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -1,22 +1,40 @@
-# A dockerfile which builds an image suitable for testing Synapse under
-# complement.
+# This dockerfile builds on top of 'docker/Dockerfile-worker' in matrix-org/synapse
+# by including a built-in postgres instance, as well as setting up the homeserver so
+# that it is ready for testing via Complement.
+#
+# Instructions for building this image from those it depends on is detailed in this guide:
+# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
+FROM matrixdotorg/synapse-workers
+
+# Install postgresql
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13
+
+# Configure a user and create a database for Synapse
+RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
+ \"ALTER USER postgres PASSWORD 'somesecret'; \
+ CREATE DATABASE synapse \
+  ENCODING 'UTF8' \
+  LC_COLLATE='C' \
+  LC_CTYPE='C' \
+  template=template0;\" | psql" && pg_ctlcluster 13 main stop
+
+# Modify the shared homeserver config with postgres support, certificate setup
+# and the disabling of rate-limiting
+COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml
 
-ARG SYNAPSE_VERSION=latest
-
-FROM matrixdotorg/synapse:${SYNAPSE_VERSION}
-
-ENV SERVER_NAME=localhost
-
-COPY conf/* /conf/
+WORKDIR /data
 
-# generate a signing key
-RUN generate_signing_key -o /conf/server.signing.key
+COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
 
-WORKDIR /data
+# Copy the entrypoint
+COPY conf-workers/start-complement-synapse-workers.sh /
 
+# Expose nginx's listener ports
 EXPOSE 8008 8448
 
-ENTRYPOINT ["/conf/start.sh"]
+ENTRYPOINT ["/start-complement-synapse-workers.sh"]
 
+# Update the healthcheck to have a shorter check interval
 HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
-    CMD curl -fSs http://localhost:8008/health || exit 1
+    CMD /bin/sh /healthcheck.sh
diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile
deleted file mode 100644
index 99a09cbc2bab..000000000000
--- a/docker/complement/SynapseWorkers.Dockerfile
+++ /dev/null
@@ -1,40 +0,0 @@
-# This dockerfile builds on top of 'docker/Dockerfile-worker' in matrix-org/synapse
-# by including a built-in postgres instance, as well as setting up the homeserver so
-# that it is ready for testing via Complement.
-#
-# Instructions for building this image from those it depends on is detailed in this guide:
-# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
-FROM matrixdotorg/synapse-workers
-
-# Install postgresql
-RUN apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13
-
-# Configure a user and create a database for Synapse
-RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
- \"ALTER USER postgres PASSWORD 'somesecret'; \
- CREATE DATABASE synapse \
-  ENCODING 'UTF8' \
-  LC_COLLATE='C' \
-  LC_CTYPE='C' \
-  template=template0;\" | psql" && pg_ctlcluster 13 main stop
-
-# Modify the shared homeserver config with postgres support, certificate setup
-# and the disabling of rate-limiting
-COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml
-
-WORKDIR /data
-
-COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
-
-# Copy the entrypoint
-COPY conf-workers/start-complement-synapse-workers.sh /
-
-# Expose nginx's listener ports
-EXPOSE 8008 8448
-
-ENTRYPOINT ["/start-complement-synapse-workers.sh"]
-
-# Update the healthcheck to have a shorter check interval
-HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
-    CMD /bin/sh /healthcheck.sh
diff --git a/docker/complement/conf/homeserver.yaml b/docker/complement/conf/homeserver.yaml
deleted file mode 100644
index e2be540bbb9e..000000000000
--- a/docker/complement/conf/homeserver.yaml
+++ /dev/null
@@ -1,129 +0,0 @@
-## Server ##
-
-server_name: SERVER_NAME
-log_config: /conf/log_config.yaml
-report_stats: False
-signing_key_path: /conf/server.signing.key
-trusted_key_servers: []
-enable_registration: true
-enable_registration_without_verification: true
-
-## Listeners ##
-
-tls_certificate_path: /conf/server.tls.crt
-tls_private_key_path: /conf/server.tls.key
-bcrypt_rounds: 4
-registration_shared_secret: complement
-
-listeners:
-  - port: 8448
-    bind_addresses: ['::']
-    type: http
-    tls: true
-    resources:
-      - names: [federation]
-
-  - port: 8008
-    bind_addresses: ['::']
-    type: http
-
-    resources:
-      - names: [client]
-
-## Database ##
-
-database:
-  name: "sqlite3"
-  args:
-    # We avoid /data, as it is a volume and is not transferred when the container is committed,
-    # which is a fundamental necessity in complement.
-    database: "/conf/homeserver.db"
-
-## Federation ##
-
-# trust certs signed by the complement CA
-federation_custom_ca_list:
-- /complement/ca/ca.crt
-
-# unblacklist RFC1918 addresses
-ip_range_blacklist: []
-
-# Disable server rate-limiting
-rc_federation:
-  window_size: 1000
-  sleep_limit: 10
-  sleep_delay: 500
-  reject_limit: 99999
-  concurrent: 3
-
-rc_message:
-  per_second: 9999
-  burst_count: 9999
-
-rc_registration:
-  per_second: 9999
-  burst_count: 9999
-
-rc_login:
-  address:
-    per_second: 9999
-    burst_count: 9999
-  account:
-    per_second: 9999
-    burst_count: 9999
-  failed_attempts:
-    per_second: 9999
-    burst_count: 9999
-
-rc_admin_redaction:
-  per_second: 9999
-  burst_count: 9999
-
-rc_joins:
-  local:
-    per_second: 9999
-    burst_count: 9999
-  remote:
-    per_second: 9999
-    burst_count: 9999
-
-rc_3pid_validation:
-  per_second: 1000
-  burst_count: 1000
-
-rc_invites:
-  per_room:
-    per_second: 1000
-    burst_count: 1000
-  per_user:
-    per_second: 1000
-    burst_count: 1000
-
-federation_rr_transactions_per_room_per_second: 9999
-
-## API Configuration ##
-
-# A list of application service config files to use
-#
-app_service_config_files:
-AS_REGISTRATION_FILES  
-
-## Experimental Features ##
-
-experimental_features:
-  # Enable spaces support
-  spaces_enabled: true
-  # Enable history backfilling support
-  msc2716_enabled: true
-  # server-side support for partial state in /send_join responses
-  msc3706_enabled: true
-  # client-side support for partial state in /send_join responses
-  faster_joins: true
-  # Enable jump to date endpoint
-  msc3030_enabled: true
-
-server_notices:
-  system_mxid_localpart: _server
-  system_mxid_display_name: "Server Alert"
-  system_mxid_avatar_url: ""
-  room_name: "Server Alert"
diff --git a/docker/complement/conf/log_config.yaml b/docker/complement/conf/log_config.yaml
deleted file mode 100644
index c33fd6cd00a7..000000000000
--- a/docker/complement/conf/log_config.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-version: 1
-
-formatters:
-  precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-
-filters:
-  context:
-    (): synapse.logging.context.LoggingContextFilter
-    request: ""
-
-handlers:
-  console:
-    class: logging.StreamHandler
-    formatter: precise
-    filters: [context]
-    # log to stdout, for easier use with 'docker logs'
-    stream: 'ext://sys.stdout'
-
-root:
-    level: INFO
-    handlers: [console]
-
-disable_existing_loggers: false
diff --git a/docker/complement/conf-workers/postgres.supervisord.conf b/docker/complement/conf/postgres.supervisord.conf
similarity index 100%
rename from docker/complement/conf-workers/postgres.supervisord.conf
rename to docker/complement/conf/postgres.supervisord.conf
diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf/start-complement-synapse-workers.sh
similarity index 100%
rename from docker/complement/conf-workers/start-complement-synapse-workers.sh
rename to docker/complement/conf/start-complement-synapse-workers.sh
diff --git a/docker/complement/conf/start.sh b/docker/complement/conf/start.sh
deleted file mode 100755
index 5d8d0fe016cf..000000000000
--- a/docker/complement/conf/start.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/sh
-
-set -e
-
-sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /conf/homeserver.yaml
-
-# Add the application service registration files to the homeserver.yaml config
-for filename in /complement/appservice/*.yaml; do
-  [ -f "$filename" ] || break
-
-  as_id=$(basename "$filename" .yaml)
-
-  # Insert the path to the registration file and the AS_REGISTRATION_FILES marker after 
-  # so we can add the next application service in the next iteration of this for loop
-  sed -i "s/AS_REGISTRATION_FILES/  - \/complement\/appservice\/${as_id}.yaml\nAS_REGISTRATION_FILES/g" /conf/homeserver.yaml
-done
-# Remove the AS_REGISTRATION_FILES entry
-sed -i "s/AS_REGISTRATION_FILES//g" /conf/homeserver.yaml
-
-# generate an ssl key and cert for the server, signed by the complement CA
-openssl genrsa -out /conf/server.tls.key 2048
-
-openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
-  -subj "/CN=${SERVER_NAME}"
-openssl x509 -req -in /conf/server.tls.csr \
-  -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \
-  -out /conf/server.tls.crt
-
-exec python -m synapse.app.homeserver -c /conf/homeserver.yaml "$@"
-
diff --git a/docker/complement/conf-workers/workers-shared.yaml b/docker/complement/conf/workers-shared.yaml
similarity index 100%
rename from docker/complement/conf-workers/workers-shared.yaml
rename to docker/complement/conf/workers-shared.yaml

From 865fa26f2d068de6190e444c553de2cb5d267e8d Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 24 May 2022 14:43:20 +0100
Subject: [PATCH 02/27] Update references in Dockerfile

---
 docker/complement/Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 99a09cbc2bab..9c3e1c8476a2 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -21,14 +21,14 @@ RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
 
 # Modify the shared homeserver config with postgres support, certificate setup
 # and the disabling of rate-limiting
-COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml
+COPY conf/workers-shared.yaml /conf/workers/shared.yaml
 
 WORKDIR /data
 
-COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
+COPY conf/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
 
 # Copy the entrypoint
-COPY conf-workers/start-complement-synapse-workers.sh /
+COPY conf/start-complement-synapse-workers.sh /
 
 # Expose nginx's listener ports
 EXPOSE 8008 8448

From edc7fcd5a7110d376936a80358b8e25c7a1b736d Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 24 May 2022 14:43:32 +0100
Subject: [PATCH 03/27] Update complement.sh to use the unified image

---
 scripts-dev/complement.sh | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index ca476d9a5e61..4707b34bfd4a 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -43,17 +43,24 @@ fi
 # Build the base Synapse image from the local checkout
 docker build -t matrixdotorg/synapse -f "docker/Dockerfile" .
 
+# Build the workers docker image (from the base Synapse image we just built).
+docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
+
+# Build the Complement image (from the worker Synapse image we just built).
+docker build -t complement-synapse -f "docker/complement/Dockerfile" "docker/complement"
+
+export COMPLEMENT_BASE_IMAGE=complement-synapse
+
 extra_test_args=()
 
 test_tags="synapse_blacklist,msc2716,msc3030"
 
-# If we're using workers, modify the docker files slightly.
 if [[ -n "$WORKERS" ]]; then
-  # Build the workers docker image (from the base Synapse image).
-  docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
+  # Use workers.
+  export SYNAPSE_COMPLEMENT_USE_WORKERS=1
 
-  export COMPLEMENT_BASE_IMAGE=complement-synapse-workers
-  COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile
+  # Workers can only use Postgres as a database.
+  export SYNAPSE_COMPLEMENT_DATABASE=postgres
 
   # And provide some more configuration to complement.
 
@@ -65,16 +72,17 @@ if [[ -n "$WORKERS" ]]; then
   # ... and it takes longer than 10m to run the whole suite.
   extra_test_args+=("-timeout=60m")
 else
-  export COMPLEMENT_BASE_IMAGE=complement-synapse
-  COMPLEMENT_DOCKERFILE=Dockerfile
+  if [[ -n "$POSTGRES" ]]; then
+    export SYNAPSE_COMPLEMENT_DATABASE=postgres
+  else
+    export SYNAPSE_COMPLEMENT_DATABASE=sqlite
+  fi
 
   # We only test faster room joins on monoliths, because they are purposefully
   # being developed without worker support to start with.
   test_tags="$test_tags,faster_joins"
 fi
 
-# Build the Complement image from the Synapse image we just built.
-docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERFILE" "docker/complement"
 
 # Run the tests!
 echo "Images built; running complement"

From 15215bd983e1569e92b63f167ebeb1b4e5064776 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 26 May 2022 11:19:38 +0100
Subject: [PATCH 04/27] Make it possible to disable Redis with an env var

---
 docker/Dockerfile-workers               | 5 +++++
 docker/conf-workers/supervisord.conf.j2 | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
index 24b03585f9a2..849e8a6b0b4c 100644
--- a/docker/Dockerfile-workers
+++ b/docker/Dockerfile-workers
@@ -26,6 +26,11 @@ COPY ./docker/prefix-log /usr/local/bin/
 # Expose nginx listener port
 EXPOSE 8080/tcp
 
+# Default to starting Redis.
+# (This environment variable is only used in the Complement image to disable
+# Redis when workers aren't being used)
+ENV START_REDIS=true
+
 # A script to read environment variables and create the necessary
 # files to run the desired worker configuration. Will start supervisord.
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2
index ca1f7aef8e3f..112c10b585c9 100644
--- a/docker/conf-workers/supervisord.conf.j2
+++ b/docker/conf-workers/supervisord.conf.j2
@@ -28,6 +28,11 @@ stderr_logfile_maxbytes=0
 username=redis
 autorestart=true
 
+# Only start Redis if START_REDIS is true.
+# (This is only used in the Complement image to allow disabling Redis when
+# workers aren't being used.)
+autostart=%(ENV_START_REDIS)s
+
 [program:synapse_main]
 command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
 priority=10
@@ -41,4 +46,4 @@ autorestart=unexpected
 exitcodes=0
 
 # Additional process blocks
-{{ worker_config }}
\ No newline at end of file
+{{ worker_config }}

From 97699e39a8600ceae94ffd3e1e27472aa84c8eb0 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 26 May 2022 11:59:24 +0100
Subject: [PATCH 05/27] Require START_POSTGRES=1 to start Postgres

---
 docker/complement/conf/postgres.supervisord.conf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker/complement/conf/postgres.supervisord.conf b/docker/complement/conf/postgres.supervisord.conf
index 5608342d1a9e..5dae3e63309a 100644
--- a/docker/complement/conf/postgres.supervisord.conf
+++ b/docker/complement/conf/postgres.supervisord.conf
@@ -1,6 +1,9 @@
 [program:postgres]
 command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground
 
+# Only start if START_POSTGRES=1
+autostart=%(ENV_START_POSTGRES)s
+
 # Lower priority number = starts first
 priority=1
 

From 9cd96d2df1ee62dee8cf380b75139c2b7d424a5a Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 26 May 2022 12:09:38 +0100
Subject: [PATCH 06/27] Make the Complement Synapse launcher able to handle all
 3 scenarios

---
 .../conf/start-complement-synapse-workers.sh  | 71 +++++++++++++------
 1 file changed, 51 insertions(+), 20 deletions(-)

diff --git a/docker/complement/conf/start-complement-synapse-workers.sh b/docker/complement/conf/start-complement-synapse-workers.sh
index b7e24440006f..d5a5afbb5cfd 100755
--- a/docker/complement/conf/start-complement-synapse-workers.sh
+++ b/docker/complement/conf/start-complement-synapse-workers.sh
@@ -4,6 +4,10 @@
 
 set -e
 
+echo "Complement Synapse launcher"
+echo "  Args: $@"
+echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS"
+
 function log {
     d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
     echo "$d $@"
@@ -15,26 +19,53 @@ export SYNAPSE_SERVER_NAME=${SERVER_NAME}
 # No need to report stats here
 export SYNAPSE_REPORT_STATS=no
 
-# Set postgres authentication details which will be placed in the homeserver config file
-export POSTGRES_PASSWORD=somesecret
-export POSTGRES_USER=postgres
-export POSTGRES_HOST=localhost
-
-# Specify the workers to test with
-export SYNAPSE_WORKER_TYPES="\
-    event_persister, \
-    event_persister, \
-    background_worker, \
-    frontend_proxy, \
-    event_creator, \
-    user_dir, \
-    media_repository, \
-    federation_inbound, \
-    federation_reader, \
-    federation_sender, \
-    synchrotron, \
-    appservice, \
-    pusher"
+
+case "$SYNAPSE_COMPLEMENT_DATABASE" in
+  postgres)
+    # Set postgres authentication details which will be placed in the homeserver config file
+    export POSTGRES_PASSWORD=somesecret
+    export POSTGRES_USER=postgres
+    export POSTGRES_HOST=localhost
+    export START_POSTGRES=true
+    ;;
+
+  sqlite)
+    # Prevent Postgres from starting up as we don't need it to
+    export START_POSTGRES=false
+    ;;
+
+  *)
+    echo "Unknown Synapse database: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE"
+    exit 1
+    ;;
+esac
+
+
+if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
+  # Specify the workers to test with
+  export SYNAPSE_WORKER_TYPES="\
+      event_persister, \
+      event_persister, \
+      background_worker, \
+      frontend_proxy, \
+      event_creator, \
+      user_dir, \
+      media_repository, \
+      federation_inbound, \
+      federation_reader, \
+      federation_sender, \
+      synchrotron, \
+      appservice, \
+      pusher"
+
+  export START_REDIS=true
+else
+  # Empty string here means 'main process only'
+  export SYNAPSE_WORKER_TYPES=""
+  # No sense starting Redis as we won't need it or use it
+  export START_REDIS=false
+fi
+
 
 # Add Complement's appservice registration directory, if there is one
 # (It can be absent when there are no application services in this test!)

From 3a588a023d472d4e25617d72ba36e898732947cd Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 26 May 2022 12:09:58 +0100
Subject: [PATCH 07/27] Don't enable Redis for the monolith case

(It causes test errors!)
---
 docker/conf-workers/shared.yaml.j2    | 2 ++
 docker/configure_workers_and_start.py | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docker/conf-workers/shared.yaml.j2 b/docker/conf-workers/shared.yaml.j2
index 644ed788f3d5..92d25386dc34 100644
--- a/docker/conf-workers/shared.yaml.j2
+++ b/docker/conf-workers/shared.yaml.j2
@@ -3,8 +3,10 @@
 # configure_workers_and_start.py uses and amends to this file depending on the workers
 # that have been selected.
 
+{% if enable_redis %}
 redis:
     enabled: true
+{% endif %}
 
 {% if appservice_registrations is not none %}
 ## Application Services ##
diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
index b6ad14117325..f2758a186f3a 100755
--- a/docker/configure_workers_and_start.py
+++ b/docker/configure_workers_and_start.py
@@ -378,7 +378,7 @@ def generate_worker_files(
 
     # Read the desired worker configuration from the environment
     worker_types_env = environ.get("SYNAPSE_WORKER_TYPES")
-    if worker_types_env is None:
+    if worker_types_env is None or worker_types_env.strip() == "":
         # No workers, just the main process
         worker_types = []
     else:
@@ -511,6 +511,7 @@ def generate_worker_files(
         "/conf/workers/shared.yaml",
         shared_worker_config=yaml.dump(shared_config),
         appservice_registrations=appservice_registrations,
+        enable_redis=len(worker_types) > 0,
     )
 
     # Nginx config

From 7bc03a6b090f72f929e59901a7502e88fcf9b89d Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 26 May 2022 12:10:22 +0100
Subject: [PATCH 08/27] Burn env vars into the image (sad and hopefully
 temporary)

---
 docker/complement/Dockerfile |  8 ++++++++
 scripts-dev/complement.sh    | 16 ++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 9c3e1c8476a2..4437a0e7429c 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -38,3 +38,11 @@ ENTRYPOINT ["/start-complement-synapse-workers.sh"]
 # Update the healthcheck to have a shorter check interval
 HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
     CMD /bin/sh /healthcheck.sh
+
+# TODO Ideally we should be able to pass env vars straight in via Complement
+# but for now we burn them in to the image.
+# See https://github.com/matrix-org/complement/issues/6.
+ARG use_workers=true
+ENV SYNAPSE_COMPLEMENT_USE_WORKERS $use_workers
+ARG database=postgres
+ENV SYNAPSE_COMPLEMENT_DATABASE $database
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 4707b34bfd4a..a8ea7c205823 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -46,9 +46,6 @@ docker build -t matrixdotorg/synapse -f "docker/Dockerfile" .
 # Build the workers docker image (from the base Synapse image we just built).
 docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
 
-# Build the Complement image (from the worker Synapse image we just built).
-docker build -t complement-synapse -f "docker/complement/Dockerfile" "docker/complement"
-
 export COMPLEMENT_BASE_IMAGE=complement-synapse
 
 extra_test_args=()
@@ -57,7 +54,7 @@ test_tags="synapse_blacklist,msc2716,msc3030"
 
 if [[ -n "$WORKERS" ]]; then
   # Use workers.
-  export SYNAPSE_COMPLEMENT_USE_WORKERS=1
+  export SYNAPSE_COMPLEMENT_USE_WORKERS=true
 
   # Workers can only use Postgres as a database.
   export SYNAPSE_COMPLEMENT_DATABASE=postgres
@@ -72,6 +69,7 @@ if [[ -n "$WORKERS" ]]; then
   # ... and it takes longer than 10m to run the whole suite.
   extra_test_args+=("-timeout=60m")
 else
+  export SYNAPSE_COMPLEMENT_USE_WORKERS=
   if [[ -n "$POSTGRES" ]]; then
     export SYNAPSE_COMPLEMENT_DATABASE=postgres
   else
@@ -83,6 +81,16 @@ else
   test_tags="$test_tags,faster_joins"
 fi
 
+# TODO Since we can't pass env vars through Complement
+# (see https://github.com/matrix-org/complement/issues/6),
+# we burn them in to the image for now.
+
+# Build the Complement image (from the worker Synapse image we just built).
+docker build -t complement-synapse \
+  --build-arg "use_workers=$SYNAPSE_COMPLEMENT_USE_WORKERS" \
+  --build-arg "database=$SYNAPSE_COMPLEMENT_DATABASE" \
+  -f "docker/complement/Dockerfile" "docker/complement"
+
 
 # Run the tests!
 echo "Images built; running complement"

From bf37bf382e5f6d84d4b1a8666f71b4cb15f3c506 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 26 May 2022 12:31:53 +0100
Subject: [PATCH 09/27] Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
---
 changelog.d/12881.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/12881.misc

diff --git a/changelog.d/12881.misc b/changelog.d/12881.misc
new file mode 100644
index 000000000000..8a83182bd40a
--- /dev/null
+++ b/changelog.d/12881.misc
@@ -0,0 +1 @@
+Merge the Complement testing Docker images into a single, multi-purpose image.
\ No newline at end of file

From facccd0150203fc48bb2f15a4b3af65d64bfce17 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 26 May 2022 14:22:01 +0100
Subject: [PATCH 10/27] Bring back the experimental features

---
 docker/complement/conf/workers-shared.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docker/complement/conf/workers-shared.yaml b/docker/complement/conf/workers-shared.yaml
index cd7b50c65cc3..36a6933a538a 100644
--- a/docker/complement/conf/workers-shared.yaml
+++ b/docker/complement/conf/workers-shared.yaml
@@ -76,10 +76,14 @@ federation_rr_transactions_per_room_per_second: 9999
 ## Experimental Features ##
 
 experimental_features:
-  # Enable history backfilling support
-  msc2716_enabled: true
   # Enable spaces support
   spaces_enabled: true
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # server-side support for partial state in /send_join responses
+  msc3706_enabled: true
+  # client-side support for partial state in /send_join responses
+  faster_joins: true
   # Enable jump to date endpoint
   msc3030_enabled: true
 

From 7e9884a141c9b5e4ade121c4237435aabd19f696 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Mon, 30 May 2022 16:37:57 +0100
Subject: [PATCH 11/27] Apply suggestions from Rich

---
 docker/configure_workers_and_start.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
index f2758a186f3a..d07a0a444af3 100755
--- a/docker/configure_workers_and_start.py
+++ b/docker/configure_workers_and_start.py
@@ -377,8 +377,8 @@ def generate_worker_files(
     nginx_locations = {}
 
     # Read the desired worker configuration from the environment
-    worker_types_env = environ.get("SYNAPSE_WORKER_TYPES")
-    if worker_types_env is None or worker_types_env.strip() == "":
+    worker_types_env = environ.get("SYNAPSE_WORKER_TYPES", "").strip()
+    if not worker_types_env:
         # No workers, just the main process
         worker_types = []
     else:

From 423e21289194ff42f67e795f4399dd9f21c8e998 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Mon, 30 May 2022 16:40:41 +0100
Subject: [PATCH 12/27] Remove mention of 'worker' from entrypoint

But still retain the word 'Complement' so we know it's Complement-specific
---
 docker/complement/Dockerfile                                  | 4 ++--
 ...-complement-synapse-workers.sh => start_for_complement.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename docker/complement/conf/{start-complement-synapse-workers.sh => start_for_complement.sh} (100%)

diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 4437a0e7429c..4819e8db52c3 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -28,12 +28,12 @@ WORKDIR /data
 COPY conf/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
 
 # Copy the entrypoint
-COPY conf/start-complement-synapse-workers.sh /
+COPY conf/start_for_complement.sh /
 
 # Expose nginx's listener ports
 EXPOSE 8008 8448
 
-ENTRYPOINT ["/start-complement-synapse-workers.sh"]
+ENTRYPOINT ["/start_for_complement.sh"]
 
 # Update the healthcheck to have a shorter check interval
 HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
diff --git a/docker/complement/conf/start-complement-synapse-workers.sh b/docker/complement/conf/start_for_complement.sh
similarity index 100%
rename from docker/complement/conf/start-complement-synapse-workers.sh
rename to docker/complement/conf/start_for_complement.sh

From 470fabe8a4609da035c842dd7c09e86339c836cc Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Mon, 30 May 2022 16:47:35 +0100
Subject: [PATCH 13/27] Pass the environment variables straight through, with
 no burning!

---
 docker/complement/Dockerfile |  8 --------
 scripts-dev/complement.sh    | 29 +++++++++++++----------------
 2 files changed, 13 insertions(+), 24 deletions(-)

diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 4819e8db52c3..d88e449d3ca3 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -38,11 +38,3 @@ ENTRYPOINT ["/start_for_complement.sh"]
 # Update the healthcheck to have a shorter check interval
 HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
     CMD /bin/sh /healthcheck.sh
-
-# TODO Ideally we should be able to pass env vars straight in via Complement
-# but for now we burn them in to the image.
-# See https://github.com/matrix-org/complement/issues/6.
-ARG use_workers=true
-ENV SYNAPSE_COMPLEMENT_USE_WORKERS $use_workers
-ARG database=postgres
-ENV SYNAPSE_COMPLEMENT_DATABASE $database
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index a8ea7c205823..e7842e2f51bb 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -46,18 +46,26 @@ docker build -t matrixdotorg/synapse -f "docker/Dockerfile" .
 # Build the workers docker image (from the base Synapse image we just built).
 docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
 
+# Build the unified Complement image (from the worker Synapse image we just built).
+docker build -t complement-synapse \
+  -f "docker/complement/Dockerfile" "docker/complement"
+
 export COMPLEMENT_BASE_IMAGE=complement-synapse
 
 extra_test_args=()
 
 test_tags="synapse_blacklist,msc2716,msc3030"
 
+# All environment variables starting with PASS_ will be shared.
+# (The prefix is stripped off before reaching the container.)
+export COMPLEMENT_SHARE_ENV_PREFIX=PASS_
+
 if [[ -n "$WORKERS" ]]; then
   # Use workers.
-  export SYNAPSE_COMPLEMENT_USE_WORKERS=true
+  export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true
 
   # Workers can only use Postgres as a database.
-  export SYNAPSE_COMPLEMENT_DATABASE=postgres
+  export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres
 
   # And provide some more configuration to complement.
 
@@ -69,11 +77,11 @@ if [[ -n "$WORKERS" ]]; then
   # ... and it takes longer than 10m to run the whole suite.
   extra_test_args+=("-timeout=60m")
 else
-  export SYNAPSE_COMPLEMENT_USE_WORKERS=
+  export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=
   if [[ -n "$POSTGRES" ]]; then
-    export SYNAPSE_COMPLEMENT_DATABASE=postgres
+    export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres
   else
-    export SYNAPSE_COMPLEMENT_DATABASE=sqlite
+    export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
   fi
 
   # We only test faster room joins on monoliths, because they are purposefully
@@ -81,17 +89,6 @@ else
   test_tags="$test_tags,faster_joins"
 fi
 
-# TODO Since we can't pass env vars through Complement
-# (see https://github.com/matrix-org/complement/issues/6),
-# we burn them in to the image for now.
-
-# Build the Complement image (from the worker Synapse image we just built).
-docker build -t complement-synapse \
-  --build-arg "use_workers=$SYNAPSE_COMPLEMENT_USE_WORKERS" \
-  --build-arg "database=$SYNAPSE_COMPLEMENT_DATABASE" \
-  -f "docker/complement/Dockerfile" "docker/complement"
-
-
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"

From b0241e74dcc3162e1753d00c7975cf764df4914f Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Mon, 30 May 2022 17:06:52 +0100
Subject: [PATCH 14/27] Add back SYNAPSE_VERSION build arguments

---
 docker/Dockerfile-workers    | 3 ++-
 docker/complement/Dockerfile | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
index 849e8a6b0b4c..d20e1263729a 100644
--- a/docker/Dockerfile-workers
+++ b/docker/Dockerfile-workers
@@ -1,5 +1,6 @@
 # Inherit from the official Synapse docker image
-FROM matrixdotorg/synapse
+ARG SYNAPSE_VERSION=latest
+FROM matrixdotorg/synapse:$SYNAPSE_VERSION
 
 # Install deps
 RUN \
diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index d88e449d3ca3..4002cf03b0cf 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -4,7 +4,8 @@
 #
 # Instructions for building this image from those it depends on is detailed in this guide:
 # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
-FROM matrixdotorg/synapse-workers
+ARG SYNAPSE_VERSION=latest
+FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION
 
 # Install postgresql
 RUN apt-get update && \

From 35377627d112ef049740b5a4cba72b25a585d291 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 31 May 2022 10:34:40 +0100
Subject: [PATCH 15/27] Automatically disable Redis only for no-worker
 instances

---
 docker/complement/conf/start_for_complement.sh | 4 ----
 docker/conf-workers/supervisord.conf.j2        | 6 ++----
 docker/configure_workers_and_start.py          | 5 ++++-
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh
index d5a5afbb5cfd..8fb824ade72b 100755
--- a/docker/complement/conf/start_for_complement.sh
+++ b/docker/complement/conf/start_for_complement.sh
@@ -57,13 +57,9 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
       synchrotron, \
       appservice, \
       pusher"
-
-  export START_REDIS=true
 else
   # Empty string here means 'main process only'
   export SYNAPSE_WORKER_TYPES=""
-  # No sense starting Redis as we won't need it or use it
-  export START_REDIS=false
 fi
 
 
diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2
index 112c10b585c9..7afab0513310 100644
--- a/docker/conf-workers/supervisord.conf.j2
+++ b/docker/conf-workers/supervisord.conf.j2
@@ -28,10 +28,8 @@ stderr_logfile_maxbytes=0
 username=redis
 autorestart=true
 
-# Only start Redis if START_REDIS is true.
-# (This is only used in the Complement image to allow disabling Redis when
-# workers aren't being used.)
-autostart=%(ENV_START_REDIS)s
+# Redis can be disabled if the image is being used without workers
+autostart={{ enable_redis }}
 
 [program:synapse_main]
 command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
index d07a0a444af3..568f22e1d73a 100755
--- a/docker/configure_workers_and_start.py
+++ b/docker/configure_workers_and_start.py
@@ -505,13 +505,15 @@ def generate_worker_files(
             if reg_path.suffix.lower() in (".yaml", ".yml")
         ]
 
+    enable_redis = len(worker_types) > 0
+
     # Shared homeserver config
     convert(
         "/conf/shared.yaml.j2",
         "/conf/workers/shared.yaml",
         shared_worker_config=yaml.dump(shared_config),
         appservice_registrations=appservice_registrations,
-        enable_redis=len(worker_types) > 0,
+        enable_redis=enable_redis,
     )
 
     # Nginx config
@@ -531,6 +533,7 @@ def generate_worker_files(
         "/etc/supervisor/supervisord.conf",
         main_config_path=config_path,
         worker_config=supervisord_config,
+        enable_redis=enable_redis,
     )
 
     # healthcheck config

From 8ae3da751b5e857d0d7e3d075d32198030f48194 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 31 May 2022 10:46:32 +0100
Subject: [PATCH 16/27] Improve Complement documentation about 'Stuff' :)

---
 docker/complement/README.md | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/docker/complement/README.md b/docker/complement/README.md
index e075418e4a96..37c39e2dfc29 100644
--- a/docker/complement/README.md
+++ b/docker/complement/README.md
@@ -1 +1,32 @@
-Stuff for building the docker image used for testing under complement.
+# Unified Complement image for Synapse
+
+This is an image for testing Synapse with [the *Complement* integration test suite][complement].
+It contains some insecure defaults that are only suitable for testing purposes,
+so **please don't use this image for a production server**.
+
+This multi-purpose image is built on top of `Dockerfile-workers` in the parent directory
+and can be switched using environment variables between the following configurations:
+
+- Monolithic Synapse with SQLite (`SYNAPSE_COMPLEMENT_DATABASE=sqlite`)
+- Monolithic Synapse with Postgres (`SYNAPSE_COMPLEMENT_DATABASE=postgres`)
+- Workerised Synapse with Postgres (`SYNAPSE_COMPLEMENT_DATABASE=postgres` and `SYNAPSE_COMPLEMENT_USE_WORKERS=true`)
+
+The image is self-contained; it contains an integrated Postgres, Redis and Nginx.
+
+
+## How to get Complement to pass the environment variables through
+
+To pass these environment variables, use [Complement's `COMPLEMENT_SHARE_ENV_PREFIX`][complementEnv]
+variable to configure an environment prefix to pass through, then prefix the above options
+with that prefix.
+
+Example:
+```
+COMPLEMENT_SHARE_ENV_PREFIX=PASS_ PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres
+```
+
+Consult `scripts-dev/complement.sh` in the repository root for a real example.
+
+
+[complement]: https://github.com/matrix-org/complement
+[complementEnv]: https://github.com/matrix-org/complement/pull/382

From 33aeb673a681ac75a82a56e5165afde55a9b31bd Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 31 May 2022 10:50:35 +0100
Subject: [PATCH 17/27] Update the contributing guide

---
 docs/development/contributing_guide.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md
index 2b3714df66f9..c2f04a39051f 100644
--- a/docs/development/contributing_guide.md
+++ b/docs/development/contributing_guide.md
@@ -304,6 +304,11 @@ To run a specific test, you can specify the whole name structure:
 COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestImportHistoricalMessages/parallel/Historical_events_resolve_in_the_correct_order
 ```
 
+The above will run a monolithic (single-process) Synapse with SQLite as the database. For other configurations, try:
+
+- Passing `POSTGRES=1` as an environment variable to use the Postgres database instead.
+- Passing `WORKERS=1` as an environment variable to use a workerised setup instead. This option implies the use of Postgres.
+
 
 ### Access database for homeserver after Complement test runs.
 

From 9f8d317a6b95d5affce891a1ef2b76393c6f2453 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 31 May 2022 11:30:18 +0100
Subject: [PATCH 18/27] Update README-testing.md

---
 docker/README-testing.md | 97 +++++++++++++++++++---------------------
 1 file changed, 46 insertions(+), 51 deletions(-)

diff --git a/docker/README-testing.md b/docker/README-testing.md
index c38cae75308e..eea899227aed 100644
--- a/docker/README-testing.md
+++ b/docker/README-testing.md
@@ -8,79 +8,50 @@ docker images that can be run inside Complement for testing purposes.
 
 Note that running Synapse's unit tests from within the docker image is not supported.
 
-## Testing with SQLite and single-process Synapse
+## Using the Complement launch script
 
-> Note that `scripts-dev/complement.sh` is a script that will automatically build
-> and run an SQLite-based, single-process of Synapse against Complement.
+`scripts-dev/complement.sh` is a script that will automatically build
+and run Synapse against Complement.
+Consult the [contributing guide][guideComplementSh]
 
-The instructions below will set up Complement testing for a single-process,
-SQLite-based Synapse deployment.
 
-Start by building the base Synapse docker image. If you wish to run tests with the latest
-release of Synapse, instead of your current checkout, you can skip this step. From the
-root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse -f docker/Dockerfile .
-```
+[guideComplementSh]: https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-integration-tests-complement
 
-This will build an image with the tag `matrixdotorg/synapse`.
+## Building and running the images manually
 
-Next, build the Synapse image for Complement.
+Under some circumstances, you may wish to build the images manually.
+The instructions below will lead you to doing that.
 
-```sh
-docker build -t complement-synapse -f "docker/complement/Dockerfile" docker/complement
-```
-
-This will build an image with the tag `complement-synapse`, which can be handed to
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for
-how to run the tests, as well as the various available command line flags.
-
-## Testing with PostgreSQL and single or multi-process Synapse
-
-The above docker image only supports running Synapse with SQLite and in a
-single-process topology. The following instructions are used to build a Synapse image for
-Complement that supports either single or multi-process topology with a PostgreSQL
-database backend.
-
-As with the single-process image, build the base Synapse docker image. If you wish to run
-tests with the latest release of Synapse, instead of your current checkout, you can skip
-this step. From the root of the repository:
+Start by building the base Synapse docker image. If you wish to run tests with the latest
+release of Synapse, instead of your current checkout, you can skip this step. From the
+root of the repository:
 
 ```sh
 docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 ```
 
-This will build an image with the tag `matrixdotorg/synapse`.
-
-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`.
-Again, from the root of the repository:
+Next, build the workerised Synapse docker image, which is a layer over the base
+image.
 
 ```sh
 docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
 ```
 
-This will build an image with the tag` matrixdotorg/synapse-workers`.
-
-It's worth noting at this point that this image is fully functional, and
-can be used for testing against locally. See instructions for using the container
-under
-[Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
-below.
-
-Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`.
+Finally, build the multi-purpose image for Complement, which is a layer over the workers image.
 
 ```sh
-docker build -t matrixdotorg/complement-synapse-workers -f docker/complement/SynapseWorkers.Dockerfile docker/complement
+docker build -t complement-synapse -f "docker/complement/Dockerfile" docker/complement
 ```
 
-This will build an image with the tag `complement-synapse-workers`, which can be handed to
+This will build an image with the tag `complement-synapse`, which can be handed to
 Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
 [Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.
 
+See [the Complement image README](./complement/README.md) for information about the
+expected environment variables.
+
+
 ## Running the Dockerfile-worker image standalone
 
 For manual testing of a multi-process Synapse instance in Docker,
@@ -113,6 +84,9 @@ docker run -d --name synapse \
 ...substituting `POSTGRES*` variables for those that match a postgres host you have
 available (usually a running postgres docker container).
 
+
+### Workers
+
 The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
 use when running the container. All possible worker names are defined by the keys of the
 `WORKERS_CONFIG` variable in [this script](configure_workers_and_start.py), which the
@@ -125,8 +99,11 @@ type, simply specify the type multiple times in `SYNAPSE_WORKER_TYPES`
 (e.g `SYNAPSE_WORKER_TYPES=event_creator,event_creator...`).
 
 Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no workers
-(leaving only the main process). The container is configured to use redis-based worker
-mode.
+(leaving only the main process).
+The container will only be configured to use Redis-based worker mode if there are
+workers enabled.
+
+### Logging
 
 Logs for workers and the main process are logged to stdout and can be viewed with
 standard `docker logs` tooling. Worker logs contain their worker name
@@ -136,3 +113,21 @@ Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be writ
 `<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
 00, according to the container's clock. Logging for the main process must still be
 configured by modifying the homeserver's log config in your Synapse data volume.
+
+
+### Application Services
+
+Setting the `SYNAPSE_AS_REGISTRATION_DIR` environment variable to the path of
+a directory (within the container) will cause the configuration script to scan
+that directory for `.yaml`/`.yml` registration files.
+Synapse will be configured to load these configuration files.
+
+
+### TLS Termination
+
+Nginx is present in the image to route requests to the appropriate workers,
+but it does not serve TLS by default.
+
+You can configure `SYNAPSE_TLS_CERT` and `SYNAPSE_TLS_KEY` to point to a
+TLS certificate and key (respectively), both in PEM (textual) format.
+In this case, Nginx will additionally serve using HTTPS on port 8448.

From 9531da611f1b0da6a00f563dfaa0506646e85f41 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 31 May 2022 11:31:54 +0100
Subject: [PATCH 19/27] Fix sentence that dropped off

---
 docker/README-testing.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/README-testing.md b/docker/README-testing.md
index eea899227aed..7ad0cfda64bc 100644
--- a/docker/README-testing.md
+++ b/docker/README-testing.md
@@ -12,7 +12,7 @@ Note that running Synapse's unit tests from within the docker image is not suppo
 
 `scripts-dev/complement.sh` is a script that will automatically build
 and run Synapse against Complement.
-Consult the [contributing guide][guideComplementSh]
+Consult the [contributing guide][guideComplementSh] for instructions on how to use it.
 
 
 [guideComplementSh]: https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-integration-tests-complement

From 2beae8fb5375fa063c0e9e05a6ce70c332220ddf Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Tue, 31 May 2022 12:18:47 +0100
Subject: [PATCH 20/27] Update docker/README-testing.md

Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 docker/README-testing.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/README-testing.md b/docker/README-testing.md
index 7ad0cfda64bc..1f0423f09b91 100644
--- a/docker/README-testing.md
+++ b/docker/README-testing.md
@@ -40,7 +40,7 @@ docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
 Finally, build the multi-purpose image for Complement, which is a layer over the workers image.
 
 ```sh
-docker build -t complement-synapse -f "docker/complement/Dockerfile" docker/complement
+docker build -t complement-synapse -f docker/complement/Dockerfile docker/complement
 ```
 
 This will build an image with the tag `complement-synapse`, which can be handed to

From bc5d5296330c7d7b048e7f06b1f324ac0d498197 Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Tue, 31 May 2022 12:21:34 +0100
Subject: [PATCH 21/27] Apply suggestions from code review

Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 docker/complement/Dockerfile                   | 4 ++--
 docker/complement/conf/start_for_complement.sh | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 4002cf03b0cf..95c6267bc8f6 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -1,4 +1,4 @@
-# This dockerfile builds on top of 'docker/Dockerfile-worker' in matrix-org/synapse
+# This dockerfile builds on top of 'docker/Dockerfile-workers' in matrix-org/synapse
 # by including a built-in postgres instance, as well as setting up the homeserver so
 # that it is ready for testing via Complement.
 #
@@ -20,7 +20,7 @@ RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
   LC_CTYPE='C' \
   template=template0;\" | psql" && pg_ctlcluster 13 main stop
 
-# Modify the shared homeserver config with postgres support, certificate setup
+# Replace the shared homeserver config to include postgres support, certificate setup
 # and the disabling of rate-limiting
 COPY conf/workers-shared.yaml /conf/workers/shared.yaml
 
diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh
index 8fb824ade72b..b9c97ab687f7 100755
--- a/docker/complement/conf/start_for_complement.sh
+++ b/docker/complement/conf/start_for_complement.sh
@@ -26,16 +26,18 @@ case "$SYNAPSE_COMPLEMENT_DATABASE" in
     export POSTGRES_PASSWORD=somesecret
     export POSTGRES_USER=postgres
     export POSTGRES_HOST=localhost
+
+    # configure supervisord to start postgres
     export START_POSTGRES=true
     ;;
 
   sqlite)
-    # Prevent Postgres from starting up as we don't need it to
+    # Configure supervisord not to start Postgres, as we don't need it
     export START_POSTGRES=false
     ;;
 
   *)
-    echo "Unknown Synapse database: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE"
+    echo "Unknown Synapse database: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE" >&2
     exit 1
     ;;
 esac

From 166b046c31d8515c9776bfd5bab3cb3a9ac30fb0 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 31 May 2022 12:15:40 +0100
Subject: [PATCH 22/27] Remove obsolete START_REDIS

---
 docker/Dockerfile-workers | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
index d20e1263729a..83db0a95b9fd 100644
--- a/docker/Dockerfile-workers
+++ b/docker/Dockerfile-workers
@@ -27,11 +27,6 @@ COPY ./docker/prefix-log /usr/local/bin/
 # Expose nginx listener port
 EXPOSE 8080/tcp
 
-# Default to starting Redis.
-# (This environment variable is only used in the Complement image to disable
-# Redis when workers aren't being used)
-ENV START_REDIS=true
-
 # A script to read environment variables and create the necessary
 # files to run the desired worker configuration. Will start supervisord.
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py

From 4a739cabb94ef937b131cb03fae9d5daf410ab5e Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Tue, 31 May 2022 12:25:17 +0100
Subject: [PATCH 23/27] Fix comment explaining the overwritten config file

---
 docker/complement/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 95c6267bc8f6..88e1c82ca4f0 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -20,8 +20,9 @@ RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
   LC_CTYPE='C' \
   template=template0;\" | psql" && pg_ctlcluster 13 main stop
 
-# Replace the shared homeserver config to include postgres support, certificate setup
-# and the disabling of rate-limiting
+# Replace the shared homeserver config to disable rate-limiting,
+# set Complement's static shared secret, enable registration, amongst other
+# tweaks to get Synapse ready for testing.
 COPY conf/workers-shared.yaml /conf/workers/shared.yaml
 
 WORKDIR /data

From 1af38ef0a11453c6f1c8cac52df44e53fcc1db26 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 1 Jun 2022 16:28:57 +0100
Subject: [PATCH 24/27] Convert workers-shared.yaml to Jinja2 and import the
 original file rather than appending the files

---
 docker/complement/Dockerfile                  |  7 +++++--
 ...ared.yaml => workers-shared-extra.yaml.j2} |  4 ++++
 docker/configure_workers_and_start.py         | 21 +++++++++++--------
 3 files changed, 21 insertions(+), 11 deletions(-)
 rename docker/complement/conf/{workers-shared.yaml => workers-shared-extra.yaml.j2} (95%)

diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 88e1c82ca4f0..50684c956de6 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -20,10 +20,13 @@ RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
   LC_CTYPE='C' \
   template=template0;\" | psql" && pg_ctlcluster 13 main stop
 
-# Replace the shared homeserver config to disable rate-limiting,
+# Extend the shared homeserver config to disable rate-limiting,
 # set Complement's static shared secret, enable registration, amongst other
 # tweaks to get Synapse ready for testing.
-COPY conf/workers-shared.yaml /conf/workers/shared.yaml
+# To do this, we copy the old template out of the way and then include it
+# with Jinja2.
+RUN mv /conf/shared.yaml.j2 /conf/shared-orig.yaml.j2
+COPY conf/workers-shared-extra.yaml.j2 /conf/shared.yaml.j2
 
 WORKDIR /data
 
diff --git a/docker/complement/conf/workers-shared.yaml b/docker/complement/conf/workers-shared-extra.yaml.j2
similarity index 95%
rename from docker/complement/conf/workers-shared.yaml
rename to docker/complement/conf/workers-shared-extra.yaml.j2
index 36a6933a538a..3eae5dba30b6 100644
--- a/docker/complement/conf/workers-shared.yaml
+++ b/docker/complement/conf/workers-shared-extra.yaml.j2
@@ -82,8 +82,10 @@ experimental_features:
   msc2716_enabled: true
   # server-side support for partial state in /send_join responses
   msc3706_enabled: true
+  {% if not workers_in_use %}
   # client-side support for partial state in /send_join responses
   faster_joins: true
+  {% endif %}
   # Enable jump to date endpoint
   msc3030_enabled: true
 
@@ -92,3 +94,5 @@ server_notices:
   system_mxid_display_name: "Server Alert"
   system_mxid_avatar_url: ""
   room_name: "Server Alert"
+
+{% include "shared-orig.yaml.j2" %}
diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
index 568f22e1d73a..5a7250c29da2 100755
--- a/docker/configure_workers_and_start.py
+++ b/docker/configure_workers_and_start.py
@@ -37,8 +37,8 @@
 from pathlib import Path
 from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set
 
-import jinja2
 import yaml
+from jinja2 import Environment, FileSystemLoader
 
 MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 
@@ -234,13 +234,15 @@ def convert(src: str, dst: str, **template_vars: object) -> None:
         dst: Path to write to.
         template_vars: The arguments to replace placeholder variables in the template with.
     """
+
     # Read the template file
-    with open(src) as infile:
-        template = infile.read()
+    # We disable autoescape to prevent template variables from being escaped,
+    # as we're not using HTML.
+    env = Environment(loader=FileSystemLoader(os.path.dirname(src)), autoescape=False)
+    template = env.get_template(os.path.basename(src))
 
-    # Generate a string from the template. We disable autoescape to prevent template
-    # variables from being escaped.
-    rendered = jinja2.Template(template, autoescape=False).render(**template_vars)
+    # Generate a string from the template.
+    rendered = template.render(**template_vars)
 
     # Write the generated contents to a file
     #
@@ -505,7 +507,7 @@ def generate_worker_files(
             if reg_path.suffix.lower() in (".yaml", ".yml")
         ]
 
-    enable_redis = len(worker_types) > 0
+    workers_in_use = len(worker_types) > 0
 
     # Shared homeserver config
     convert(
@@ -513,7 +515,8 @@ def generate_worker_files(
         "/conf/workers/shared.yaml",
         shared_worker_config=yaml.dump(shared_config),
         appservice_registrations=appservice_registrations,
-        enable_redis=enable_redis,
+        enable_redis=workers_in_use,
+        workers_in_use=workers_in_use,
     )
 
     # Nginx config
@@ -533,7 +536,7 @@ def generate_worker_files(
         "/etc/supervisor/supervisord.conf",
         main_config_path=config_path,
         worker_config=supervisord_config,
-        enable_redis=enable_redis,
+        enable_redis=workers_in_use,
     )
 
     # healthcheck config

From 6daac8aefbec02ebbc8ea20974826b49844b766b Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Mon, 6 Jun 2022 12:27:51 +0100
Subject: [PATCH 25/27] Add comment about how it fits in

---
 docker/complement/conf/workers-shared-extra.yaml.j2 | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2
index 3eae5dba30b6..c15e309d61bc 100644
--- a/docker/complement/conf/workers-shared-extra.yaml.j2
+++ b/docker/complement/conf/workers-shared-extra.yaml.j2
@@ -95,4 +95,12 @@ server_notices:
   system_mxid_avatar_url: ""
   room_name: "Server Alert"
 
+{#
+  This file is intended to extend the configuration with Complement-specific
+  tweaks, so here we include the original section of configuration that does
+  not have any Complement-specific values.
+
+  The base configuration is moved out of the default path to `shared-orig.yaml.j2`
+  in the Complement Dockerfile.
+#}
 {% include "shared-orig.yaml.j2" %}

From 2fd415b800754c24eea49861d8d5d5c188fd853a Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Wed, 8 Jun 2022 10:08:13 +0100
Subject: [PATCH 26/27] Update docker/configure_workers_and_start.py

Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 docker/configure_workers_and_start.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
index ccfec989531a..64697e03548a 100755
--- a/docker/configure_workers_and_start.py
+++ b/docker/configure_workers_and_start.py
@@ -235,7 +235,6 @@ def convert(src: str, dst: str, **template_vars: object) -> None:
         dst: Path to write to.
         template_vars: The arguments to replace placeholder variables in the template with.
     """
-
     # Read the template file
     # We disable autoescape to prevent template variables from being escaped,
     # as we're not using HTML.

From b5481329eba214ebc4e851ba8cedd0a9632f9170 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 8 Jun 2022 10:09:50 +0100
Subject: [PATCH 27/27] Move comment to top of file and rephrase slightly

---
 .../complement/conf/workers-shared-extra.yaml.j2 | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2
index c15e309d61bc..a5b1b6bb8bf0 100644
--- a/docker/complement/conf/workers-shared-extra.yaml.j2
+++ b/docker/complement/conf/workers-shared-extra.yaml.j2
@@ -1,3 +1,11 @@
+{#
+  This file extends the default 'shared' configuration file (from the 'synapse-workers'
+  docker image) with Complement-specific  tweak.
+
+  The base configuration is moved out of the default path to `shared-orig.yaml.j2`
+  in the Complement Dockerfile and below we include that original file.
+#}
+
 ## Server ##
 report_stats: False
 trusted_key_servers: []
@@ -95,12 +103,4 @@ server_notices:
   system_mxid_avatar_url: ""
   room_name: "Server Alert"
 
-{#
-  This file is intended to extend the configuration with Complement-specific
-  tweaks, so here we include the original section of configuration that does
-  not have any Complement-specific values.
-
-  The base configuration is moved out of the default path to `shared-orig.yaml.j2`
-  in the Complement Dockerfile.
-#}
 {% include "shared-orig.yaml.j2" %}