From 51e11a45466c2f54141cbd8e5cc0d8f7194fff4b Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Wed, 13 Jul 2022 22:22:35 +0300 Subject: [PATCH 01/10] Create create-multiple-stream-writers.md --- .../create-multiple-stream-writers.md | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 contrib/workers-bash-scripts/create-multiple-stream-writers.md diff --git a/contrib/workers-bash-scripts/create-multiple-stream-writers.md b/contrib/workers-bash-scripts/create-multiple-stream-writers.md new file mode 100644 index 000000000000..6074d93af287 --- /dev/null +++ b/contrib/workers-bash-scripts/create-multiple-stream-writers.md @@ -0,0 +1,80 @@ +# Creating multiple stream writers with a bash script + +This script creates multiple [stream writer](https://github.com/matrix-org/synapse/blob/develop/docs/workers.md#stream-writers) workers. + +Stream writers require both replication and HTTP listeners. + +It also prints out the example lines for Synapse main configuration file. + +Remember to route necessary endpoints directly to a worker associated with it. + +If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, "safe" listener starting from 8044. If you don't need all of stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array. + +```sh +#!/bin/bash + +# Start with these replication and http ports. +# The script loop starts with the exact port and then increments it by one. +REP_START_PORT=8034 +HTTP_START_PORT=8044 + +# Stream writer workers to generate. Feel free to add or remove them as you wish. +# Event persister ("events") isn't included here as it does not require its +# own HTTP listener. + +STREAM_WRITERS+=( "presence" "typing" "receipts" "to_device" "account_data" ) + +NUM_WRITERS=$(expr ${#STREAM_WRITERS[@]} - 1) + +i=0 + +while [ $i -le "$NUM_WRITERS" ] +do +cat << EOF > ${STREAM_WRITERS[$i]}_stream_writer.yaml +worker_app: synapse.app.generic_worker +worker_name: ${STREAM_WRITERS[$i]}_stream_writer + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: $(expr $REP_START_PORT + $i) + resources: + - names: [replication] + + - type: http + port: $(expr $HTTP_START_PORT + $i) + resources: + - names: [client] + +worker_log_config: /etc/matrix-synapse/stream-writer-log.yaml +EOF +HOMESERVER_YAML_INSTANCE_MAP+=$" ${STREAM_WRITERS[$i]}_stream_writer: + host: 127.0.0.1 + port: $(expr $REP_START_PORT + $i) +" + +HOMESERVER_YAML_STREAM_WRITERS+=$" ${STREAM_WRITERS[$i]}: ${STREAM_WRITERS[$i]}_stream_writer +" + +((i++)) +done + +echo "# Add these lines to your homeserver.yaml." +echo "# Don't forget to configure your reverse proxy and" +echo "# necessary endpoints to their respective worker." +echo "" +echo "# See https://github.com/matrix-org/synapse/blob/develop/docs/workers.md" +echo "# for more information" +echo "" +echo "# Remember: Under NO circumstances should the replication" +echo "# listener be exposed to the public internet;" +echo "# it has no authentication and is unencrypted." +echo "" +echo "instance_map:" +echo "$HOMESERVER_YAML_INSTANCE_MAP" +echo "stream_writers:" +echo "$HOMESERVER_YAML_STREAM_WRITERS" +``` From 650902e8931b4e28696e5feb1754d3512f5ee63a Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Wed, 13 Jul 2022 22:24:12 +0300 Subject: [PATCH 02/10] Renaming for distinctness --- ...e-multiple-workers.md => create-multiple-generic-workers.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename contrib/workers-bash-scripts/{create-multiple-workers.md => create-multiple-generic-workers.md} (93%) diff --git a/contrib/workers-bash-scripts/create-multiple-workers.md b/contrib/workers-bash-scripts/create-multiple-generic-workers.md similarity index 93% rename from contrib/workers-bash-scripts/create-multiple-workers.md rename to contrib/workers-bash-scripts/create-multiple-generic-workers.md index ad5142fe15e7..d30310142946 100644 --- a/contrib/workers-bash-scripts/create-multiple-workers.md +++ b/contrib/workers-bash-scripts/create-multiple-generic-workers.md @@ -1,4 +1,4 @@ -# Creating multiple workers with a bash script +# Creating multiple generic workers with a bash script Setting up multiple worker configuration files manually can be time-consuming. You can alternatively create multiple worker configuration files with a simple `bash` script. For example: From 23e9ad9cf818ac8e20b7335177f11231cdfff33f Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Wed, 13 Jul 2022 22:44:13 +0300 Subject: [PATCH 03/10] Create 13271.doc --- changelog.d/13271.doc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/13271.doc diff --git a/changelog.d/13271.doc b/changelog.d/13271.doc new file mode 100644 index 000000000000..a59a25321ee2 --- /dev/null +++ b/changelog.d/13271.doc @@ -0,0 +1 @@ +Added a script to contrib directory for creating multiple different stream writers. Renamed the script related to generic workers. Contributed by @villepeh. From 8e3c92d3d5a2366bf525693fd73b7838a7ac653f Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Wed, 13 Jul 2022 23:14:49 +0300 Subject: [PATCH 04/10] Remove unnecessary counting Remove unnecessary counting as it can be done with the loop directly --- .../workers-bash-scripts/create-multiple-stream-writers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/workers-bash-scripts/create-multiple-stream-writers.md b/contrib/workers-bash-scripts/create-multiple-stream-writers.md index 6074d93af287..f07f60ee7e26 100644 --- a/contrib/workers-bash-scripts/create-multiple-stream-writers.md +++ b/contrib/workers-bash-scripts/create-multiple-stream-writers.md @@ -24,11 +24,11 @@ HTTP_START_PORT=8044 STREAM_WRITERS+=( "presence" "typing" "receipts" "to_device" "account_data" ) -NUM_WRITERS=$(expr ${#STREAM_WRITERS[@]} - 1) +NUM_WRITERS=$(expr ${#STREAM_WRITERS[@]}) i=0 -while [ $i -le "$NUM_WRITERS" ] +while [ $i -lt "$NUM_WRITERS" ] do cat << EOF > ${STREAM_WRITERS[$i]}_stream_writer.yaml worker_app: synapse.app.generic_worker From fadf95316e720039859fb52560312257ab1de426 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:23:41 +0300 Subject: [PATCH 05/10] Replace echo with cat. Added more instructions. --- .../create-multiple-stream-writers.md | 91 ++++++++++++++++--- 1 file changed, 76 insertions(+), 15 deletions(-) diff --git a/contrib/workers-bash-scripts/create-multiple-stream-writers.md b/contrib/workers-bash-scripts/create-multiple-stream-writers.md index f07f60ee7e26..bf248e67c722 100644 --- a/contrib/workers-bash-scripts/create-multiple-stream-writers.md +++ b/contrib/workers-bash-scripts/create-multiple-stream-writers.md @@ -62,19 +62,80 @@ HOMESERVER_YAML_STREAM_WRITERS+=$" ${STREAM_WRITERS[$i]}: ${STREAM_WRITERS[$i]} ((i++)) done -echo "# Add these lines to your homeserver.yaml." -echo "# Don't forget to configure your reverse proxy and" -echo "# necessary endpoints to their respective worker." -echo "" -echo "# See https://github.com/matrix-org/synapse/blob/develop/docs/workers.md" -echo "# for more information" -echo "" -echo "# Remember: Under NO circumstances should the replication" -echo "# listener be exposed to the public internet;" -echo "# it has no authentication and is unencrypted." -echo "" -echo "instance_map:" -echo "$HOMESERVER_YAML_INSTANCE_MAP" -echo "stream_writers:" -echo "$HOMESERVER_YAML_STREAM_WRITERS" +cat << EXAMPLECONFIG +# Add these lines to your homeserver.yaml. +# Don't forget to configure your reverse proxy and +# necessary endpoints to their respective worker. + +# See https://github.com/matrix-org/synapse/blob/develop/docs/workers.md +# for more information. + +# Remember: Under NO circumstances should the replication +# listener be exposed to the public internet; +# it has no authentication and is unencrypted. + +instance_map: +$HOMESERVER_YAML_INSTANCE_MAP +stream_writers: +$HOMESERVER_YAML_STREAM_WRITERS +EXAMPLECONFIG +``` + +Copy the code above save it to a file ```create_stream_writers.sh``` (for example). + +Simply run the script to create YAML files in the current folder and print out the required configuration for ```homeserver.yaml```. + +```console +$ ./create_stream_writers.sh + +# Add these lines to your homeserver.yaml. +# Don't forget to configure your reverse proxy and +# necessary endpoints to their respective worker. + +# See https://github.com/matrix-org/synapse/blob/develop/docs/workers.md +# for more information + +# Remember: Under NO circumstances should the replication +# listener be exposed to the public internet; +# it has no authentication and is unencrypted. + +instance_map: + presence_stream_writer: + host: 127.0.0.1 + port: 8034 + typing_stream_writer: + host: 127.0.0.1 + port: 8035 + receipts_stream_writer: + host: 127.0.0.1 + port: 8036 + to_device_stream_writer: + host: 127.0.0.1 + port: 8037 + account_data_stream_writer: + host: 127.0.0.1 + port: 8038 + +stream_writers: + presence: presence_stream_writer + typing: typing_stream_writer + receipts: receipts_stream_writer + to_device: to_device_stream_writer + account_data: account_data_stream_writer +``` + +Simply copy-and-paste this the output to an appropriate place in your Synapse main configuration file. + +## Write directly to Synapse configuration file + +You could also write the output directly to homeserver main configuration file. **This, however, is not recommended** as even a small typo (such as replacing >> with >) can erase the entire ```homeserver.yaml```. + +If you do this, back up your original configuration file first: + +```console +# Back up homeserver.yaml first +cp /etc/matrix-synapse/homeserver.yaml /etc/matrix-synapse/homeserver.yaml.bak + +# Create workers and write output to your homeserver.yaml +./create_stream_writers.sh >> /etc/matrix-synapse/homeserver.yaml ``` From 59cacc7df426a4c484eee34670175c298fe7dd2f Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:29:23 +0300 Subject: [PATCH 06/10] Fixed typo, added another title --- .../workers-bash-scripts/create-multiple-stream-writers.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/workers-bash-scripts/create-multiple-stream-writers.md b/contrib/workers-bash-scripts/create-multiple-stream-writers.md index bf248e67c722..7bc014b46b33 100644 --- a/contrib/workers-bash-scripts/create-multiple-stream-writers.md +++ b/contrib/workers-bash-scripts/create-multiple-stream-writers.md @@ -83,6 +83,8 @@ EXAMPLECONFIG Copy the code above save it to a file ```create_stream_writers.sh``` (for example). +## Run the script to create workers and print out a sample configuration + Simply run the script to create YAML files in the current folder and print out the required configuration for ```homeserver.yaml```. ```console @@ -124,7 +126,7 @@ stream_writers: account_data: account_data_stream_writer ``` -Simply copy-and-paste this the output to an appropriate place in your Synapse main configuration file. +Simply copy-and-paste the output to an appropriate place in your Synapse main configuration file. ## Write directly to Synapse configuration file From 8dd87081bc19e2717054bee7c9aa0ec9ce71a9c0 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:50:09 +0300 Subject: [PATCH 07/10] Wording --- contrib/workers-bash-scripts/create-multiple-stream-writers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/workers-bash-scripts/create-multiple-stream-writers.md b/contrib/workers-bash-scripts/create-multiple-stream-writers.md index 7bc014b46b33..7ba287aa84a0 100644 --- a/contrib/workers-bash-scripts/create-multiple-stream-writers.md +++ b/contrib/workers-bash-scripts/create-multiple-stream-writers.md @@ -8,7 +8,7 @@ It also prints out the example lines for Synapse main configuration file. Remember to route necessary endpoints directly to a worker associated with it. -If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, "safe" listener starting from 8044. If you don't need all of stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array. +If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, regular http listener starting from 8044. If you don't need all of stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array. ```sh #!/bin/bash From ae6bd96dc5d3a5d790505982443ebf996a6e6ff7 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:50:25 +0300 Subject: [PATCH 08/10] Update changelog.d/13271.doc Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- changelog.d/13271.doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/13271.doc b/changelog.d/13271.doc index a59a25321ee2..b50e60d02962 100644 --- a/changelog.d/13271.doc +++ b/changelog.d/13271.doc @@ -1 +1 @@ -Added a script to contrib directory for creating multiple different stream writers. Renamed the script related to generic workers. Contributed by @villepeh. +Add another `contrib` script to help set up worker processes. Contributed by @villepeh. From 59c7695f04df4c7d7c808f2ae2d37d02cb0effa2 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Tue, 19 Jul 2022 02:43:54 +0300 Subject: [PATCH 09/10] Noting that the script needs to be executable Final fine-tuning. Even non-beginner bash script guides generally instruct running chmod +x when creating new script. --- contrib/workers-bash-scripts/create-multiple-stream-writers.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/workers-bash-scripts/create-multiple-stream-writers.md b/contrib/workers-bash-scripts/create-multiple-stream-writers.md index 7ba287aa84a0..ee58a41750ad 100644 --- a/contrib/workers-bash-scripts/create-multiple-stream-writers.md +++ b/contrib/workers-bash-scripts/create-multiple-stream-writers.md @@ -83,6 +83,8 @@ EXAMPLECONFIG Copy the code above save it to a file ```create_stream_writers.sh``` (for example). +Make the script executable by running ```chmod +x create_stream_writers.sh```. + ## Run the script to create workers and print out a sample configuration Simply run the script to create YAML files in the current folder and print out the required configuration for ```homeserver.yaml```. From 6ae37d1bd34823061fd6e34f8a553c7ca91de3d9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:05:16 +0100 Subject: [PATCH 10/10] Update contrib/workers-bash-scripts/create-multiple-stream-writers.md --- contrib/workers-bash-scripts/create-multiple-stream-writers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/workers-bash-scripts/create-multiple-stream-writers.md b/contrib/workers-bash-scripts/create-multiple-stream-writers.md index ee58a41750ad..0d2ca780a6a3 100644 --- a/contrib/workers-bash-scripts/create-multiple-stream-writers.md +++ b/contrib/workers-bash-scripts/create-multiple-stream-writers.md @@ -8,7 +8,7 @@ It also prints out the example lines for Synapse main configuration file. Remember to route necessary endpoints directly to a worker associated with it. -If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, regular http listener starting from 8044. If you don't need all of stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array. +If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, regular http listener starting from 8044. If you don't need all of the stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array. ```sh #!/bin/bash