[FS-1940] Start sending backend notifications through rabbitMQ and consuming them

.envrc

@@ -37,3 +37,7 @@ path_add "PYTHONPATH" "./hack/python"
 # Locale
 export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
+
+# RabbitMQ
+export RABBITMQ_USERNAME=guest
+export RABBITMQ_PASSWORD=alpaca-grapefruit

cabal.project

@@ -31,6 +31,7 @@ packages:
   , libs/wire-api-federation/
   , libs/wire-message-proto-lens/
   , libs/zauth/
+  , services/background-worker/
   , services/brig/
   , services/cannon/
   , services/cargohold/
@@ -66,6 +67,8 @@ package assets
     ghc-options: -Werror
 package auto-whitelist
     ghc-options: -Werror
+package background-worker
+    ghc-options: -Werror
 package bilge
     ghc-options: -Werror
 package billing-team-member-backfill

changelog.d/0-release-notes/background-worker

@@ -0,0 +1,38 @@
+This release introduces a new component: background-worker. This is currently
+only used to forward notifications to federated backends. Enabling federation in
+the wire-server helm chart automatically installs this component.
+
+When federation is enabled, wire-server will require running RabbitMQ. The helm
+chart in `rabbitmq` can be used to install RabbitMQ. Please refer to the
+documentation at https://docs.wire.com to install RabbitMQ in Kubernetes. These
+new configurations are required:
+
+```yaml
+brig:
+  config:
+    rabbitmq:
+      host: rabbitmq
+      port: 5672
+      vHost: /
+  secrets:
+    rabbitmq:
+      username: <YOUR_USERNAME>
+      password: <YOUR_PASSWORD>
+background-worker:
+  config:
+    rabbitmq:
+      host: rabbitmq
+      port: 5672
+      vHost: /
+    remoteDomains: []
+  secrets:
+    rabbitmq:
+      username: <YOUR_USERNAME>
+      password: <YOUR_PASSWORD>
+```
+
+The above are the default values (except for secrets, which do not have
+defaults), if they work they are not required to be configured.
+`background-worker.config.remoteDomains` should contain all the remote domains
+with which the wire-server instance allows federating. This change is
+incompatible with open-federation.

charts/background-worker/templates/deployment.yaml

@@ -9,12 +9,11 @@ metadata:
     heritage: {{ .Release.Service }}
 spec:
   replicas: {{ .Values.replicaCount }}
-  # TODO(elland): Review this
   strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 0
-      maxSurge: {{ .Values.replicaCount }}
+    # Ensures only one version of the background worker is running at any given
+    # moment. This means small downtime, but the background workers should be
+    # able to catch up.
+    type: Recreate
   selector:
     matchLabels:
       app: background-worker

charts/background-worker/values.yaml

@@ -9,12 +9,11 @@ resources:
     cpu: "100m"
   limits:
     memory: "512Mi"
-# TODO(elland): Create issue for a metrics endpoint
+# FUTUREWORK: Implement metrics
 # metrics:
 #   serviceMonitor:
 #     enabled: false
 config:
-  # TODO(elland): Proper logging
   logLevel: Info
   logFormat: StructuredJSON
   rabbitmq:

charts/integration/templates/integration-integration.yaml

@@ -71,6 +71,12 @@ spec:
     - name: AWS_REGION
       value: "eu-west-1"
     - name: RABBITMQ_USERNAME
-      value: "guest"
+      valueFrom:
+        secretKeyRef:
+          name: brig
+          key: rabbitmqUsername
     - name: RABBITMQ_PASSWORD
-      value: "guest"
+      valueFrom:
+        secretKeyRef:
+          name: brig
+          key: rabbitmqPassword

charts/wire-server/requirements.yaml

@@ -111,6 +111,14 @@ dependencies:
     - federation
     - haskellServices
     - services
+- name: background-worker
+  version: "0.0.42"
+  repository: "file://../background-worker"
+  tags:
+    - background-worker
+    - federation
+    - haskellServices
+    - services
 - name: sftd
   version: "0.0.42"
   repository: "file://../sftd"

docs/src/how-to/install/helm-prod.md

@@ -155,6 +155,32 @@ cp values/wire-server/prod-secrets.example.yaml my-wire-server/secrets.yaml
 cp values/wire-server/prod-values.example.yaml my-wire-server/values.yaml
 ```
 
+## How to install RabbitMQ
+
+This is only required when federation needs to be enabled.
+
+1. Generate password for rabbitmq:
+
+   ```shell
+   openssl rand -base64 64 | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 42 > my-wire-server/rabbitmq-password
+   ```
+
+2. Copy example values
+
+   ```shell
+   cp values/rabbitmq/prod-secrets.example.yaml values/rabbitmq/secrets.yaml
+   cp values/rabbitmq/prod-values.example.yaml values/rabbitmq/values.yaml
+   ```
+
+3. Add the generated secret from `my-wire-server/rabbitmq-password` to
+   `values/rabbitmq/secrets.yaml` under `rabbitmq.auth.password`.
+
+4. Install the helm chart using:
+
+   ```shell
+   helm upgrade --install rabbitmq wire/rabbitmq -f values/rabbitmq/values.yaml -f values/rabbitmq/secrets.yaml
+   ```
+
 ## How to configure real SMTP (email) services
 
 In order for users to interact with their wire account, they need to receive mail from your wire server.
@@ -189,9 +215,10 @@ apt install docker-ce
 sudo docker run --rm quay.io/wire/alpine-intermediate /dist/zauth -m gen-keypair -i 1 > my-wire-server/zauth.txt
 ```
 
-1. Add the generated secret from my-wire-server/restund.txt to my-wire-serwer/secrets.yaml under `brig.secrets.turn.secret`
-2. add **both** the public and private parts from zauth.txt to secrets.yaml under `brig.secrets.zAuth`
-3. Add the public key from zauth.txt to secrets.yaml under `nginz.secrets.zAuth.publicKeys`
+1. Add the generated secret from `my-wire-server/restund.txt` to `my-wire-server/secrets.yaml` under `brig.secrets.turn.secret`.
+2. add **both** the public and private parts from `my-wire-server/zauth.txt` to `my-wire-server/secrets.yaml` under `brig.secrets.zAuth`.
+3. Add the public key from `my-wire-server/zauth.txt` to `my-wire-server/secrets.yaml` under `nginz.secrets.zAuth.publicKeys`.
+4. Add the generated secret from my-wire-server/rabbitmq-password to `my-wire-server/secerts.yaml` under `brig.secrets.rabbitmq.password` and `background-worker.secrets.rabbitmq.password`.
 
 Great, now try the installation:
 

hack/bin/set-wire-server-image-version.sh

@@ -6,7 +6,7 @@ target_version=${1?$USAGE}
 TOP_LEVEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
 CHARTS_DIR="$TOP_LEVEL/.local/charts"
 
-charts=(brig cannon galley gundeck spar cargohold proxy cassandra-migrations elasticsearch-index federator backoffice integration)
+charts=(brig cannon galley gundeck spar cargohold proxy cassandra-migrations elasticsearch-index federator backoffice background-worker integration)
 
 for chart in "${charts[@]}"; do
     sed -i "s/^  tag: .*/  tag: $target_version/g" "$CHARTS_DIR/$chart/values.yaml"

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -300,3 +300,16 @@ federator:
       federationStrategy:
         allowAll: true
       useSystemCAStore: false
+
+background-worker:
+  replicaCount: 1
+  resources:
+    requests: {}
+  imagePullPolicy: {{ .Values.imagePullPolicy }}
+  config:
+    # See helmfile for the real value
+    remoteDomains: []
+  secrets:
+    rabbitmq:
+      username: {{ .Values.rabbitmqUsername }}
+      password: {{ .Values.rabbitmqPassword }}

hack/helmfile.yaml

@@ -141,6 +141,9 @@ releases:
         value: {{ .Values.federationDomain1 }}
       - name: brig.config.optSettings.setFederationDomainConfigs[0].domain
         value: {{ .Values.federationDomain2 }}
+      - name: background-worker.config.remoteDomains
+        values:
+        - {{ .Values.federationDomain2 }}
     needs:
       - 'databases-ephemeral'
 
@@ -159,5 +162,8 @@ releases:
         value: {{ .Values.federationDomain2 }}
       - name: brig.config.optSettings.setFederationDomainConfigs[0].domain
         value: {{ .Values.federationDomain1 }}
+      - name: background-worker.config.remoteDomains
+        values:
+        - {{ .Values.federationDomain1 }}
     needs:
       - 'databases-ephemeral'

libs/extended/default.nix

@@ -4,6 +4,7 @@
 # dependencies are added or removed.
 { mkDerivation
 , aeson
+, amqp
 , base
 , bytestring
 , cassandra-util
@@ -18,13 +19,16 @@
 , imports
 , lib
 , metrics-wai
+, monad-control
 , optparse-applicative
 , resourcet
+, retry
 , servant
 , servant-server
 , servant-swagger
 , string-conversions
 , temporary
+, text
 , tinylog
 , wai
 }:
@@ -34,6 +38,7 @@ mkDerivation {
   src = gitignoreSource ./.;
   libraryHaskellDepends = [
     aeson
+    amqp
     base
     bytestring
     cassandra-util
@@ -44,12 +49,15 @@ mkDerivation {
     http-types
     imports
     metrics-wai
+    monad-control
     optparse-applicative
     resourcet
+    retry
     servant
     servant-server
     servant-swagger
     string-conversions
+    text
     tinylog
     wai
   ];

libs/extended/extended.cabal

@@ -17,7 +17,9 @@ license-file:  LICENSE
 build-type:    Simple
 
 library
+  -- cabal-fmt: expand src
   exposed-modules:
+    Network.AMQP.Extended
     Options.Applicative.Extended
     Servant.API.Extended
     Servant.API.Extended.RawM
@@ -74,6 +76,7 @@ library
 
   build-depends:
       aeson
+    , amqp
     , base
     , bytestring
     , cassandra-util
@@ -84,12 +87,15 @@ library
     , http-types
     , imports
     , metrics-wai
+    , monad-control
     , optparse-applicative
     , resourcet
+    , retry
     , servant
     , servant-server
     , servant-swagger
     , string-conversions
+    , text
     , tinylog
     , wai