[FS-1845] Deploy RabbitMQ on k8s for CI

.envrc

@@ -37,3 +37,6 @@ path_add "PYTHONPATH" "./hack/python"
 # Locale
 export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
+
+export RABBITMQ_USERNAME=guest
+export RABBITMQ_PASSWORD=alpaca-grapefruit

Makefile

@@ -7,13 +7,13 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
 # this list could be generated from the folder names under ./charts/ like so:
 # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts)
-CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster databases-ephemeral	\
+CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster rabbitmq databases-ephemeral	\
 fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\

cabal.project

@@ -31,6 +31,7 @@ packages:
   , libs/wire-api-federation/
   , libs/wire-message-proto-lens/
   , libs/zauth/
+  , services/background-worker/
   , services/brig/
   , services/cannon/
   , services/cargohold/
@@ -66,6 +67,8 @@ package assets
     ghc-options: -Werror
 package auto-whitelist
     ghc-options: -Werror
+package background-worker
+    ghc-options: -Werror
 package bilge
     ghc-options: -Werror
 package billing-team-member-backfill
@@ -155,4 +158,4 @@ package wire-api-federation
 package wire-message-proto-lens
     ghc-options: -Werror
 package zauth
-    ghc-options: -Werror
+    ghc-options: -Werror

changelog.d/0-release-notes/helm-tag-rename

@@ -0,0 +1,33 @@
+A few helm values related to federation have been renamed, no action is required if federation was disabled. 
+If federation was enabled these values must be renamed in the wire-server chart:
+- tags.federator -> tags.federation
+- brig.enableFederator -> brig.enableFederation
+- galley.enableFederator -> galley.enableFederation
+- cargohold.enableFederator -> galley.enableFederation
+
+So, an old config which looked like this:
+
+```yaml
+tags:
+  federator: true
+brig:
+  enableFederator: true
+galley:
+  enableFederator: true
+cargohold:
+  enableFederator: true
+```
+
+would now look like this:
+
+```yaml
+tags:
+  federation: true
+brig:
+  enableFederation: true
+galley:
+  enableFederation: true
+cargohold:
+  enableFederation: true
+```
+

charts/background-worker/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: Backend notification pusheer
+name: background-worker
+version: 0.0.42

charts/background-worker/README.md

@@ -0,0 +1,5 @@
+Note that background-worker depends on some provisioned storage, namely:
+
+- rabbitmq
+
+These are dealt with independently from this chart.

charts/background-worker/templates/configmap.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "background-worker"
+  labels:
+    app: background-worker
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  {{- with .Values.config }}
+  background-worker.yaml: |
+    logFormat: {{ .logFormat }}
+    logLevel: {{ .logLevel }}
+
+    federatorInternal:
+      host: federator
+      port: 8080
+    rabbitmq:
+{{toYaml .rabbitmq | indent 6 }}
+    remoteDomains:
+{{toYaml .remoteDomains | indent 6 }}
+  {{- end }}

charts/background-worker/templates/deployment.yaml

@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: background-worker
+  labels:
+    app: background-worker
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  # TODO(elland): Review this
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: background-worker
+  template:
+    metadata:
+      labels:
+        app: background-worker
+        release: {{ .Release.Name }}
+      annotations:
+        # An annotation of the configmap checksum ensures changes to the configmap cause a redeployment upon `helm upgrade`
+        checksum/configmap: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/secret: {{ include (print .Template.BasePath "/secret.yaml") . | sha256sum }}
+        fluentbit.io/parser: json
+    spec:
+      serviceAccountName: {{ .Values.serviceAccount.name }}
+      volumes:
+        - name: "background-worker-config"
+          configMap:
+            name: "background-worker"
+        - name: "background-worker-secrets"
+          secret:
+            secretName: "background-worker"
+      containers:
+        - name: background-worker
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
+          volumeMounts:
+          - name: "background-worker-config"
+            mountPath: "/etc/wire/background-worker/conf"
+          env:
+          - name: RABBITMQ_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: background-worker
+                key: rabbitmqUsername
+          - name: RABBITMQ_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: background-worker
+                key: rabbitmqPassword
+          resources:
+{{ toYaml .Values.resources | indent 12 }}

charts/background-worker/templates/secret.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: background-worker
+  labels:
+    app: background-worker
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+type: Opaque
+data:
+  {{/* for_helm_linting is necessary only since the 'with' block below does not throw an error upon an empty .Values.secrets */}}
+  for_helm_linting: {{ required "No .secrets found in configuration. Did you forget to helm <command> -f path/to/secrets.yaml ?" .Values.secrets | quote | b64enc | quote }}
+
+  {{- with .Values.secrets }}
+  rabbitmqUsername: {{ .rabbitmq.username | b64enc | quote }}
+  rabbitmqPassword: {{ .rabbitmq.password | b64enc | quote }}
+  {{- end }}

charts/background-worker/templates/serviceaccount.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.serviceAccount.name }}
+  labels:
+    app: brig
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
+{{- end }}

charts/background-worker/values.yaml

@@ -0,0 +1,35 @@
+replicaCount: 1
+image:
+  repository: quay.io/wire/background-worker
+  tag: do-not-use
+# FUTUREWORK: Review these values when we have some experience
+resources:
+  requests:
+    memory: "200Mi"
+    cpu: "100m"
+  limits:
+    memory: "512Mi"
+# TODO(elland): Create issue for a metrics endpoint
+# metrics:
+#   serviceMonitor:
+#     enabled: false
+config:
+  # TODO(elland): Proper logging
+  logLevel: Info
+  logFormat: StructuredJSON
+  rabbitmq:
+    host: rabbitmq
+    port: 5672
+    vHost: /
+  remoteDomains: []
+
+serviceAccount:
+  # When setting this to 'false', either make sure that a service account named
+  # 'background-worker' exists or change the 'name' field to 'default'
+  create: true
+  name: background-worker
+  annotations: {}
+  automountServiceAccountToken: true
+
+
+secrets: {}

charts/brig/templates/configmap.yaml

@@ -48,7 +48,7 @@ data:
       host: gundeck
       port: 8080
 
-    {{- if .enableFederator }}
+    {{- if .enableFederation }}
     # TODO remove this
     federator:
       host: federator
@@ -57,6 +57,8 @@ data:
     federatorInternal:
       host: federator
       port: 8080
+    rabbitmq:
+{{ toYaml .rabbitmq | indent 6}}
     {{- end }}
 
     {{- with .aws }}

charts/brig/templates/deployment.yaml

@@ -130,7 +130,19 @@ spec:
           - name: NO_PROXY
             value: {{ join "," .noProxyList | quote }}
           {{- end }}
-          {{- end }} 
+          {{- end }}
+          {{- if .Values.config.enableFederation }}
+          - name: RABBITMQ_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: brig
+                key: rabbitmqUsername
+          - name: RABBITMQ_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: brig
+                key: rabbitmqPassword
+          {{- end }}
           ports:
             - containerPort: {{ .Values.service.internalPort }}
           startupProbe:

charts/brig/templates/secret.yaml

@@ -30,6 +30,9 @@ data:
   {{- end }}
   {{- if .oauthJwkKeyPair }}
   oauth_ed25519.jwk: {{ .oauthJwkKeyPair | b64enc | quote }}
-  {{- end }}  
   {{- end }}
-
+  {{- if $.Values.config.enableFederation }}
+  rabbitmqUsername: {{ .rabbitmq.username | b64enc | quote }}
+  rabbitmqPassword: {{ .rabbitmq.password | b64enc | quote }}
+  {{- end }}
+  {{- end }}

charts/brig/templates/tests/brig-integration.yaml

@@ -84,6 +84,18 @@ spec:
     - name: INTEGRATION_FEDERATION_TESTS
       value: "1"
     {{- end }}
+    {{- if .Values.config.enableFederation }}
+    - name: RABBITMQ_USERNAME
+      valueFrom:
+        secretKeyRef:
+          name: brig
+          key: rabbitmqUsername
+    - name: RABBITMQ_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: brig
+          key: rabbitmqPassword
+    {{- end }}
     resources:
       requests:
         memory: "512Mi"

charts/brig/values.yaml

@@ -32,7 +32,12 @@ config:
   # -- If set to false,  'dynamoDBEndpoint' _must_ be set.
   randomPrekeys: true
   useSES: true
-  enableFederator: false # keep enableFederator default in sync with galley and cargohold chart's config.enableFederator as well as wire-server chart's tag.federator
+  enableFederation: false # keep enableFederation default in sync with galley and cargohold chart's config.enableFederation as well as wire-server chart's tags.federation
+  # Not used if enableFederation is false
+  rabbitmq:
+    host: rabbitmq
+    port: 5672
+    vHost: /
   emailSMS:
     general:
       templateBranding:

charts/cargohold/templates/configmap.yaml

@@ -12,7 +12,7 @@ data:
       host: 0.0.0.0
       port: {{ .Values.service.internalPort }}
 
-    {{- if .Values.config.enableFederator }}
+    {{- if .Values.config.enableFederation }}
     federator:
       host: federator
       port: 8080

charts/cargohold/values.yaml

@@ -18,7 +18,7 @@ config:
   logLevel: Info
   logFormat: StructuredJSON
   logNetStrings: false
-  enableFederator: false # keep enableFederator default in sync with brig and galley chart's config.enableFederator as well as wire-server chart's tag.federator
+  enableFederation: false # keep enableFederation default in sync with brig and galley chart's config.enableFederation as well as wire-server chart's tags.federation
   aws:
     region: "eu-west-1"
     s3Bucket: assets

charts/galley/templates/configmap.yaml

@@ -34,7 +34,7 @@ data:
       host: spar
       port: 8080
 
-    {{- if .enableFederator }}
+    {{- if .enableFederation }}
     federator:
       host: federator
       port: 8080

charts/galley/values.yaml

@@ -22,7 +22,7 @@ config:
   cassandra:
     host: aws-cassandra
     replicaCount: 3
-  enableFederator: false # keep enableFederator default in sync with brig and cargohold chart's config.enableFederator as well as wire-server chart's tag.federator
+  enableFederation: false # keep enableFederation default in sync with brig and cargohold chart's config.enableFederation as well as wire-server chart's tags.federation
   settings:
     httpPoolSize: 128
     maxTeamSize: 10000

charts/rabbitmq/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: Wrapper chart for bitnami/rabbitmq
+name: rabbitmq
+version: 0.0.42

charts/rabbitmq/requirements.yaml

@@ -0,0 +1,4 @@
+dependencies:
+- name: rabbitmq
+  version: 11.13.0
+  repository: https://charts.bitnami.com/bitnami

charts/wire-server/requirements.yaml

@@ -108,6 +108,15 @@ dependencies:
   repository: "file://../federator"
   tags:
     - federator
+    - federation
+    - haskellServices
+    - services
+- name: background-worker
+  version: "0.0.42"
+  repository: "file://../background-worker"
+  tags:
+    - background-worker
+    - federation
     - haskellServices
     - services
 - name: sftd