integration: Add test to verify behaviour with offline backends

charts/background-worker/values.yaml

@@ -24,7 +24,8 @@ config:
     vHost: /
     adminPort: 15672
   backendNotificationPusher:
-    remotesRefreshInterval: 60 # seconds
+    pushBackoffMinWait: 10000 # in microseconds, so 10ms
+    pushBackoffMaxWait: 300000000 # microseconds, so 300s
 
 serviceAccount:
   # When setting this to 'false', either make sure that a service account named

deploy/dockerephemeral/init_vhosts.sh

@@ -6,6 +6,8 @@ exec_until_ready() {
 
 echo 'Creating RabbitMQ resources'
 
+exec_until_ready "curl -u $RABBITMQ_USERNAME:$RABBITMQ_PASSWORD -X PUT http://rabbitmq:15672/api/vhosts/backendA"
+exec_until_ready "curl -u $RABBITMQ_USERNAME:$RABBITMQ_PASSWORD -X PUT http://rabbitmq:15672/api/vhosts/backendB"
 exec_until_ready "curl -u $RABBITMQ_USERNAME:$RABBITMQ_PASSWORD -X PUT http://rabbitmq:15672/api/vhosts/d1.example.com"
 exec_until_ready "curl -u $RABBITMQ_USERNAME:$RABBITMQ_PASSWORD -X PUT http://rabbitmq:15672/api/vhosts/d2.example.com"
 exec_until_ready "curl -u $RABBITMQ_USERNAME:$RABBITMQ_PASSWORD -X PUT http://rabbitmq:15672/api/vhosts/d3.example.com"

hack/helm_vars/common.yaml.gotmpl

@@ -5,3 +5,7 @@ federationDomain2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
 ingressChart: {{ requiredEnv "INGRESS_CHART" }}
 rabbitmqUsername: guest
 rabbitmqPassword: guest
+
+dynBackendDomain1: dynamic-backend-1.{{ requiredEnv "NAMESPACE_1" }}.svc.cluster.local
+dynBackendDomain2: dynamic-backend-2.{{ requiredEnv "NAMESPACE_1" }}.svc.cluster.local
+dynBackendDomain3: dynamic-backend-3.{{ requiredEnv "NAMESPACE_1" }}.svc.cluster.local

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -85,6 +85,13 @@ brig:
           search_policy: full_search
         - domain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
           search_policy: full_search
+        # Remove these after fixing https://wearezeta.atlassian.net/browse/WPB-3796
+        - domain: dyn-backend-1
+          search_policy: full_search
+        - domain: dyn-backend-2
+          search_policy: full_search
+        - domain: dyn-backend-3
+          search_policy: full_search
       setFederationStrategy: allowAll
       setFederationDomainConfigsUpdateFreq: 10
       set2FACodeGenerationDelaySecs: 5
@@ -312,7 +319,8 @@ background-worker:
   imagePullPolicy: {{ .Values.imagePullPolicy }}
   config:
     backendNotificationPusher:
-      remotesRefreshInterval: 1
+      pushBackoffMinWait: 1000 # 1ms
+      pushBackoffMaxWait: 500000 # 0.5s
   secrets:
     rabbitmq:
       username: {{ .Values.rabbitmqUsername }}

hack/helmfile.yaml

@@ -129,6 +129,12 @@ releases:
         value: {{ .Values.federationDomain1 }}
       - name: brig.config.optSettings.setFederationDomainConfigs[0].domain
         value: {{ .Values.federationDomain2 }}
+      - name: brig.config.optSettings.setFederationDomainConfigs[2].domain
+        value: {{ .Values.dynBackendDomain1 }}
+      - name: brig.config.optSettings.setFederationDomainConfigs[3].domain
+        value: {{ .Values.dynBackendDomain2 }}
+      - name: brig.config.optSettings.setFederationDomainConfigs[4].domain
+        value: {{ .Values.dynBackendDomain3 }}
     needs:
       - 'databases-ephemeral'
 
@@ -147,5 +153,11 @@ releases:
         value: {{ .Values.federationDomain2 }}
       - name: brig.config.optSettings.setFederationDomainConfigs[0].domain
         value: {{ .Values.federationDomain1 }}
+      - name: brig.config.optSettings.setFederationDomainConfigs[2].domain
+        value: {{ .Values.dynBackendDomain1 }}
+      - name: brig.config.optSettings.setFederationDomainConfigs[3].domain
+        value: {{ .Values.dynBackendDomain2 }}
+      - name: brig.config.optSettings.setFederationDomainConfigs[4].domain
+        value: {{ .Values.dynBackendDomain3 }}
     needs:
       - 'databases-ephemeral'

integration/default.nix

@@ -35,6 +35,7 @@
 , network-uri
 , optparse-applicative
 , process
+, proto-lens
 , random
 , raw-strings-qq
 , retry
@@ -53,6 +54,7 @@
 , uuid
 , vector
 , websockets
+, wire-message-proto-lens
 , yaml
 }:
 mkDerivation {
@@ -92,6 +94,7 @@ mkDerivation {
     network-uri
     optparse-applicative
     process
+    proto-lens
     random
     raw-strings-qq
     retry
@@ -110,6 +113,7 @@ mkDerivation {
     uuid
     vector
     websockets
+    wire-message-proto-lens
     yaml
   ];
   license = lib.licenses.agpl3Only;

integration/integration.cabal

@@ -96,6 +96,7 @@ library
     API.GundeckInternal
     API.Nginz
     MLS.Util
+    Notifications
     RunAllTests
     SetupHelpers
     Test.AssetDownload
@@ -105,6 +106,7 @@ library
     Test.Conversation
     Test.Defederation
     Test.Demo
+    Test.Federation
     Test.Federator
     Test.Notifications
     Test.Presence
@@ -156,6 +158,7 @@ library
     , network-uri
     , optparse-applicative
     , process
+    , proto-lens
     , random
     , raw-strings-qq
     , retry
@@ -174,4 +177,5 @@ library
     , uuid
     , vector
     , websockets
+    , wire-message-proto-lens
     , yaml

integration/test/API/Brig.hs

@@ -32,6 +32,12 @@ getClient u cli = do
       joinHttpPath ["clients", c]
   submit "GET" req
 
+deleteUser :: (HasCallStack, MakesValue user) => user -> App Response
+deleteUser user = do
+  req <- baseRequest user Brig Versioned "/self"
+  submit "DELETE" $
+    req & addJSONObject ["password" .= defPassword]
+
 data AddClient = AddClient
   { ctype :: String,
     internal :: Bool,

integration/test/API/Galley.hs

@@ -1,6 +1,8 @@
 module API.Galley where
 
 import Data.Aeson qualified as Aeson
+import Data.ProtoLens qualified as Proto
+import Proto.Otr
 import Testlib.Prelude
 
 data CreateConv = CreateConv
@@ -152,6 +154,14 @@ postMLSCommitBundle cid msg = do
   req <- baseRequest cid Galley Versioned "/mls/commit-bundles"
   submit "POST" (addMLS msg req)
 
+postProteusMessage :: (HasCallStack, MakesValue user, MakesValue conv) => user -> conv -> QualifiedNewOtrMessage -> App Response
+postProteusMessage user conv msgs = do
+  convDomain <- objDomain conv
+  convId <- objId conv
+  let bytes = Proto.encodeMessage msgs
+  req <- baseRequest user Galley Versioned ("/conversations/" <> convDomain <> "/" <> convId <> "/proteus/messages")
+  submit "POST" (addProtobuf bytes req)
+
 getGroupInfo ::
   (HasCallStack, MakesValue user, MakesValue conv) =>
   user ->
@@ -167,7 +177,15 @@ getGroupInfo user conv = do
   submit "GET" req
 
 addMembers :: (HasCallStack, MakesValue user, MakesValue conv) => user -> conv -> [Value] -> App Response
-addMembers usr qcnv qUsers = do
+addMembers usr qcnv newMembers = do
   (convDomain, convId) <- objQid qcnv
+  qUsers <- mapM objQidObject newMembers
   req <- baseRequest usr Galley Versioned (joinHttpPath ["conversations", convDomain, convId, "members"])
   submit "POST" (req & addJSONObject ["qualified_users" .= qUsers])
+
+removeMember :: (HasCallStack, MakesValue remover, MakesValue conv, MakesValue removed) => remover -> conv -> removed -> App Response
+removeMember remover qcnv removed = do
+  (convDomain, convId) <- objQid qcnv
+  (removedDomain, removedId) <- objQid removed
+  req <- baseRequest remover Galley Versioned (joinHttpPath ["conversations", convDomain, convId, "members", removedDomain, removedId])
+  submit "DELETE" req

integration/test/Notifications.hs

@@ -0,0 +1,53 @@
+module Notifications where
+
+import API.Gundeck
+import Control.Monad.Extra
+import Testlib.Prelude
+import UnliftIO.Concurrent
+
+awaitNotifications ::
+  (HasCallStack, MakesValue user, MakesValue client) =>
+  user ->
+  client ->
+  Maybe String ->
+  -- | Timeout in seconds
+  Int ->
+  -- | Max no. of notifications
+  Int ->
+  -- | Selection function. Should not throw any exceptions
+  (Value -> App Bool) ->
+  App [Value]
+awaitNotifications user client since0 tSecs n selector =
+  assertAwaitResult =<< go tSecs since0 (AwaitResult False n [] [])
+  where
+    go 0 _ res = pure res
+    go timeRemaining since res0 = do
+      notifs <- bindResponse (getNotifications user client (GetNotifications since Nothing)) $ \resp -> asList (resp.json %. "notifications")
+      lastNotifId <- case notifs of
+        [] -> pure since
+        _ -> Just <$> objId (last notifs)
+      (matching, notMatching) <- partitionM selector notifs
+      let matchesSoFar = res0.matches <> matching
+          res =
+            res0
+              { matches = matchesSoFar,
+                nonMatches = res0.nonMatches <> notMatching,
+                success = length matchesSoFar >= res0.nMatchesExpected
+              }
+      if res.success
+        then pure res
+        else do
+          threadDelay (1_000_000)
+          go (timeRemaining - 1) lastNotifId res
+
+awaitNotification ::
+  (HasCallStack, MakesValue user, MakesValue client, MakesValue lastNotifId) =>
+  user ->
+  client ->
+  Maybe lastNotifId ->
+  Int ->
+  (Value -> App Bool) ->
+  App Value
+awaitNotification user client lastNotifId tSecs selector = do
+  since0 <- mapM objId lastNotifId
+  head <$> awaitNotifications user client since0 tSecs 1 selector

integration/test/SetupHelpers.hs

@@ -1,6 +1,6 @@
 module SetupHelpers where
 
-import API.Brig qualified as Public
+import API.Brig qualified as Brig
 import API.BrigInternal qualified as Internal
 import API.Galley
 import Control.Concurrent (threadDelay)
@@ -25,6 +25,10 @@ randomUser domain cu = bindResponse (Internal.createUser domain cu) $ \resp -> d
   resp.status `shouldMatchInt` 201
   resp.json
 
+deleteUser :: (HasCallStack, MakesValue user) => user -> App ()
+deleteUser user = bindResponse (Brig.deleteUser user) $ \resp -> do
+  resp.status `shouldMatchInt` 200
+
 -- | returns (user, team id)
 createTeam :: (HasCallStack, MakesValue domain) => domain -> App (Value, String)
 createTeam domain = do
@@ -45,8 +49,8 @@ connectUsers ::
   bob ->
   App ()
 connectUsers alice bob = do
-  bindResponse (Public.postConnection alice bob) (\resp -> resp.status `shouldMatchInt` 201)
-  bindResponse (Public.putConnection bob alice "accepted") (\resp -> resp.status `shouldMatchInt` 200)
+  bindResponse (Brig.postConnection alice bob) (\resp -> resp.status `shouldMatchInt` 201)
+  bindResponse (Brig.putConnection bob alice "accepted") (\resp -> resp.status `shouldMatchInt` 200)
 
 createAndConnectUsers :: (HasCallStack, MakesValue domain) => [domain] -> App [Value]
 createAndConnectUsers domains = do