From 37eacf18146d5b8036dd71c2152bc5c9fbfb54b0 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 19 Apr 2023 18:28:11 +1000
Subject: [PATCH 001/220] FS-1115: Initial effort to get a cross-thread mutable
 variable for Env.

---
 services/federator/federator.cabal            |  1 +
 .../federator/src/Federator/ExternalServer.hs |  7 +--
 .../federator/src/Federator/InternalServer.hs |  9 ++--
 services/federator/src/Federator/Response.hs  | 17 ++++---
 services/federator/src/Federator/Run.hs       |  6 ++-
 .../federator/src/Federator/Validation.hs     | 48 +++++++++++++++----
 6 files changed, 64 insertions(+), 24 deletions(-)

diff --git a/services/federator/federator.cabal b/services/federator/federator.cabal
index 47498f145f..243fd096c9 100644
--- a/services/federator/federator.cabal
+++ b/services/federator/federator.cabal
@@ -138,6 +138,7 @@ library
     , retry
     , servant
     , servant-client-core
+    , stm
     , streaming-commons
     , string-conversions
     , text
diff --git a/services/federator/src/Federator/ExternalServer.hs b/services/federator/src/Federator/ExternalServer.hs
index 65444d252e..51dd2f8c7a 100644
--- a/services/federator/src/Federator/ExternalServer.hs
+++ b/services/federator/src/Federator/ExternalServer.hs
@@ -53,9 +53,10 @@ callInward ::
     Member (Error ServerError) r,
     Member (Input RunSettings) r
   ) =>
+  TVar Env ->
   Wai.Request ->
   Sem r Wai.Response
-callInward wreq = do
+callInward tvar wreq = do
   req <- parseRequestData wreq
   Log.debug $
     Log.msg ("Inward Request" :: ByteString)
@@ -63,7 +64,7 @@ callInward wreq = do
       . Log.field "component" (show (rdComponent req))
       . Log.field "rpc" (rdRPC req)
 
-  validatedDomain <- validateDomain (rdCertificate req) (rdOriginDomain req)
+  validatedDomain <- validateDomain tvar (rdCertificate req) (rdOriginDomain req)
 
   let path = LBS.toStrict (toLazyByteString (HTTP.encodePathSegments ["federation", rdRPC req]))
 
@@ -138,7 +139,7 @@ parseRequestData req = do
 isAllowedRPCChar :: Char -> Bool
 isAllowedRPCChar c = isAsciiLower c || isAsciiUpper c || isNumber c || c == '_' || c == '-'
 
-serveInward :: Env -> Int -> IO ()
+serveInward :: TVar Env -> Int -> IO ()
 serveInward = serve callInward
 
 lookupCertificate :: Wai.Request -> Maybe ByteString
diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs
index 76d0fdd443..0ef26ac08c 100644
--- a/services/federator/src/Federator/InternalServer.hs
+++ b/services/federator/src/Federator/InternalServer.hs
@@ -84,12 +84,15 @@ callOutward ::
     Member (Error ServerError) r,
     Member (Input RunSettings) r
   ) =>
+  TVar Env ->
   Wai.Request ->
   Sem r Wai.Response
-callOutward req = do
+callOutward tvar req = do
   rd <- parseRequestData req
   domain <- parseDomainText (rdTargetDomain rd)
-  ensureCanFederateWith domain
+  -- This call will check for new domains
+  -- when it encounters something that it doesn't know about.
+  ensureCanFederateWith tvar domain
   resp <-
     discoverAndCall
       domain
@@ -99,5 +102,5 @@ callOutward req = do
       (fromLazyByteString (rdBody rd))
   pure $ streamingResponseToWai resp
 
-serveOutward :: Env -> Int -> IO ()
+serveOutward :: TVar Env -> Int -> IO ()
 serveOutward = serve callOutward
diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index 0c6c30b649..19b9fe99e9 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -99,18 +99,19 @@ runWaiError =
       throw e
 
 serve ::
-  (Wai.Request -> Sem AllEffects Wai.Response) ->
-  Env ->
+  (TVar Env -> Wai.Request -> Sem AllEffects Wai.Response) ->
+  TVar Env ->
   Int ->
   IO ()
-serve action env port =
+serve action tvar port = do
+  env <- readTVarIO tvar
   Warp.run port
     . Wai.catchErrors (view applog env) []
     $ app
   where
     app :: Wai.Application
     app req respond =
-      runCodensity (runFederator env (action req)) respond
+      runCodensity (runFederator tvar (action tvar req)) respond
 
 type AllEffects =
   '[ Remote,
@@ -131,9 +132,10 @@ type AllEffects =
 
 -- | Run Sem action containing HTTP handlers. All errors have to been handled
 -- already by this point.
-runFederator :: Env -> Sem AllEffects Wai.Response -> Codensity IO Wai.Response
-runFederator env =
-  runM
+runFederator :: TVar Env -> Sem AllEffects Wai.Response -> Codensity IO Wai.Response
+runFederator tvar resp = do
+  env <- liftIO $ readTVarIO tvar
+  resp & runM
     . runEmbedded @IO @(Codensity IO) liftIO
     . loggerToTinyLogReqId (view requestId env) (view applog env)
     . runWaiErrors
@@ -144,6 +146,7 @@ runFederator env =
         ]
     . runInputConst env
     . runInputSem (embed @IO (readIORef (view sslContext env)))
+    -- This is the point at which federation settings are extracted
     . runInputConst (view runSettings env)
     . interpretServiceHTTP
     . runDNSLookupWithResolver (view dnsResolver env)
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 8251db5fce..07f4bcf2e2 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -62,8 +62,10 @@ run opts = do
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res ->
     bracket (newEnv opts res) closeEnv $ \env -> do
-      let externalServer = serveInward env portExternal
-          internalServer = serveOutward env portInternal
+      -- Build a new TVar holding the state we want for the initial environment.
+      tEnv <- newTVarIO env
+      let externalServer = serveInward tEnv portExternal
+          internalServer = serveOutward tEnv portInternal
       withMonitor (env ^. applog) (env ^. sslContext) (optSettings opts) $ do
         internalServerThread <- async internalServer
         externalServerThread <- async externalServer
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index 27a5245e28..357cbdc987 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -25,6 +25,7 @@ module Federator.Validation
   )
 where
 
+import Control.Lens (over)
 import qualified Data.ByteString.Char8 as B8
 import Data.ByteString.Conversion
 import Data.Domain
@@ -46,6 +47,7 @@ import Polysemy
 import Polysemy.Error
 import Polysemy.Input
 import Wire.Network.DNS.SRV (SrvTarget (..))
+import Federator.Env (Env, runSettings)
 
 data ValidationError
   = NoClientCertificate
@@ -89,21 +91,47 @@ validationErrorStatus :: ValidationError -> HTTP.Status
 validationErrorStatus (FederationDenied _) = HTTP.status400
 validationErrorStatus _ = HTTP.status403
 
+
+ensureCanFederateWith'
+  :: Member (Error ValidationError) r =>
+  FederationStrategy ->
+  Domain ->
+  Sem r ()
+ensureCanFederateWith' strategy targetDomain =
+  case strategy of
+    AllowAll -> pure ()
+    AllowList (AllowedDomains domains) ->
+      unless (targetDomain `elem` domains) $
+        throw (FederationDenied targetDomain)
+
 -- | Validates an already-parsed domain against the allowList using the federator
--- startup configuration.
+-- startup configuration, and can update the allowList from the DB at runtime.
 ensureCanFederateWith ::
   ( Member (Input RunSettings) r,
-    Member (Error ValidationError) r
+    Member (Error ValidationError) r,
+    Member (Embed IO) r
   ) =>
+  TVar Env ->
   Domain ->
   Sem r ()
-ensureCanFederateWith targetDomain = do
+ensureCanFederateWith tvar targetDomain = do
   strategy <- inputs federationStrategy
   case strategy of
     AllowAll -> pure ()
     AllowList (AllowedDomains domains) ->
-      unless (targetDomain `elem` domains) $
-        throw (FederationDenied targetDomain)
+      -- First domain check. If the domain is in this list then we
+      -- can exit early without having to do a bunch of IO
+      unless (targetDomain `elem` domains) $ do
+        -- If the domain is unrecognised, refresh from cassandra, update the TVar, and check again for a final response.
+        -- Shadowing the strategy value above would be nice, but there is no pragma to tell GHC that it is
+        -- ok in this instance. So we just need to be careful to not accidently reuse it.
+        -- TODO: Remove this undefined.
+        strat <- undefined
+        let updateDomains :: Env -> Env
+            updateDomains = over Federator.Env.runSettings (\rs -> rs { federationStrategy = strat })
+        liftIO $ atomically $ modifyTVar tvar $ updateDomains
+        -- Rerun the federation check.
+        ensureCanFederateWith' strat targetDomain
 
 decodeCertificate ::
   Member (Error String) r =>
@@ -142,15 +170,17 @@ validateDomain ::
   ( Member (Input RunSettings) r,
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
-    Member DiscoverFederator r
+    Member DiscoverFederator r,
+    Member (Embed IO) r
   ) =>
+  TVar Env ->
   Maybe ByteString ->
   ByteString ->
   Sem r Domain
-validateDomain Nothing _ = throw NoClientCertificate
-validateDomain (Just encodedCertificate) unparsedDomain = do
+validateDomain _ Nothing _ = throw NoClientCertificate
+validateDomain tvar (Just encodedCertificate) unparsedDomain = do
   targetDomain <- parseDomain unparsedDomain
-  ensureCanFederateWith targetDomain
+  ensureCanFederateWith tvar targetDomain
 
   -- run discovery to find the hostname of the client federator
   certificate <-

From cac0a7a9518709867eeb17fd70f85923f4abd21a Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 19 Apr 2023 19:35:38 +1000
Subject: [PATCH 002/220] FS-1115: Simplifing TVar usage

---
 .../federator/src/Federator/ExternalServer.hs |  5 +-
 .../federator/src/Federator/InternalServer.hs |  5 +-
 services/federator/src/Federator/Response.hs  |  9 ++--
 services/federator/src/Federator/Run.hs       |  1 +
 .../federator/src/Federator/Validation.hs     | 46 ++++---------------
 5 files changed, 19 insertions(+), 47 deletions(-)

diff --git a/services/federator/src/Federator/ExternalServer.hs b/services/federator/src/Federator/ExternalServer.hs
index 51dd2f8c7a..a3244c94d3 100644
--- a/services/federator/src/Federator/ExternalServer.hs
+++ b/services/federator/src/Federator/ExternalServer.hs
@@ -53,10 +53,9 @@ callInward ::
     Member (Error ServerError) r,
     Member (Input RunSettings) r
   ) =>
-  TVar Env ->
   Wai.Request ->
   Sem r Wai.Response
-callInward tvar wreq = do
+callInward wreq = do
   req <- parseRequestData wreq
   Log.debug $
     Log.msg ("Inward Request" :: ByteString)
@@ -64,7 +63,7 @@ callInward tvar wreq = do
       . Log.field "component" (show (rdComponent req))
       . Log.field "rpc" (rdRPC req)
 
-  validatedDomain <- validateDomain tvar (rdCertificate req) (rdOriginDomain req)
+  validatedDomain <- validateDomain (rdCertificate req) (rdOriginDomain req)
 
   let path = LBS.toStrict (toLazyByteString (HTTP.encodePathSegments ["federation", rdRPC req]))
 
diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs
index 0ef26ac08c..66b953c730 100644
--- a/services/federator/src/Federator/InternalServer.hs
+++ b/services/federator/src/Federator/InternalServer.hs
@@ -84,15 +84,14 @@ callOutward ::
     Member (Error ServerError) r,
     Member (Input RunSettings) r
   ) =>
-  TVar Env ->
   Wai.Request ->
   Sem r Wai.Response
-callOutward tvar req = do
+callOutward req = do
   rd <- parseRequestData req
   domain <- parseDomainText (rdTargetDomain rd)
   -- This call will check for new domains
   -- when it encounters something that it doesn't know about.
-  ensureCanFederateWith tvar domain
+  ensureCanFederateWith domain
   resp <-
     discoverAndCall
       domain
diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index 19b9fe99e9..a5540fcd67 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -99,7 +99,7 @@ runWaiError =
       throw e
 
 serve ::
-  (TVar Env -> Wai.Request -> Sem AllEffects Wai.Response) ->
+  (Wai.Request -> Sem AllEffects Wai.Response) ->
   TVar Env ->
   Int ->
   IO ()
@@ -111,7 +111,7 @@ serve action tvar port = do
   where
     app :: Wai.Application
     app req respond =
-      runCodensity (runFederator tvar (action tvar req)) respond
+      runCodensity (runFederator tvar (action req)) respond
 
 type AllEffects =
   '[ Remote,
@@ -147,7 +147,10 @@ runFederator tvar resp = do
     . runInputConst env
     . runInputSem (embed @IO (readIORef (view sslContext env)))
     -- This is the point at which federation settings are extracted
-    . runInputConst (view runSettings env)
+    -- For each request, extract a fresh copy of the runSettings. This allows us
+    -- to independently update the settings and have them be used as requests
+    -- come in.
+    . runInputSem (embed @IO $ fmap (view runSettings) . liftIO $ readTVarIO tvar)
     . interpretServiceHTTP
     . runDNSLookupWithResolver (view dnsResolver env)
     . runFederatorDiscovery
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 07f4bcf2e2..e684700bd3 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -64,6 +64,7 @@ run opts = do
     bracket (newEnv opts res) closeEnv $ \env -> do
       -- Build a new TVar holding the state we want for the initial environment.
       tEnv <- newTVarIO env
+      -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
       let externalServer = serveInward tEnv portExternal
           internalServer = serveOutward tEnv portInternal
       withMonitor (env ^. applog) (env ^. sslContext) (optSettings opts) $ do
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index 357cbdc987..2dcf0c4742 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -25,7 +25,6 @@ module Federator.Validation
   )
 where
 
-import Control.Lens (over)
 import qualified Data.ByteString.Char8 as B8
 import Data.ByteString.Conversion
 import Data.Domain
@@ -47,7 +46,6 @@ import Polysemy
 import Polysemy.Error
 import Polysemy.Input
 import Wire.Network.DNS.SRV (SrvTarget (..))
-import Federator.Env (Env, runSettings)
 
 data ValidationError
   = NoClientCertificate
@@ -91,47 +89,21 @@ validationErrorStatus :: ValidationError -> HTTP.Status
 validationErrorStatus (FederationDenied _) = HTTP.status400
 validationErrorStatus _ = HTTP.status403
 
-
-ensureCanFederateWith'
-  :: Member (Error ValidationError) r =>
-  FederationStrategy ->
-  Domain ->
-  Sem r ()
-ensureCanFederateWith' strategy targetDomain =
-  case strategy of
-    AllowAll -> pure ()
-    AllowList (AllowedDomains domains) ->
-      unless (targetDomain `elem` domains) $
-        throw (FederationDenied targetDomain)
-
 -- | Validates an already-parsed domain against the allowList using the federator
 -- startup configuration, and can update the allowList from the DB at runtime.
 ensureCanFederateWith ::
   ( Member (Input RunSettings) r,
-    Member (Error ValidationError) r,
-    Member (Embed IO) r
+    Member (Error ValidationError) r
   ) =>
-  TVar Env ->
   Domain ->
   Sem r ()
-ensureCanFederateWith tvar targetDomain = do
+ensureCanFederateWith targetDomain = do
   strategy <- inputs federationStrategy
   case strategy of
     AllowAll -> pure ()
     AllowList (AllowedDomains domains) ->
-      -- First domain check. If the domain is in this list then we
-      -- can exit early without having to do a bunch of IO
-      unless (targetDomain `elem` domains) $ do
-        -- If the domain is unrecognised, refresh from cassandra, update the TVar, and check again for a final response.
-        -- Shadowing the strategy value above would be nice, but there is no pragma to tell GHC that it is
-        -- ok in this instance. So we just need to be careful to not accidently reuse it.
-        -- TODO: Remove this undefined.
-        strat <- undefined
-        let updateDomains :: Env -> Env
-            updateDomains = over Federator.Env.runSettings (\rs -> rs { federationStrategy = strat })
-        liftIO $ atomically $ modifyTVar tvar $ updateDomains
-        -- Rerun the federation check.
-        ensureCanFederateWith' strat targetDomain
+      unless (targetDomain `elem` domains) $
+        throw (FederationDenied targetDomain)
 
 decodeCertificate ::
   Member (Error String) r =>
@@ -170,17 +142,15 @@ validateDomain ::
   ( Member (Input RunSettings) r,
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
-    Member DiscoverFederator r,
-    Member (Embed IO) r
+    Member DiscoverFederator r
   ) =>
-  TVar Env ->
   Maybe ByteString ->
   ByteString ->
   Sem r Domain
-validateDomain _ Nothing _ = throw NoClientCertificate
-validateDomain tvar (Just encodedCertificate) unparsedDomain = do
+validateDomain Nothing _ = throw NoClientCertificate
+validateDomain (Just encodedCertificate) unparsedDomain = do
   targetDomain <- parseDomain unparsedDomain
-  ensureCanFederateWith tvar targetDomain
+  ensureCanFederateWith targetDomain
 
   -- run discovery to find the hostname of the client federator
   certificate <-

From e63bcfe55cfde56d3a940fe618af079f480b7023 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 19 Apr 2023 16:00:01 +0200
Subject: [PATCH 003/220] Cassandra schema update.

---
 services/brig/brig.cabal                      |  1 +
 services/brig/schema/src/Main.hs              |  4 +-
 .../brig/schema/src/V76_FederationRemotes.hs  | 37 +++++++++++++++++++
 services/brig/src/Brig/App.hs                 |  2 +-
 4 files changed, 42 insertions(+), 2 deletions(-)
 create mode 100644 services/brig/schema/src/V76_FederationRemotes.hs

diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 00a464df5c..23f55ee609 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -668,6 +668,7 @@ executable brig-schema
     V73_ReplaceNonceTable
     V74_AddOAuthTables
     V75_AddOAuthCodeChallenge
+    V76_FederationRemotes
     V_FUTUREWORK
 
   hs-source-dirs:     schema/src
diff --git a/services/brig/schema/src/Main.hs b/services/brig/schema/src/Main.hs
index f1f35ccd37..db41e48eab 100644
--- a/services/brig/schema/src/Main.hs
+++ b/services/brig/schema/src/Main.hs
@@ -55,6 +55,7 @@ import qualified V72_AddNonceTable
 import qualified V73_ReplaceNonceTable
 import qualified V74_AddOAuthTables
 import qualified V75_AddOAuthCodeChallenge
+import qualified V76_FederationRemotes
 
 main :: IO ()
 main = do
@@ -97,7 +98,8 @@ main = do
       V72_AddNonceTable.migration,
       V73_ReplaceNonceTable.migration,
       V74_AddOAuthTables.migration,
-      V75_AddOAuthCodeChallenge.migration
+      V75_AddOAuthCodeChallenge.migration,
+      V76_FederationRemotes.migration
       -- When adding migrations here, don't forget to update
       -- 'schemaVersion' in Brig.App
 
diff --git a/services/brig/schema/src/V76_FederationRemotes.hs b/services/brig/schema/src/V76_FederationRemotes.hs
new file mode 100644
index 0000000000..b1cc1db35c
--- /dev/null
+++ b/services/brig/schema/src/V76_FederationRemotes.hs
@@ -0,0 +1,37 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module V76_FederationRemotes
+  ( migration,
+  )
+where
+
+import Cassandra.Schema
+import Imports
+import Text.RawString.QQ
+
+migration :: Migration
+migration =
+  Migration 76 "Table for keeping track of instances we federate with" $
+    schema'
+      [r| CREATE TABLE federation_remotes (
+            domain text PRIMARY KEY,
+            search_policy int
+          )
+     |]
diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs
index 584eb4deaf..00fa0ea438 100644
--- a/services/brig/src/Brig/App.hs
+++ b/services/brig/src/Brig/App.hs
@@ -152,7 +152,7 @@ import Wire.API.User.Identity (Email)
 import Wire.API.User.Profile (Locale)
 
 schemaVersion :: Int32
-schemaVersion = 75
+schemaVersion = 76
 
 -------------------------------------------------------------------------------
 -- Environment

From 2bf18a5fb752b8fef3921d10ac8302618bd49e13 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 19 Apr 2023 16:32:06 +0200
Subject: [PATCH 004/220] Internal routes for CRUDding the new table.

---
 .../src/Wire/API/Routes/Internal/Brig.hs      | 26 +++++++++++++++++++
 libs/wire-api/wire-api.cabal                  |  1 +
 services/brig/src/Brig/API/Internal.hs        |  7 +++++
 services/brig/src/Brig/API/Util.hs            |  8 +++---
 services/brig/src/Brig/Options.hs             | 19 +++-----------
 5 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index e6de219115..2445cec75f 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -23,6 +23,7 @@ module Wire.API.Routes.Internal.Brig
     TeamsAPI,
     UserAPI,
     AuthAPI,
+    FederationRemotesAPI,
     EJPDRequest,
     GetAccountConferenceCallingConfig,
     PutAccountConferenceCallingConfig,
@@ -38,6 +39,7 @@ where
 import Control.Lens ((.~))
 import Data.Aeson (FromJSON, ToJSON)
 import qualified Data.Code as Code
+import Data.Domain (Domain)
 import Data.Id as Id
 import Data.Qualified (Qualified)
 import Data.Schema hiding (swaggerDoc)
@@ -53,6 +55,7 @@ import Wire.API.Error.Brig
 import Wire.API.MLS.Credential
 import Wire.API.MLS.KeyPackage
 import Wire.API.MakesFederatedCall
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.Internal.Brig.Connection
 import Wire.API.Routes.Internal.Brig.EJPD
 import Wire.API.Routes.Internal.Brig.OAuth (OAuthAPI)
@@ -327,6 +330,7 @@ type API =
            :<|> UserAPI
            :<|> AuthAPI
            :<|> OAuthAPI
+           :<|> FederationRemotesAPI
        )
 
 type TeamsAPI =
@@ -397,6 +401,28 @@ type AuthAPI =
                :> MultiVerb1 'GET '[JSON] (RespondEmpty 200 "OK")
            )
 
+-- | This is located in brig, not in federator, because brig has a cassandra instance.  This
+-- is not ideal, but since all services have a local in-ram copy of this table and keep track
+-- of changes via rabbitmq, we argue it's "fine" for federators to ask brig once on startup.
+type FederationRemotesAPI =
+  Named
+    "get-federator-remotes"
+    ( "federator-remotes"
+        :> Get '[JSON] [FederationDomainConfig]
+    )
+    :<|> Named
+           "add-federator-remote"
+           ( "federator-remotes"
+               :> ReqBody '[JSON] FederationDomainConfig
+               :> Post '[JSON] ()
+           )
+    :<|> Named
+           "delete-federator-remote"
+           ( "federator-remotes"
+               :> Capture "domain" Domain
+               :> Delete '[JSON] ()
+           )
+
 swaggerDoc :: Swagger
 swaggerDoc =
   toSwagger (Proxy @API)
diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal
index 5e7afe420f..cc6e76f5e9 100644
--- a/libs/wire-api/wire-api.cabal
+++ b/libs/wire-api/wire-api.cabal
@@ -79,6 +79,7 @@ library
     Wire.API.Routes.ClientAlgebra
     Wire.API.Routes.Cookies
     Wire.API.Routes.CSV
+    Wire.API.Routes.FederationDomainConfig
     Wire.API.Routes.Internal.Brig
     Wire.API.Routes.Internal.Brig.Connection
     Wire.API.Routes.Internal.Brig.EJPD
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index f08c9cd651..5aeb32016c 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -119,6 +119,7 @@ servantSitemap =
     :<|> userAPI
     :<|> authAPI
     :<|> internalOauthAPI
+    :<|> federationRemotesAPI
 
 ejpdAPI ::
   Member GalleyProvider r =>
@@ -171,6 +172,12 @@ authAPI =
     :<|> Named @"login-code" getLoginCode
     :<|> Named @"reauthenticate" reauthenticate
 
+federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
+federationRemotesAPI =
+  Named @"get-federator-remotes" undefined
+    :<|> Named @"add-federator-remote" undefined
+    :<|> Named @"delete-federator-remote" undefined
+
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)
 getAccountConferenceCallingConfig uid =
diff --git a/services/brig/src/Brig/API/Util.hs b/services/brig/src/Brig/API/Util.hs
index 46f286d6a1..cfa7a4f700 100644
--- a/services/brig/src/Brig/API/Util.hs
+++ b/services/brig/src/Brig/API/Util.hs
@@ -40,8 +40,7 @@ import Brig.API.Types
 import Brig.App
 import qualified Brig.Code as Code
 import qualified Brig.Data.User as Data
-import Brig.Options (FederationDomainConfig, federationDomainConfigs, set2FACodeGenerationDelaySecs)
-import qualified Brig.Options as Opts
+import Brig.Options (federationDomainConfigs, set2FACodeGenerationDelaySecs)
 import Brig.Types.Intra (accountUser)
 import Control.Lens (view)
 import Control.Monad.Catch (throwM)
@@ -65,6 +64,7 @@ import Util.Logging (sha256String)
 import Wire.API.Error
 import Wire.API.Error.Brig
 import Wire.API.Federation.Error
+import Wire.API.Routes.FederationDomainConfig as FederationDomainConfig
 import Wire.API.User
 import Wire.API.User.Search (FederatedUserSearchPolicy (NoSearch))
 import qualified Wire.Sem.Concurrency as C
@@ -171,11 +171,11 @@ exceptTToMaybe = (pure . either Just (const Nothing)) <=< runExceptT
 lookupDomainConfig :: MonadReader Env m => Domain -> m (Maybe FederationDomainConfig)
 lookupDomainConfig domain = do
   domainConfigs <- fromMaybe [] <$> view (settings . federationDomainConfigs)
-  pure $ find ((== domain) . Opts.domain) domainConfigs
+  pure $ find ((== domain) . FederationDomainConfig.domain) domainConfigs
 
 -- | If domain is not configured fall back to `FullSearch`
 lookupSearchPolicy :: MonadReader Env m => Domain -> m FederatedUserSearchPolicy
-lookupSearchPolicy domain = fromMaybe NoSearch <$> (Opts.cfgSearchPolicy <$$> lookupDomainConfig domain)
+lookupSearchPolicy domain = fromMaybe NoSearch <$> (FederationDomainConfig.cfgSearchPolicy <$$> lookupDomainConfig domain)
 
 -- | Convert a qualified value into a local one. Throw if the value is not actually local.
 ensureLocal :: Qualified a -> AppT r (Local a)
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index 66904d3ce0..89b49b8dfa 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -54,10 +54,10 @@ import Imports
 import qualified Network.DNS as DNS
 import System.Logger.Extended (Level, LogFormat)
 import Util.Options
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.Version
 import qualified Wire.API.Team.Feature as Public
 import Wire.API.User
-import Wire.API.User.Search (FederatedUserSearchPolicy)
 import Wire.Arbitrary (Arbitrary, arbitrary)
 
 newtype Timeout = Timeout
@@ -400,20 +400,6 @@ instance ToSchema ListAllSFTServers where
           element "disabled" HideAllSFTServers
         ]
 
-data FederationDomainConfig = FederationDomainConfig
-  { domain :: Domain,
-    cfgSearchPolicy :: FederatedUserSearchPolicy
-  }
-  deriving (Show, Generic)
-  deriving (ToJSON, FromJSON) via Schema FederationDomainConfig
-
-instance ToSchema FederationDomainConfig where
-  schema =
-    object "FederationDomainConfig" $
-      FederationDomainConfig
-        <$> domain .= field "domain" schema
-        <*> cfgSearchPolicy .= field "search_policy" schema
-
 -- | Options that are consumed on startup
 data Opts = Opts
   -- services
@@ -562,7 +548,8 @@ data Settings = Settings
     --     - wire.com
     --     - example.com
     setFederationDomain :: !Domain,
-    setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]),
+    setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]), -- TODO: deprecate this in docs and config file samples.
+
     -- | The amount of time in milliseconds to wait after reading from an SQS queue
     -- returns no message, before asking for messages from SQS again.
     -- defaults to 'defSqsThrottleMillis'.

From 01c32e4bbb9cf1c7479f4a53b0a8a0e2aa1adc5c Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 19 Apr 2023 16:00:01 +0200
Subject: [PATCH 005/220] Cassandra schema update.

---
 services/brig/brig.cabal                      |  1 +
 services/brig/schema/src/Main.hs              |  4 +-
 .../brig/schema/src/V76_FederationRemotes.hs  | 37 +++++++++++++++++++
 services/brig/src/Brig/App.hs                 |  2 +-
 4 files changed, 42 insertions(+), 2 deletions(-)
 create mode 100644 services/brig/schema/src/V76_FederationRemotes.hs

diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 00a464df5c..23f55ee609 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -668,6 +668,7 @@ executable brig-schema
     V73_ReplaceNonceTable
     V74_AddOAuthTables
     V75_AddOAuthCodeChallenge
+    V76_FederationRemotes
     V_FUTUREWORK
 
   hs-source-dirs:     schema/src
diff --git a/services/brig/schema/src/Main.hs b/services/brig/schema/src/Main.hs
index f1f35ccd37..db41e48eab 100644
--- a/services/brig/schema/src/Main.hs
+++ b/services/brig/schema/src/Main.hs
@@ -55,6 +55,7 @@ import qualified V72_AddNonceTable
 import qualified V73_ReplaceNonceTable
 import qualified V74_AddOAuthTables
 import qualified V75_AddOAuthCodeChallenge
+import qualified V76_FederationRemotes
 
 main :: IO ()
 main = do
@@ -97,7 +98,8 @@ main = do
       V72_AddNonceTable.migration,
       V73_ReplaceNonceTable.migration,
       V74_AddOAuthTables.migration,
-      V75_AddOAuthCodeChallenge.migration
+      V75_AddOAuthCodeChallenge.migration,
+      V76_FederationRemotes.migration
       -- When adding migrations here, don't forget to update
       -- 'schemaVersion' in Brig.App
 
diff --git a/services/brig/schema/src/V76_FederationRemotes.hs b/services/brig/schema/src/V76_FederationRemotes.hs
new file mode 100644
index 0000000000..b1cc1db35c
--- /dev/null
+++ b/services/brig/schema/src/V76_FederationRemotes.hs
@@ -0,0 +1,37 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module V76_FederationRemotes
+  ( migration,
+  )
+where
+
+import Cassandra.Schema
+import Imports
+import Text.RawString.QQ
+
+migration :: Migration
+migration =
+  Migration 76 "Table for keeping track of instances we federate with" $
+    schema'
+      [r| CREATE TABLE federation_remotes (
+            domain text PRIMARY KEY,
+            search_policy int
+          )
+     |]
diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs
index 584eb4deaf..00fa0ea438 100644
--- a/services/brig/src/Brig/App.hs
+++ b/services/brig/src/Brig/App.hs
@@ -152,7 +152,7 @@ import Wire.API.User.Identity (Email)
 import Wire.API.User.Profile (Locale)
 
 schemaVersion :: Int32
-schemaVersion = 75
+schemaVersion = 76
 
 -------------------------------------------------------------------------------
 -- Environment

From 2bd4c6c5de37684a71864e9f761196bf229b5cec Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 19 Apr 2023 16:32:06 +0200
Subject: [PATCH 006/220] Internal routes for CRUDding the new table.

---
 .../src/Wire/API/Routes/Internal/Brig.hs      | 26 +++++++++++++++++++
 libs/wire-api/wire-api.cabal                  |  1 +
 services/brig/src/Brig/API/Internal.hs        |  7 +++++
 services/brig/src/Brig/API/Util.hs            |  8 +++---
 services/brig/src/Brig/Options.hs             | 19 +++-----------
 5 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index e6de219115..2445cec75f 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -23,6 +23,7 @@ module Wire.API.Routes.Internal.Brig
     TeamsAPI,
     UserAPI,
     AuthAPI,
+    FederationRemotesAPI,
     EJPDRequest,
     GetAccountConferenceCallingConfig,
     PutAccountConferenceCallingConfig,
@@ -38,6 +39,7 @@ where
 import Control.Lens ((.~))
 import Data.Aeson (FromJSON, ToJSON)
 import qualified Data.Code as Code
+import Data.Domain (Domain)
 import Data.Id as Id
 import Data.Qualified (Qualified)
 import Data.Schema hiding (swaggerDoc)
@@ -53,6 +55,7 @@ import Wire.API.Error.Brig
 import Wire.API.MLS.Credential
 import Wire.API.MLS.KeyPackage
 import Wire.API.MakesFederatedCall
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.Internal.Brig.Connection
 import Wire.API.Routes.Internal.Brig.EJPD
 import Wire.API.Routes.Internal.Brig.OAuth (OAuthAPI)
@@ -327,6 +330,7 @@ type API =
            :<|> UserAPI
            :<|> AuthAPI
            :<|> OAuthAPI
+           :<|> FederationRemotesAPI
        )
 
 type TeamsAPI =
@@ -397,6 +401,28 @@ type AuthAPI =
                :> MultiVerb1 'GET '[JSON] (RespondEmpty 200 "OK")
            )
 
+-- | This is located in brig, not in federator, because brig has a cassandra instance.  This
+-- is not ideal, but since all services have a local in-ram copy of this table and keep track
+-- of changes via rabbitmq, we argue it's "fine" for federators to ask brig once on startup.
+type FederationRemotesAPI =
+  Named
+    "get-federator-remotes"
+    ( "federator-remotes"
+        :> Get '[JSON] [FederationDomainConfig]
+    )
+    :<|> Named
+           "add-federator-remote"
+           ( "federator-remotes"
+               :> ReqBody '[JSON] FederationDomainConfig
+               :> Post '[JSON] ()
+           )
+    :<|> Named
+           "delete-federator-remote"
+           ( "federator-remotes"
+               :> Capture "domain" Domain
+               :> Delete '[JSON] ()
+           )
+
 swaggerDoc :: Swagger
 swaggerDoc =
   toSwagger (Proxy @API)
diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal
index 5e7afe420f..cc6e76f5e9 100644
--- a/libs/wire-api/wire-api.cabal
+++ b/libs/wire-api/wire-api.cabal
@@ -79,6 +79,7 @@ library
     Wire.API.Routes.ClientAlgebra
     Wire.API.Routes.Cookies
     Wire.API.Routes.CSV
+    Wire.API.Routes.FederationDomainConfig
     Wire.API.Routes.Internal.Brig
     Wire.API.Routes.Internal.Brig.Connection
     Wire.API.Routes.Internal.Brig.EJPD
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index f08c9cd651..5aeb32016c 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -119,6 +119,7 @@ servantSitemap =
     :<|> userAPI
     :<|> authAPI
     :<|> internalOauthAPI
+    :<|> federationRemotesAPI
 
 ejpdAPI ::
   Member GalleyProvider r =>
@@ -171,6 +172,12 @@ authAPI =
     :<|> Named @"login-code" getLoginCode
     :<|> Named @"reauthenticate" reauthenticate
 
+federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
+federationRemotesAPI =
+  Named @"get-federator-remotes" undefined
+    :<|> Named @"add-federator-remote" undefined
+    :<|> Named @"delete-federator-remote" undefined
+
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)
 getAccountConferenceCallingConfig uid =
diff --git a/services/brig/src/Brig/API/Util.hs b/services/brig/src/Brig/API/Util.hs
index 46f286d6a1..cfa7a4f700 100644
--- a/services/brig/src/Brig/API/Util.hs
+++ b/services/brig/src/Brig/API/Util.hs
@@ -40,8 +40,7 @@ import Brig.API.Types
 import Brig.App
 import qualified Brig.Code as Code
 import qualified Brig.Data.User as Data
-import Brig.Options (FederationDomainConfig, federationDomainConfigs, set2FACodeGenerationDelaySecs)
-import qualified Brig.Options as Opts
+import Brig.Options (federationDomainConfigs, set2FACodeGenerationDelaySecs)
 import Brig.Types.Intra (accountUser)
 import Control.Lens (view)
 import Control.Monad.Catch (throwM)
@@ -65,6 +64,7 @@ import Util.Logging (sha256String)
 import Wire.API.Error
 import Wire.API.Error.Brig
 import Wire.API.Federation.Error
+import Wire.API.Routes.FederationDomainConfig as FederationDomainConfig
 import Wire.API.User
 import Wire.API.User.Search (FederatedUserSearchPolicy (NoSearch))
 import qualified Wire.Sem.Concurrency as C
@@ -171,11 +171,11 @@ exceptTToMaybe = (pure . either Just (const Nothing)) <=< runExceptT
 lookupDomainConfig :: MonadReader Env m => Domain -> m (Maybe FederationDomainConfig)
 lookupDomainConfig domain = do
   domainConfigs <- fromMaybe [] <$> view (settings . federationDomainConfigs)
-  pure $ find ((== domain) . Opts.domain) domainConfigs
+  pure $ find ((== domain) . FederationDomainConfig.domain) domainConfigs
 
 -- | If domain is not configured fall back to `FullSearch`
 lookupSearchPolicy :: MonadReader Env m => Domain -> m FederatedUserSearchPolicy
-lookupSearchPolicy domain = fromMaybe NoSearch <$> (Opts.cfgSearchPolicy <$$> lookupDomainConfig domain)
+lookupSearchPolicy domain = fromMaybe NoSearch <$> (FederationDomainConfig.cfgSearchPolicy <$$> lookupDomainConfig domain)
 
 -- | Convert a qualified value into a local one. Throw if the value is not actually local.
 ensureLocal :: Qualified a -> AppT r (Local a)
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index 66904d3ce0..89b49b8dfa 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -54,10 +54,10 @@ import Imports
 import qualified Network.DNS as DNS
 import System.Logger.Extended (Level, LogFormat)
 import Util.Options
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.Version
 import qualified Wire.API.Team.Feature as Public
 import Wire.API.User
-import Wire.API.User.Search (FederatedUserSearchPolicy)
 import Wire.Arbitrary (Arbitrary, arbitrary)
 
 newtype Timeout = Timeout
@@ -400,20 +400,6 @@ instance ToSchema ListAllSFTServers where
           element "disabled" HideAllSFTServers
         ]
 
-data FederationDomainConfig = FederationDomainConfig
-  { domain :: Domain,
-    cfgSearchPolicy :: FederatedUserSearchPolicy
-  }
-  deriving (Show, Generic)
-  deriving (ToJSON, FromJSON) via Schema FederationDomainConfig
-
-instance ToSchema FederationDomainConfig where
-  schema =
-    object "FederationDomainConfig" $
-      FederationDomainConfig
-        <$> domain .= field "domain" schema
-        <*> cfgSearchPolicy .= field "search_policy" schema
-
 -- | Options that are consumed on startup
 data Opts = Opts
   -- services
@@ -562,7 +548,8 @@ data Settings = Settings
     --     - wire.com
     --     - example.com
     setFederationDomain :: !Domain,
-    setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]),
+    setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]), -- TODO: deprecate this in docs and config file samples.
+
     -- | The amount of time in milliseconds to wait after reading from an SQS queue
     -- returns no message, before asking for messages from SQS again.
     -- defaults to 'defSqsThrottleMillis'.

From 6bb1d270c3797382c37deeb8a94e3e6907550631 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 20 Apr 2023 09:39:44 +0200
Subject: [PATCH 007/220] Fixup

---
 .../Wire/API/Routes/FederationDomainConfig.hs | 61 +++++++++++++++++++
 .../src/Wire/API/Routes/Internal/Brig.hs      | 17 +++---
 libs/wire-api/src/Wire/API/User/Search.hs     |  2 +-
 services/brig/src/Brig/API/Internal.hs        |  6 +-
 services/brig/src/Brig/API/Util.hs            |  6 +-
 .../brig/test/integration/API/Federation.hs   | 15 ++---
 6 files changed, 86 insertions(+), 21 deletions(-)
 create mode 100644 libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs

diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
new file mode 100644
index 0000000000..1149843856
--- /dev/null
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -0,0 +1,61 @@
+{- LANGUAGE DeriveAnyClass #-}
+{- LANGUAGE GeneralizedNewtypeDeriving #-}
+{- LANGUAGE TemplateHaskell #-}
+-- Disabling to stop errors on Getters
+{- OPTIONS_GHC -Wno-redundant-constraints #-}
+
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Wire.API.Routes.FederationDomainConfig
+  ( FederationDomainConfig (..),
+    FederationDomainConfigs (..),
+  )
+where
+
+import Data.Aeson (FromJSON, ToJSON)
+import Data.Domain (Domain)
+import Data.Schema
+import qualified Data.Swagger as S
+import GHC.Generics
+import Imports
+import Wire.API.User.Search (FederatedUserSearchPolicy)
+
+data FederationDomainConfig = FederationDomainConfig
+  { domain :: Domain,
+    cfgSearchPolicy :: FederatedUserSearchPolicy
+  }
+  deriving (Eq, Ord, Show, Generic)
+  deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationDomainConfig
+
+instance ToSchema FederationDomainConfig where
+  schema =
+    object "FederationDomainConfig" $
+      FederationDomainConfig
+        <$> domain .= field "domain" schema
+        <*> cfgSearchPolicy .= field "search_policy" schema
+
+newtype FederationDomainConfigs = FederationDomainConfigs
+  {fromFederationDomainConfigs :: [FederationDomainConfig]}
+  deriving (Show, Generic)
+  deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationDomainConfigs
+
+instance ToSchema FederationDomainConfigs where
+  schema =
+    object "FederationDomainConfigs" $
+      FederationDomainConfigs
+        <$> fromFederationDomainConfigs .= field "remotes" (array schema)
diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index 2445cec75f..3e9330a377 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -406,19 +406,22 @@ type AuthAPI =
 -- of changes via rabbitmq, we argue it's "fine" for federators to ask brig once on startup.
 type FederationRemotesAPI =
   Named
-    "get-federator-remotes"
-    ( "federator-remotes"
-        :> Get '[JSON] [FederationDomainConfig]
+    "get-federation-remotes"
+    ( "federation"
+        :> "remotes"
+        :> Get '[JSON] FederationDomainConfigs
     )
     :<|> Named
-           "add-federator-remote"
-           ( "federator-remotes"
+           "add-federation-remotes"
+           ( "federation"
+               :> "remotes"
                :> ReqBody '[JSON] FederationDomainConfig
                :> Post '[JSON] ()
            )
     :<|> Named
-           "delete-federator-remote"
-           ( "federator-remotes"
+           "delete-federation-remotes"
+           ( "federation"
+               :> "remotes"
                :> Capture "domain" Domain
                :> Delete '[JSON] ()
            )
diff --git a/libs/wire-api/src/Wire/API/User/Search.hs b/libs/wire-api/src/Wire/API/User/Search.hs
index 90cddf0cea..7398625dbb 100644
--- a/libs/wire-api/src/Wire/API/User/Search.hs
+++ b/libs/wire-api/src/Wire/API/User/Search.hs
@@ -307,7 +307,7 @@ data FederatedUserSearchPolicy
   = NoSearch
   | ExactHandleSearch
   | FullSearch
-  deriving (Show, Eq, Generic, Enum, Bounded)
+  deriving (Show, Eq, Ord, Generic, Enum, Bounded)
   deriving (Arbitrary) via (GenericUniform FederatedUserSearchPolicy)
   deriving (ToJSON, FromJSON) via (Schema FederatedUserSearchPolicy)
 
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 5aeb32016c..0cab9eca1a 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -174,9 +174,9 @@ authAPI =
 
 federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
 federationRemotesAPI =
-  Named @"get-federator-remotes" undefined
-    :<|> Named @"add-federator-remote" undefined
-    :<|> Named @"delete-federator-remote" undefined
+  Named @"get-federation-remotes" undefined
+    :<|> Named @"add-federation-remotes" undefined
+    :<|> Named @"delete-federation-remotes" undefined
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)
diff --git a/services/brig/src/Brig/API/Util.hs b/services/brig/src/Brig/API/Util.hs
index cfa7a4f700..7fe181c157 100644
--- a/services/brig/src/Brig/API/Util.hs
+++ b/services/brig/src/Brig/API/Util.hs
@@ -64,7 +64,7 @@ import Util.Logging (sha256String)
 import Wire.API.Error
 import Wire.API.Error.Brig
 import Wire.API.Federation.Error
-import Wire.API.Routes.FederationDomainConfig as FederationDomainConfig
+import Wire.API.Routes.FederationDomainConfig as FD
 import Wire.API.User
 import Wire.API.User.Search (FederatedUserSearchPolicy (NoSearch))
 import qualified Wire.Sem.Concurrency as C
@@ -171,11 +171,11 @@ exceptTToMaybe = (pure . either Just (const Nothing)) <=< runExceptT
 lookupDomainConfig :: MonadReader Env m => Domain -> m (Maybe FederationDomainConfig)
 lookupDomainConfig domain = do
   domainConfigs <- fromMaybe [] <$> view (settings . federationDomainConfigs)
-  pure $ find ((== domain) . FederationDomainConfig.domain) domainConfigs
+  pure $ find ((== domain) . FD.domain) domainConfigs
 
 -- | If domain is not configured fall back to `FullSearch`
 lookupSearchPolicy :: MonadReader Env m => Domain -> m FederatedUserSearchPolicy
-lookupSearchPolicy domain = fromMaybe NoSearch <$> (FederationDomainConfig.cfgSearchPolicy <$$> lookupDomainConfig domain)
+lookupSearchPolicy domain = fromMaybe NoSearch <$> (FD.cfgSearchPolicy <$$> lookupDomainConfig domain)
 
 -- | Convert a qualified value into a local one. Throw if the value is not actually local.
 ensureLocal :: Qualified a -> AppT r (Local a)
diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index af65db356b..27f01e2bf0 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -56,6 +56,7 @@ import Wire.API.Federation.Component
 import Wire.API.Federation.Version
 import Wire.API.MLS.Credential
 import Wire.API.MLS.KeyPackage
+import Wire.API.Routes.FederationDomainConfig as FD
 import Wire.API.User
 import Wire.API.User.Client
 import Wire.API.User.Client.Prekey
@@ -93,7 +94,7 @@ tests m opts brig cannon fedBrigClient =
 
 allowFullSearch :: Domain -> Opt.Opts -> Opt.Opts
 allowFullSearch domain opts =
-  opts & Opt.optionSettings . Opt.federationDomainConfigs ?~ [Opt.FederationDomainConfig domain FullSearch]
+  opts & Opt.optionSettings . Opt.federationDomainConfigs ?~ [FD.FederationDomainConfig domain FullSearch]
 
 testSearchSuccess :: Opt.Opts -> Brig -> Http ()
 testSearchSuccess opts brig = do
@@ -192,9 +193,9 @@ testSearchRestrictions opts brig = do
   let opts' =
         opts
           & Opt.optionSettings . Opt.federationDomainConfigs
-            ?~ [ Opt.FederationDomainConfig domainNoSearch NoSearch,
-                 Opt.FederationDomainConfig domainExactHandle ExactHandleSearch,
-                 Opt.FederationDomainConfig domainFullSearch FullSearch
+            ?~ [ FD.FederationDomainConfig domainNoSearch NoSearch,
+                 FD.FederationDomainConfig domainExactHandle ExactHandleSearch,
+                 FD.FederationDomainConfig domainFullSearch FullSearch
                ]
 
   let expectSearch :: HasCallStack => Domain -> Text -> [Qualified UserId] -> FederatedUserSearchPolicy -> WaiTest.Session ()
@@ -228,9 +229,9 @@ testGetUserByHandleRestrictions opts brig = do
   let opts' =
         opts
           & Opt.optionSettings . Opt.federationDomainConfigs
-            ?~ [ Opt.FederationDomainConfig domainNoSearch NoSearch,
-                 Opt.FederationDomainConfig domainExactHandle ExactHandleSearch,
-                 Opt.FederationDomainConfig domainFullSearch FullSearch
+            ?~ [ FD.FederationDomainConfig domainNoSearch NoSearch,
+                 FD.FederationDomainConfig domainExactHandle ExactHandleSearch,
+                 FD.FederationDomainConfig domainFullSearch FullSearch
                ]
 
   let expectSearch domain expectedUser = do

From 59dcbf0c789931c93528fbb5fec7abac3de1f7d4 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 20 Apr 2023 10:41:58 +0200
Subject: [PATCH 008/220] Integration tests.

---
 .../brig/test/integration/API/Federation.hs   | 29 ++++++++++++++++++-
 services/brig/test/integration/Util.hs        | 19 ++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index 27f01e2bf0..cece13f5d0 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -56,6 +56,7 @@ import Wire.API.Federation.Component
 import Wire.API.Federation.Version
 import Wire.API.MLS.Credential
 import Wire.API.MLS.KeyPackage
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.FederationDomainConfig as FD
 import Wire.API.User
 import Wire.API.User.Client
@@ -89,7 +90,8 @@ tests m opts brig cannon fedBrigClient =
         test m "POST /federation/on-user-deleted-connections : 200" (testRemoteUserGetsDeleted opts brig cannon fedBrigClient),
         test m "POST /federation/api-version : 200" (testAPIVersion brig fedBrigClient),
         test m "POST /federation/claim-key-packages : 200" (testClaimKeyPackages brig fedBrigClient),
-        test m "POST /federation/claim-key-packages (MLS disabled) : 200" (testClaimKeyPackagesMLSDisabled opts brig)
+        test m "POST /federation/claim-key-packages (MLS disabled) : 200" (testClaimKeyPackagesMLSDisabled opts brig),
+        test m "CRUD /i/federation/remotes" (crudFederationRemotes opts brig)
       ]
 
 allowFullSearch :: Domain -> Opt.Opts -> Opt.Opts
@@ -458,3 +460,28 @@ testClaimKeyPackagesMLSDisabled opts brig = do
           ClaimKeyPackageRequest (qUnqualified alice) (qUnqualified bob)
 
   liftIO $ mbundle @?= Nothing
+
+crudFederationRemotes :: HasCallStack => Opt.Opts -> Brig -> Http ()
+crudFederationRemotes _opts brig = do
+  resetFederationRemotes brig
+
+  res1 <- getFederationRemotes brig
+  liftIO $ assertEqual "should return nothing" [] res1
+
+  let remote1 = FederationDomainConfig (Domain "good.example.com") NoSearch
+  addFederationRemote brig remote1
+  res2 <- getFederationRemotes brig
+  liftIO $ assertEqual "should return good.example.com" [remote1] res2
+
+  let remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
+  addFederationRemote brig remote2
+  res3 <- getFederationRemotes brig
+  liftIO $ assertEqual "should return {good,evil}.example.com" (sort [remote1, remote2]) (sort res3)
+
+  deleteFederationRemote brig (domain remote1)
+  res4 <- getFederationRemotes brig
+  liftIO $ assertEqual "should return evil.example.com" (sort [remote2]) (sort res4)
+
+  -- TODO: how do we test that the TVar is updated in all services?  some fancy unit test?
+  -- duplicate internal end-point to all services, and implement the hanlers in a library?
+  pure ()
diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs
index dd8368dba2..51f525ef8a 100644
--- a/services/brig/test/integration/Util.hs
+++ b/services/brig/test/integration/Util.hs
@@ -111,6 +111,7 @@ import Wire.API.Conversation.Role (roleNameWireAdmin)
 import Wire.API.Federation.API
 import Wire.API.Federation.Domain
 import Wire.API.Internal.Notification
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.MultiTablePaging
 import Wire.API.Team.Member hiding (userId)
 import Wire.API.User
@@ -1074,6 +1075,24 @@ withDomainsBlockedForRegistration opts domains sess = do
       unsafeMkDomain = either error id . mkDomain
   withSettingsOverrides opts' sess
 
+getFederationRemotes :: Brig -> Http [FederationDomainConfig]
+getFederationRemotes brig =
+  fromFederationDomainConfigs . responseJsonUnsafe <$> do
+    get (brig . paths ["i", "federation", "remotes"] . contentJson . expect2xx)
+
+addFederationRemote :: Brig -> FederationDomainConfig -> Http ()
+addFederationRemote brig remote =
+  void $ post (brig . paths ["i", "federation", "remotes"] . contentJson . json remote . expect2xx)
+
+deleteFederationRemote :: Brig -> Domain -> Http ()
+deleteFederationRemote brig rdom =
+  void $ delete (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . expect2xx)
+
+resetFederationRemotes :: Brig -> Http ()
+resetFederationRemotes brig = do
+  rs <- getFederationRemotes brig
+  forM_ rs $ \(FederationDomainConfig rdom _) -> deleteFederationRemote brig rdom
+
 -- | Run a probe several times, until a "good" value materializes or until patience runs out
 aFewTimes ::
   (HasCallStack, MonadIO m) =>

From d369479a0facdb25598c337eaa3d61a5aee43437 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 20 Apr 2023 11:41:06 +0200
Subject: [PATCH 009/220] Simple handlers.

---
 services/brig/brig.cabal                 |  1 +
 services/brig/src/Brig/API/Internal.hs   |  8 +++++---
 services/brig/src/Brig/Data/Instances.hs | 13 +++++++++++++
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 23f55ee609..d0ae7a57ee 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -51,6 +51,7 @@ library
     Brig.Data.Activation
     Brig.Data.Client
     Brig.Data.Connection
+    Brig.Data.Federation
     Brig.Data.Instances
     Brig.Data.LoginCode
     Brig.Data.MLS.KeyPackage
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 0cab9eca1a..035f30ee0d 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -38,6 +38,7 @@ import qualified Brig.Code as Code
 import Brig.Data.Activation
 import qualified Brig.Data.Client as Data
 import qualified Brig.Data.Connection as Data
+import qualified Brig.Data.Federation as Data
 import qualified Brig.Data.MLS.KeyPackage as Data
 import qualified Brig.Data.User as Data
 import Brig.Effects.BlacklistPhonePrefixStore (BlacklistPhonePrefixStore)
@@ -89,6 +90,7 @@ import Wire.API.Federation.API
 import Wire.API.MLS.Credential
 import Wire.API.MLS.KeyPackage
 import Wire.API.MLS.Serialisation
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.Internal.Brig
 import qualified Wire.API.Routes.Internal.Brig as BrigIRoutes
 import Wire.API.Routes.Internal.Brig.Connection
@@ -174,9 +176,9 @@ authAPI =
 
 federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
 federationRemotesAPI =
-  Named @"get-federation-remotes" undefined
-    :<|> Named @"add-federation-remotes" undefined
-    :<|> Named @"delete-federation-remotes" undefined
+  Named @"get-federation-remotes" (lift $ FederationDomainConfigs <$> wrapClient Data.getFederationRemotes) -- TODO: get this from TVar!  also merge in config file!
+    :<|> Named @"add-federation-remotes" (lift . wrapClient . Data.addFederationRemote)
+    :<|> Named @"delete-federation-remotes" (lift . wrapClient . Data.deleteFederationRemote)
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)
diff --git a/services/brig/src/Brig/Data/Instances.hs b/services/brig/src/Brig/Data/Instances.hs
index 4ec4e3890c..0e3dba9276 100644
--- a/services/brig/src/Brig/Data/Instances.hs
+++ b/services/brig/src/Brig/Data/Instances.hs
@@ -47,6 +47,7 @@ import Wire.API.User.Activation
 import Wire.API.User.Client
 import Wire.API.User.Password
 import Wire.API.User.RichInfo
+import Wire.API.User.Search
 
 deriving instance Cql Name
 
@@ -288,3 +289,15 @@ instance Cql SearchVisibilityInbound where
   fromCql (CqlInt 0) = pure SearchableByOwnTeam
   fromCql (CqlInt 1) = pure SearchableByAllTeams
   fromCql n = Left $ "Unexpected SearchVisibilityInbound: " ++ show n
+
+instance Cql FederatedUserSearchPolicy where
+  ctype = Tagged IntColumn
+
+  toCql NoSearch = CqlInt 0
+  toCql ExactHandleSearch = CqlInt 1
+  toCql FullSearch = CqlInt 2
+
+  fromCql (CqlInt 0) = pure NoSearch
+  fromCql (CqlInt 1) = pure ExactHandleSearch
+  fromCql (CqlInt 2) = pure FullSearch
+  fromCql n = Left $ "Unexpected SearchVisibilityInbound: " ++ show n

From f8d8b5831b385b4fcf77ed048b45dcf7f0bd3147 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 20 Apr 2023 12:06:34 +0200
Subject: [PATCH 010/220] Fixup

---
 cassandra-schema.cql                          | 68 ++++++++++++-------
 .../Wire/API/Routes/FederationDomainConfig.hs |  6 --
 .../brig/test/integration/API/Federation.hs   |  1 -
 3 files changed, 43 insertions(+), 32 deletions(-)

diff --git a/cassandra-schema.cql b/cassandra-schema.cql
index a3b1e345be..f0c76947c0 100644
--- a/cassandra-schema.cql
+++ b/cassandra-schema.cql
@@ -1290,31 +1290,6 @@ CREATE TABLE brig_test.service_team (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.invitation (
-    inviter uuid,
-    id uuid,
-    code ascii,
-    created_at timestamp,
-    email text,
-    name text,
-    phone text,
-    PRIMARY KEY (inviter, id)
-) WITH CLUSTERING ORDER BY (id ASC)
-    AND bloom_filter_fp_chance = 0.01
-    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
-    AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
-    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
-    AND crc_check_chance = 1.0
-    AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 0
-    AND gc_grace_seconds = 864000
-    AND max_index_interval = 2048
-    AND memtable_flush_period_in_ms = 0
-    AND min_index_interval = 128
-    AND read_repair_chance = 0.0
-    AND speculative_retry = '99PERCENTILE';
-
 CREATE TABLE brig_test.blacklist (
     key text PRIMARY KEY
 ) WITH bloom_filter_fp_chance = 0.1
@@ -1665,6 +1640,49 @@ CREATE TABLE brig_test.password_reset (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE brig_test.federation_remotes (
+    domain text PRIMARY KEY,
+    search_policy int
+) WITH bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
+CREATE TABLE brig_test.invitation (
+    inviter uuid,
+    id uuid,
+    code ascii,
+    created_at timestamp,
+    email text,
+    name text,
+    phone text,
+    PRIMARY KEY (inviter, id)
+) WITH CLUSTERING ORDER BY (id ASC)
+    AND bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 CREATE TABLE brig_test.activation_keys (
     key ascii PRIMARY KEY,
     challenge ascii,
diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 1149843856..725a48b71e 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -1,9 +1,3 @@
-{- LANGUAGE DeriveAnyClass #-}
-{- LANGUAGE GeneralizedNewtypeDeriving #-}
-{- LANGUAGE TemplateHaskell #-}
--- Disabling to stop errors on Getters
-{- OPTIONS_GHC -Wno-redundant-constraints #-}
-
 -- This file is part of the Wire Server implementation.
 --
 -- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index cece13f5d0..9c01cd1aa7 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -56,7 +56,6 @@ import Wire.API.Federation.Component
 import Wire.API.Federation.Version
 import Wire.API.MLS.Credential
 import Wire.API.MLS.KeyPackage
-import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.FederationDomainConfig as FD
 import Wire.API.User
 import Wire.API.User.Client

From d80cef96a8fe773b502345b0a779810abeae51de Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 20 Apr 2023 13:12:54 +0200
Subject: [PATCH 011/220] Fixup

---
 services/brig/src/Brig/Data/Federation.hs | 53 +++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 services/brig/src/Brig/Data/Federation.hs

diff --git a/services/brig/src/Brig/Data/Federation.hs b/services/brig/src/Brig/Data/Federation.hs
new file mode 100644
index 0000000000..f2c0d4912a
--- /dev/null
+++ b/services/brig/src/Brig/Data/Federation.hs
@@ -0,0 +1,53 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Brig.Data.Federation
+  ( getFederationRemotes,
+    addFederationRemote,
+    deleteFederationRemote,
+  )
+where
+
+import Brig.Data.Instances ()
+import Cassandra
+import Data.Domain
+import Imports
+import Wire.API.Routes.FederationDomainConfig
+import Wire.API.User.Search
+
+getFederationRemotes :: forall m. MonadClient m => m [FederationDomainConfig]
+getFederationRemotes = uncurry FederationDomainConfig <$$> qry
+  where
+    qry :: m [(Domain, FederatedUserSearchPolicy)]
+    qry = retry x1 . query get $ params LocalQuorum ()
+
+    get :: PrepQuery R () (Domain, FederatedUserSearchPolicy)
+    get = "SELECT domain, search_policy FROM federation_remotes"
+
+addFederationRemote :: MonadClient m => FederationDomainConfig -> m ()
+addFederationRemote (FederationDomainConfig rdom searchpolicy) =
+  retry x5 $ write add (params LocalQuorum (rdom, searchpolicy))
+  where
+    add :: PrepQuery W (Domain, FederatedUserSearchPolicy) ()
+    add = "INSERT INTO federation_remotes (domain, search_policy) VALUES (?, ?)"
+
+deleteFederationRemote :: MonadClient m => Domain -> m ()
+deleteFederationRemote rdom =
+  retry x1 $ write delete (params LocalQuorum (Identity rdom))
+  where
+    delete :: PrepQuery W (Identity Domain) ()
+    delete = "DELETE FROM federation_remotes WHERE domain = ?"

From c553c0efc7b8ad1282af689abfaf6aa88299e5ff Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 21 Apr 2023 12:06:07 +0200
Subject: [PATCH 012/220] Attempt at making C* full table scan acceptable.

---
 services/brig/src/Brig/Data/Federation.hs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/services/brig/src/Brig/Data/Federation.hs b/services/brig/src/Brig/Data/Federation.hs
index f2c0d4912a..23bb1f2e86 100644
--- a/services/brig/src/Brig/Data/Federation.hs
+++ b/services/brig/src/Brig/Data/Federation.hs
@@ -29,6 +29,9 @@ import Imports
 import Wire.API.Routes.FederationDomainConfig
 import Wire.API.User.Search
 
+maxKnownNodes :: Int
+maxKnownNodes = 10000
+
 getFederationRemotes :: forall m. MonadClient m => m [FederationDomainConfig]
 getFederationRemotes = uncurry FederationDomainConfig <$$> qry
   where
@@ -36,10 +39,12 @@ getFederationRemotes = uncurry FederationDomainConfig <$$> qry
     qry = retry x1 . query get $ params LocalQuorum ()
 
     get :: PrepQuery R () (Domain, FederatedUserSearchPolicy)
-    get = "SELECT domain, search_policy FROM federation_remotes"
+    get = fromString $ "SELECT domain, search_policy FROM federation_remotes LIMIT " <> show maxKnownNodes
 
 addFederationRemote :: MonadClient m => FederationDomainConfig -> m ()
-addFederationRemote (FederationDomainConfig rdom searchpolicy) =
+addFederationRemote (FederationDomainConfig rdom searchpolicy) = do
+  l <- length <$> getFederationRemotes
+  when (l >= maxKnownNodes) $ error "TODO: make this error better"
   retry x5 $ write add (params LocalQuorum (rdom, searchpolicy))
   where
     add :: PrepQuery W (Domain, FederatedUserSearchPolicy) ()

From 68deac545c760ed49fcb2d0edb3260da7806cc21 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 24 Apr 2023 14:08:49 +1000
Subject: [PATCH 013/220] FS-1115: Setting up AMQP messaging for Federator to
 receive updates

---
 services/federator/federator.cabal            |  2 +-
 .../federator/src/Federator/EnvUpdater.hs     | 16 --------
 services/federator/src/Federator/Run.hs       | 39 +++++++++++++------
 3 files changed, 28 insertions(+), 29 deletions(-)
 delete mode 100644 services/federator/src/Federator/EnvUpdater.hs

diff --git a/services/federator/federator.cabal b/services/federator/federator.cabal
index ecb6c7d975..a29b6a1484 100644
--- a/services/federator/federator.cabal
+++ b/services/federator/federator.cabal
@@ -37,7 +37,6 @@ library
     Federator.App
     Federator.Discovery
     Federator.Env
-    Federator.EnvUpdater
     Federator.Error
     Federator.Error.ServerError
     Federator.ExternalServer
@@ -103,6 +102,7 @@ library
 
   build-depends:
       aeson
+    , amqp
     , async
     , base
     , bilge
diff --git a/services/federator/src/Federator/EnvUpdater.hs b/services/federator/src/Federator/EnvUpdater.hs
deleted file mode 100644
index 6fe3ee6152..0000000000
--- a/services/federator/src/Federator/EnvUpdater.hs
+++ /dev/null
@@ -1,16 +0,0 @@
-module Federator.EnvUpdater
-  ( envUpdater
-  ) where
-
-import Control.Concurrent.STM (TVar)
-import Federator.Env (Env)
-import Prelude (IO, pure, ($))
-import Control.Monad (forever)
-
-envUpdater :: TVar Env -> IO ()
-envUpdater _tEnv = do
-  -- TODO: This needs to update the TVar with a new Env value on some signal.
-  -- Presumably, configuration for how this can be done will be available in
-  -- the Env values being initially passed in.
-  forever $ do
-    pure ()
\ No newline at end of file
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index fb214b712c..ab0868ed4e 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -36,11 +36,10 @@ where
 
 import Control.Concurrent.Async
 import Control.Exception (bracket)
-import Control.Lens ((^.))
+import Control.Lens ((^.), (.~), (%~))
 import Data.Default (def)
 import qualified Data.Metrics.Middleware as Metrics
 import Federator.Env
-import Federator.EnvUpdater (envUpdater)
 import Federator.ExternalServer (serveInward)
 import Federator.InternalServer (serveOutward)
 import Federator.Monitor
@@ -53,6 +52,7 @@ import qualified System.Logger.Extended as LogExt
 import Util.Options
 import Wire.API.Federation.Component
 import qualified Wire.Network.DNS.Helper as DNS
+import Network.AMQP (openConnection, closeChannel, closeConnection, openChannel, consumeMsgs, Ack (Ack), Envelope, Message (msgBody), ackEnv)
 
 ------------------------------------------------------------------------------
 -- run/app
@@ -63,16 +63,31 @@ run opts = do
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res ->
     bracket (newEnv opts res) closeEnv $ \env -> do
-      -- Build a new TVar holding the state we want for the initial environment.
-      tEnv <- newTVarIO env
-      -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
-      let externalServer = serveInward tEnv portExternal
-          internalServer = serveOutward tEnv portInternal
-      withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
-        envUpdateThread <- async $ envUpdater tEnv
-        internalServerThread <- async internalServer
-        externalServerThread <- async externalServer
-        void $ waitAnyCancel [envUpdateThread, internalServerThread, externalServerThread]
+      -- TODO pull these values from Env
+      let host = undefined
+          vhost = undefined
+          user = undefined
+          pass = undefined
+          queue = undefined
+      bracket (openConnection host vhost user pass) closeConnection $ \amqpConn -> do
+        bracket (openChannel amqpConn) closeChannel $ \amqpChan -> do
+          -- Build a new TVar holding the state we want for the initial environment.
+          tEnv <- newTVarIO env
+          let
+              callback :: (Message, Envelope) -> IO ()
+              callback (message, envelope) = do
+                -- TODO: parse out the message body and update the tEnv
+                strat <- undefined $ msgBody message
+                atomically $ modifyTVar tEnv (Federator.Env.runSettings %~ \s -> s { federationStrategy = strat })
+                ackEnv envelope
+          -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
+          let externalServer = serveInward tEnv portExternal
+              internalServer = serveOutward tEnv portInternal
+          withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
+            envUpdateThread <- async . void $ consumeMsgs amqpChan queue Ack callback
+            internalServerThread <- async internalServer
+            externalServerThread <- async externalServer
+            void $ waitAnyCancel [envUpdateThread, internalServerThread, externalServerThread]
   where
     endpointInternal = federatorInternal opts
     portInternal = fromIntegral $ endpointInternal ^. epPort

From 7b531bac4d7afb25691a685a5d07c9abb5601391 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 24 Apr 2023 14:48:32 +1000
Subject: [PATCH 014/220] FS-1115: Adding env file values for message queues

---
 services/federator/federator.integration.yaml |  8 ++++++++
 services/federator/src/Federator/Options.hs   | 14 +++++++++++++-
 services/federator/src/Federator/Run.hs       |  9 ++-------
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/services/federator/federator.integration.yaml b/services/federator/federator.integration.yaml
index 9562b697e1..fa19cdd4ce 100644
--- a/services/federator/federator.integration.yaml
+++ b/services/federator/federator.integration.yaml
@@ -40,3 +40,11 @@ optSettings:
   clientPrivateKey: "test/resources/integration-leaf-key.pem"
   dnsHost: "127.0.0.1"
   dnsPort: 9053
+
+mqSettings:
+  host: "some.mq.host"
+  # https://www.rabbitmq.com/vhosts.html
+  vhost: "some.mq.host"
+  user: "username"
+  pass: "password"
+  queue: "queue name"
\ No newline at end of file
diff --git a/services/federator/src/Federator/Options.hs b/services/federator/src/Federator/Options.hs
index 803faa8d2a..69db0321e9 100644
--- a/services/federator/src/Federator/Options.hs
+++ b/services/federator/src/Federator/Options.hs
@@ -72,6 +72,16 @@ data RunSettings = RunSettings
 
 instance FromJSON RunSettings
 
+-- | Options for connecting to the message queue system
+data MessageQueueSettings = MessageQueueSettings
+  { host  :: String
+  , vhost :: Text
+  , user  :: Text
+  , pass  :: Text
+  , queue :: Text
+  } deriving (Show, Generic)
+instance FromJSON MessageQueueSettings
+
 data Opts = Opts
   { -- | Host and port for endpoint reachable only by other wire-server
     -- components in the same private network
@@ -92,7 +102,9 @@ data Opts = Opts
     -- | Logformat to use
     logFormat :: !(Maybe (Last LogFormat)),
     -- | Runtime settings
-    optSettings :: !RunSettings
+    optSettings :: !RunSettings,
+    -- | Message Queue settings
+    mqSettings :: !MessageQueueSettings
   }
   deriving (Show, Generic)
 
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index ab0868ed4e..8aac7eb45a 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -36,7 +36,7 @@ where
 
 import Control.Concurrent.Async
 import Control.Exception (bracket)
-import Control.Lens ((^.), (.~), (%~))
+import Control.Lens ((^.), (%~))
 import Data.Default (def)
 import qualified Data.Metrics.Middleware as Metrics
 import Federator.Env
@@ -63,12 +63,7 @@ run opts = do
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res ->
     bracket (newEnv opts res) closeEnv $ \env -> do
-      -- TODO pull these values from Env
-      let host = undefined
-          vhost = undefined
-          user = undefined
-          pass = undefined
-          queue = undefined
+      let MessageQueueSettings {host, vhost, user, pass, queue} = mqSettings opts
       bracket (openConnection host vhost user pass) closeConnection $ \amqpConn -> do
         bracket (openChannel amqpConn) closeChannel $ \amqpChan -> do
           -- Build a new TVar holding the state we want for the initial environment.

From f95c66661b2255156a15a275fbf14760208ab1d3 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 24 Apr 2023 18:35:08 +1000
Subject: [PATCH 015/220] wip

---
 libs/types-common/src/Data/MessageQueue.hs    | 18 ++++++++++++++++++
 libs/types-common/types-common.cabal          |  1 +
 services/brig/brig.cabal                      |  1 +
 services/brig/src/Brig/API/Internal.hs        | 17 ++++++++++++++---
 services/brig/src/Brig/Options.hs             |  7 +++++--
 services/federator/federator.integration.yaml | 10 +++++-----
 services/federator/src/Federator/Options.hs   | 11 +----------
 services/federator/src/Federator/Run.hs       |  7 ++++---
 8 files changed, 49 insertions(+), 23 deletions(-)
 create mode 100644 libs/types-common/src/Data/MessageQueue.hs

diff --git a/libs/types-common/src/Data/MessageQueue.hs b/libs/types-common/src/Data/MessageQueue.hs
new file mode 100644
index 0000000000..8348544f95
--- /dev/null
+++ b/libs/types-common/src/Data/MessageQueue.hs
@@ -0,0 +1,18 @@
+module Data.MessageQueue
+  ( MessageQueueSettings (..)
+  ) where
+
+import Prelude (Show, String)
+import Data.Text (Text)
+import Data.Aeson (FromJSON)
+import GHC.Generics (Generic)
+
+-- | Options for connecting to the message queue system
+data MessageQueueSettings = MessageQueueSettings
+  { mqHost  :: String
+  , mqVHost :: Text
+  , mqUser  :: Text
+  , mqPass  :: Text
+  , mqQueue :: Text
+  } deriving (Show, Generic)
+instance FromJSON MessageQueueSettings
\ No newline at end of file
diff --git a/libs/types-common/types-common.cabal b/libs/types-common/types-common.cabal
index 471765b90b..28e0ed80cc 100644
--- a/libs/types-common/types-common.cabal
+++ b/libs/types-common/types-common.cabal
@@ -22,6 +22,7 @@ library
     Data.Json.Util
     Data.LegalHold
     Data.List1
+    Data.MessageQueue
     Data.Misc
     Data.Nonce
     Data.PEMKeys
diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index d0ae7a57ee..3bf4fef572 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -191,6 +191,7 @@ library
     , amazonka-dynamodb          >=2
     , amazonka-ses               >=2
     , amazonka-sqs               >=2
+    , amqp
     , async                      >=2.1
     , attoparsec                 >=0.12
     , auto-update                >=0.1
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 035f30ee0d..5d55584345 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -101,6 +101,7 @@ import Wire.API.User.Activation
 import Wire.API.User.Client
 import Wire.API.User.Password
 import Wire.API.User.RichInfo
+import Data.Domain (Domain)
 
 ---------------------------------------------------------------------------
 -- Sitemap (servant)
@@ -176,9 +177,19 @@ authAPI =
 
 federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
 federationRemotesAPI =
-  Named @"get-federation-remotes" (lift $ FederationDomainConfigs <$> wrapClient Data.getFederationRemotes) -- TODO: get this from TVar!  also merge in config file!
-    :<|> Named @"add-federation-remotes" (lift . wrapClient . Data.addFederationRemote)
-    :<|> Named @"delete-federation-remotes" (lift . wrapClient . Data.deleteFederationRemote)
+  Named @"get-federation-remotes" getFederationRemotes -- TODO: get this from TVar!  also merge in config file!
+    :<|> Named @"add-federation-remotes" addFederationRemote
+    :<|> Named @"delete-federation-remotes" deleteFederationRemotes
+
+addFederationRemote :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
+addFederationRemote fedDomConf = do
+  lift . wrapClient $ Data.addFederationRemote fedDomConf
+
+getFederationRemotes :: ExceptT Brig.API.Error.Error (AppT r) FederationDomainConfigs
+getFederationRemotes = lift $ FederationDomainConfigs <$> wrapClient Data.getFederationRemotes
+
+deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
+deleteFederationRemotes = lift . wrapClient . Data.deleteFederationRemote
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index 89b49b8dfa..d1d019c3e2 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -59,6 +59,7 @@ import Wire.API.Routes.Version
 import qualified Wire.API.Team.Feature as Public
 import Wire.API.User
 import Wire.Arbitrary (Arbitrary, arbitrary)
+import Data.MessageQueue
 
 newtype Timeout = Timeout
   { timeoutDiff :: NominalDiffTime
@@ -458,7 +459,9 @@ data Opts = Opts
     -- | SFT Settings
     sft :: !(Maybe SFTOptions),
     -- | Runtime settings
-    optSettings :: !Settings
+    optSettings :: !Settings,
+    -- | Message Queue settings
+    mqSettings :: !MessageQueueSettings
   }
   deriving (Show, Generic)
 
@@ -924,4 +927,4 @@ Lens.makeLensesFor
 
 Lens.makeLensesFor [("sftBaseDomain", "sftBaseDomainL")] ''SFTOptions
 
-Lens.makeLensesFor [("serversSource", "serversSourceL")] ''TurnOpts
+Lens.makeLensesFor [("serversSource", "serversSourceL")] ''TurnOpts
\ No newline at end of file
diff --git a/services/federator/federator.integration.yaml b/services/federator/federator.integration.yaml
index fa19cdd4ce..0a99456314 100644
--- a/services/federator/federator.integration.yaml
+++ b/services/federator/federator.integration.yaml
@@ -42,9 +42,9 @@ optSettings:
   dnsPort: 9053
 
 mqSettings:
-  host: "some.mq.host"
+  mqHost: "some.mq.host"
   # https://www.rabbitmq.com/vhosts.html
-  vhost: "some.mq.host"
-  user: "username"
-  pass: "password"
-  queue: "queue name"
\ No newline at end of file
+  mqVHost: "some.mq.host"
+  mqUser: "username"
+  mqPass: "password"
+  mqQueue: "queue name"
\ No newline at end of file
diff --git a/services/federator/src/Federator/Options.hs b/services/federator/src/Federator/Options.hs
index 69db0321e9..a40a3421a5 100644
--- a/services/federator/src/Federator/Options.hs
+++ b/services/federator/src/Federator/Options.hs
@@ -25,6 +25,7 @@ import Data.Domain (Domain ())
 import Imports
 import System.Logger.Extended (Level, LogFormat)
 import Util.Options
+import Data.MessageQueue
 
 newtype AllowedDomains = AllowedDomains {allowedDomains :: [Domain]}
   deriving (Eq, Show, Generic)
@@ -72,16 +73,6 @@ data RunSettings = RunSettings
 
 instance FromJSON RunSettings
 
--- | Options for connecting to the message queue system
-data MessageQueueSettings = MessageQueueSettings
-  { host  :: String
-  , vhost :: Text
-  , user  :: Text
-  , pass  :: Text
-  , queue :: Text
-  } deriving (Show, Generic)
-instance FromJSON MessageQueueSettings
-
 data Opts = Opts
   { -- | Host and port for endpoint reachable only by other wire-server
     -- components in the same private network
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 8aac7eb45a..05b19d0a24 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -53,6 +53,7 @@ import Util.Options
 import Wire.API.Federation.Component
 import qualified Wire.Network.DNS.Helper as DNS
 import Network.AMQP (openConnection, closeChannel, closeConnection, openChannel, consumeMsgs, Ack (Ack), Envelope, Message (msgBody), ackEnv)
+import Data.MessageQueue
 
 ------------------------------------------------------------------------------
 -- run/app
@@ -63,8 +64,8 @@ run opts = do
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res ->
     bracket (newEnv opts res) closeEnv $ \env -> do
-      let MessageQueueSettings {host, vhost, user, pass, queue} = mqSettings opts
-      bracket (openConnection host vhost user pass) closeConnection $ \amqpConn -> do
+      let MessageQueueSettings {mqHost, mqVHost, mqUser, mqPass, mqQueue} = mqSettings opts
+      bracket (openConnection mqHost mqVHost mqUser mqPass) closeConnection $ \amqpConn -> do
         bracket (openChannel amqpConn) closeChannel $ \amqpChan -> do
           -- Build a new TVar holding the state we want for the initial environment.
           tEnv <- newTVarIO env
@@ -79,7 +80,7 @@ run opts = do
           let externalServer = serveInward tEnv portExternal
               internalServer = serveOutward tEnv portInternal
           withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
-            envUpdateThread <- async . void $ consumeMsgs amqpChan queue Ack callback
+            envUpdateThread <- async . void $ consumeMsgs amqpChan mqQueue Ack callback
             internalServerThread <- async internalServer
             externalServerThread <- async externalServer
             void $ waitAnyCancel [envUpdateThread, internalServerThread, externalServerThread]

From d5e171bc3523754b94fc97f79d0e87fa930796d2 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 24 Apr 2023 19:00:14 +1000
Subject: [PATCH 016/220] FS-1115: Ripping out AMQP code and replacing it with
 a http loop

---
 services/federator/federator.cabal          |  2 +-
 services/federator/src/Federator/Options.hs |  5 +-
 services/federator/src/Federator/Run.hs     | 65 ++++++++++++++-------
 3 files changed, 46 insertions(+), 26 deletions(-)

diff --git a/services/federator/federator.cabal b/services/federator/federator.cabal
index a29b6a1484..20af5dd57f 100644
--- a/services/federator/federator.cabal
+++ b/services/federator/federator.cabal
@@ -102,7 +102,6 @@ library
 
   build-depends:
       aeson
-    , amqp
     , async
     , base
     , bilge
@@ -139,6 +138,7 @@ library
     , polysemy-wire-zoo
     , retry
     , servant
+    , servant-client
     , servant-client-core
     , stm
     , streaming-commons
diff --git a/services/federator/src/Federator/Options.hs b/services/federator/src/Federator/Options.hs
index a40a3421a5..809e8374f1 100644
--- a/services/federator/src/Federator/Options.hs
+++ b/services/federator/src/Federator/Options.hs
@@ -25,7 +25,6 @@ import Data.Domain (Domain ())
 import Imports
 import System.Logger.Extended (Level, LogFormat)
 import Util.Options
-import Data.MessageQueue
 
 newtype AllowedDomains = AllowedDomains {allowedDomains :: [Domain]}
   deriving (Eq, Show, Generic)
@@ -94,8 +93,8 @@ data Opts = Opts
     logFormat :: !(Maybe (Last LogFormat)),
     -- | Runtime settings
     optSettings :: !RunSettings,
-    -- | Message Queue settings
-    mqSettings :: !MessageQueueSettings
+    -- | Domain update interval (microseconds)
+    domainUpdateInterval :: Int
   }
   deriving (Show, Generic)
 
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 05b19d0a24..ff9b25868f 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -52,8 +52,12 @@ import qualified System.Logger.Extended as LogExt
 import Util.Options
 import Wire.API.Federation.Component
 import qualified Wire.Network.DNS.Helper as DNS
-import Network.AMQP (openConnection, closeChannel, closeConnection, openChannel, consumeMsgs, Ack (Ack), Envelope, Message (msgBody), ackEnv)
-import Data.MessageQueue
+import qualified Wire.API.Routes.Internal.Brig as IAPI
+import Servant.Client
+import Wire.API.Routes.FederationDomainConfig
+import Network.HTTP.Client
+import Data.Text
+import Wire.API.Routes.Named
 
 ------------------------------------------------------------------------------
 -- run/app
@@ -64,27 +68,31 @@ run opts = do
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res ->
     bracket (newEnv opts res) closeEnv $ \env -> do
-      let MessageQueueSettings {mqHost, mqVHost, mqUser, mqPass, mqQueue} = mqSettings opts
-      bracket (openConnection mqHost mqVHost mqUser mqPass) closeConnection $ \amqpConn -> do
-        bracket (openChannel amqpConn) closeChannel $ \amqpChan -> do
-          -- Build a new TVar holding the state we want for the initial environment.
-          tEnv <- newTVarIO env
-          let
-              callback :: (Message, Envelope) -> IO ()
-              callback (message, envelope) = do
-                -- TODO: parse out the message body and update the tEnv
-                strat <- undefined $ msgBody message
-                atomically $ modifyTVar tEnv (Federator.Env.runSettings %~ \s -> s { federationStrategy = strat })
-                ackEnv envelope
-          -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
-          let externalServer = serveInward tEnv portExternal
-              internalServer = serveOutward tEnv portInternal
-          withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
-            envUpdateThread <- async . void $ consumeMsgs amqpChan mqQueue Ack callback
-            internalServerThread <- async internalServer
-            externalServerThread <- async externalServer
-            void $ waitAnyCancel [envUpdateThread, internalServerThread, externalServerThread]
+      -- Build a new TVar holding the state we want for the initial environment.
+      -- This needs to contact Brig before accepting other requests
+      manager <- newManager defaultManagerSettings
+      let Endpoint host port = brig opts
+          baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
+          clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
+      -- Explode if things went sideways. TODO make better
+      fedStrat <- either (error . show) pure =<< runClientM getFedRemotes clientEnv
+      tEnv <- newTVarIO $ updateFedStrat fedStrat env
+      let
+          callback :: FederationDomainConfigs -> IO ()
+          callback strat = do
+            atomically $ modifyTVar tEnv $ updateFedStrat strat
+      -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
+      let externalServer = serveInward tEnv portExternal
+          internalServer = serveOutward tEnv portInternal
+      withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
+        envUpdateThread <- async $ updateDomains clientEnv callback
+        internalServerThread <- async internalServer
+        externalServerThread <- async externalServer
+        void $ waitAnyCancel [envUpdateThread, internalServerThread, externalServerThread]
   where
+    updateFedStrat :: FederationDomainConfigs -> Env -> Env
+    updateFedStrat fedDomConfigs = Federator.Env.runSettings %~ \s -> s { federationStrategy = AllowList $ AllowedDomains $ domain <$> fromFederationDomainConfigs fedDomConfigs }
+
     endpointInternal = federatorInternal opts
     portInternal = fromIntegral $ endpointInternal ^. epPort
 
@@ -100,6 +108,19 @@ run opts = do
           conf {DNS.resolvInfo = DNS.RCHostPort host (fromIntegral port)}
         (_, _) -> conf
 
+    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
+
+    updateDomains :: ClientEnv -> (FederationDomainConfigs -> IO ()) -> IO ()
+    updateDomains clientEnv update = forever $ do
+      strat <- runClientM getFedRemotes clientEnv
+      either
+        print
+        update
+        strat
+      threadDelay $ domainUpdateInterval opts
+        
+        
+
 -------------------------------------------------------------------------------
 -- Environment
 

From e5935106739e0b891e934638bd0df8027f05595b Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 26 Apr 2023 20:25:36 +1000
Subject: [PATCH 017/220] FS-1115: Adding smarts to galley.

TODOs abound in the code, but the main outline is there. Discussions
about how we are going to remove conversations and add links for new
federation servers will need to be had and documented.
---
 .../Wire/API/Routes/FederationDomainConfig.hs |  2 +-
 services/federator/src/Federator/Options.hs   |  2 +-
 services/galley/galley.integration.yaml       |  2 +
 services/galley/src/Galley/App.hs             | 18 ++++++++-
 services/galley/src/Galley/Env.hs             |  4 +-
 services/galley/src/Galley/Monad.hs           |  2 +-
 services/galley/src/Galley/Options.hs         |  5 ++-
 services/galley/src/Galley/Run.hs             | 37 +++++++++++++++++++
 8 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 725a48b71e..fc5ce971a5 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -45,7 +45,7 @@ instance ToSchema FederationDomainConfig where
 
 newtype FederationDomainConfigs = FederationDomainConfigs
   {fromFederationDomainConfigs :: [FederationDomainConfig]}
-  deriving (Show, Generic)
+  deriving (Show, Generic, Eq)
   deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationDomainConfigs
 
 instance ToSchema FederationDomainConfigs where
diff --git a/services/federator/src/Federator/Options.hs b/services/federator/src/Federator/Options.hs
index 809e8374f1..19a006a176 100644
--- a/services/federator/src/Federator/Options.hs
+++ b/services/federator/src/Federator/Options.hs
@@ -94,7 +94,7 @@ data Opts = Opts
     -- | Runtime settings
     optSettings :: !RunSettings,
     -- | Domain update interval (microseconds)
-    domainUpdateInterval :: Int
+    domainUpdateInterval :: !Int
   }
   deriving (Show, Generic)
 
diff --git a/services/galley/galley.integration.yaml b/services/galley/galley.integration.yaml
index 35426d4ed3..cd6685f6f6 100644
--- a/services/galley/galley.integration.yaml
+++ b/services/galley/galley.integration.yaml
@@ -87,3 +87,5 @@ journal: # if set, journals; if not set, disables journaling
   queueName: integration-team-events.fifo
   endpoint: http://localhost:4568 # https://sqs.eu-west-1.amazonaws.com
   region: eu-west-1
+
+domainUpdateInterval: 1000000
\ No newline at end of file
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index b40bc462a6..4a3322db02 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -107,6 +107,9 @@ import Util.Options
 import Wire.API.Error
 import Wire.API.Federation.Error
 import qualified Wire.Sem.Logger
+import qualified Servant.Client as SC
+import qualified Wire.API.Routes.Internal.Brig as IAPI
+import Wire.API.Routes.Named (namedClient)
 
 -- Effects needed by the interpretation of other effects
 type GalleyEffects0 =
@@ -156,11 +159,24 @@ createEnv m o = do
   mgr <- initHttpManager o
   h2mgr <- initHttp2Manager
   validateOptions l o
-  Env def m o l mgr h2mgr (o ^. optFederator) (o ^. optBrig) cass
+  
+  -- Fetch the initial federation domain list so we always start with
+  -- a known update to date dataset.
+  
+  let brigEndpoint = o ^. optBrig
+      Endpoint h p = brigEndpoint
+      baseUrl = SC.BaseUrl SC.Http (unpack h) (fromIntegral p) ""
+      clientEnv = SC.ClientEnv mgr baseUrl Nothing SC.defaultMakeClientRequest
+      getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
+  strat <- either fedDomainError pure =<< SC.runClientM getFedRemotes clientEnv
+  Env def m o l mgr h2mgr (o ^. optFederator) brigEndpoint cass
     <$> Q.new 16000
     <*> initExtEnv
     <*> maybe (pure Nothing) (fmap Just . Aws.mkEnv l mgr) (o ^. optJournal)
     <*> loadAllMLSKeys (fold (o ^. optSettings . setMlsPrivateKeyPaths))
+    <*> newTVarIO strat
+  where
+    fedDomainError e = error $ "Could not retrieve the latest list of federation domains from Brig: " <> show e
 
 initCassandra :: Opts -> Logger -> IO ClientState
 initCassandra o l = do
diff --git a/services/galley/src/Galley/Env.hs b/services/galley/src/Galley/Env.hs
index 39e648c1a0..dc460be470 100644
--- a/services/galley/src/Galley/Env.hs
+++ b/services/galley/src/Galley/Env.hs
@@ -41,6 +41,7 @@ import Util.Options
 import Wire.API.MLS.Credential
 import Wire.API.MLS.Keys
 import Wire.API.Team.Member
+import Wire.API.Routes.FederationDomainConfig (FederationDomainConfigs)
 
 data DeleteItem = TeamItem TeamId UserId (Maybe ConnId)
   deriving (Eq, Ord, Show)
@@ -59,7 +60,8 @@ data Env = Env
     _deleteQueue :: Q.Queue DeleteItem,
     _extEnv :: ExtEnv,
     _aEnv :: Maybe Aws.Env,
-    _mlsKeys :: SignaturePurpose -> MLSKeys
+    _mlsKeys :: SignaturePurpose -> MLSKeys,
+    _fedDomains :: TVar FederationDomainConfigs
   }
 
 -- | Environment specific to the communication with external
diff --git a/services/galley/src/Galley/Monad.hs b/services/galley/src/Galley/Monad.hs
index 83cb34e5ed..ad9ef51c6a 100644
--- a/services/galley/src/Galley/Monad.hs
+++ b/services/galley/src/Galley/Monad.hs
@@ -32,7 +32,7 @@ import Polysemy.Input
 import System.Logger
 import qualified System.Logger.Class as LC
 
-newtype App a = App {unApp :: ReaderT Env IO a}
+newtype App a = App {unApp ::ReaderT Env IO a}
   deriving
     ( Functor,
       Applicative,
diff --git a/services/galley/src/Galley/Options.hs b/services/galley/src/Galley/Options.hs
index d9776451eb..8f9197a425 100644
--- a/services/galley/src/Galley/Options.hs
+++ b/services/galley/src/Galley/Options.hs
@@ -52,6 +52,7 @@ module Galley.Options
     optLogLevel,
     optLogNetStrings,
     optLogFormat,
+    optDomainUpdateInterval
   )
 where
 
@@ -172,7 +173,9 @@ data Opts = Opts
     --  <http://cr.yp.to/proto/netstrings.txt>
     _optLogNetStrings :: !(Maybe (Last Bool)),
     -- | What log format to use
-    _optLogFormat :: !(Maybe (Last LogFormat))
+    _optLogFormat :: !(Maybe (Last LogFormat)),
+    -- | Domain update interval (microseconds)
+    _optDomainUpdateInterval :: !Int
   }
 
 deriveFromJSON toOptionFieldName ''Opts
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index b528a6c054..df0a413c99 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -48,6 +48,7 @@ import Galley.App
 import qualified Galley.App as App
 import Galley.Aws (awsEnv)
 import Galley.Cassandra
+import Galley.Env (fedDomains)
 import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
@@ -59,11 +60,21 @@ import qualified Network.Wai.Middleware.Gunzip as GZip
 import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
 import Servant hiding (route)
+import Servant.Client
+  ( BaseUrl (BaseUrl),
+    ClientEnv (ClientEnv),
+    Scheme (Http),
+    defaultMakeClientRequest,
+    runClientM,
+  )
 import qualified System.Logger as Log
 import Util.Options
 import Wire.API.Routes.API
+import qualified Wire.API.Routes.Internal.Brig as IAPI
+import Wire.API.Routes.Named (namedClient)
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
+import qualified System.Logger.Class as L
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -79,8 +90,10 @@ run opts = lowerCodensity $ do
 
   forM_ (env ^. aEnv) $ \aws ->
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
+  void $ Codensity $ Async.withAsync $ runApp env updateFedDomains
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
+  void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
 mkApp :: Opts -> Codensity IO (Application, Env)
@@ -168,3 +181,27 @@ collectAuthMetrics m env = do
       mbRemaining <- readAuthExpiration env
       gaugeTokenRemaing m mbRemaining
       threadDelay 1_000_000
+
+updateFedDomains :: App ()
+updateFedDomains = do
+  updateInterval <- view $ options . optDomainUpdateInterval
+  tvar <- view fedDomains
+  manager' <- view manager
+  Endpoint host port <- view brig
+  let baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
+      clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
+  forever $ do
+    previous <- liftIO $ readTVarIO tvar
+    strat <- liftIO $ runClientM getFedRemotes clientEnv
+    case strat of
+      Left e -> L.err . L.msg $ "Could not retrieve federation domains from brig: " <> show e
+      Right s -> when (s /= previous) $ do
+        -- Perform updates before rewriting the tvar
+        -- This means that if the update fails on a
+        -- particular invocation, it can be run again
+        -- on the next firing as it isn't likely that
+        -- the domain list is changing frequently.
+        liftIO $ atomically $ writeTVar tvar s
+    threadDelay updateInterval
+  where
+    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
\ No newline at end of file

From 8a3f4db7ebf486a9372c25f5d2d3fe0437b7edd3 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 27 Apr 2023 18:53:13 +1000
Subject: [PATCH 018/220] WIP

---
 services/galley/src/Galley/Run.hs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index df0a413c99..4e9abb0e6b 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -75,6 +75,7 @@ import Wire.API.Routes.Named (namedClient)
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
 import qualified System.Logger.Class as L
+import qualified Data.Set as Set
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -193,14 +194,19 @@ updateFedDomains = do
   forever $ do
     previous <- liftIO $ readTVarIO tvar
     strat <- liftIO $ runClientM getFedRemotes clientEnv
+    let domainListsEqual s =
+          Set.fromList (fromFederationDomainConfigs s) ==
+          Set.fromList (fromFederationDomainConfigs previous)
     case strat of
       Left e -> L.err . L.msg $ "Could not retrieve federation domains from brig: " <> show e
-      Right s -> when (s /= previous) $ do
+      -- Using Set to do the comparison, as it will handle the lists being in different orders.
+      Right s -> unless (domainListsEqual s) $ do
         -- Perform updates before rewriting the tvar
         -- This means that if the update fails on a
         -- particular invocation, it can be run again
         -- on the next firing as it isn't likely that
         -- the domain list is changing frequently.
+        -- FS-1179 is handling this part.
         liftIO $ atomically $ writeTVar tvar s
     threadDelay updateInterval
   where

From 6703241c35a111270e10979c4b553434f3454038 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Fri, 28 Apr 2023 18:18:00 +1000
Subject: [PATCH 019/220] FS-1115: Fixes for some tests

---
 services/brig/brig.integration.yaml           |  6 ++++++
 services/federator/federator.integration.yaml |  3 ++-
 services/federator/src/Federator/Run.hs       | 17 +++++++++++++----
 services/galley/src/Galley/App.hs             | 14 +++++++++++---
 services/galley/src/Galley/Run.hs             |  3 ++-
 5 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml
index 8d8b67ba30..1d6ef0b859 100644
--- a/services/brig/brig.integration.yaml
+++ b/services/brig/brig.integration.yaml
@@ -209,3 +209,9 @@ optSettings:
 
 logLevel: Warn
 logNetStrings: false
+mqSettings:
+  mqHost: ""
+  mqVHost: ""
+  mqUser: ""
+  mqPass: ""
+  mqQueue: ""
\ No newline at end of file
diff --git a/services/federator/federator.integration.yaml b/services/federator/federator.integration.yaml
index 0a99456314..593dda9b5f 100644
--- a/services/federator/federator.integration.yaml
+++ b/services/federator/federator.integration.yaml
@@ -47,4 +47,5 @@ mqSettings:
   mqVHost: "some.mq.host"
   mqUser: "username"
   mqPass: "password"
-  mqQueue: "queue name"
\ No newline at end of file
+  mqQueue: "queue name"
+domainUpdateInterval: 1000000
\ No newline at end of file
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index ff9b25868f..28677af406 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -74,13 +74,22 @@ run opts = do
       let Endpoint host port = brig opts
           baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
           clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-      -- Explode if things went sideways. TODO make better
-      fedStrat <- either (error . show) pure =<< runClientM getFedRemotes clientEnv
+          -- Loop the request until we get an answer. This is helpful during integration
+          -- tests where services are being brought up in parallel.
+          getInitialFedDomains = do
+            runClientM getFedRemotes clientEnv >>= \case
+              Right s -> pure s
+              Left e -> do
+                print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e
+                threadDelay $ domainUpdateInterval opts
+                getInitialFedDomains
+      fedStrat <- getInitialFedDomains
       tEnv <- newTVarIO $ updateFedStrat fedStrat env
       let
           callback :: FederationDomainConfigs -> IO ()
           callback strat = do
             atomically $ modifyTVar tEnv $ updateFedStrat strat
+            print strat
       -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
       let externalServer = serveInward tEnv portExternal
           internalServer = serveOutward tEnv portInternal
@@ -112,12 +121,12 @@ run opts = do
 
     updateDomains :: ClientEnv -> (FederationDomainConfigs -> IO ()) -> IO ()
     updateDomains clientEnv update = forever $ do
+      threadDelay $ domainUpdateInterval opts
       strat <- runClientM getFedRemotes clientEnv
       either
         print
         update
-        strat
-      threadDelay $ domainUpdateInterval opts
+        strat      
         
         
 
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index 4a3322db02..e0d96da0ee 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -1,4 +1,5 @@
 {-# LANGUAGE StrictData #-}
+{-# LANGUAGE LambdaCase #-}
 
 -- This file is part of the Wire Server implementation.
 --
@@ -168,15 +169,22 @@ createEnv m o = do
       baseUrl = SC.BaseUrl SC.Http (unpack h) (fromIntegral p) ""
       clientEnv = SC.ClientEnv mgr baseUrl Nothing SC.defaultMakeClientRequest
       getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
-  strat <- either fedDomainError pure =<< SC.runClientM getFedRemotes clientEnv
+      -- Loop the request until we get an answer. This is helpful during integration
+      -- tests where services are being brought up in parallel.
+      getInitalFedDomains = do
+        SC.runClientM getFedRemotes clientEnv >>= \case
+          Right s -> pure s
+          Left e -> do
+            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e
+            threadDelay $ o ^. optDomainUpdateInterval
+            getInitalFedDomains
+  strat <- getInitalFedDomains
   Env def m o l mgr h2mgr (o ^. optFederator) brigEndpoint cass
     <$> Q.new 16000
     <*> initExtEnv
     <*> maybe (pure Nothing) (fmap Just . Aws.mkEnv l mgr) (o ^. optJournal)
     <*> loadAllMLSKeys (fold (o ^. optSettings . setMlsPrivateKeyPaths))
     <*> newTVarIO strat
-  where
-    fedDomainError e = error $ "Could not retrieve the latest list of federation domains from Brig: " <> show e
 
 initCassandra :: Opts -> Logger -> IO ClientState
 initCassandra o l = do
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 4e9abb0e6b..1f92e41a51 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -76,6 +76,7 @@ import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
 import qualified System.Logger.Class as L
 import qualified Data.Set as Set
+import Wire.API.Routes.FederationDomainConfig
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -192,6 +193,7 @@ updateFedDomains = do
   let baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
   forever $ do
+    threadDelay updateInterval
     previous <- liftIO $ readTVarIO tvar
     strat <- liftIO $ runClientM getFedRemotes clientEnv
     let domainListsEqual s =
@@ -208,6 +210,5 @@ updateFedDomains = do
         -- the domain list is changing frequently.
         -- FS-1179 is handling this part.
         liftIO $ atomically $ writeTVar tvar s
-    threadDelay updateInterval
   where
     getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
\ No newline at end of file

From 6dc1d4e6b74ca07666a74a35e66b76d98f0e99dd Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 1 May 2023 15:40:12 +0200
Subject: [PATCH 020/220] Merge config file in with db in `GET
 /federation/remotes`.

---
 services/brig/src/Brig/API/Internal.hs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 5d55584345..6057e058ac 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -66,6 +66,7 @@ import Control.Lens (view)
 import Data.Aeson hiding (json)
 import Data.ByteString.Conversion
 import qualified Data.ByteString.Conversion as List
+import Data.Domain (Domain)
 import Data.Handle
 import Data.Id as Id
 import qualified Data.Map.Strict as Map
@@ -101,7 +102,6 @@ import Wire.API.User.Activation
 import Wire.API.User.Client
 import Wire.API.User.Password
 import Wire.API.User.RichInfo
-import Data.Domain (Domain)
 
 ---------------------------------------------------------------------------
 -- Sitemap (servant)
@@ -177,7 +177,7 @@ authAPI =
 
 federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
 federationRemotesAPI =
-  Named @"get-federation-remotes" getFederationRemotes -- TODO: get this from TVar!  also merge in config file!
+  Named @"get-federation-remotes" getFederationRemotes -- TODO: get this from TVar!
     :<|> Named @"add-federation-remotes" addFederationRemote
     :<|> Named @"delete-federation-remotes" deleteFederationRemotes
 
@@ -186,7 +186,12 @@ addFederationRemote fedDomConf = do
   lift . wrapClient $ Data.addFederationRemote fedDomConf
 
 getFederationRemotes :: ExceptT Brig.API.Error.Error (AppT r) FederationDomainConfigs
-getFederationRemotes = lift $ FederationDomainConfigs <$> wrapClient Data.getFederationRemotes
+getFederationRemotes = lift $ do
+  db <- wrapClient Data.getFederationRemotes
+  cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
+  -- FUTUREWORK: we should solely rely on `db` in the future; `cfg` is just for an easier,
+  -- more robust migration path.
+  pure . FederationDomainConfigs . nub $ db <> cfg
 
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes = lift . wrapClient . Data.deleteFederationRemote

From af9c127abcfe7049c56e8e3ed8e4eddb568ddc50 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 2 May 2023 18:09:14 +1000
Subject: [PATCH 021/220] FS-1115: Fixing build errors, updating tests,
 formatting

---
 libs/types-common/src/Data/MessageQueue.hs    | 25 ++++++-----
 services/brig/brig.cabal                      |  1 -
 services/brig/src/Brig/Options.hs             |  4 +-
 .../brig/test/integration/API/Federation.hs   | 15 ++++---
 .../brig/test/integration/API/User/Account.hs |  4 +-
 .../brig/test/integration/API/User/Client.hs  |  4 +-
 .../test/integration/API/User/Connection.hs   |  2 +-
 services/federator/default.nix                |  2 +
 services/federator/src/Federator/Response.hs  | 41 ++++++++++---------
 services/federator/src/Federator/Run.hs       | 21 ++++------
 services/galley/src/Galley/App.hs             | 10 ++---
 services/galley/src/Galley/Env.hs             |  2 +-
 services/galley/src/Galley/Monad.hs           |  2 +-
 services/galley/src/Galley/Options.hs         |  2 +-
 services/galley/src/Galley/Run.hs             | 12 +++---
 services/galley/test/integration/API.hs       |  2 +-
 16 files changed, 78 insertions(+), 71 deletions(-)

diff --git a/libs/types-common/src/Data/MessageQueue.hs b/libs/types-common/src/Data/MessageQueue.hs
index 8348544f95..6d90f1f509 100644
--- a/libs/types-common/src/Data/MessageQueue.hs
+++ b/libs/types-common/src/Data/MessageQueue.hs
@@ -1,18 +1,21 @@
 module Data.MessageQueue
-  ( MessageQueueSettings (..)
-  ) where
+  ( MessageQueueSettings (..),
+  )
+where
 
-import Prelude (Show, String)
-import Data.Text (Text)
 import Data.Aeson (FromJSON)
+import Data.Text (Text)
 import GHC.Generics (Generic)
+import Prelude (Show, String)
 
 -- | Options for connecting to the message queue system
 data MessageQueueSettings = MessageQueueSettings
-  { mqHost  :: String
-  , mqVHost :: Text
-  , mqUser  :: Text
-  , mqPass  :: Text
-  , mqQueue :: Text
-  } deriving (Show, Generic)
-instance FromJSON MessageQueueSettings
\ No newline at end of file
+  { mqHost :: String,
+    mqVHost :: Text,
+    mqUser :: Text,
+    mqPass :: Text,
+    mqQueue :: Text
+  }
+  deriving (Show, Generic)
+
+instance FromJSON MessageQueueSettings
diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 2744145bef..69d02377fa 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -192,7 +192,6 @@ library
     , amazonka-dynamodb          >=2
     , amazonka-ses               >=2
     , amazonka-sqs               >=2
-    , amqp
     , async                      >=2.1
     , auto-update                >=0.1
     , base                       >=4       && <5
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index d1d019c3e2..6ec49300d3 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -39,6 +39,7 @@ import qualified Data.Code as Code
 import Data.Domain (Domain (..))
 import Data.Id
 import Data.LanguageCodes (ISO639_1 (EN))
+import Data.MessageQueue
 import Data.Misc (HttpsUrl)
 import Data.Nonce
 import Data.Range
@@ -59,7 +60,6 @@ import Wire.API.Routes.Version
 import qualified Wire.API.Team.Feature as Public
 import Wire.API.User
 import Wire.Arbitrary (Arbitrary, arbitrary)
-import Data.MessageQueue
 
 newtype Timeout = Timeout
   { timeoutDiff :: NominalDiffTime
@@ -927,4 +927,4 @@ Lens.makeLensesFor
 
 Lens.makeLensesFor [("sftBaseDomain", "sftBaseDomainL")] ''SFTOptions
 
-Lens.makeLensesFor [("serversSource", "serversSourceL")] ''TurnOpts
\ No newline at end of file
+Lens.makeLensesFor [("serversSource", "serversSourceL")] ''TurnOpts
diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index 9c01cd1aa7..c747a10354 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -461,26 +461,31 @@ testClaimKeyPackagesMLSDisabled opts brig = do
   liftIO $ mbundle @?= Nothing
 
 crudFederationRemotes :: HasCallStack => Opt.Opts -> Brig -> Http ()
-crudFederationRemotes _opts brig = do
+crudFederationRemotes opts brig = do
+  -- Delete the remotes from the database
+  -- This doesn't do anything with the remotes
+  -- defined in config files.
   resetFederationRemotes brig
 
   res1 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return nothing" [] res1
+  liftIO $ assertEqual "should return config values" cfgRemotes res1
 
   let remote1 = FederationDomainConfig (Domain "good.example.com") NoSearch
   addFederationRemote brig remote1
   res2 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return good.example.com" [remote1] res2
+  liftIO $ assertEqual "should return config values and good.example.com" (nub $ sort $ cfgRemotes <> [remote1]) (sort res2)
 
   let remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
   addFederationRemote brig remote2
   res3 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return {good,evil}.example.com" (sort [remote1, remote2]) (sort res3)
+  liftIO $ assertEqual "should return config values and {good,evil}.example.com" (nub $ sort $ cfgRemotes <> [remote1, remote2]) (sort res3)
 
   deleteFederationRemote brig (domain remote1)
   res4 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return evil.example.com" (sort [remote2]) (sort res4)
+  liftIO $ assertEqual "should return config values and evil.example.com" (nub $ sort $ cfgRemotes <> [remote2]) (sort res4)
 
   -- TODO: how do we test that the TVar is updated in all services?  some fancy unit test?
   -- duplicate internal end-point to all services, and implement the hanlers in a library?
   pure ()
+  where
+    cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
diff --git a/services/brig/test/integration/API/User/Account.hs b/services/brig/test/integration/API/User/Account.hs
index 664a17d964..7840f45b46 100644
--- a/services/brig/test/integration/API/User/Account.hs
+++ b/services/brig/test/integration/API/User/Account.hs
@@ -124,7 +124,7 @@ tests _ at opts p b c ch g aws userJournalWatcher =
       test p "post /activate - 200/204 + expiry" $ testActivateWithExpiry opts b at,
       test p "get /users/:uid - 404" $ testNonExistingUserUnqualified b,
       test p "get /users/<localdomain>/:uid - 404" $ testNonExistingUser b,
-      test p "get /users/:domain/:uid - 422" $ testUserInvalidDomain b,
+      test p "get /users/:domain/:uid - 400" $ testUserInvalidDomain b,
       test p "get /users/:uid - 200" $ testExistingUserUnqualified b,
       test p "get /users/<localdomain>/:uid - 200" $ testExistingUser b,
       test p "get /users?:id=.... - 200" $ testMultipleUsersUnqualified b,
@@ -637,7 +637,7 @@ testUserInvalidDomain brig = do
   let uid = qUnqualified qself
   get (brig . paths ["users", "invalid.example.com", toByteString' uid] . zUser uid)
     !!! do
-      const 422 === statusCode
+      const 400 === statusCode
       const (Just "/federation/api-version")
         === preview (ix "data" . ix "path") . responseJsonUnsafe @Value
       const (Just "invalid.example.com")
diff --git a/services/brig/test/integration/API/User/Client.hs b/services/brig/test/integration/API/User/Client.hs
index 7fb3d7658d..55af16195e 100644
--- a/services/brig/test/integration/API/User/Client.hs
+++ b/services/brig/test/integration/API/User/Client.hs
@@ -92,7 +92,7 @@ tests e _cl _at opts p db b c g =
       test p "get /users/<localdomain>/:uid/clients - 200" $ testGetUserClientsQualified opts b,
       test p "get /users/:uid/prekeys - 200" $ testGetUserPrekeys b,
       test p "get /users/<localdomain>/:uid/prekeys - 200" $ testGetUserPrekeysQualified b opts,
-      test p "get /users/:domain/:uid/prekeys - 422" $ testGetUserPrekeysInvalidDomain b,
+      test p "get /users/:domain/:uid/prekeys - 400" $ testGetUserPrekeysInvalidDomain b,
       test p "get /users/:uid/prekeys/:client - 200" $ testGetClientPrekey b,
       test p "get /users/<localdomain>/:uid/prekeys/:client - 200" $ testGetClientPrekeyQualified b opts,
       test p "post /users/prekeys" $ testMultiUserGetPrekeys b,
@@ -783,7 +783,7 @@ testGetUserPrekeysInvalidDomain :: Brig -> Http ()
 testGetUserPrekeysInvalidDomain brig = do
   [(uid, _c, _lpk, _)] <- generateClients 1 brig
   get (brig . paths ["users", "invalid.example.com", toByteString' uid, "prekeys"] . zUser uid) !!! do
-    const 422 === statusCode
+    const 400 === statusCode
 
 testGetClientPrekey :: Brig -> Http ()
 testGetClientPrekey brig = do
diff --git a/services/brig/test/integration/API/User/Connection.hs b/services/brig/test/integration/API/User/Connection.hs
index 1e1353d1df..d95df5e553 100644
--- a/services/brig/test/integration/API/User/Connection.hs
+++ b/services/brig/test/integration/API/User/Connection.hs
@@ -722,7 +722,7 @@ testConnectFederationNotAvailable :: Brig -> Http ()
 testConnectFederationNotAvailable brig = do
   (uid1, quid2) <- localAndRemoteUser brig
   postConnectionQualified brig uid1 quid2
-    !!! const 422 === statusCode
+    !!! const 400 === statusCode
 
 testConnectOK :: Brig -> Galley -> FedClient 'Brig -> Http ()
 testConnectOK brig galley fedBrigClient = do
diff --git a/services/federator/default.nix b/services/federator/default.nix
index 28fc86498b..ccdb55ccf6 100644
--- a/services/federator/default.nix
+++ b/services/federator/default.nix
@@ -106,6 +106,7 @@ mkDerivation {
     polysemy
     polysemy-wire-zoo
     servant
+    servant-client
     servant-client-core
     string-conversions
     text
@@ -116,6 +117,7 @@ mkDerivation {
     wai
     wai-utilities
     warp
+    wire-api
     wire-api-federation
     x509
     x509-validation
diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index 5d1befd987..86ccf0ae2c 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -135,26 +135,27 @@ type AllEffects =
 runFederator :: TVar Env -> Sem AllEffects Wai.Response -> Codensity IO Wai.Response
 runFederator tvar resp = do
   env <- liftIO $ readTVarIO tvar
-  resp & runM
-    . runEmbedded @IO @(Codensity IO) liftIO
-    . loggerToTinyLogReqId (view requestId env) (view applog env)
-    . runWaiErrors
-      @'[ ValidationError,
-          RemoteError,
-          ServerError,
-          DiscoveryFailure
-        ]
-    . runInputConst env
-    . runInputSem (embed @IO (readIORef (view http2Manager env)))
-    -- This is the point at which federation settings are extracted
-    -- For each request, extract a fresh copy of the runSettings. This allows us
-    -- to independently update the settings and have them be used as requests
-    -- come in.
-    . runInputSem (embed @IO $ fmap (view runSettings) . liftIO $ readTVarIO tvar)
-    . interpretServiceHTTP
-    . runDNSLookupWithResolver (view dnsResolver env)
-    . runFederatorDiscovery
-    . interpretRemote
+  resp
+    & runM
+      . runEmbedded @IO @(Codensity IO) liftIO
+      . loggerToTinyLogReqId (view requestId env) (view applog env)
+      . runWaiErrors
+        @'[ ValidationError,
+            RemoteError,
+            ServerError,
+            DiscoveryFailure
+          ]
+      . runInputConst env
+      . runInputSem (embed @IO (readIORef (view http2Manager env)))
+      -- This is the point at which federation settings are extracted
+      -- For each request, extract a fresh copy of the runSettings. This allows us
+      -- to independently update the settings and have them be used as requests
+      -- come in.
+      . runInputSem (embed @IO $ fmap (view runSettings) . liftIO $ readTVarIO tvar)
+      . interpretServiceHTTP
+      . runDNSLookupWithResolver (view dnsResolver env)
+      . runFederatorDiscovery
+      . interpretRemote
 
 streamingResponseToWai :: StreamingResponse -> Wai.Response
 streamingResponseToWai resp =
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 28677af406..f4be91cebf 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -36,9 +36,10 @@ where
 
 import Control.Concurrent.Async
 import Control.Exception (bracket)
-import Control.Lens ((^.), (%~))
+import Control.Lens ((%~), (^.))
 import Data.Default (def)
 import qualified Data.Metrics.Middleware as Metrics
+import Data.Text
 import Federator.Env
 import Federator.ExternalServer (serveInward)
 import Federator.InternalServer (serveOutward)
@@ -46,18 +47,17 @@ import Federator.Monitor
 import Federator.Options as Opt
 import Imports
 import qualified Network.DNS as DNS
+import Network.HTTP.Client
 import qualified Network.HTTP.Client as HTTP
+import Servant.Client
 import qualified System.Logger.Class as Log
 import qualified System.Logger.Extended as LogExt
 import Util.Options
 import Wire.API.Federation.Component
-import qualified Wire.Network.DNS.Helper as DNS
-import qualified Wire.API.Routes.Internal.Brig as IAPI
-import Servant.Client
 import Wire.API.Routes.FederationDomainConfig
-import Network.HTTP.Client
-import Data.Text
+import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named
+import qualified Wire.Network.DNS.Helper as DNS
 
 ------------------------------------------------------------------------------
 -- run/app
@@ -85,8 +85,7 @@ run opts = do
                 getInitialFedDomains
       fedStrat <- getInitialFedDomains
       tEnv <- newTVarIO $ updateFedStrat fedStrat env
-      let
-          callback :: FederationDomainConfigs -> IO ()
+      let callback :: FederationDomainConfigs -> IO ()
           callback strat = do
             atomically $ modifyTVar tEnv $ updateFedStrat strat
             print strat
@@ -100,7 +99,7 @@ run opts = do
         void $ waitAnyCancel [envUpdateThread, internalServerThread, externalServerThread]
   where
     updateFedStrat :: FederationDomainConfigs -> Env -> Env
-    updateFedStrat fedDomConfigs = Federator.Env.runSettings %~ \s -> s { federationStrategy = AllowList $ AllowedDomains $ domain <$> fromFederationDomainConfigs fedDomConfigs }
+    updateFedStrat fedDomConfigs = Federator.Env.runSettings %~ \s -> s {federationStrategy = AllowList $ AllowedDomains $ domain <$> fromFederationDomainConfigs fedDomConfigs}
 
     endpointInternal = federatorInternal opts
     portInternal = fromIntegral $ endpointInternal ^. epPort
@@ -126,9 +125,7 @@ run opts = do
       either
         print
         update
-        strat      
-        
-        
+        strat
 
 -------------------------------------------------------------------------------
 -- Environment
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index e0d96da0ee..343d9e6d10 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -1,5 +1,5 @@
-{-# LANGUAGE StrictData #-}
 {-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE StrictData #-}
 
 -- This file is part of the Wire Server implementation.
 --
@@ -99,6 +99,7 @@ import Polysemy.Internal (Append)
 import Polysemy.Resource
 import qualified Polysemy.TinyLog as P
 import qualified Servant
+import qualified Servant.Client as SC
 import Ssl.Util
 import qualified System.Logger as Log
 import System.Logger.Class
@@ -107,10 +108,9 @@ import qualified UnliftIO.Exception as UnliftIO
 import Util.Options
 import Wire.API.Error
 import Wire.API.Federation.Error
-import qualified Wire.Sem.Logger
-import qualified Servant.Client as SC
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
+import qualified Wire.Sem.Logger
 
 -- Effects needed by the interpretation of other effects
 type GalleyEffects0 =
@@ -160,10 +160,10 @@ createEnv m o = do
   mgr <- initHttpManager o
   h2mgr <- initHttp2Manager
   validateOptions l o
-  
+
   -- Fetch the initial federation domain list so we always start with
   -- a known update to date dataset.
-  
+
   let brigEndpoint = o ^. optBrig
       Endpoint h p = brigEndpoint
       baseUrl = SC.BaseUrl SC.Http (unpack h) (fromIntegral p) ""
diff --git a/services/galley/src/Galley/Env.hs b/services/galley/src/Galley/Env.hs
index dc460be470..0232fd4f17 100644
--- a/services/galley/src/Galley/Env.hs
+++ b/services/galley/src/Galley/Env.hs
@@ -40,8 +40,8 @@ import System.Logger
 import Util.Options
 import Wire.API.MLS.Credential
 import Wire.API.MLS.Keys
-import Wire.API.Team.Member
 import Wire.API.Routes.FederationDomainConfig (FederationDomainConfigs)
+import Wire.API.Team.Member
 
 data DeleteItem = TeamItem TeamId UserId (Maybe ConnId)
   deriving (Eq, Ord, Show)
diff --git a/services/galley/src/Galley/Monad.hs b/services/galley/src/Galley/Monad.hs
index ad9ef51c6a..83cb34e5ed 100644
--- a/services/galley/src/Galley/Monad.hs
+++ b/services/galley/src/Galley/Monad.hs
@@ -32,7 +32,7 @@ import Polysemy.Input
 import System.Logger
 import qualified System.Logger.Class as LC
 
-newtype App a = App {unApp ::ReaderT Env IO a}
+newtype App a = App {unApp :: ReaderT Env IO a}
   deriving
     ( Functor,
       Applicative,
diff --git a/services/galley/src/Galley/Options.hs b/services/galley/src/Galley/Options.hs
index 8f9197a425..d40bd4b2ed 100644
--- a/services/galley/src/Galley/Options.hs
+++ b/services/galley/src/Galley/Options.hs
@@ -52,7 +52,7 @@ module Galley.Options
     optLogLevel,
     optLogNetStrings,
     optLogFormat,
-    optDomainUpdateInterval
+    optDomainUpdateInterval,
   )
 where
 
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 1f92e41a51..11a5d54e9f 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -39,6 +39,7 @@ import Data.Metrics.AWS (gaugeTokenRemaing)
 import qualified Data.Metrics.Middleware as M
 import Data.Metrics.Servant (servantPlusWAIPrometheusMiddleware)
 import Data.Misc (portNumber)
+import qualified Data.Set as Set
 import Data.String.Conversions (cs)
 import Data.Text (unpack)
 import qualified Galley.API as API
@@ -68,15 +69,14 @@ import Servant.Client
     runClientM,
   )
 import qualified System.Logger as Log
+import qualified System.Logger.Class as L
 import Util.Options
 import Wire.API.Routes.API
+import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
-import qualified System.Logger.Class as L
-import qualified Data.Set as Set
-import Wire.API.Routes.FederationDomainConfig
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -197,8 +197,8 @@ updateFedDomains = do
     previous <- liftIO $ readTVarIO tvar
     strat <- liftIO $ runClientM getFedRemotes clientEnv
     let domainListsEqual s =
-          Set.fromList (fromFederationDomainConfigs s) ==
-          Set.fromList (fromFederationDomainConfigs previous)
+          Set.fromList (fromFederationDomainConfigs s)
+            == Set.fromList (fromFederationDomainConfigs previous)
     case strat of
       Left e -> L.err . L.msg $ "Could not retrieve federation domains from brig: " <> show e
       -- Using Set to do the comparison, as it will handle the lists being in different orders.
@@ -211,4 +211,4 @@ updateFedDomains = do
         -- FS-1179 is handling this part.
         liftIO $ atomically $ writeTVar tvar s
   where
-    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
\ No newline at end of file
+    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs
index 5689797051..a29345a79c 100644
--- a/services/galley/test/integration/API.hs
+++ b/services/galley/test/integration/API.hs
@@ -2969,7 +2969,7 @@ testAddRemoteMemberInvalidDomain = do
 
   postQualifiedMembers alice (remoteBob :| []) convId
     !!! do
-      const 422 === statusCode
+      const 400 === statusCode
       const (Just "/federation/api-version")
         === preview (ix "data" . ix "path") . responseJsonUnsafe @Value
       const (Just "invalid.example.com")

From bfa663e42bdd60955c7c1122f6e4db190dfaea4b Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 2 May 2023 10:46:45 +0200
Subject: [PATCH 022/220] Don't allow removing remote domains from config file
 via rest api.

---
 .../wire-api-federation/src/Wire/API/Federation/Error.hs | 5 ++++-
 services/brig/src/Brig/API/Internal.hs                   | 9 ++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/libs/wire-api-federation/src/Wire/API/Federation/Error.hs b/libs/wire-api-federation/src/Wire/API/Federation/Error.hs
index 1f6f297e80..f141b8430f 100644
--- a/libs/wire-api-federation/src/Wire/API/Federation/Error.hs
+++ b/libs/wire-api-federation/src/Wire/API/Federation/Error.hs
@@ -149,7 +149,10 @@ data FederationError
     -- indicate a bug in either backend, or an incompatibility in the
     -- server-to-server API.
     FederationUnexpectedBody Text
-  | -- | Federator client got an unexpected error response from remote backend
+  | -- | Federator client got an unexpected error response from remote backend.
+    -- Also used for error conditions that will go away in a future release,
+    -- like "can't delete remote domains from config file", which is only
+    -- needed until we start disregarding the config file.
     FederationUnexpectedError Text
   deriving (Show, Typeable)
 
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 6057e058ac..f6e5c1a6a2 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -194,7 +194,14 @@ getFederationRemotes = lift $ do
   pure . FederationDomainConfigs . nub $ db <> cfg
 
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
-deleteFederationRemotes = lift . wrapClient . Data.deleteFederationRemote
+deleteFederationRemotes dom = do
+  cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
+  when (dom `elem` (domain <$> cfg)) $ do
+    throwStd . fedError . FederationUnexpectedError $
+      "keeping track of remote domains in the brig config file is deprecated, but as long as we \
+      \do that, removing items listed in the config file is not allowed."
+  -- FUTUREWORK: see 'getFederationRemotes'.
+  lift . wrapClient . Data.deleteFederationRemote $ dom
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)

From d9b30c3e43f350ef2978d77ffddc35c63e8ae843 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 2 May 2023 19:02:08 +1000
Subject: [PATCH 023/220] FS-1115: Fixing compile errors on a new test

---
 services/brig/src/Brig/API/Internal.hs           | 3 ++-
 services/brig/test/integration/API/Federation.hs | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index f6e5c1a6a2..23bd8f7d9e 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -88,6 +88,7 @@ import Wire.API.Connection
 import Wire.API.Error
 import qualified Wire.API.Error.Brig as E
 import Wire.API.Federation.API
+import Wire.API.Federation.Error (FederationError (..))
 import Wire.API.MLS.Credential
 import Wire.API.MLS.KeyPackage
 import Wire.API.MLS.Serialisation
@@ -197,7 +198,7 @@ deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
   cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
   when (dom `elem` (domain <$> cfg)) $ do
-    throwStd . fedError . FederationUnexpectedError $
+    throwError . fedError . FederationUnexpectedError $
       "keeping track of remote domains in the brig config file is deprecated, but as long as we \
       \do that, removing items listed in the config file is not allowed."
   -- FUTUREWORK: see 'getFederationRemotes'.
diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index c747a10354..e670173461 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -479,7 +479,7 @@ crudFederationRemotes opts brig = do
   addFederationRemote brig remote2
   res3 <- getFederationRemotes brig
   liftIO $ assertEqual "should return config values and {good,evil}.example.com" (nub $ sort $ cfgRemotes <> [remote1, remote2]) (sort res3)
-
+ 
   deleteFederationRemote brig (domain remote1)
   res4 <- getFederationRemotes brig
   liftIO $ assertEqual "should return config values and evil.example.com" (nub $ sort $ cfgRemotes <> [remote2]) (sort res4)

From 8c7b646a021d9bbaf3163d8686fd01d6c819a897 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 3 May 2023 13:54:03 +1000
Subject: [PATCH 024/220] FS-1115: Fixing the federation CRUD test

---
 services/brig/test/integration/API/Federation.hs |  4 ++--
 services/brig/test/integration/Util.hs           | 12 +++++++++---
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index e670173461..dc108eb438 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -465,7 +465,7 @@ crudFederationRemotes opts brig = do
   -- Delete the remotes from the database
   -- This doesn't do anything with the remotes
   -- defined in config files.
-  resetFederationRemotes brig
+  resetFederationRemotes opts brig
 
   res1 <- getFederationRemotes brig
   liftIO $ assertEqual "should return config values" cfgRemotes res1
@@ -479,7 +479,7 @@ crudFederationRemotes opts brig = do
   addFederationRemote brig remote2
   res3 <- getFederationRemotes brig
   liftIO $ assertEqual "should return config values and {good,evil}.example.com" (nub $ sort $ cfgRemotes <> [remote1, remote2]) (sort res3)
- 
+
   deleteFederationRemote brig (domain remote1)
   res4 <- getFederationRemotes brig
   liftIO $ assertEqual "should return config values and evil.example.com" (nub $ sort $ cfgRemotes <> [remote2]) (sort res4)
diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs
index 51f525ef8a..f90c331667 100644
--- a/services/brig/test/integration/Util.hs
+++ b/services/brig/test/integration/Util.hs
@@ -29,6 +29,7 @@ import Brig.AWS.Types
 import Brig.App (applog, fsWatcher, sftEnv, turnEnv)
 import Brig.Calling as Calling
 import qualified Brig.Code as Code
+import Brig.Options (Opts)
 import qualified Brig.Options as Opt
 import qualified Brig.Options as Opts
 import qualified Brig.Run as Run
@@ -1088,10 +1089,15 @@ deleteFederationRemote :: Brig -> Domain -> Http ()
 deleteFederationRemote brig rdom =
   void $ delete (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . expect2xx)
 
-resetFederationRemotes :: Brig -> Http ()
-resetFederationRemotes brig = do
+resetFederationRemotes :: Opts -> Brig -> Http ()
+resetFederationRemotes opts brig = do
   rs <- getFederationRemotes brig
-  forM_ rs $ \(FederationDomainConfig rdom _) -> deleteFederationRemote brig rdom
+  -- Filter out domains that are in the config file.
+  -- These values can't be deleted yet, so don't even try.
+  forM_ (notCfgRemotes rs) $ \(FederationDomainConfig rdom _) -> deleteFederationRemote brig rdom
+  where
+    cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
+    notCfgRemotes = filter (`notElem` cfgRemotes)
 
 -- | Run a probe several times, until a "good" value materializes or until patience runs out
 aFewTimes ::

From d9f429856fc48540d4354fd7c247d0068a4a3a62 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 3 May 2023 16:49:07 +1000
Subject: [PATCH 025/220] FS-1115: Adding federation domain updates to brig and
 cannon

---
 services/cannon/cannon.cabal               |  1 +
 services/cannon/cannon.integration.yaml    |  5 +++
 services/cannon/cannon2.integration.yaml   |  5 +++
 services/cannon/default.nix                |  2 ++
 services/cannon/src/Cannon/Options.hs      | 17 +++++++++-
 services/cannon/src/Cannon/Run.hs          | 39 +++++++++++++++++++++-
 services/cargohold/test/integration/API.hs |  2 +-
 services/federator/src/Federator/Run.hs    |  4 +--
 services/gundeck/default.nix               |  2 ++
 services/gundeck/gundeck.cabal             |  1 +
 services/gundeck/gundeck.integration.yaml  |  6 ++++
 services/gundeck/src/Gundeck/Options.hs    |  3 ++
 services/gundeck/src/Gundeck/Run.hs        | 38 +++++++++++++++++++++
 13 files changed, 119 insertions(+), 6 deletions(-)

diff --git a/services/cannon/cannon.cabal b/services/cannon/cannon.cabal
index 75c1e6188c..2be10d9594 100644
--- a/services/cannon/cannon.cabal
+++ b/services/cannon/cannon.cabal
@@ -100,6 +100,7 @@ library
     , mwc-random             >=0.13
     , retry                  >=0.7
     , safe-exceptions
+    , servant-client
     , servant-conduit
     , servant-server
     , strict                 >=0.3.2
diff --git a/services/cannon/cannon.integration.yaml b/services/cannon/cannon.integration.yaml
index f64f3c104f..e2d3e243c2 100644
--- a/services/cannon/cannon.integration.yaml
+++ b/services/cannon/cannon.integration.yaml
@@ -16,6 +16,11 @@ gundeck:
   host: 127.0.0.1
   port: 8086
 
+brig:
+  host: 0.0.0.0
+  port: 8082
+domainUpdateInterval: 1000000
+
 drainOpts:
   gracePeriodSeconds: 1
   millisecondsBetweenBatches: 500
diff --git a/services/cannon/cannon2.integration.yaml b/services/cannon/cannon2.integration.yaml
index 5c25937652..3baf6a8100 100644
--- a/services/cannon/cannon2.integration.yaml
+++ b/services/cannon/cannon2.integration.yaml
@@ -16,6 +16,11 @@ gundeck:
   host: 127.0.0.1
   port: 8086
 
+brig:
+  host: 0.0.0.0
+  port: 8082
+domainUpdateInterval: 1000000
+
 drainOpts:
   gracePeriodSeconds: 1
   millisecondsBetweenBatches: 5
diff --git a/services/cannon/default.nix b/services/cannon/default.nix
index 1032b92eb1..746e0d09e0 100644
--- a/services/cannon/default.nix
+++ b/services/cannon/default.nix
@@ -31,6 +31,7 @@
 , random
 , retry
 , safe-exceptions
+, servant-client
 , servant-conduit
 , servant-server
 , strict
@@ -81,6 +82,7 @@ mkDerivation {
     mwc-random
     retry
     safe-exceptions
+    servant-client
     servant-conduit
     servant-server
     strict
diff --git a/services/cannon/src/Cannon/Options.hs b/services/cannon/src/Cannon/Options.hs
index bce5cba50c..0eab5e3908 100644
--- a/services/cannon/src/Cannon/Options.hs
+++ b/services/cannon/src/Cannon/Options.hs
@@ -24,6 +24,8 @@ module Cannon.Options
     port,
     cannon,
     gundeck,
+    brig,
+    Brig (..),
     externalHost,
     externalHostFile,
     logLevel,
@@ -35,6 +37,7 @@ module Cannon.Options
     millisecondsBetweenBatches,
     minBatchSize,
     disabledAPIVersions,
+    domainUpdateInterval,
     DrainOpts,
   )
 where
@@ -67,6 +70,15 @@ makeFields ''Gundeck
 
 deriveApiFieldJSON ''Gundeck
 
+data Brig = Brig
+  { _brigHost :: !Text,
+    _brigPort :: !Word16
+  }
+  deriving (Eq, Show, Generic)
+
+makeFields ''Brig
+deriveApiFieldJSON ''Brig
+
 data DrainOpts = DrainOpts
   { -- | Maximum amount of time draining should take. Must not be set to 0.
     _drainOptsGracePeriodSeconds :: Word64,
@@ -87,11 +99,14 @@ deriveApiFieldJSON ''DrainOpts
 data Opts = Opts
   { _optsCannon :: !Cannon,
     _optsGundeck :: !Gundeck,
+    _optsBrig :: !Brig,
     _optsLogLevel :: !Level,
     _optsLogNetStrings :: !(Maybe (Last Bool)),
     _optsLogFormat :: !(Maybe (Last LogFormat)),
     _optsDrainOpts :: DrainOpts,
-    _optsDisabledAPIVersions :: Maybe (Set Version)
+    _optsDisabledAPIVersions :: Maybe (Set Version),
+    -- | Domain update interval (microseconds)
+    _optsDomainUpdateInterval :: !Int
   }
   deriving (Eq, Show, Generic)
 
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index 8fb26e7e4f..22967b08a0 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -37,7 +37,7 @@ import Data.Metrics.Middleware (gaugeSet, path)
 import qualified Data.Metrics.Middleware as Middleware
 import Data.Metrics.Servant
 import Data.Proxy
-import Data.Text (pack, strip)
+import Data.Text (pack, strip, unpack)
 import Data.Text.Encoding (encodeUtf8)
 import Imports hiding (head)
 import qualified Network.Wai as Wai
@@ -45,6 +45,7 @@ import Network.Wai.Handler.Warp hiding (run)
 import qualified Network.Wai.Middleware.Gzip as Gzip
 import Network.Wai.Utilities.Server
 import Servant
+import Servant.Client
 import qualified System.IO.Strict as Strict
 import qualified System.Logger.Class as LC
 import qualified System.Logger.Extended as L
@@ -52,7 +53,10 @@ import System.Posix.Signals
 import qualified System.Posix.Signals as Signals
 import System.Random.MWC (createSystemRandom)
 import UnliftIO.Concurrent (myThreadId, throwTo)
+import Wire.API.Routes.FederationDomainConfig
+import qualified Wire.API.Routes.Internal.Brig as IAPI
 import qualified Wire.API.Routes.Internal.Cannon as Internal
+import Wire.API.Routes.Named (namedClient)
 import Wire.API.Routes.Public.Cannon
 import Wire.API.Routes.Version.Wai
 
@@ -75,6 +79,27 @@ run o = do
       <*> mkClock
   refreshMetricsThread <- Async.async $ runCannon' e refreshMetrics
   s <- newSettings $ Server (o ^. cannon . host) (o ^. cannon . port) (applog e) m (Just idleTimeout)
+
+  -- Get the federaion domain list from Brig and start the updater loop
+  manager <- newManager defaultManagerSettings
+  let Brig bh bp = o ^. brig
+      baseUrl = BaseUrl Http (unpack bh) (fromIntegral bp) ""
+      clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
+      -- Loop the request until we get an answer. This is helpful during integration
+      -- tests where services are being brought up in parallel.
+      getInitialFedDomains = do
+        runClientM getFedRemotes clientEnv >>= \case
+          Right strat -> pure strat
+          Left err -> do
+            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show err
+            threadDelay $ o ^. domainUpdateInterval
+            getInitialFedDomains
+  fedStrat <- getInitialFedDomains
+  tEnv <- newTVarIO fedStrat
+  let callback :: FederationDomainConfigs -> IO ()
+      callback = atomically . writeTVar tEnv
+  updateDomainsThread <- Async.async $ updateDomains clientEnv callback
+
   let middleware :: Wai.Middleware
       middleware =
         versionMiddleware (fold (o ^. disabledAPIVersions))
@@ -96,6 +121,7 @@ run o = do
     -- the same time and then calling the drain script. I suspect this might be due to some
     -- cleanup in wai.  this needs to be tested very carefully when touched.
     Async.cancel refreshMetricsThread
+    Async.cancel updateDomainsThread
     L.close (applog e)
   where
     idleTimeout = fromIntegral $ maxPingInterval + 3
@@ -108,6 +134,17 @@ run o = do
     readExternal :: FilePath -> IO ByteString
     readExternal f = encodeUtf8 . strip . pack <$> Strict.readFile f
 
+    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
+
+    updateDomains :: ClientEnv -> (FederationDomainConfigs -> IO ()) -> IO ()
+    updateDomains clientEnv update = forever $ do
+      threadDelay $ o ^. domainUpdateInterval
+      strat <- runClientM getFedRemotes clientEnv
+      either
+        print
+        update
+        strat
+
 signalHandler :: Env -> ThreadId -> Signals.Handler
 signalHandler e mainThread = CatchOnce $ do
   runWS e drain
diff --git a/services/cargohold/test/integration/API.hs b/services/cargohold/test/integration/API.hs
index 88f733e763..5d4ff3ad89 100644
--- a/services/cargohold/test/integration/API.hs
+++ b/services/cargohold/test/integration/API.hs
@@ -308,7 +308,7 @@ testRemoteDownloadWrongDomain = do
   let key = AssetKeyV3 assetId AssetPersistent
       qkey = Qualified key (Domain "invalid.example.com")
   downloadAsset uid qkey () !!! do
-    const 422 === statusCode
+    const 400 === statusCode
 
 testRemoteDownloadNoAsset :: TestM ()
 testRemoteDownloadNoAsset = do
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index f4be91cebf..e71ea9e5e9 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -86,9 +86,7 @@ run opts = do
       fedStrat <- getInitialFedDomains
       tEnv <- newTVarIO $ updateFedStrat fedStrat env
       let callback :: FederationDomainConfigs -> IO ()
-          callback strat = do
-            atomically $ modifyTVar tEnv $ updateFedStrat strat
-            print strat
+          callback = atomically . modifyTVar tEnv . updateFedStrat
       -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
       let externalServer = serveInward tEnv portExternal
           internalServer = serveOutward tEnv portInternal
diff --git a/services/gundeck/default.nix b/services/gundeck/default.nix
index 8c74c72e95..2d76650bff 100644
--- a/services/gundeck/default.nix
+++ b/services/gundeck/default.nix
@@ -56,6 +56,7 @@
 , safe
 , safe-exceptions
 , scientific
+, servant-client
 , servant-server
 , string-conversions
 , tagged
@@ -123,6 +124,7 @@ mkDerivation {
     resourcet
     retry
     safe-exceptions
+    servant-client
     servant-server
     text
     time
diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal
index 8f586043cd..a7465feca1 100644
--- a/services/gundeck/gundeck.cabal
+++ b/services/gundeck/gundeck.cabal
@@ -133,6 +133,7 @@ library
     , resourcet              >=1.1
     , retry                  >=0.5
     , safe-exceptions
+    , servant-client
     , servant-server
     , text                   >=1.1
     , time                   >=1.4
diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml
index 1d56cfdf88..dde2717184 100644
--- a/services/gundeck/gundeck.integration.yaml
+++ b/services/gundeck/gundeck.integration.yaml
@@ -2,6 +2,10 @@ gundeck:
   host: 0.0.0.0
   port: 8086
 
+brig:
+  host: 0.0.0.0
+  port: 8082
+
 cassandra:
   endpoint:
     host: 127.0.0.1
@@ -37,5 +41,7 @@ settings:
     hard: 30  # more than this number of threads will not be allowed
     soft: 10  # more than this number of threads will be warned about
 
+domainUpdateInterval: 1000000
+
 logLevel: Warn
 logNetStrings: false
diff --git a/services/gundeck/src/Gundeck/Options.hs b/services/gundeck/src/Gundeck/Options.hs
index 8e23457e9d..8971f1c3b5 100644
--- a/services/gundeck/src/Gundeck/Options.hs
+++ b/services/gundeck/src/Gundeck/Options.hs
@@ -116,12 +116,15 @@ deriveFromJSON toOptionFieldName ''Settings
 data Opts = Opts
   { -- | Hostname and port to bind to
     _optGundeck :: !Endpoint,
+    _optBrig :: !Endpoint,
     _optCassandra :: !CassandraOpts,
     _optRedis :: !RedisEndpoint,
     _optRedisAdditionalWrite :: !(Maybe RedisEndpoint),
     _optAws :: !AWSOpts,
     _optDiscoUrl :: !(Maybe Text),
     _optSettings :: !Settings,
+    -- | Domain update interval (microseconds)
+    _optDomainUpdateInterval :: !Int,
     -- Logging
 
     -- | Log level (Debug, Info, etc)
diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs
index 66062f96e6..a55fa5ee7b 100644
--- a/services/gundeck/src/Gundeck/Run.hs
+++ b/services/gundeck/src/Gundeck/Run.hs
@@ -43,15 +43,20 @@ import Gundeck.Options
 import Gundeck.React
 import Gundeck.ThreadBudget
 import Imports hiding (head)
+import Network.HTTP.Client (defaultManagerSettings, newManager)
 import Network.Wai as Wai
 import qualified Network.Wai.Middleware.Gunzip as GZip
 import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server hiding (serverPort)
 import Servant (Handler (Handler), (:<|>) (..))
 import qualified Servant
+import Servant.Client
 import qualified System.Logger as Log
 import qualified UnliftIO.Async as Async
 import Util.Options
+import Wire.API.Routes.FederationDomainConfig
+import qualified Wire.API.Routes.Internal.Brig as IAPI
+import Wire.API.Routes.Named (namedClient)
 import Wire.API.Routes.Public.Gundeck (GundeckAPI)
 import Wire.API.Routes.Version.Wai
 
@@ -64,6 +69,27 @@ run o = do
   let l = e ^. applog
   s <- newSettings $ defaultServer (unpack $ o ^. optGundeck . epHost) (o ^. optGundeck . epPort) l m
   let throttleMillis = fromMaybe defSqsThrottleMillis $ o ^. (optSettings . setSqsThrottleMillis)
+
+  -- Get the federaion domain list from Brig and start the updater loop
+  mgr <- newManager defaultManagerSettings
+  let Endpoint host port = o ^. optBrig
+      baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
+      clientEnv = ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
+      -- Loop the request until we get an answer. This is helpful during integration
+      -- tests where services are being brought up in parallel.
+      getInitialFedDomains = do
+        runClientM getFedRemotes clientEnv >>= \case
+          Right strat -> pure strat
+          Left err -> do
+            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show err
+            threadDelay $ o ^. optDomainUpdateInterval
+            getInitialFedDomains
+  fedStrat <- getInitialFedDomains
+  tEnv <- newTVarIO fedStrat
+  let callback :: FederationDomainConfigs -> IO ()
+      callback = atomically . writeTVar tEnv
+  updateDomainsThread <- Async.async $ updateDomains clientEnv callback
+
   lst <- Async.async $ Aws.execute (e ^. awsEnv) (Aws.listen throttleMillis (runDirect e . onEvent))
   wtbs <- forM (e ^. threadBudgetState) $ \tbs -> Async.async $ runDirect e $ watchThreadBudgetState m tbs 10
   wCollectAuth <- Async.async (collectAuthMetrics m (Aws._awsEnv (Env._awsEnv e)))
@@ -72,6 +98,7 @@ run o = do
     shutdown (e ^. cstate)
     Async.cancel lst
     Async.cancel wCollectAuth
+    Async.cancel updateDomainsThread
     forM_ wtbs Async.cancel
     forM_ rThreads Async.cancel
     Redis.disconnect =<< takeMVar (e ^. rstate)
@@ -86,6 +113,17 @@ run o = do
         . GZip.gzip GZip.def
         . catchErrors (e ^. applog) [Right $ e ^. monitor]
 
+    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
+
+    updateDomains :: ClientEnv -> (FederationDomainConfigs -> IO ()) -> IO ()
+    updateDomains clientEnv update = forever $ do
+      threadDelay $ o ^. optDomainUpdateInterval
+      strat <- runClientM getFedRemotes clientEnv
+      either
+        print
+        update
+        strat
+
 type CombinedAPI = GundeckAPI :<|> Servant.Raw
 
 mkApp :: Env -> Wai.Application

From a8156520ee49b02faf19271ebe68b255e622f58b Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 4 May 2023 23:07:51 +0200
Subject: [PATCH 026/220] Docs (draft), release notes.

---
 changelog.d/0-release-notes/pr-3260           |  1 +
 .../developer/federation-design-aspects.md    | 30 ++++++++++++++++
 .../federation/backend-communication.md       | 36 +++++++++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 changelog.d/0-release-notes/pr-3260
 create mode 100644 docs/src/developer/developer/federation-design-aspects.md

diff --git a/changelog.d/0-release-notes/pr-3260 b/changelog.d/0-release-notes/pr-3260
new file mode 100644
index 0000000000..7b836ffe2a
--- /dev/null
+++ b/changelog.d/0-release-notes/pr-3260
@@ -0,0 +1 @@
+Federation only: from this release on, remote connections should be configured via an internal REST API; the config files will be honored for a transition period, but will be ignored starting in a future release.  [Details in the docs.](https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
\ No newline at end of file
diff --git a/docs/src/developer/developer/federation-design-aspects.md b/docs/src/developer/developer/federation-design-aspects.md
new file mode 100644
index 0000000000..2232efd7d7
--- /dev/null
+++ b/docs/src/developer/developer/federation-design-aspects.md
@@ -0,0 +1,30 @@
+# Federation Design Aspects
+
+(configuring-remote-connections-dev-perspective)=
+
+## keeping track of federator remotes
+
+Federation can start and end.  These events need handlers to be called
+(like remove remote users from local conv), plus it is not convenient
+to edit and re-deploy config files every time that happens.  Hence
+remotes are stored in cassandra in brig, and every pod of every
+service keeps a cache in a `TVar` (this information is needed in many
+end-points).
+
+This secion elaborates on the implementation.  See
+{ref}`configuring-remote-connections` for the administrator's point of
+view.
+
+The state is persistent in cassandra table `brig.federation_remotes`
+brig itself for performance keeps a `TVar` that it updates at regular
+intervals.  Plus provides the contents of the `TVar` via an internal
+[CRUD API](TODO: swagger docs).
+
+Update intervals could be made configurable in config files, but we
+chose to hard-wire this for now: values are [TODO].
+
+Transition from config file to cassandra table: we consider the union
+for now, and don't allow removing remote hosts that are (also) given
+in the config file.  A future release will stop honoring the config
+file altogether.  By then you'll have to be done getting the data into
+cassandra.
diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index a71c6e158b..ddf02e8d10 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -153,3 +153,39 @@ search request from *Alice*, one of its clients.
 :width: 100%
 :align: center
 ```
+
+(configuring-remote-connections)=
+
+## Configuring Remote Connections
+
+Up to release 4.36.0, the config file statically contains information
+about the remote connections in the configs of all services that need
+to know.  **TODO: elaborate.  also, this has probably been documented
+elsewhere?  maybe move this section there, or at least link?**
+
+Since release 4.36.0, there is an internal REST API for adding remote
+wire-server instances:
+
+* [get](TODO: swagger urls...)
+* [post](TODO: swagger urls...)
+* [delete](TODO: swagger urls...)
+
+Changing the configuration of existing edges is not implemented at the
+moment.
+
+See /developer/developer/federation-design-aspects.html for details
+See {ref}`configuring-remote-connections-dev-perspective` for the
+developer's point of view on this topic.
+
+### Transitioning from config file to database state
+
+As of release 4.36.0, federation config file info about remote
+backends are ignored, and brig info is used instead.  For a transition
+period, brig reports with the union of its config file data and
+cassandra data.  It is not allowed to dynamically delete a remote
+backend that is contained in the config file.
+
+In the future, wire-server will stop honoring the config file data at
+all, and solely rely on the cassandra data.  From that point onward,
+you can delete any connection.  Watch out for the release notes for
+when this happens.

From 0ac36828b1087747bf3b74251f507b62b92e6c84 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 4 May 2023 23:16:08 +0200
Subject: [PATCH 027/220] (Slightly) better api error on fail-to-delete.

---
 services/brig/src/Brig/API/Internal.hs    |  8 +++++++-
 services/brig/src/Brig/Data/Federation.hs | 10 +++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 23bd8f7d9e..b8f75eb35c 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -184,7 +184,13 @@ federationRemotesAPI =
 
 addFederationRemote :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 addFederationRemote fedDomConf = do
-  lift . wrapClient $ Data.addFederationRemote fedDomConf
+  result <- lift . wrapClient $ Data.addFederationRemote fedDomConf
+  case result of
+    Data.AddFederationRemoteSuccess -> pure ()
+    Data.AddFederationRemoteMaxRemotesReached ->
+      throwError . fedError . FederationUnexpectedError $
+        "Maximum number of remote backends reached.  If you need to create more connections, \
+        \please contact wire.com."
 
 getFederationRemotes :: ExceptT Brig.API.Error.Error (AppT r) FederationDomainConfigs
 getFederationRemotes = lift $ do
diff --git a/services/brig/src/Brig/Data/Federation.hs b/services/brig/src/Brig/Data/Federation.hs
index 23bb1f2e86..ce2deb5da7 100644
--- a/services/brig/src/Brig/Data/Federation.hs
+++ b/services/brig/src/Brig/Data/Federation.hs
@@ -19,6 +19,7 @@ module Brig.Data.Federation
   ( getFederationRemotes,
     addFederationRemote,
     deleteFederationRemote,
+    AddFederationRemoteResult (..),
   )
 where
 
@@ -41,11 +42,14 @@ getFederationRemotes = uncurry FederationDomainConfig <$$> qry
     get :: PrepQuery R () (Domain, FederatedUserSearchPolicy)
     get = fromString $ "SELECT domain, search_policy FROM federation_remotes LIMIT " <> show maxKnownNodes
 
-addFederationRemote :: MonadClient m => FederationDomainConfig -> m ()
+data AddFederationRemoteResult = AddFederationRemoteSuccess | AddFederationRemoteMaxRemotesReached
+
+addFederationRemote :: MonadClient m => FederationDomainConfig -> m AddFederationRemoteResult
 addFederationRemote (FederationDomainConfig rdom searchpolicy) = do
   l <- length <$> getFederationRemotes
-  when (l >= maxKnownNodes) $ error "TODO: make this error better"
-  retry x5 $ write add (params LocalQuorum (rdom, searchpolicy))
+  if l >= maxKnownNodes
+    then pure AddFederationRemoteMaxRemotesReached
+    else AddFederationRemoteSuccess <$ retry x5 (write add (params LocalQuorum (rdom, searchpolicy)))
   where
     add :: PrepQuery W (Domain, FederatedUserSearchPolicy) ()
     add = "INSERT INTO federation_remotes (domain, search_policy) VALUES (?, ?)"

From a478868ac0cded6c3c65d518d1421072d2caef95 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 4 May 2023 23:17:15 +0200
Subject: [PATCH 028/220] Breadcrumbs.

---
 charts/brig/templates/configmap.yaml           | 3 +++
 docs/src/developer/reference/config-options.md | 5 +++++
 hack/helm_vars/wire-server/values.yaml.gotmpl  | 7 +++++--
 services/brig/src/Brig/API/Internal.hs         | 4 +++-
 services/brig/src/Brig/Options.hs              | 6 ++++--
 5 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml
index 781f90c9f1..82fc34246b 100644
--- a/charts/brig/templates/configmap.yaml
+++ b/charts/brig/templates/configmap.yaml
@@ -246,6 +246,9 @@ data:
       setDeleteThrottleMillis: {{ .setDeleteThrottleMillis }}
       setFederationDomain: {{ .setFederationDomain }}
       {{- if .setFederationDomainConfigs }}
+      # 'setFederationDomainConfigs' is deprecated as of release 4.36.0.  See
+      # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
+      # for details.
       setFederationDomainConfigs: {{ toYaml .setFederationDomainConfigs | nindent 8 }}
       {{- end }}
       {{- if .setSearchSameTeamOnly }}
diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md
index 98f3770431..7f40ce53a0 100644
--- a/docs/src/developer/reference/config-options.md
+++ b/docs/src/developer/reference/config-options.md
@@ -611,6 +611,11 @@ any key package whose expiry date is set further than 15 days after upload time
 
 
 ### Federated domain specific configuration settings
+
+**This section is deprecated as of Release 4.36.0.  See
+https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
+for details.**
+
 #### Restrict user search
 
 The lookup and search of users on a wire instance can be configured. This can be done per federated domain.
diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl
index 82f965a399..0e08e4a1bb 100644
--- a/hack/helm_vars/wire-server/values.yaml.gotmpl
+++ b/hack/helm_vars/wire-server/values.yaml.gotmpl
@@ -77,6 +77,9 @@ brig:
       setMaxConvSize: 16
       setFederationDomain: integration.example.com
       setFederationDomainConfigs:
+        # 'setFederationDomainConfigs' is deprecated as of release 4.36.0.  See
+        # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
+        # for details.
         - domain: integration.example.com
           search_policy: full_search
         - domain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
@@ -132,7 +135,7 @@ brig:
         "crv": "Ed25519",
         "x": "mhP-NgFw3ifIXGZqJVB0kemt9L3BtD5P8q4Gah4Iklc",
         "d": "R8-pV2-sPN7dykV8HFJ73S64F3kMHTNnJiSN8UdWk_o"
-      }    
+      }
   tests:
     enableFederationTests: true
 cannon:
@@ -245,7 +248,7 @@ nginz:
           "kty": "OKP",
           "crv": "Ed25519",
           "x": "mhP-NgFw3ifIXGZqJVB0kemt9L3BtD5P8q4Gah4Iklc"
-        }       
+        }
 proxy:
   replicaCount: 1
   imagePullPolicy: {{ .Values.imagePullPolicy }}
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index b8f75eb35c..5ebb16681e 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -197,7 +197,9 @@ getFederationRemotes = lift $ do
   db <- wrapClient Data.getFederationRemotes
   cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
   -- FUTUREWORK: we should solely rely on `db` in the future; `cfg` is just for an easier,
-  -- more robust migration path.
+  -- more robust migration path.  See
+  -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections,
+  -- http://docs.wire.com/developer/developer/federation-design-aspects.html#configuring-remote-connections-dev-perspective
   pure . FederationDomainConfigs . nub $ db <> cfg
 
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index 6ec49300d3..3cc2325314 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -551,8 +551,10 @@ data Settings = Settings
     --     - wire.com
     --     - example.com
     setFederationDomain :: !Domain,
-    setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]), -- TODO: deprecate this in docs and config file samples.
-
+    -- | 'setFederationDomainConfigs' is deprecated as of release 4.36.0.  See
+    -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
+    -- for details.
+    setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]),
     -- | The amount of time in milliseconds to wait after reading from an SQS queue
     -- returns no message, before asking for messages from SQS again.
     -- defaults to 'defSqsThrottleMillis'.

From 66e6969d7999b7b829a51b24e50b6d52462a8ee2 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 12:32:39 +0200
Subject: [PATCH 029/220] Better breadcrumbs.

We don't know which release this will land in yet, PR is more future-proof.
---
 charts/brig/templates/configmap.yaml           | 2 +-
 docs/src/developer/reference/config-options.md | 2 +-
 hack/helm_vars/wire-server/values.yaml.gotmpl  | 2 +-
 services/brig/src/Brig/Options.hs              | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml
index 776e297ab2..60cffe82ad 100644
--- a/charts/brig/templates/configmap.yaml
+++ b/charts/brig/templates/configmap.yaml
@@ -248,7 +248,7 @@ data:
       setDeleteThrottleMillis: {{ .setDeleteThrottleMillis }}
       setFederationDomain: {{ .setFederationDomain }}
       {{- if .setFederationDomainConfigs }}
-      # 'setFederationDomainConfigs' is deprecated as of release 4.36.0.  See
+      # 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
       # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
       # for details.
       setFederationDomainConfigs: {{ toYaml .setFederationDomainConfigs | nindent 8 }}
diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md
index 7f40ce53a0..18b3a2de3c 100644
--- a/docs/src/developer/reference/config-options.md
+++ b/docs/src/developer/reference/config-options.md
@@ -612,7 +612,7 @@ any key package whose expiry date is set further than 15 days after upload time
 
 ### Federated domain specific configuration settings
 
-**This section is deprecated as of Release 4.36.0.  See
+**This section is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
 https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
 for details.**
 
diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl
index 693615e83e..e260b2f254 100644
--- a/hack/helm_vars/wire-server/values.yaml.gotmpl
+++ b/hack/helm_vars/wire-server/values.yaml.gotmpl
@@ -78,7 +78,7 @@ brig:
       # See helmfile for the real value
       setFederationDomain: integration.example.com
       setFederationDomainConfigs:
-        # 'setFederationDomainConfigs' is deprecated as of release 4.36.0.  See
+        # 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
         # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
         # for details.
         - domain: integration.example.com
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index 3cc2325314..7a959264d8 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -551,7 +551,7 @@ data Settings = Settings
     --     - wire.com
     --     - example.com
     setFederationDomain :: !Domain,
-    -- | 'setFederationDomainConfigs' is deprecated as of release 4.36.0.  See
+    -- | 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
     -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
     -- for details.
     setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]),

From cdf36503d981da8f543f873c0f9b0a7187b830f4 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 13:48:28 +0200
Subject: [PATCH 030/220] Polish docs.

---
 .../developer/federation-design-aspects.md    | 14 ++--
 .../src/developer/reference/config-options.md |  5 ++
 .../federation/backend-communication.md       | 68 ++++++++++++-------
 .../src/Wire/API/Routes/Internal/Brig.hs      | 17 ++++-
 4 files changed, 67 insertions(+), 37 deletions(-)

diff --git a/docs/src/developer/developer/federation-design-aspects.md b/docs/src/developer/developer/federation-design-aspects.md
index 2232efd7d7..54e50cd152 100644
--- a/docs/src/developer/developer/federation-design-aspects.md
+++ b/docs/src/developer/developer/federation-design-aspects.md
@@ -13,18 +13,14 @@ end-points).
 
 This secion elaborates on the implementation.  See
 {ref}`configuring-remote-connections` for the administrator's point of
-view.
+view.  Go read that section now!
 
 The state is persistent in cassandra table `brig.federation_remotes`
 brig itself for performance keeps a `TVar` that it updates at regular
 intervals.  Plus provides the contents of the `TVar` via an internal
-[CRUD API](TODO: swagger docs).
+CRUD API (see {ref}`configuring-remote-connections` for the links).
 
-Update intervals could be made configurable in config files, but we
-chose to hard-wire this for now: values are [TODO].
+Update intervals are currently hard-wired into the code.
 
-Transition from config file to cassandra table: we consider the union
-for now, and don't allow removing remote hosts that are (also) given
-in the config file.  A future release will stop honoring the config
-file altogether.  By then you'll have to be done getting the data into
-cassandra.
+Introduced in
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260).
diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md
index 18b3a2de3c..9bd159261a 100644
--- a/docs/src/developer/reference/config-options.md
+++ b/docs/src/developer/reference/config-options.md
@@ -618,6 +618,11 @@ for details.**
 
 #### Restrict user search
 
+TODO: deprecate this, also rename this section.  it's about federation now.
+
+TODO: should we consider the federation strategy from federator in the
+union returned by brig for a transition period as well?
+
 The lookup and search of users on a wire instance can be configured. This can be done per federated domain.
 
 ```yaml
diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index ddf02e8d10..536ebaa268 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -158,34 +158,52 @@ search request from *Alice*, one of its clients.
 
 ## Configuring Remote Connections
 
-Up to release 4.36.0, the config file statically contains information
-about the remote connections in the configs of all services that need
-to know.  **TODO: elaborate.  also, this has probably been documented
-elsewhere?  maybe move this section there, or at least link?**
+Up to the release containing
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260), the
+config file statically contains information about the remote
+connections in the configs of all services that need to know.  Since
+then, there is an internal REST API for adding remote wire-server
+instances:
+
+* [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
+* [`POST`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/post_i_federation_remotes)
+* [`DELETE`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/delete_i_federation_remotes__domain_)
+
+Changing the configuration of existing edges via `PUT` is not
+implemented at the moment, if you need to do that, delete the
+connection and add it again.
+
+If you delete a connection, all users from that remote will be removed
+from local conversations, and all conversations hosted by that remote
+will be removed from the local backend.  Connections between local and
+remote users that are removed will be archived, and can be
+re-established should you decide to add the same backend later.
 
-Since release 4.36.0, there is an internal REST API for adding remote
-wire-server instances:
-
-* [get](TODO: swagger urls...)
-* [post](TODO: swagger urls...)
-* [delete](TODO: swagger urls...)
-
-Changing the configuration of existing edges is not implemented at the
-moment.
-
-See /developer/developer/federation-design-aspects.html for details
 See {ref}`configuring-remote-connections-dev-perspective` for the
 developer's point of view on this topic.
 
 ### Transitioning from config file to database state
 
-As of release 4.36.0, federation config file info about remote
-backends are ignored, and brig info is used instead.  For a transition
-period, brig reports with the union of its config file data and
-cassandra data.  It is not allowed to dynamically delete a remote
-backend that is contained in the config file.
-
-In the future, wire-server will stop honoring the config file data at
-all, and solely rely on the cassandra data.  From that point onward,
-you can delete any connection.  Watch out for the release notes for
-when this happens.
+As of the release containing
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260),
+[`federationStrategy`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/federator/values.yaml#L44-L45)
+in the federation config file is ignored, and brig's cassandra is used
+instead.  Furthermore, for a transition period,
+[`setFederationDomainConfigs`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/brig/templates/configmap.yaml#L250-L252)
+from the brig config file also remains being honored.  Attempting to
+delete entries that occur in the config file will trigger an error;
+delete from the config file first, then from cassandra.
+
+In the future, wire-server will stop honoring the config file data,
+and solely rely on brig's cassandra.  From that point onward, you can
+delete any connection, whether listed in the config file or not.
+Watch out for the release notes to learn when this will happen.
+(Something like *"[Federation only] support for remote configuration
+in config file is discontinued.  Before upgrading to this release,
+upgrade to the release containing
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260) first.
+After upgrading to this release, `setFederationDomainConfigs` in brig's
+config file will be ignored, and you should remove it at your
+convenience.  See
+[docs](https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
+for details."*)
diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index 3e9330a377..07cabed987 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -407,25 +407,36 @@ type AuthAPI =
 type FederationRemotesAPI =
   Named
     "get-federation-remotes"
-    ( "federation"
+    ( Description FederationRemotesAPIDescription
+        :> "federation"
         :> "remotes"
         :> Get '[JSON] FederationDomainConfigs
     )
     :<|> Named
            "add-federation-remotes"
-           ( "federation"
+           ( Description FederationRemotesAPIDescription
+               :> "federation"
                :> "remotes"
                :> ReqBody '[JSON] FederationDomainConfig
                :> Post '[JSON] ()
            )
     :<|> Named
            "delete-federation-remotes"
-           ( "federation"
+           ( Description FederationRemotesAPIDeleteDescription
+               :> "federation"
                :> "remotes"
                :> Capture "domain" Domain
                :> Delete '[JSON] ()
            )
 
+type FederationRemotesAPIDescription =
+  "See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections for background."
+
+type FederationRemotesAPIDeleteDescription =
+  "WARNING!  If you remove a remote connection, all users from that remote will be removed from local conversations, and all \
+  \group conversations hosted by that remote will be removed from the local backend.  This cannot be reverted!  See \
+  \https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections for background."
+
 swaggerDoc :: Swagger
 swaggerDoc =
   toSwagger (Proxy @API)

From ef149e3167dfb7cb44f3673513f9456446e55d73 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 13:58:22 +0200
Subject: [PATCH 031/220] afterthought.

---
 docs/src/developer/reference/config-options.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md
index 9bd159261a..5aa8a2b680 100644
--- a/docs/src/developer/reference/config-options.md
+++ b/docs/src/developer/reference/config-options.md
@@ -621,7 +621,10 @@ for details.**
 TODO: deprecate this, also rename this section.  it's about federation now.
 
 TODO: should we consider the federation strategy from federator in the
-union returned by brig for a transition period as well?
+union returned by brig for a transition period as well?  (if not, we
+need to insist on updating brig's config before this upgrade.  no
+remote backend may be unlisted and use the search policy default.  we
+should also crash on startup when somebody tries that.)
 
 The lookup and search of users on a wire instance can be configured. This can be done per federated domain.
 

From f9f0e874a53fb36ccf67557b8449489f3768a2ad Mon Sep 17 00:00:00 2001
From: fisx <mf@zerobuzz.net>
Date: Mon, 8 May 2023 14:03:33 +0200
Subject: [PATCH 032/220] Update
 services/brig/test/integration/API/Federation.hs

---
 services/brig/test/integration/API/Federation.hs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index dc108eb438..37a92cedf5 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -475,6 +475,11 @@ crudFederationRemotes opts brig = do
   res2 <- getFederationRemotes brig
   liftIO $ assertEqual "should return config values and good.example.com" (nub $ sort $ cfgRemotes <> [remote1]) (sort res2)
 
+  -- idempotency
+  addFederationRemote brig remote1
+  res2' <- getFederationRemotes brig
+  liftIO $ assertEqual "should return config values and good.example.com" (nub $ sort $ cfgRemotes <> [remote1]) (sort res2')
+  
   let remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
   addFederationRemote brig remote2
   res3 <- getFederationRemotes brig

From 10441245b2df9c7f93ed7d6d54dfcd456a54cbd1 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 14:57:55 +0200
Subject: [PATCH 033/220] More tests.

---
 services/brig/test/integration/API/Federation.hs | 5 ++++-
 services/brig/test/integration/Util.hs           | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index 37a92cedf5..ebc3b1d128 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -479,7 +479,7 @@ crudFederationRemotes opts brig = do
   addFederationRemote brig remote1
   res2' <- getFederationRemotes brig
   liftIO $ assertEqual "should return config values and good.example.com" (nub $ sort $ cfgRemotes <> [remote1]) (sort res2')
-  
+
   let remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
   addFederationRemote brig remote2
   res3 <- getFederationRemotes brig
@@ -489,6 +489,9 @@ crudFederationRemotes opts brig = do
   res4 <- getFederationRemotes brig
   liftIO $ assertEqual "should return config values and evil.example.com" (nub $ sort $ cfgRemotes <> [remote2]) (sort res4)
 
+  -- deleting from the config file triggers an error
+  deleteFederationRemote' id brig (domain $ head $ cfgRemotes) !!! const 533 === statusCode
+
   -- TODO: how do we test that the TVar is updated in all services?  some fancy unit test?
   -- duplicate internal end-point to all services, and implement the hanlers in a library?
   pure ()
diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs
index f90c331667..4ea86efa23 100644
--- a/services/brig/test/integration/Util.hs
+++ b/services/brig/test/integration/Util.hs
@@ -1087,7 +1087,11 @@ addFederationRemote brig remote =
 
 deleteFederationRemote :: Brig -> Domain -> Http ()
 deleteFederationRemote brig rdom =
-  void $ delete (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . expect2xx)
+  void $ deleteFederationRemote' expect2xx brig rdom
+
+deleteFederationRemote' :: (Request -> Request) -> Brig -> Domain -> Http ResponseLBS
+deleteFederationRemote' mods brig rdom =
+  delete (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . mods)
 
 resetFederationRemotes :: Opts -> Brig -> Http ()
 resetFederationRemotes opts brig = do

From 5c229383f43a97c22bbe5c2b07b8fc94337ee9a3 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 15:33:13 +0200
Subject: [PATCH 034/220] Remove a lying comment.

---
 services/federator/src/Federator/InternalServer.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs
index 66b953c730..801bc079cf 100644
--- a/services/federator/src/Federator/InternalServer.hs
+++ b/services/federator/src/Federator/InternalServer.hs
@@ -89,8 +89,6 @@ callOutward ::
 callOutward req = do
   rd <- parseRequestData req
   domain <- parseDomainText (rdTargetDomain rd)
-  -- This call will check for new domains
-  -- when it encounters something that it doesn't know about.
   ensureCanFederateWith domain
   resp <-
     discoverAndCall

From 02c1432e78eef8d2f9c48195e0f2407ddf1e106f Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 16:12:04 +0200
Subject: [PATCH 035/220] Refactor: keep the semantics of `_runSettings`
 intact.

(This is part of the `Opts` data structure read from a file.  it's
very confusing to start updating part of that on the fly, and ignoring
whatever comes from the config file.)
---
 services/federator/src/Federator/Env.hs       |  7 +++
 .../federator/src/Federator/ExternalServer.hs |  3 +-
 .../federator/src/Federator/InternalServer.hs |  3 +-
 services/federator/src/Federator/Options.hs   | 19 ++++----
 services/federator/src/Federator/Run.hs       | 43 ++++++++++---------
 .../federator/src/Federator/Validation.hs     | 11 +++--
 6 files changed, 46 insertions(+), 40 deletions(-)

diff --git a/services/federator/src/Federator/Env.hs b/services/federator/src/Federator/Env.hs
index f6b8e79053..b3ecbcfe2f 100644
--- a/services/federator/src/Federator/Env.hs
+++ b/services/federator/src/Federator/Env.hs
@@ -23,6 +23,8 @@ module Federator.Env where
 
 import Bilge (RequestId)
 import Control.Lens (makeLenses)
+import Data.Aeson (FromJSON, ToJSON)
+import Data.Domain (Domain ())
 import Data.Metrics (Metrics)
 import Federator.Options (RunSettings)
 import HTTP2.Client.Manager
@@ -34,12 +36,17 @@ import qualified System.Logger.Class as LC
 import Util.Options
 import Wire.API.Federation.Component
 
+newtype AllowedDomains = AllowedDomains {allowedDomains :: [Domain]}
+  deriving (Eq, Show, Generic)
+  deriving newtype (FromJSON, ToJSON)
+
 data Env = Env
   { _metrics :: Metrics,
     _applog :: LC.Logger,
     _requestId :: RequestId,
     _dnsResolver :: Resolver,
     _runSettings :: RunSettings,
+    _allowedRemoteDomains :: AllowedDomains,
     _service :: Component -> Endpoint,
     _httpManager :: HTTP.Manager,
     _http2Manager :: IORef Http2Manager
diff --git a/services/federator/src/Federator/ExternalServer.hs b/services/federator/src/Federator/ExternalServer.hs
index a3244c94d3..fc37ba7333 100644
--- a/services/federator/src/Federator/ExternalServer.hs
+++ b/services/federator/src/Federator/ExternalServer.hs
@@ -25,7 +25,6 @@ import qualified Data.Text as Text
 import Federator.Discovery
 import Federator.Env
 import Federator.Error.ServerError
-import Federator.Options (RunSettings)
 import Federator.Response
 import Federator.Service
 import Federator.Validation
@@ -51,7 +50,7 @@ callInward ::
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
     Member (Error ServerError) r,
-    Member (Input RunSettings) r
+    Member (Input Env) r
   ) =>
   Wai.Request ->
   Sem r Wai.Response
diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs
index 801bc079cf..e7721b2476 100644
--- a/services/federator/src/Federator/InternalServer.hs
+++ b/services/federator/src/Federator/InternalServer.hs
@@ -25,7 +25,6 @@ import qualified Data.ByteString as BS
 import qualified Data.Text as Text
 import Federator.Env
 import Federator.Error.ServerError
-import Federator.Options (RunSettings)
 import Federator.Remote
 import Federator.Response
 import Federator.Validation
@@ -82,7 +81,7 @@ callOutward ::
     Member (Embed IO) r,
     Member (Error ValidationError) r,
     Member (Error ServerError) r,
-    Member (Input RunSettings) r
+    Member (Input Env) r
   ) =>
   Wai.Request ->
   Sem r Wai.Response
diff --git a/services/federator/src/Federator/Options.hs b/services/federator/src/Federator/Options.hs
index 19a006a176..e180f2accc 100644
--- a/services/federator/src/Federator/Options.hs
+++ b/services/federator/src/Federator/Options.hs
@@ -21,20 +21,15 @@
 module Federator.Options where
 
 import Data.Aeson
-import Data.Domain (Domain ())
 import Imports
 import System.Logger.Extended (Level, LogFormat)
 import Util.Options
 
-newtype AllowedDomains = AllowedDomains {allowedDomains :: [Domain]}
-  deriving (Eq, Show, Generic)
-  deriving newtype (FromJSON, ToJSON)
-
 data FederationStrategy
   = -- | This backend allows federating with any other Wire-Server backend
     AllowAll
-  | -- | Any backend explicitly configured in a FederationAllowList
-    AllowList AllowedDomains
+  | -- | Any backend explicitly configured in table `brig.federation_remotes`.
+    AllowList
   deriving (Eq, Show, Generic)
 
 instance ToJSON FederationStrategy where
@@ -42,19 +37,21 @@ instance ToJSON FederationStrategy where
     object
       [ "allowAll" .= object []
       ]
-  toJSON (AllowList domains) =
+  toJSON AllowList =
     object
-      [ "allowedDomains" .= domains
+      [ "allowedDomains" .= object []
       ]
 
+-- | This parser is a bit odd: for historical reasons, we support a list of sub-items (for
+-- allowlist), but we don't keep that any more.
 instance FromJSON FederationStrategy where
   parseJSON = withObject "FederationStrategy" $ \o -> do
     -- Only inspect field content once we committed to one, for better error messages.
     allowAll :: Maybe Value <- o .:! "allowAll"
     allowList :: Maybe Value <- o .:! "allowedDomains"
     case (allowAll, allowList) of
-      (Just _, Nothing) -> pure AllowAll -- accept any content
-      (Nothing, Just l) -> AllowList <$> parseJSON l
+      (Just _, Nothing) -> pure AllowAll
+      (Nothing, Just _) -> pure AllowList
       _ -> fail "invalid FederationStrategy: expected either allowAll or allowedDomains"
 
 -- | Options that persist as runtime settings.
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index e71ea9e5e9..81524de790 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -36,7 +36,7 @@ where
 
 import Control.Concurrent.Async
 import Control.Exception (bracket)
-import Control.Lens ((%~), (^.))
+import Control.Lens ((.~), (^.))
 import Data.Default (def)
 import qualified Data.Metrics.Middleware as Metrics
 import Data.Text
@@ -65,26 +65,27 @@ import qualified Wire.Network.DNS.Helper as DNS
 -- FUTUREWORK(federation): Add metrics and status endpoints
 run :: Opts -> IO ()
 run opts = do
+  manager <- newManager defaultManagerSettings
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
-  DNS.withCachingResolver resolvConf $ \res ->
-    bracket (newEnv opts res) closeEnv $ \env -> do
       -- Build a new TVar holding the state we want for the initial environment.
       -- This needs to contact Brig before accepting other requests
-      manager <- newManager defaultManagerSettings
-      let Endpoint host port = brig opts
-          baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
-          clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-          -- Loop the request until we get an answer. This is helpful during integration
-          -- tests where services are being brought up in parallel.
-          getInitialFedDomains = do
-            runClientM getFedRemotes clientEnv >>= \case
-              Right s -> pure s
-              Left e -> do
-                print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e
-                threadDelay $ domainUpdateInterval opts
-                getInitialFedDomains
-      fedStrat <- getInitialFedDomains
-      tEnv <- newTVarIO $ updateFedStrat fedStrat env
+      Endpoint host port = brig opts
+      baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
+      clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
+      getInitialFedDomains = do
+        runClientM getFedRemotes clientEnv >>= \case
+          Right s -> pure s
+          Left e -> do
+            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e
+            threadDelay $ domainUpdateInterval opts
+            getInitialFedDomains
+  okRemoteDomains <- (AllowedDomains . fmap domain . fromFederationDomainConfigs) <$> getInitialFedDomains
+
+  DNS.withCachingResolver resolvConf $ \res ->
+    bracket (newEnv opts res okRemoteDomains) closeEnv $ \env -> do
+      -- Loop the request until we get an answer. This is helpful during integration
+      -- tests where services are being brought up in parallel.
+      tEnv <- newTVarIO env
       let callback :: FederationDomainConfigs -> IO ()
           callback = atomically . modifyTVar tEnv . updateFedStrat
       -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
@@ -97,7 +98,7 @@ run opts = do
         void $ waitAnyCancel [envUpdateThread, internalServerThread, externalServerThread]
   where
     updateFedStrat :: FederationDomainConfigs -> Env -> Env
-    updateFedStrat fedDomConfigs = Federator.Env.runSettings %~ \s -> s {federationStrategy = AllowList $ AllowedDomains $ domain <$> fromFederationDomainConfigs fedDomConfigs}
+    updateFedStrat fedDomConfigs = Federator.Env.allowedRemoteDomains .~ AllowedDomains (domain <$> fromFederationDomainConfigs fedDomConfigs)
 
     endpointInternal = federatorInternal opts
     portInternal = fromIntegral $ endpointInternal ^. epPort
@@ -128,8 +129,8 @@ run opts = do
 -------------------------------------------------------------------------------
 -- Environment
 
-newEnv :: Opts -> DNS.Resolver -> IO Env
-newEnv o _dnsResolver = do
+newEnv :: Opts -> DNS.Resolver -> AllowedDomains -> IO Env
+newEnv o _dnsResolver _allowedRemoteDomains = do
   _metrics <- Metrics.metrics
   _applog <- LogExt.mkLogger (Opt.logLevel o) (Opt.logNetStrings o) (Opt.logFormat o)
   let _requestId = def
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index 2dcf0c4742..ebf6986d76 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -25,6 +25,7 @@ module Federator.Validation
   )
 where
 
+import Control.Lens (view)
 import qualified Data.ByteString.Char8 as B8
 import Data.ByteString.Conversion
 import Data.Domain
@@ -37,6 +38,7 @@ import qualified Data.Text.Lazy as LText
 import qualified Data.X509 as X509
 import qualified Data.X509.Validation as X509
 import Federator.Discovery
+import Federator.Env
 import Federator.Error
 import Federator.Options
 import Imports
@@ -92,16 +94,17 @@ validationErrorStatus _ = HTTP.status403
 -- | Validates an already-parsed domain against the allowList using the federator
 -- startup configuration, and can update the allowList from the DB at runtime.
 ensureCanFederateWith ::
-  ( Member (Input RunSettings) r,
+  ( Member (Input Env) r,
     Member (Error ValidationError) r
   ) =>
   Domain ->
   Sem r ()
 ensureCanFederateWith targetDomain = do
-  strategy <- inputs federationStrategy
+  strategy <- inputs (federationStrategy . view runSettings)
   case strategy of
     AllowAll -> pure ()
-    AllowList (AllowedDomains domains) ->
+    AllowList -> do
+      AllowedDomains domains <- inputs (view allowedRemoteDomains)
       unless (targetDomain `elem` domains) $
         throw (FederationDenied targetDomain)
 
@@ -139,7 +142,7 @@ parseDomainText domain =
 -- federator startup configuration and checks that it matches the names reported
 -- by the client certificate
 validateDomain ::
-  ( Member (Input RunSettings) r,
+  ( Member (Input Env) r,
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
     Member DiscoverFederator r

From 7dfb4e8dc9094ebda494d6852215af4c44a74709 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 16:13:31 +0200
Subject: [PATCH 036/220] Remove a lying comment.

---
 services/federator/src/Federator/Validation.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index ebf6986d76..785d20cf86 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -91,8 +91,8 @@ validationErrorStatus :: ValidationError -> HTTP.Status
 validationErrorStatus (FederationDenied _) = HTTP.status400
 validationErrorStatus _ = HTTP.status403
 
--- | Validates an already-parsed domain against the allowList using the federator
--- startup configuration, and can update the allowList from the DB at runtime.
+-- | Validates an already-parsed domain against the allowList (stored in
+-- `brig.federation_remotes`, cached in `Env`).
 ensureCanFederateWith ::
   ( Member (Input Env) r,
     Member (Error ValidationError) r

From 6f9b9cf699f5b0d7b8270b7bce44d450550d0a0f Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 16:51:28 +0200
Subject: [PATCH 037/220] Fixup / WIP

---
 services/federator/src/Federator/Response.hs | 49 +++++++++++---------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index 86ccf0ae2c..ac646585aa 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -132,30 +132,33 @@ type AllEffects =
 
 -- | Run Sem action containing HTTP handlers. All errors have to been handled
 -- already by this point.
+--
+-- The `Env` is extracted from the `TVar` for each request.  This allows us to independently
+-- update the settings and have them be used as requests come in.
 runFederator :: TVar Env -> Sem AllEffects Wai.Response -> Codensity IO Wai.Response
-runFederator tvar resp = do
-  env <- liftIO $ readTVarIO tvar
-  resp
-    & runM
-      . runEmbedded @IO @(Codensity IO) liftIO
-      . loggerToTinyLogReqId (view requestId env) (view applog env)
-      . runWaiErrors
-        @'[ ValidationError,
-            RemoteError,
-            ServerError,
-            DiscoveryFailure
-          ]
-      . runInputConst env
-      . runInputSem (embed @IO (readIORef (view http2Manager env)))
-      -- This is the point at which federation settings are extracted
-      -- For each request, extract a fresh copy of the runSettings. This allows us
-      -- to independently update the settings and have them be used as requests
-      -- come in.
-      . runInputSem (embed @IO $ fmap (view runSettings) . liftIO $ readTVarIO tvar)
-      . interpretServiceHTTP
-      . runDNSLookupWithResolver (view dnsResolver env)
-      . runFederatorDiscovery
-      . interpretRemote
+runFederator tvar =
+  runM
+    . runEmbedded @IO @(Codensity IO) liftIO
+    . f tvar (\env -> loggerToTinyLogReqId (view requestId env) (view applog env))
+    . runWaiErrors
+      @'[ ValidationError,
+          RemoteError,
+          ServerError,
+          DiscoveryFailure
+        ]
+    . f tvar runInputConst
+    . f tvar (\env -> runInputSem (embed @IO (readIORef (view http2Manager env))))
+    . f tvar (runInputConst . view runSettings)
+    . interpretServiceHTTP
+    . f tvar (runDNSLookupWithResolver . view dnsResolver)
+    . runFederatorDiscovery
+    . interpretRemote
+
+f ::
+  TVar Env ->
+  (Env -> Sem r1 Wai.Response -> Sem r2 Wai.Response) ->
+  (Sem r1 Wai.Response -> Sem r2 Wai.Response)
+f = undefined
 
 streamingResponseToWai :: StreamingResponse -> Wai.Response
 streamingResponseToWai resp =

From beba2087518c9504b8a11f8edf342a689c24ce7f Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 8 May 2023 20:59:30 +0200
Subject: [PATCH 038/220] Fixup

---
 services/federator/src/Federator/Response.hs | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index ac646585aa..27aa59f345 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -139,26 +139,30 @@ runFederator :: TVar Env -> Sem AllEffects Wai.Response -> Codensity IO Wai.Resp
 runFederator tvar =
   runM
     . runEmbedded @IO @(Codensity IO) liftIO
-    . f tvar (\env -> loggerToTinyLogReqId (view requestId env) (view applog env))
+    . withEnv tvar (\env -> loggerToTinyLogReqId (view requestId env) (view applog env))
     . runWaiErrors
       @'[ ValidationError,
           RemoteError,
           ServerError,
           DiscoveryFailure
         ]
-    . f tvar runInputConst
-    . f tvar (\env -> runInputSem (embed @IO (readIORef (view http2Manager env))))
-    . f tvar (runInputConst . view runSettings)
+    . withEnv tvar runInputConst
+    . withEnv tvar (\env -> runInputSem (embed @IO (readIORef (view http2Manager env))))
+    . withEnv tvar (runInputConst . view runSettings)
     . interpretServiceHTTP
-    . f tvar (runDNSLookupWithResolver . view dnsResolver)
+    . withEnv tvar (runDNSLookupWithResolver . view dnsResolver)
     . runFederatorDiscovery
     . interpretRemote
 
-f ::
+withEnv ::
+  forall r1 r2.
+  Member (Embed IO) r2 =>
   TVar Env ->
   (Env -> Sem r1 Wai.Response -> Sem r2 Wai.Response) ->
   (Sem r1 Wai.Response -> Sem r2 Wai.Response)
-f = undefined
+withEnv tvar action cont = do
+  env <- embed @IO (liftIO (readTVarIO tvar))
+  action env cont
 
 streamingResponseToWai :: StreamingResponse -> Wai.Response
 streamingResponseToWai resp =

From 3b8b3837f03f2f8f11b71b306b3de3d5b0aa7d18 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 9 May 2023 17:22:29 +0200
Subject: [PATCH 039/220] I take it back!

---
 services/federator/src/Federator/Env.hs       |  2 +-
 .../federator/src/Federator/ExternalServer.hs |  6 +-
 .../federator/src/Federator/InternalServer.hs |  6 +-
 services/federator/src/Federator/Response.hs  | 36 ++++------
 services/federator/src/Federator/Run.hs       | 67 +++++++++----------
 .../federator/src/Federator/Validation.hs     | 11 +--
 6 files changed, 57 insertions(+), 71 deletions(-)

diff --git a/services/federator/src/Federator/Env.hs b/services/federator/src/Federator/Env.hs
index b3ecbcfe2f..9afa49614e 100644
--- a/services/federator/src/Federator/Env.hs
+++ b/services/federator/src/Federator/Env.hs
@@ -46,7 +46,7 @@ data Env = Env
     _requestId :: RequestId,
     _dnsResolver :: Resolver,
     _runSettings :: RunSettings,
-    _allowedRemoteDomains :: AllowedDomains,
+    _allowedRemoteDomains :: IORef AllowedDomains,
     _service :: Component -> Endpoint,
     _httpManager :: HTTP.Manager,
     _http2Manager :: IORef Http2Manager
diff --git a/services/federator/src/Federator/ExternalServer.hs b/services/federator/src/Federator/ExternalServer.hs
index fc37ba7333..1e4275f1e2 100644
--- a/services/federator/src/Federator/ExternalServer.hs
+++ b/services/federator/src/Federator/ExternalServer.hs
@@ -25,6 +25,7 @@ import qualified Data.Text as Text
 import Federator.Discovery
 import Federator.Env
 import Federator.Error.ServerError
+import Federator.Options (RunSettings)
 import Federator.Response
 import Federator.Service
 import Federator.Validation
@@ -50,7 +51,8 @@ callInward ::
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
     Member (Error ServerError) r,
-    Member (Input Env) r
+    Member (Input RunSettings) r,
+    Member (Input AllowedDomains) r
   ) =>
   Wai.Request ->
   Sem r Wai.Response
@@ -137,7 +139,7 @@ parseRequestData req = do
 isAllowedRPCChar :: Char -> Bool
 isAllowedRPCChar c = isAsciiLower c || isAsciiUpper c || isNumber c || c == '_' || c == '-'
 
-serveInward :: TVar Env -> Int -> IO ()
+serveInward :: Env -> Int -> IO ()
 serveInward = serve callInward
 
 lookupCertificate :: Wai.Request -> Maybe ByteString
diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs
index e7721b2476..636096217f 100644
--- a/services/federator/src/Federator/InternalServer.hs
+++ b/services/federator/src/Federator/InternalServer.hs
@@ -25,6 +25,7 @@ import qualified Data.ByteString as BS
 import qualified Data.Text as Text
 import Federator.Env
 import Federator.Error.ServerError
+import Federator.Options (RunSettings)
 import Federator.Remote
 import Federator.Response
 import Federator.Validation
@@ -81,7 +82,8 @@ callOutward ::
     Member (Embed IO) r,
     Member (Error ValidationError) r,
     Member (Error ServerError) r,
-    Member (Input Env) r
+    Member (Input RunSettings) r,
+    Member (Input AllowedDomains) r
   ) =>
   Wai.Request ->
   Sem r Wai.Response
@@ -98,5 +100,5 @@ callOutward req = do
       (fromLazyByteString (rdBody rd))
   pure $ streamingResponseToWai resp
 
-serveOutward :: TVar Env -> Int -> IO ()
+serveOutward :: Env -> Int -> IO ()
 serveOutward = serve callOutward
diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index 27aa59f345..efa2c7d421 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -100,18 +100,17 @@ runWaiError =
 
 serve ::
   (Wai.Request -> Sem AllEffects Wai.Response) ->
-  TVar Env ->
+  Env ->
   Int ->
   IO ()
-serve action tvar port = do
-  env <- readTVarIO tvar
+serve action env port =
   Warp.run port
     . Wai.catchErrors (view applog env) []
     $ app
   where
     app :: Wai.Application
     app req respond =
-      runCodensity (runFederator tvar (action req)) respond
+      runCodensity (runFederator env (action req)) respond
 
 type AllEffects =
   '[ Remote,
@@ -120,6 +119,7 @@ type AllEffects =
      ServiceStreaming,
      Input RunSettings,
      Input Http2Manager, -- needed by Remote
+     Input AllowedDomains, -- needed for `FederationStrategy` `AllowList`.
      Input Env, -- needed by Service
      Error ValidationError,
      Error RemoteError,
@@ -132,38 +132,26 @@ type AllEffects =
 
 -- | Run Sem action containing HTTP handlers. All errors have to been handled
 -- already by this point.
---
--- The `Env` is extracted from the `TVar` for each request.  This allows us to independently
--- update the settings and have them be used as requests come in.
-runFederator :: TVar Env -> Sem AllEffects Wai.Response -> Codensity IO Wai.Response
-runFederator tvar =
+runFederator :: Env -> Sem AllEffects Wai.Response -> Codensity IO Wai.Response
+runFederator env =
   runM
     . runEmbedded @IO @(Codensity IO) liftIO
-    . withEnv tvar (\env -> loggerToTinyLogReqId (view requestId env) (view applog env))
+    . loggerToTinyLogReqId (view requestId env) (view applog env)
     . runWaiErrors
       @'[ ValidationError,
           RemoteError,
           ServerError,
           DiscoveryFailure
         ]
-    . withEnv tvar runInputConst
-    . withEnv tvar (\env -> runInputSem (embed @IO (readIORef (view http2Manager env))))
-    . withEnv tvar (runInputConst . view runSettings)
+    . runInputConst env
+    . runInputSem (embed @IO (readIORef (view allowedRemoteDomains env)))
+    . runInputSem (embed @IO (readIORef (view http2Manager env)))
+    . runInputConst (view runSettings env)
     . interpretServiceHTTP
-    . withEnv tvar (runDNSLookupWithResolver . view dnsResolver)
+    . runDNSLookupWithResolver (view dnsResolver env)
     . runFederatorDiscovery
     . interpretRemote
 
-withEnv ::
-  forall r1 r2.
-  Member (Embed IO) r2 =>
-  TVar Env ->
-  (Env -> Sem r1 Wai.Response -> Sem r2 Wai.Response) ->
-  (Sem r1 Wai.Response -> Sem r2 Wai.Response)
-withEnv tvar action cont = do
-  env <- embed @IO (liftIO (readTVarIO tvar))
-  action env cont
-
 streamingResponseToWai :: StreamingResponse -> Wai.Response
 streamingResponseToWai resp =
   let headers = toList (responseHeaders resp)
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 81524de790..c2ee894918 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -36,7 +36,7 @@ where
 
 import Control.Concurrent.Async
 import Control.Exception (bracket)
-import Control.Lens ((.~), (^.))
+import Control.Lens (view, (^.))
 import Data.Default (def)
 import qualified Data.Metrics.Middleware as Metrics
 import Data.Text
@@ -67,39 +67,42 @@ run :: Opts -> IO ()
 run opts = do
   manager <- newManager defaultManagerSettings
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
-      -- Build a new TVar holding the state we want for the initial environment.
-      -- This needs to contact Brig before accepting other requests
-      Endpoint host port = brig opts
+  let Endpoint host port = brig opts
       baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-      getInitialFedDomains = do
-        runClientM getFedRemotes clientEnv >>= \case
-          Right s -> pure s
-          Left e -> do
-            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e
-            threadDelay $ domainUpdateInterval opts
-            getInitialFedDomains
-  okRemoteDomains <- (AllowedDomains . fmap domain . fromFederationDomainConfigs) <$> getInitialFedDomains
+
+      getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
+
+      getAllowedDomainsOnce :: IO AllowedDomains
+      getAllowedDomainsOnce =
+        (AllowedDomains . fmap domain . fromFederationDomainConfigs)
+          <$> let go :: IO FederationDomainConfigs
+                  go = do
+                    runClientM getFedRemotes clientEnv >>= \case
+                      Right s -> pure s
+                      Left e -> do
+                        print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e -- TODO: log error or critical!
+                        threadDelay $ domainUpdateInterval opts
+                        go
+               in go
+
+      getAllowedDomainsLoop :: Env -> IO ()
+      getAllowedDomainsLoop env = forever $ do
+        threadDelay $ domainUpdateInterval opts
+        atomicWriteIORef (view allowedRemoteDomains env) =<< getAllowedDomainsOnce
+
+  okRemoteDomains <- getAllowedDomainsOnce
 
   DNS.withCachingResolver resolvConf $ \res ->
     bracket (newEnv opts res okRemoteDomains) closeEnv $ \env -> do
-      -- Loop the request until we get an answer. This is helpful during integration
-      -- tests where services are being brought up in parallel.
-      tEnv <- newTVarIO env
-      let callback :: FederationDomainConfigs -> IO ()
-          callback = atomically . modifyTVar tEnv . updateFedStrat
-      -- We need a watcher/listener for updating this TVar to flow values through to the handlers.
-      let externalServer = serveInward tEnv portExternal
-          internalServer = serveOutward tEnv portInternal
+      let externalServer = serveInward env portExternal
+          internalServer = serveOutward env portInternal
       withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
-        envUpdateThread <- async $ updateDomains clientEnv callback
+        updateAllowedDomainsThread <- async (getAllowedDomainsLoop env)
         internalServerThread <- async internalServer
         externalServerThread <- async externalServer
-        void $ waitAnyCancel [envUpdateThread, internalServerThread, externalServerThread]
+        void $ waitAnyCancel [updateAllowedDomainsThread, internalServerThread, externalServerThread]
   where
-    updateFedStrat :: FederationDomainConfigs -> Env -> Env
-    updateFedStrat fedDomConfigs = Federator.Env.allowedRemoteDomains .~ AllowedDomains (domain <$> fromFederationDomainConfigs fedDomConfigs)
-
     endpointInternal = federatorInternal opts
     portInternal = fromIntegral $ endpointInternal ^. epPort
 
@@ -115,22 +118,11 @@ run opts = do
           conf {DNS.resolvInfo = DNS.RCHostPort host (fromIntegral port)}
         (_, _) -> conf
 
-    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
-
-    updateDomains :: ClientEnv -> (FederationDomainConfigs -> IO ()) -> IO ()
-    updateDomains clientEnv update = forever $ do
-      threadDelay $ domainUpdateInterval opts
-      strat <- runClientM getFedRemotes clientEnv
-      either
-        print
-        update
-        strat
-
 -------------------------------------------------------------------------------
 -- Environment
 
 newEnv :: Opts -> DNS.Resolver -> AllowedDomains -> IO Env
-newEnv o _dnsResolver _allowedRemoteDomains = do
+newEnv o _dnsResolver okRemoteDomains = do
   _metrics <- Metrics.metrics
   _applog <- LogExt.mkLogger (Opt.logLevel o) (Opt.logNetStrings o) (Opt.logFormat o)
   let _requestId = def
@@ -139,6 +131,7 @@ newEnv o _dnsResolver _allowedRemoteDomains = do
       _service Galley = Opt.galley o
       _service Cargohold = Opt.cargohold o
   _httpManager <- initHttpManager
+  _allowedRemoteDomains <- newIORef okRemoteDomains
   sslContext <- mkTLSSettingsOrThrow _runSettings
   _http2Manager <- newIORef =<< mkHttp2Manager sslContext
   pure Env {..}
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index 785d20cf86..3b55fc8498 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -25,7 +25,6 @@ module Federator.Validation
   )
 where
 
-import Control.Lens (view)
 import qualified Data.ByteString.Char8 as B8
 import Data.ByteString.Conversion
 import Data.Domain
@@ -94,17 +93,18 @@ validationErrorStatus _ = HTTP.status403
 -- | Validates an already-parsed domain against the allowList (stored in
 -- `brig.federation_remotes`, cached in `Env`).
 ensureCanFederateWith ::
-  ( Member (Input Env) r,
+  ( Member (Input AllowedDomains) r,
+    Member (Input RunSettings) r,
     Member (Error ValidationError) r
   ) =>
   Domain ->
   Sem r ()
 ensureCanFederateWith targetDomain = do
-  strategy <- inputs (federationStrategy . view runSettings)
+  strategy <- inputs federationStrategy
   case strategy of
     AllowAll -> pure ()
     AllowList -> do
-      AllowedDomains domains <- inputs (view allowedRemoteDomains)
+      AllowedDomains domains <- input
       unless (targetDomain `elem` domains) $
         throw (FederationDenied targetDomain)
 
@@ -142,7 +142,8 @@ parseDomainText domain =
 -- federator startup configuration and checks that it matches the names reported
 -- by the client certificate
 validateDomain ::
-  ( Member (Input Env) r,
+  ( Member (Input RunSettings) r,
+    Member (Input AllowedDomains) r,
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
     Member DiscoverFederator r

From d508cb9bcf1c001b10a73d580362579b1058e2fb Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 9 May 2023 17:23:00 +0200
Subject: [PATCH 040/220] more docs.

---
 changelog.d/0-release-notes/pr-3260           |  2 +-
 .../federation/backend-communication.md       | 36 ++++++++++++++-----
 .../Wire/API/Routes/FederationDomainConfig.hs |  3 ++
 3 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/changelog.d/0-release-notes/pr-3260 b/changelog.d/0-release-notes/pr-3260
index 7b836ffe2a..5b566bff43 100644
--- a/changelog.d/0-release-notes/pr-3260
+++ b/changelog.d/0-release-notes/pr-3260
@@ -1 +1 @@
-Federation only: from this release on, remote connections should be configured via an internal REST API; the config files will be honored for a transition period, but will be ignored starting in a future release.  [Details in the docs.](https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
\ No newline at end of file
+Federation only: from this release on, remote connections should be configured via an internal REST API; the config files will be honored for a transition period, but will be ignored starting in a future release.  YOU NEED TO UPDATE YOUR BRIG HELM CHART BEFORE DEPLOYING THIS RELEASE.  [Details in the docs.](https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
\ No newline at end of file
diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index 536ebaa268..66834c8527 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -160,30 +160,48 @@ search request from *Alice*, one of its clients.
 
 Up to the release containing
 [PR#3260](https://github.com/wireapp/wire-server/pull/3260), the
-config file statically contains information about the remote
-connections in the configs of all services that need to know.  Since
-then, there is an internal REST API for adding remote wire-server
-instances:
+config files of the individual services statically contained
+information about the remote connections.  Starting with this release,
+this information is stored in the database, and there is an internal
+REST API for adding and removing remotes:
 
 * [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
 * [`POST`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/post_i_federation_remotes)
 * [`DELETE`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/delete_i_federation_remotes__domain_)
 
+**WARNING:** If you delete a connection, all users from that remote
+will be removed from local conversations, and all conversations hosted
+by that remote will be removed from the local backend.  Connections
+between local and remote users that are removed will be archived, and
+can be re-established should you decide to add the same backend later.
+
 Changing the configuration of existing edges via `PUT` is not
 implemented at the moment, if you need to do that, delete the
 connection and add it again.
 
-If you delete a connection, all users from that remote will be removed
-from local conversations, and all conversations hosted by that remote
-will be removed from the local backend.  Connections between local and
-remote users that are removed will be archived, and can be
-re-established should you decide to add the same backend later.
+{-
+TODO: this paragraph still annoys me.  move strategy to brig, too?  or
+at least to a different syntax, and force admin to use both old and
+new syntax until transition period is over?  just to avoid the
+confusing bogus `:` at the end of the flag.
+
+The federation strategy (allow all or allow list) is still configured
+in federator, only the list of allowed hosts is ignored; if you select
+"allow all" (or if you disable federation), the list of known backends
+maintained by brig is mostly ignored, but e.g., search policy is still
+considered by brig itself.
+-}
 
 See {ref}`configuring-remote-connections-dev-perspective` for the
 developer's point of view on this topic.
 
 ### Transitioning from config file to database state
 
+TODO: you need to update config files!
+  - complete list of search policies, no more defaults
+  - new fed strategy syntax (keep the old, just copy)
+  - later, remove the old syntax in brig, federator.
+
 As of the release containing
 [PR#3260](https://github.com/wireapp/wire-server/pull/3260),
 [`federationStrategy`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/federator/values.yaml#L44-L45)
diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index fc5ce971a5..6dd136cbc3 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -29,6 +29,9 @@ import GHC.Generics
 import Imports
 import Wire.API.User.Search (FederatedUserSearchPolicy)
 
+-- | Everything we need to know about a remote instance in order to federate with it.  Comes
+-- in `AllowedDomains` if `AllowStrategy` is `AllowList`.  If `AllowAll`, we still use this
+-- information for search policy.
 data FederationDomainConfig = FederationDomainConfig
   { domain :: Domain,
     cfgSearchPolicy :: FederatedUserSearchPolicy

From 167083a63d376859cfd67e3b076d88a4ed90f3db Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 10 May 2023 09:55:21 +0200
Subject: [PATCH 041/220] rm bogus TODO, add a non-bogus one :)

---
 docs/src/understand/federation/backend-communication.md | 9 +++++++++
 services/brig/src/Brig/API/Internal.hs                  | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index 66834c8527..0b23610c72 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -192,6 +192,15 @@ maintained by brig is mostly ignored, but e.g., search policy is still
 considered by brig itself.
 -}
 
+{-
+
+TODO: explain how brig doesn't cache, but always read from the
+database, and that if you have update cycles of <10? secs, and/or
+clusters with >100? pods, you should monitor the load a little after
+upgrade.
+
+-}
+
 See {ref}`configuring-remote-connections-dev-perspective` for the
 developer's point of view on this topic.
 
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 5ebb16681e..7778f79526 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -178,7 +178,7 @@ authAPI =
 
 federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
 federationRemotesAPI =
-  Named @"get-federation-remotes" getFederationRemotes -- TODO: get this from TVar!
+  Named @"get-federation-remotes" getFederationRemotes
     :<|> Named @"add-federation-remotes" addFederationRemote
     :<|> Named @"delete-federation-remotes" deleteFederationRemotes
 

From 8cd69f1ea08a81c7dd72910c80b1dcfa6c310800 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 10 May 2023 10:14:14 +0200
Subject: [PATCH 042/220] revert changes of status code in integration tests.

(Double-check if we can't keep the prod code behavior intact to avoid
upgrade / fed-to-fed api / client api issues.)
---
 services/brig/test/integration/API/User/Account.hs    | 4 ++--
 services/brig/test/integration/API/User/Client.hs     | 4 ++--
 services/brig/test/integration/API/User/Connection.hs | 2 +-
 services/cargohold/test/integration/API.hs            | 2 +-
 services/galley/test/integration/API.hs               | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/services/brig/test/integration/API/User/Account.hs b/services/brig/test/integration/API/User/Account.hs
index 7840f45b46..e283d365cf 100644
--- a/services/brig/test/integration/API/User/Account.hs
+++ b/services/brig/test/integration/API/User/Account.hs
@@ -124,7 +124,7 @@ tests _ at opts p b c ch g aws userJournalWatcher =
       test p "post /activate - 200/204 + expiry" $ testActivateWithExpiry opts b at,
       test p "get /users/:uid - 404" $ testNonExistingUserUnqualified b,
       test p "get /users/<localdomain>/:uid - 404" $ testNonExistingUser b,
-      test p "get /users/:domain/:uid - 400" $ testUserInvalidDomain b,
+      test p "get /users/:domain/:uid - 4xx" $ testUserInvalidDomain b,
       test p "get /users/:uid - 200" $ testExistingUserUnqualified b,
       test p "get /users/<localdomain>/:uid - 200" $ testExistingUser b,
       test p "get /users?:id=.... - 200" $ testMultipleUsersUnqualified b,
@@ -637,7 +637,7 @@ testUserInvalidDomain brig = do
   let uid = qUnqualified qself
   get (brig . paths ["users", "invalid.example.com", toByteString' uid] . zUser uid)
     !!! do
-      const 400 === statusCode
+      const 422 === statusCode
       const (Just "/federation/api-version")
         === preview (ix "data" . ix "path") . responseJsonUnsafe @Value
       const (Just "invalid.example.com")
diff --git a/services/brig/test/integration/API/User/Client.hs b/services/brig/test/integration/API/User/Client.hs
index 35184d380c..4b82a1943f 100644
--- a/services/brig/test/integration/API/User/Client.hs
+++ b/services/brig/test/integration/API/User/Client.hs
@@ -92,7 +92,7 @@ tests _cl _at opts p db n b c g =
       test p "get /users/<localdomain>/:uid/clients - 200" $ testGetUserClientsQualified opts b,
       test p "get /users/:uid/prekeys - 200" $ testGetUserPrekeys b,
       test p "get /users/<localdomain>/:uid/prekeys - 200" $ testGetUserPrekeysQualified b opts,
-      test p "get /users/:domain/:uid/prekeys - 400" $ testGetUserPrekeysInvalidDomain b,
+      test p "get /users/:domain/:uid/prekeys - 4xx" $ testGetUserPrekeysInvalidDomain b,
       test p "get /users/:uid/prekeys/:client - 200" $ testGetClientPrekey b,
       test p "get /users/<localdomain>/:uid/prekeys/:client - 200" $ testGetClientPrekeyQualified b opts,
       test p "post /users/prekeys" $ testMultiUserGetPrekeys b,
@@ -783,7 +783,7 @@ testGetUserPrekeysInvalidDomain :: Brig -> Http ()
 testGetUserPrekeysInvalidDomain brig = do
   [(uid, _c, _lpk, _)] <- generateClients 1 brig
   get (brig . paths ["users", "invalid.example.com", toByteString' uid, "prekeys"] . zUser uid) !!! do
-    const 400 === statusCode
+    const 422 === statusCode
 
 testGetClientPrekey :: Brig -> Http ()
 testGetClientPrekey brig = do
diff --git a/services/brig/test/integration/API/User/Connection.hs b/services/brig/test/integration/API/User/Connection.hs
index d95df5e553..1e1353d1df 100644
--- a/services/brig/test/integration/API/User/Connection.hs
+++ b/services/brig/test/integration/API/User/Connection.hs
@@ -722,7 +722,7 @@ testConnectFederationNotAvailable :: Brig -> Http ()
 testConnectFederationNotAvailable brig = do
   (uid1, quid2) <- localAndRemoteUser brig
   postConnectionQualified brig uid1 quid2
-    !!! const 400 === statusCode
+    !!! const 422 === statusCode
 
 testConnectOK :: Brig -> Galley -> FedClient 'Brig -> Http ()
 testConnectOK brig galley fedBrigClient = do
diff --git a/services/cargohold/test/integration/API.hs b/services/cargohold/test/integration/API.hs
index 5d4ff3ad89..88f733e763 100644
--- a/services/cargohold/test/integration/API.hs
+++ b/services/cargohold/test/integration/API.hs
@@ -308,7 +308,7 @@ testRemoteDownloadWrongDomain = do
   let key = AssetKeyV3 assetId AssetPersistent
       qkey = Qualified key (Domain "invalid.example.com")
   downloadAsset uid qkey () !!! do
-    const 400 === statusCode
+    const 422 === statusCode
 
 testRemoteDownloadNoAsset :: TestM ()
 testRemoteDownloadNoAsset = do
diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs
index f8c99873f5..4d2d6242ac 100644
--- a/services/galley/test/integration/API.hs
+++ b/services/galley/test/integration/API.hs
@@ -3056,7 +3056,7 @@ testAddRemoteMemberInvalidDomain = do
 
   postQualifiedMembers alice (remoteBob :| []) convId
     !!! do
-      const 400 === statusCode
+      const 422 === statusCode
       const (Just "/federation/api-version")
         === preview (ix "data" . ix "path") . responseJsonUnsafe @Value
       const (Just "invalid.example.com")

From 0ede02d5ceeb1c6c9d7d5cc35323f67fd893e5f6 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 10 May 2023 18:35:52 +1000
Subject: [PATCH 043/220] Various updates, mainly moving some common code into
 wire-api and minimising types that are used in several common paths.

---
 libs/types-common/src/Data/MessageQueue.hs    | 10 +++---
 .../wire-api/src/Wire/API/FederationUpdate.hs | 36 +++++++++++++++++++
 .../Wire/API/Routes/FederationDomainConfig.hs |  7 ++--
 libs/wire-api/wire-api.cabal                  |  1 +
 services/brig/brig.integration.yaml           |  8 +----
 services/brig/src/Brig/API/Internal.hs        |  5 ++-
 services/brig/src/Brig/Options.hs             |  2 +-
 services/cannon/src/Cannon/Run.hs             | 32 +++--------------
 services/federator/federator.integration.yaml | 11 +-----
 services/federator/src/Federator/Env.hs       |  9 ++---
 .../federator/src/Federator/ExternalServer.hs |  3 +-
 .../federator/src/Federator/InternalServer.hs |  3 +-
 services/federator/src/Federator/Options.hs   |  4 +--
 services/federator/src/Federator/Response.hs  |  3 +-
 services/federator/src/Federator/Run.hs       | 33 +++--------------
 .../federator/src/Federator/Validation.hs     | 10 +++---
 .../unit/Test/Federator/ExternalServer.hs     |  9 +++++
 .../unit/Test/Federator/InternalServer.hs     | 14 +++++---
 .../test/unit/Test/Federator/Options.hs       | 14 +++-----
 .../test/unit/Test/Federator/Validation.hs    | 31 +++++++++++-----
 20 files changed, 123 insertions(+), 122 deletions(-)
 create mode 100644 libs/wire-api/src/Wire/API/FederationUpdate.hs

diff --git a/libs/types-common/src/Data/MessageQueue.hs b/libs/types-common/src/Data/MessageQueue.hs
index 6d90f1f509..eccc6b77d0 100644
--- a/libs/types-common/src/Data/MessageQueue.hs
+++ b/libs/types-common/src/Data/MessageQueue.hs
@@ -10,11 +10,11 @@ import Prelude (Show, String)
 
 -- | Options for connecting to the message queue system
 data MessageQueueSettings = MessageQueueSettings
-  { mqHost :: String,
-    mqVHost :: Text,
-    mqUser :: Text,
-    mqPass :: Text,
-    mqQueue :: Text
+  { host :: String,
+    vHost :: Text,
+    user :: Text,
+    pass :: Text,
+    queue :: Text
   }
   deriving (Show, Generic)
 
diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
new file mode 100644
index 0000000000..49f49e6fcd
--- /dev/null
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -0,0 +1,36 @@
+module Wire.API.FederationUpdate where
+
+import Imports
+import Wire.API.Routes.FederationDomainConfig (FederationDomainConfigs (updateInterval))
+import qualified Wire.API.Routes.Internal.Brig as IAPI
+import Servant.Client (runClientM, ClientError, ClientEnv)
+import Wire.API.Routes.Named (namedClient)
+import Servant.Client.Internal.HttpClient (ClientM)
+
+getFedRemotes :: ClientM FederationDomainConfigs
+getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
+
+-- Initial function for getting the set of domains from brig, and an update interval
+getAllowedDomainsInitial :: ClientEnv -> IO FederationDomainConfigs
+getAllowedDomainsInitial clientEnv =
+  let oneSec = 1000000 -- microsends
+      go :: IO FederationDomainConfigs
+      go = do
+        getAllowedDomains clientEnv >>= \case
+          Right s -> pure s
+          Left e -> do
+            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e -- TODO: log error or critical!
+            threadDelay oneSec
+            go
+  in go
+
+getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
+getAllowedDomains = runClientM getFedRemotes
+
+getAllowedDomainsLoop :: ClientEnv -> IORef FederationDomainConfigs -> IO ()
+getAllowedDomainsLoop clientEnv env = forever $ do
+  getAllowedDomains clientEnv >>= \case
+    Left e -> print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e -- TODO: log error or critical!
+    Right cfg -> atomicWriteIORef env cfg
+  delay <- updateInterval <$> readIORef env
+  threadDelay delay
\ No newline at end of file
diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 6dd136cbc3..00cfe35a62 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -46,8 +46,10 @@ instance ToSchema FederationDomainConfig where
         <$> domain .= field "domain" schema
         <*> cfgSearchPolicy .= field "search_policy" schema
 
-newtype FederationDomainConfigs = FederationDomainConfigs
-  {fromFederationDomainConfigs :: [FederationDomainConfig]}
+data FederationDomainConfigs = FederationDomainConfigs
+  { fromFederationDomainConfigs :: [FederationDomainConfig]
+  , updateInterval :: Int
+  }
   deriving (Show, Generic, Eq)
   deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationDomainConfigs
 
@@ -56,3 +58,4 @@ instance ToSchema FederationDomainConfigs where
     object "FederationDomainConfigs" $
       FederationDomainConfigs
         <$> fromFederationDomainConfigs .= field "remotes" (array schema)
+        <*> updateInterval .= field "updateInterval" schema
\ No newline at end of file
diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal
index 0c45cedcec..ddc2ba8c10 100644
--- a/libs/wire-api/wire-api.cabal
+++ b/libs/wire-api/wire-api.cabal
@@ -38,6 +38,7 @@ library
     Wire.API.Event.Conversation
     Wire.API.Event.FeatureConfig
     Wire.API.Event.Team
+    Wire.API.FederationUpdate
     Wire.API.Internal.BulkPush
     Wire.API.Internal.Notification
     Wire.API.MakesFederatedCall
diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml
index 91f8722239..253fe9ff84 100644
--- a/services/brig/brig.integration.yaml
+++ b/services/brig/brig.integration.yaml
@@ -213,10 +213,4 @@ optSettings:
   setOAuthMaxActiveRefreshTokens: 10
 
 logLevel: Warn
-logNetStrings: false
-mqSettings:
-  mqHost: ""
-  mqVHost: ""
-  mqUser: ""
-  mqPass: ""
-  mqQueue: ""
\ No newline at end of file
+logNetStrings: false
\ No newline at end of file
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 7778f79526..3d4673bddc 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -200,7 +200,10 @@ getFederationRemotes = lift $ do
   -- more robust migration path.  See
   -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections,
   -- http://docs.wire.com/developer/developer/federation-design-aspects.html#configuring-remote-connections-dev-perspective
-  pure . FederationDomainConfigs . nub $ db <> cfg
+  pure $ FederationDomainConfigs 
+    { fromFederationDomainConfigs = nub $ db <> cfg
+    , updateInterval = 1000000 -- TODO FIX ME!
+    }
 
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index 7a959264d8..ae69755429 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -461,7 +461,7 @@ data Opts = Opts
     -- | Runtime settings
     optSettings :: !Settings,
     -- | Message Queue settings
-    mqSettings :: !MessageQueueSettings
+    mqSettings :: !(Maybe MessageQueueSettings)
   }
   deriving (Show, Generic)
 
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index 22967b08a0..1930ff08e8 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -53,12 +53,10 @@ import System.Posix.Signals
 import qualified System.Posix.Signals as Signals
 import System.Random.MWC (createSystemRandom)
 import UnliftIO.Concurrent (myThreadId, throwTo)
-import Wire.API.Routes.FederationDomainConfig
-import qualified Wire.API.Routes.Internal.Brig as IAPI
 import qualified Wire.API.Routes.Internal.Cannon as Internal
-import Wire.API.Routes.Named (namedClient)
 import Wire.API.Routes.Public.Cannon
 import Wire.API.Routes.Version.Wai
+import Wire.API.FederationUpdate (getAllowedDomainsLoop, getAllowedDomainsInitial)
 
 type CombinedAPI = PublicAPI :<|> Internal.API
 
@@ -85,20 +83,9 @@ run o = do
   let Brig bh bp = o ^. brig
       baseUrl = BaseUrl Http (unpack bh) (fromIntegral bp) ""
       clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-      -- Loop the request until we get an answer. This is helpful during integration
-      -- tests where services are being brought up in parallel.
-      getInitialFedDomains = do
-        runClientM getFedRemotes clientEnv >>= \case
-          Right strat -> pure strat
-          Left err -> do
-            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show err
-            threadDelay $ o ^. domainUpdateInterval
-            getInitialFedDomains
-  fedStrat <- getInitialFedDomains
-  tEnv <- newTVarIO fedStrat
-  let callback :: FederationDomainConfigs -> IO ()
-      callback = atomically . writeTVar tEnv
-  updateDomainsThread <- Async.async $ updateDomains clientEnv callback
+  fedStrat <- getAllowedDomainsInitial clientEnv
+  ioref <- newIORef fedStrat
+  updateDomainsThread <- Async.async $ getAllowedDomainsLoop clientEnv ioref
 
   let middleware :: Wai.Middleware
       middleware =
@@ -134,17 +121,6 @@ run o = do
     readExternal :: FilePath -> IO ByteString
     readExternal f = encodeUtf8 . strip . pack <$> Strict.readFile f
 
-    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
-
-    updateDomains :: ClientEnv -> (FederationDomainConfigs -> IO ()) -> IO ()
-    updateDomains clientEnv update = forever $ do
-      threadDelay $ o ^. domainUpdateInterval
-      strat <- runClientM getFedRemotes clientEnv
-      either
-        print
-        update
-        strat
-
 signalHandler :: Env -> ThreadId -> Signals.Handler
 signalHandler e mainThread = CatchOnce $ do
   runWS e drain
diff --git a/services/federator/federator.integration.yaml b/services/federator/federator.integration.yaml
index 593dda9b5f..06a341d4f0 100644
--- a/services/federator/federator.integration.yaml
+++ b/services/federator/federator.integration.yaml
@@ -39,13 +39,4 @@ optSettings:
   clientCertificate: "test/resources/integration-leaf.pem"
   clientPrivateKey: "test/resources/integration-leaf-key.pem"
   dnsHost: "127.0.0.1"
-  dnsPort: 9053
-
-mqSettings:
-  mqHost: "some.mq.host"
-  # https://www.rabbitmq.com/vhosts.html
-  mqVHost: "some.mq.host"
-  mqUser: "username"
-  mqPass: "password"
-  mqQueue: "queue name"
-domainUpdateInterval: 1000000
\ No newline at end of file
+  dnsPort: 9053
\ No newline at end of file
diff --git a/services/federator/src/Federator/Env.hs b/services/federator/src/Federator/Env.hs
index 9afa49614e..b14f4212b2 100644
--- a/services/federator/src/Federator/Env.hs
+++ b/services/federator/src/Federator/Env.hs
@@ -23,8 +23,6 @@ module Federator.Env where
 
 import Bilge (RequestId)
 import Control.Lens (makeLenses)
-import Data.Aeson (FromJSON, ToJSON)
-import Data.Domain (Domain ())
 import Data.Metrics (Metrics)
 import Federator.Options (RunSettings)
 import HTTP2.Client.Manager
@@ -35,10 +33,7 @@ import OpenSSL.Session (SSLContext)
 import qualified System.Logger.Class as LC
 import Util.Options
 import Wire.API.Federation.Component
-
-newtype AllowedDomains = AllowedDomains {allowedDomains :: [Domain]}
-  deriving (Eq, Show, Generic)
-  deriving newtype (FromJSON, ToJSON)
+import Wire.API.Routes.FederationDomainConfig (FederationDomainConfigs)
 
 data Env = Env
   { _metrics :: Metrics,
@@ -46,7 +41,7 @@ data Env = Env
     _requestId :: RequestId,
     _dnsResolver :: Resolver,
     _runSettings :: RunSettings,
-    _allowedRemoteDomains :: IORef AllowedDomains,
+    _allowedRemoteDomains :: IORef FederationDomainConfigs,
     _service :: Component -> Endpoint,
     _httpManager :: HTTP.Manager,
     _http2Manager :: IORef Http2Manager
diff --git a/services/federator/src/Federator/ExternalServer.hs b/services/federator/src/Federator/ExternalServer.hs
index 1e4275f1e2..22fabfa279 100644
--- a/services/federator/src/Federator/ExternalServer.hs
+++ b/services/federator/src/Federator/ExternalServer.hs
@@ -41,6 +41,7 @@ import Servant.Client.Core
 import qualified System.Logger.Message as Log
 import Wire.API.Federation.Component
 import Wire.API.Federation.Domain
+import Wire.API.Routes.FederationDomainConfig
 
 -- FUTUREWORK(federation): Versioning of the federation API.
 callInward ::
@@ -52,7 +53,7 @@ callInward ::
     Member (Error DiscoveryFailure) r,
     Member (Error ServerError) r,
     Member (Input RunSettings) r,
-    Member (Input AllowedDomains) r
+    Member (Input FederationDomainConfigs) r
   ) =>
   Wai.Request ->
   Sem r Wai.Response
diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs
index 636096217f..1e82471a33 100644
--- a/services/federator/src/Federator/InternalServer.hs
+++ b/services/federator/src/Federator/InternalServer.hs
@@ -36,6 +36,7 @@ import Polysemy
 import Polysemy.Error
 import Polysemy.Input
 import Wire.API.Federation.Component
+import Wire.API.Routes.FederationDomainConfig
 
 data RequestData = RequestData
   { rdTargetDomain :: Text,
@@ -83,7 +84,7 @@ callOutward ::
     Member (Error ValidationError) r,
     Member (Error ServerError) r,
     Member (Input RunSettings) r,
-    Member (Input AllowedDomains) r
+    Member (Input FederationDomainConfigs) r
   ) =>
   Wai.Request ->
   Sem r Wai.Response
diff --git a/services/federator/src/Federator/Options.hs b/services/federator/src/Federator/Options.hs
index e180f2accc..f66bbd4392 100644
--- a/services/federator/src/Federator/Options.hs
+++ b/services/federator/src/Federator/Options.hs
@@ -89,9 +89,7 @@ data Opts = Opts
     -- | Logformat to use
     logFormat :: !(Maybe (Last LogFormat)),
     -- | Runtime settings
-    optSettings :: !RunSettings,
-    -- | Domain update interval (microseconds)
-    domainUpdateInterval :: !Int
+    optSettings :: !RunSettings
   }
   deriving (Show, Generic)
 
diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index efa2c7d421..0bc472b1c5 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -53,6 +53,7 @@ import Servant.Client.Core
 import Servant.Types.SourceT
 import Wire.Network.DNS.Effect
 import Wire.Sem.Logger.TinyLog
+import Wire.API.Routes.FederationDomainConfig
 
 defaultHeaders :: [HTTP.Header]
 defaultHeaders = [("Content-Type", "application/json")]
@@ -119,7 +120,7 @@ type AllEffects =
      ServiceStreaming,
      Input RunSettings,
      Input Http2Manager, -- needed by Remote
-     Input AllowedDomains, -- needed for `FederationStrategy` `AllowList`.
+     Input FederationDomainConfigs, -- needed for the domain list.
      Input Env, -- needed by Service
      Error ValidationError,
      Error RemoteError,
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index c2ee894918..d320356c7d 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -55,9 +55,8 @@ import qualified System.Logger.Extended as LogExt
 import Util.Options
 import Wire.API.Federation.Component
 import Wire.API.Routes.FederationDomainConfig
-import qualified Wire.API.Routes.Internal.Brig as IAPI
-import Wire.API.Routes.Named
 import qualified Wire.Network.DNS.Helper as DNS
+import Wire.API.FederationUpdate
 
 ------------------------------------------------------------------------------
 -- run/app
@@ -67,38 +66,16 @@ run :: Opts -> IO ()
 run opts = do
   manager <- newManager defaultManagerSettings
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
-  let Endpoint host port = brig opts
+      Endpoint host port = brig opts
       baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-
-      getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
-
-      getAllowedDomainsOnce :: IO AllowedDomains
-      getAllowedDomainsOnce =
-        (AllowedDomains . fmap domain . fromFederationDomainConfigs)
-          <$> let go :: IO FederationDomainConfigs
-                  go = do
-                    runClientM getFedRemotes clientEnv >>= \case
-                      Right s -> pure s
-                      Left e -> do
-                        print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e -- TODO: log error or critical!
-                        threadDelay $ domainUpdateInterval opts
-                        go
-               in go
-
-      getAllowedDomainsLoop :: Env -> IO ()
-      getAllowedDomainsLoop env = forever $ do
-        threadDelay $ domainUpdateInterval opts
-        atomicWriteIORef (view allowedRemoteDomains env) =<< getAllowedDomainsOnce
-
-  okRemoteDomains <- getAllowedDomainsOnce
-
+  okRemoteDomains <- getAllowedDomainsInitial clientEnv
   DNS.withCachingResolver resolvConf $ \res ->
     bracket (newEnv opts res okRemoteDomains) closeEnv $ \env -> do
       let externalServer = serveInward env portExternal
           internalServer = serveOutward env portInternal
       withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
-        updateAllowedDomainsThread <- async (getAllowedDomainsLoop env)
+        updateAllowedDomainsThread <- async (getAllowedDomainsLoop clientEnv $ view allowedRemoteDomains env)
         internalServerThread <- async internalServer
         externalServerThread <- async externalServer
         void $ waitAnyCancel [updateAllowedDomainsThread, internalServerThread, externalServerThread]
@@ -121,7 +98,7 @@ run opts = do
 -------------------------------------------------------------------------------
 -- Environment
 
-newEnv :: Opts -> DNS.Resolver -> AllowedDomains -> IO Env
+newEnv :: Opts -> DNS.Resolver -> FederationDomainConfigs -> IO Env
 newEnv o _dnsResolver okRemoteDomains = do
   _metrics <- Metrics.metrics
   _applog <- LogExt.mkLogger (Opt.logLevel o) (Opt.logNetStrings o) (Opt.logFormat o)
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index 3b55fc8498..114178c290 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -37,7 +37,6 @@ import qualified Data.Text.Lazy as LText
 import qualified Data.X509 as X509
 import qualified Data.X509.Validation as X509
 import Federator.Discovery
-import Federator.Env
 import Federator.Error
 import Federator.Options
 import Imports
@@ -47,6 +46,7 @@ import Polysemy
 import Polysemy.Error
 import Polysemy.Input
 import Wire.Network.DNS.SRV (SrvTarget (..))
+import Wire.API.Routes.FederationDomainConfig
 
 data ValidationError
   = NoClientCertificate
@@ -93,7 +93,7 @@ validationErrorStatus _ = HTTP.status403
 -- | Validates an already-parsed domain against the allowList (stored in
 -- `brig.federation_remotes`, cached in `Env`).
 ensureCanFederateWith ::
-  ( Member (Input AllowedDomains) r,
+  ( Member (Input FederationDomainConfigs) r,
     Member (Input RunSettings) r,
     Member (Error ValidationError) r
   ) =>
@@ -104,8 +104,8 @@ ensureCanFederateWith targetDomain = do
   case strategy of
     AllowAll -> pure ()
     AllowList -> do
-      AllowedDomains domains <- input
-      unless (targetDomain `elem` domains) $
+      FederationDomainConfigs domains _ <- input
+      unless (targetDomain `elem` fmap domain domains) $
         throw (FederationDenied targetDomain)
 
 decodeCertificate ::
@@ -143,7 +143,7 @@ parseDomainText domain =
 -- by the client certificate
 validateDomain ::
   ( Member (Input RunSettings) r,
-    Member (Input AllowedDomains) r,
+    Member (Input FederationDomainConfigs) r,
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
     Member DiscoverFederator r
diff --git a/services/federator/test/unit/Test/Federator/ExternalServer.hs b/services/federator/test/unit/Test/Federator/ExternalServer.hs
index 8ecb1184d8..6e0d9776e4 100644
--- a/services/federator/test/unit/Test/Federator/ExternalServer.hs
+++ b/services/federator/test/unit/Test/Federator/ExternalServer.hs
@@ -46,6 +46,7 @@ import Test.Tasty
 import Test.Tasty.HUnit
 import Wire.API.Federation.Component
 import Wire.Sem.Logger.TinyLog
+import Wire.API.Routes.FederationDomainConfig
 
 tests :: TestTree
 tests =
@@ -113,6 +114,7 @@ requestBrigSuccess =
         . discardTinyLogs
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ callInward request
     let expectedCall = Call Brig "/federation/get-user-by-handle" "\"foo\"" aValidDomain
     assertEqual "one call to brig should be made" [expectedCall] actualCalls
@@ -138,6 +140,7 @@ requestBrigFailure =
         . discardTinyLogs
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ callInward request
 
     let expectedCall = Call Brig "/federation/get-user-by-handle" "\"foo\"" aValidDomain
@@ -164,6 +167,7 @@ requestGalleySuccess =
           . discardTinyLogs
           . mockDiscoveryTrivial
           . runInputConst noClientCertSettings
+          . runInputConst (FederationDomainConfigs [] 0)
           $ callInward request
       let expectedCall = Call Galley "/federation/get-conversations" "\"foo\"" aValidDomain
       embed $ assertEqual "one call to galley should be made" [expectedCall] actualCalls
@@ -192,6 +196,7 @@ requestNoDomain =
           . discardTinyLogs
           . mockDiscoveryTrivial
           . runInputConst noClientCertSettings
+          . runInputConst (FederationDomainConfigs [] 0)
           $ callInward request
 
       embed $ assertEqual "no calls to services should be made" [] actualCalls
@@ -217,6 +222,7 @@ requestNoCertificate =
         . discardTinyLogs
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ callInward request
 
     assertEqual "no calls to services should be made" [] actualCalls
@@ -268,6 +274,7 @@ testInvalidPaths = do
           . discardTinyLogs
           . mockDiscoveryTrivial
           . runInputConst noClientCertSettings
+          . runInputConst (FederationDomainConfigs [] 0)
           $ callInward request
 
       assertEqual ("Expected request with path \"" <> cs invalidPath <> "\" to fail") (Left InvalidRoute) (void res)
@@ -291,6 +298,7 @@ testInvalidComponent =
         . discardTinyLogs
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ callInward request
 
     void res @?= Left (UnknownComponent "mast")
@@ -319,6 +327,7 @@ testMethod =
           . discardTinyLogs
           . mockDiscoveryTrivial
           . runInputConst noClientCertSettings
+          . runInputConst (FederationDomainConfigs [] 0)
           $ callInward request
       void res @?= Left InvalidRoute
 
diff --git a/services/federator/test/unit/Test/Federator/InternalServer.hs b/services/federator/test/unit/Test/Federator/InternalServer.hs
index f4d791bfc8..2d56a2cd88 100644
--- a/services/federator/test/unit/Test/Federator/InternalServer.hs
+++ b/services/federator/test/unit/Test/Federator/InternalServer.hs
@@ -25,7 +25,7 @@ import Data.Default
 import Data.Domain
 import Federator.Error.ServerError
 import Federator.InternalServer (callOutward)
-import Federator.Options (AllowedDomains (..), FederationStrategy (..), RunSettings (..))
+import Federator.Options (FederationStrategy (..), RunSettings (..))
 import Federator.Remote
 import Federator.Validation
 import Imports
@@ -44,6 +44,8 @@ import Test.Tasty.HUnit
 import Wire.API.Federation.Component
 import Wire.API.Federation.Domain
 import Wire.Sem.Logger.TinyLog
+import Wire.API.Routes.FederationDomainConfig
+import Wire.API.User.Search
 
 tests :: TestTree
 tests =
@@ -56,9 +58,9 @@ tests =
         ]
     ]
 
-settingsWithAllowList :: [Domain] -> RunSettings
-settingsWithAllowList domains =
-  noClientCertSettings {federationStrategy = AllowList (AllowedDomains domains)}
+settingsWithAllowList :: RunSettings
+settingsWithAllowList =
+  noClientCertSettings {federationStrategy = AllowList}
 
 federatedRequestSuccess :: TestTree
 federatedRequestSuccess =
@@ -95,6 +97,7 @@ federatedRequestSuccess =
         . assertNoError @ServerError
         . discardTinyLogs
         . runInputConst settings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ callOutward request
     Wai.responseStatus res @?= HTTP.status200
     body <- Wai.lazyResponseBody res
@@ -106,7 +109,7 @@ federatedRequestSuccess =
 federatedRequestFailureAllowList :: TestTree
 federatedRequestFailureAllowList =
   testCase "should not make a call when target domain not in the allowList" $ do
-    let settings = settingsWithAllowList [Domain "hello.world"]
+    let settings = settingsWithAllowList
     let targetDomain = Domain "target.example.com"
         headers = [(originDomainHeaderName, "origin.example.com")]
     request <-
@@ -136,6 +139,7 @@ federatedRequestFailureAllowList =
         . assertNoError @ServerError
         . discardTinyLogs
         . runInputConst settings
+        . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "hello.world") FullSearch] 0)
         $ callOutward request
     eith @?= Left (FederationDenied targetDomain)
 
diff --git a/services/federator/test/unit/Test/Federator/Options.hs b/services/federator/test/unit/Test/Federator/Options.hs
index 4c3b62e2b9..b197c310a8 100644
--- a/services/federator/test/unit/Test/Federator/Options.hs
+++ b/services/federator/test/unit/Test/Federator/Options.hs
@@ -23,10 +23,7 @@ module Test.Federator.Options where
 
 import Control.Exception (try)
 import Data.Aeson (FromJSON)
-import qualified Data.Aeson as Aeson
 import qualified Data.ByteString.Char8 as B8
-import Data.ByteString.Lazy (toStrict)
-import Data.Domain (Domain (..), mkDomain)
 import Data.String.Interpolate as QQ
 import qualified Data.Yaml as Yaml
 import Federator.Options
@@ -54,10 +51,11 @@ tests :: TestTree
 tests =
   testGroup
     "Options"
-    [ parseFederationStrategy,
+    [ -- parseFederationStrategy,
       testSettings
     ]
 
+{- TODO fixme
 parseFederationStrategy :: TestTree
 parseFederationStrategy =
   testCase "parse FederationStrategy examples" $ do
@@ -83,7 +81,8 @@ parseFederationStrategy =
     assertParsesAs allowWire $ allowedDom
   where
     withAllowList =
-      AllowList . AllowedDomains . map (either error id . mkDomain)
+      AllowedDomains . map (either error id . mkDomain)
+-}
 
 testSettings :: TestTree
 testSettings =
@@ -103,10 +102,7 @@ testSettings =
       testCase "parse configuration example (closed federation)" $ do
         let settings =
               (defRunSettings "client.pem" "client-key.pem")
-                { federationStrategy =
-                    AllowList
-                      ( AllowedDomains [Domain "server2.example.com"]
-                      ),
+                { federationStrategy = AllowList,
                   useSystemCAStore = False
                 }
         assertParsesAs settings . B8.pack $
diff --git a/services/federator/test/unit/Test/Federator/Validation.hs b/services/federator/test/unit/Test/Federator/Validation.hs
index 95c842f676..57615d5da4 100644
--- a/services/federator/test/unit/Test/Federator/Validation.hs
+++ b/services/federator/test/unit/Test/Federator/Validation.hs
@@ -38,6 +38,8 @@ import Test.Federator.Util
 import Test.Tasty
 import Test.Tasty.HUnit
 import Wire.Network.DNS.SRV (SrvTarget (..))
+import Wire.API.Routes.FederationDomainConfig
+import Wire.API.User.Search
 
 mockDiscoveryTrivial :: Sem (DiscoverFederator ': r) x -> Sem r x
 mockDiscoveryTrivial = Polysemy.interpret $ \case
@@ -86,20 +88,22 @@ tests =
 federateWithAllowListSuccess :: TestTree
 federateWithAllowListSuccess =
   testCase "should give True when target domain is in the list" $ do
-    let settings = settingsWithAllowList [Domain "hello.world"]
+    let settings = settingsWithAllowList 
     runM
       . assertNoError @ValidationError
       . runInputConst settings
+      . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "hello.world") FullSearch] 0)
       $ ensureCanFederateWith (Domain "hello.world")
 
 federateWithAllowListFail :: TestTree
 federateWithAllowListFail =
   testCase "should give False when target domain is not in the list" $ do
-    let settings = settingsWithAllowList [Domain "only.other.domain"]
+    let settings = settingsWithAllowList
     eith :: Either ValidationError () <-
       runM
         . runError @ValidationError
         . runInputConst settings
+        . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ ensureCanFederateWith (Domain "hello.world")
     assertBool "federating should not be allowed" (isLeft eith)
 
@@ -107,13 +111,14 @@ validateDomainAllowListFailSemantic :: TestTree
 validateDomainAllowListFailSemantic =
   testCase "semantic validation" $ do
     exampleCert <- BS.readFile "test/resources/unit/localhost.pem"
-    let settings = settingsWithAllowList [Domain "only.other.domain"]
+    let settings = settingsWithAllowList
     res <-
       runM
         . runError
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
+        . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ validateDomain (Just exampleCert) "invalid//.><-semantic-&@-domain"
     res @?= Left (DomainParseError "invalid//.><-semantic-&@-domain")
 
@@ -124,13 +129,14 @@ validateDomainAllowListFail :: TestTree
 validateDomainAllowListFail =
   testCase "allow list validation" $ do
     exampleCert <- BS.readFile "test/resources/unit/localhost.example.com.pem"
-    let settings = settingsWithAllowList [Domain "only.other.domain"]
+    let settings = settingsWithAllowList
     res <-
       runM
         . runError
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
+        . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ validateDomain (Just exampleCert) "localhost.example.com"
     res @?= Left (FederationDenied (Domain "localhost.example.com"))
 
@@ -141,13 +147,14 @@ validateDomainAllowListSuccess =
   testCase "should give parsed domain if in the allow list" $ do
     exampleCert <- BS.readFile "test/resources/unit/localhost.example.com.pem"
     let domain = Domain "localhost.example.com"
-        settings = settingsWithAllowList [domain]
+        settings = settingsWithAllowList
     res <-
       runM
         . assertNoError @ValidationError
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
+        . runInputConst (FederationDomainConfigs [FederationDomainConfig domain FullSearch] 0)
         $ validateDomain (Just exampleCert) (toByteString' domain)
     assertEqual "validateDomain should give 'localhost.example.com' as domain" domain res
 
@@ -160,6 +167,7 @@ validateDomainCertMissing =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ validateDomain Nothing "foo.example.com"
     res @?= Left NoClientCertificate
 
@@ -174,6 +182,7 @@ validateDomainCertInvalid =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ validateDomain (Just "not a certificate") "foo.example.com"
     res @?= Left (CertificateParseError "no certificate found")
 
@@ -193,6 +202,7 @@ validateDomainCertWrongDomain =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ validateDomain (Just exampleCert) "foo.example.com"
     res @?= Left (AuthenticationFailure (pure [X509.NameMismatch "foo.example.com"]))
 
@@ -209,6 +219,7 @@ validateDomainCertCN =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ validateDomain (Just exampleCert) (toByteString' domain)
     res @?= domain
 
@@ -223,6 +234,7 @@ validateDomainCertSAN =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ validateDomain (Just exampleCert) (toByteString' domain)
     res @?= domain
 
@@ -237,6 +249,7 @@ validateDomainMultipleFederators =
             . assertNoError @DiscoveryFailure
             . mockDiscoveryMapping domain ("localhost.example.com" :| ["second-federator.example.com"])
             . runInputConst noClientCertSettings
+            . runInputConst (FederationDomainConfigs [] 0)
         domain = Domain "foo.example.com"
     resFirst <-
       runValidation $
@@ -258,6 +271,7 @@ validateDomainDiscoveryFailed =
         . assertNoError @ValidationError
         . mockDiscoveryFailure
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ validateDomain (Just exampleCert) "example.com"
     res @?= Left (DiscoveryFailureDNSError "mock DNS error")
 
@@ -272,9 +286,10 @@ validateDomainNonIdentitySRV =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryMapping domain ("localhost.example.com" :| [])
         . runInputConst noClientCertSettings
+        . runInputConst (FederationDomainConfigs [] 0)
         $ validateDomain (Just exampleCert) (toByteString' domain)
     res @?= domain
 
-settingsWithAllowList :: [Domain] -> RunSettings
-settingsWithAllowList domains =
-  noClientCertSettings {federationStrategy = AllowList (AllowedDomains domains)}
+settingsWithAllowList :: RunSettings
+settingsWithAllowList =
+  noClientCertSettings {federationStrategy = AllowList}

From adbf7d972d4a8ca94b48866fce07aa17d9361f00 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 10 May 2023 19:15:48 +1000
Subject: [PATCH 044/220] FS-1115: More common code for federation domain
 updates

---
 .../wire-api/src/Wire/API/FederationUpdate.hs | 20 ++++++++--
 services/cannon/src/Cannon/Run.hs             |  4 +-
 services/federator/src/Federator/Run.hs       |  2 +-
 services/galley/galley.integration.yaml       |  4 +-
 services/galley/src/Galley/App.hs             | 17 ++------
 services/galley/src/Galley/Env.hs             |  2 +-
 services/galley/src/Galley/Options.hs         |  5 +--
 services/galley/src/Galley/Run.hs             | 40 +++++++------------
 services/gundeck/src/Gundeck/Run.hs           | 32 ++-------------
 9 files changed, 43 insertions(+), 83 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 49f49e6fcd..9dcbdf840f 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -27,10 +27,22 @@ getAllowedDomainsInitial clientEnv =
 getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
 getAllowedDomains = runClientM getFedRemotes
 
-getAllowedDomainsLoop :: ClientEnv -> IORef FederationDomainConfigs -> IO ()
-getAllowedDomainsLoop clientEnv env = forever $ do
+type FedUpdateCallback = FederationDomainConfigs -> FederationDomainConfigs -> IO ()
+
+-- The callback takes the previous and the new values of the federation domain configs
+-- and runs a given action. This function is not called if a new config value cannot be fetched.
+getAllowedDomainsLoop :: ClientEnv -> IORef FederationDomainConfigs -> FedUpdateCallback -> IO ()
+getAllowedDomainsLoop clientEnv env callback = forever $ do
   getAllowedDomains clientEnv >>= \case
     Left e -> print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e -- TODO: log error or critical!
-    Right cfg -> atomicWriteIORef env cfg
+    Right cfg -> do
+      old <- readIORef env
+      callback old cfg
+      atomicWriteIORef env cfg
   delay <- updateInterval <$> readIORef env
-  threadDelay delay
\ No newline at end of file
+  threadDelay delay
+
+-- A version where the callback isn't needed. Most of the services don't care about
+-- when the list changes, just that they have the new list and can use it as-is
+getAllowedDomainsLoop' :: ClientEnv -> IORef FederationDomainConfigs -> IO ()
+getAllowedDomainsLoop' c r = getAllowedDomainsLoop c r $ \_ _ -> pure ()
\ No newline at end of file
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index 1930ff08e8..4ad246e7a3 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -56,7 +56,7 @@ import UnliftIO.Concurrent (myThreadId, throwTo)
 import qualified Wire.API.Routes.Internal.Cannon as Internal
 import Wire.API.Routes.Public.Cannon
 import Wire.API.Routes.Version.Wai
-import Wire.API.FederationUpdate (getAllowedDomainsLoop, getAllowedDomainsInitial)
+import Wire.API.FederationUpdate (getAllowedDomainsInitial, getAllowedDomainsLoop')
 
 type CombinedAPI = PublicAPI :<|> Internal.API
 
@@ -85,7 +85,7 @@ run o = do
       clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
   fedStrat <- getAllowedDomainsInitial clientEnv
   ioref <- newIORef fedStrat
-  updateDomainsThread <- Async.async $ getAllowedDomainsLoop clientEnv ioref
+  updateDomainsThread <- Async.async $ getAllowedDomainsLoop' clientEnv ioref
 
   let middleware :: Wai.Middleware
       middleware =
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index d320356c7d..aa7cc82ab3 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -75,7 +75,7 @@ run opts = do
       let externalServer = serveInward env portExternal
           internalServer = serveOutward env portInternal
       withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
-        updateAllowedDomainsThread <- async (getAllowedDomainsLoop clientEnv $ view allowedRemoteDomains env)
+        updateAllowedDomainsThread <- async (getAllowedDomainsLoop' clientEnv $ view allowedRemoteDomains env)
         internalServerThread <- async internalServer
         externalServerThread <- async externalServer
         void $ waitAnyCancel [updateAllowedDomainsThread, internalServerThread, externalServerThread]
diff --git a/services/galley/galley.integration.yaml b/services/galley/galley.integration.yaml
index cd6685f6f6..f81722cc86 100644
--- a/services/galley/galley.integration.yaml
+++ b/services/galley/galley.integration.yaml
@@ -86,6 +86,4 @@ logNetStrings: false
 journal: # if set, journals; if not set, disables journaling
   queueName: integration-team-events.fifo
   endpoint: http://localhost:4568 # https://sqs.eu-west-1.amazonaws.com
-  region: eu-west-1
-
-domainUpdateInterval: 1000000
\ No newline at end of file
+  region: eu-west-1
\ No newline at end of file
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index 343d9e6d10..7bf7520e05 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -108,9 +108,8 @@ import qualified UnliftIO.Exception as UnliftIO
 import Util.Options
 import Wire.API.Error
 import Wire.API.Federation.Error
-import qualified Wire.API.Routes.Internal.Brig as IAPI
-import Wire.API.Routes.Named (namedClient)
 import qualified Wire.Sem.Logger
+import Wire.API.FederationUpdate
 
 -- Effects needed by the interpretation of other effects
 type GalleyEffects0 =
@@ -168,23 +167,13 @@ createEnv m o = do
       Endpoint h p = brigEndpoint
       baseUrl = SC.BaseUrl SC.Http (unpack h) (fromIntegral p) ""
       clientEnv = SC.ClientEnv mgr baseUrl Nothing SC.defaultMakeClientRequest
-      getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
-      -- Loop the request until we get an answer. This is helpful during integration
-      -- tests where services are being brought up in parallel.
-      getInitalFedDomains = do
-        SC.runClientM getFedRemotes clientEnv >>= \case
-          Right s -> pure s
-          Left e -> do
-            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e
-            threadDelay $ o ^. optDomainUpdateInterval
-            getInitalFedDomains
-  strat <- getInitalFedDomains
+  strat <- getAllowedDomainsInitial clientEnv
   Env def m o l mgr h2mgr (o ^. optFederator) brigEndpoint cass
     <$> Q.new 16000
     <*> initExtEnv
     <*> maybe (pure Nothing) (fmap Just . Aws.mkEnv l mgr) (o ^. optJournal)
     <*> loadAllMLSKeys (fold (o ^. optSettings . setMlsPrivateKeyPaths))
-    <*> newTVarIO strat
+    <*> newIORef strat
 
 initCassandra :: Opts -> Logger -> IO ClientState
 initCassandra o l = do
diff --git a/services/galley/src/Galley/Env.hs b/services/galley/src/Galley/Env.hs
index 0232fd4f17..632a318db7 100644
--- a/services/galley/src/Galley/Env.hs
+++ b/services/galley/src/Galley/Env.hs
@@ -61,7 +61,7 @@ data Env = Env
     _extEnv :: ExtEnv,
     _aEnv :: Maybe Aws.Env,
     _mlsKeys :: SignaturePurpose -> MLSKeys,
-    _fedDomains :: TVar FederationDomainConfigs
+    _fedDomains :: IORef FederationDomainConfigs
   }
 
 -- | Environment specific to the communication with external
diff --git a/services/galley/src/Galley/Options.hs b/services/galley/src/Galley/Options.hs
index d40bd4b2ed..d9776451eb 100644
--- a/services/galley/src/Galley/Options.hs
+++ b/services/galley/src/Galley/Options.hs
@@ -52,7 +52,6 @@ module Galley.Options
     optLogLevel,
     optLogNetStrings,
     optLogFormat,
-    optDomainUpdateInterval,
   )
 where
 
@@ -173,9 +172,7 @@ data Opts = Opts
     --  <http://cr.yp.to/proto/netstrings.txt>
     _optLogNetStrings :: !(Maybe (Last Bool)),
     -- | What log format to use
-    _optLogFormat :: !(Maybe (Last LogFormat)),
-    -- | Domain update interval (microseconds)
-    _optDomainUpdateInterval :: !Int
+    _optLogFormat :: !(Maybe (Last LogFormat))
   }
 
 deriveFromJSON toOptionFieldName ''Opts
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 11a5d54e9f..b2aded9702 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -66,17 +66,14 @@ import Servant.Client
     ClientEnv (ClientEnv),
     Scheme (Http),
     defaultMakeClientRequest,
-    runClientM,
   )
 import qualified System.Logger as Log
-import qualified System.Logger.Class as L
 import Util.Options
 import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
-import qualified Wire.API.Routes.Internal.Brig as IAPI
-import Wire.API.Routes.Named (namedClient)
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
+import Wire.API.FederationUpdate
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -186,29 +183,20 @@ collectAuthMetrics m env = do
 
 updateFedDomains :: App ()
 updateFedDomains = do
-  updateInterval <- view $ options . optDomainUpdateInterval
-  tvar <- view fedDomains
+  ioref <- view fedDomains
   manager' <- view manager
   Endpoint host port <- view brig
   let baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
-  forever $ do
-    threadDelay updateInterval
-    previous <- liftIO $ readTVarIO tvar
-    strat <- liftIO $ runClientM getFedRemotes clientEnv
-    let domainListsEqual s =
-          Set.fromList (fromFederationDomainConfigs s)
-            == Set.fromList (fromFederationDomainConfigs previous)
-    case strat of
-      Left e -> L.err . L.msg $ "Could not retrieve federation domains from brig: " <> show e
-      -- Using Set to do the comparison, as it will handle the lists being in different orders.
-      Right s -> unless (domainListsEqual s) $ do
-        -- Perform updates before rewriting the tvar
-        -- This means that if the update fails on a
-        -- particular invocation, it can be run again
-        -- on the next firing as it isn't likely that
-        -- the domain list is changing frequently.
-        -- FS-1179 is handling this part.
-        liftIO $ atomically $ writeTVar tvar s
-  where
-    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
+  
+  liftIO $ do
+    okRemoteDomains <- getAllowedDomainsInitial clientEnv
+    atomicWriteIORef ioref okRemoteDomains
+    let domainListsEqual old new =
+            Set.fromList (domain <$> fromFederationDomainConfigs old)
+              == Set.fromList (domain <$> fromFederationDomainConfigs new)
+        callback old new = unless (domainListsEqual old new) $ do
+          -- TODO: perform the database updates here
+          -- This code will only run when there is a change in the domain lists
+          pure ()
+    getAllowedDomainsLoop clientEnv ioref callback
diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs
index a55fa5ee7b..d2104739a7 100644
--- a/services/gundeck/src/Gundeck/Run.hs
+++ b/services/gundeck/src/Gundeck/Run.hs
@@ -54,11 +54,9 @@ import Servant.Client
 import qualified System.Logger as Log
 import qualified UnliftIO.Async as Async
 import Util.Options
-import Wire.API.Routes.FederationDomainConfig
-import qualified Wire.API.Routes.Internal.Brig as IAPI
-import Wire.API.Routes.Named (namedClient)
 import Wire.API.Routes.Public.Gundeck (GundeckAPI)
 import Wire.API.Routes.Version.Wai
+import Wire.API.FederationUpdate
 
 run :: Opts -> IO ()
 run o = do
@@ -75,20 +73,9 @@ run o = do
   let Endpoint host port = o ^. optBrig
       baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
-      -- Loop the request until we get an answer. This is helpful during integration
-      -- tests where services are being brought up in parallel.
-      getInitialFedDomains = do
-        runClientM getFedRemotes clientEnv >>= \case
-          Right strat -> pure strat
-          Left err -> do
-            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show err
-            threadDelay $ o ^. optDomainUpdateInterval
-            getInitialFedDomains
-  fedStrat <- getInitialFedDomains
-  tEnv <- newTVarIO fedStrat
-  let callback :: FederationDomainConfigs -> IO ()
-      callback = atomically . writeTVar tEnv
-  updateDomainsThread <- Async.async $ updateDomains clientEnv callback
+  fedStrat <- getAllowedDomainsInitial clientEnv
+  ioref <- newIORef fedStrat
+  updateDomainsThread <- Async.async $ getAllowedDomainsLoop' clientEnv ioref
 
   lst <- Async.async $ Aws.execute (e ^. awsEnv) (Aws.listen throttleMillis (runDirect e . onEvent))
   wtbs <- forM (e ^. threadBudgetState) $ \tbs -> Async.async $ runDirect e $ watchThreadBudgetState m tbs 10
@@ -113,17 +100,6 @@ run o = do
         . GZip.gzip GZip.def
         . catchErrors (e ^. applog) [Right $ e ^. monitor]
 
-    getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
-
-    updateDomains :: ClientEnv -> (FederationDomainConfigs -> IO ()) -> IO ()
-    updateDomains clientEnv update = forever $ do
-      threadDelay $ o ^. optDomainUpdateInterval
-      strat <- runClientM getFedRemotes clientEnv
-      either
-        print
-        update
-        strat
-
 type CombinedAPI = GundeckAPI :<|> Servant.Raw
 
 mkApp :: Env -> Wai.Application

From 236c2fdc6efe209500cba1013e9dc5bd57ee60b3 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 10 May 2023 14:29:55 +0200
Subject: [PATCH 045/220] Use tinylog instead of `print`.

---
 .../wire-api/src/Wire/API/FederationUpdate.hs | 38 ++++++++++++-------
 libs/wire-api/wire-api.cabal                  |  1 +
 services/cannon/src/Cannon/Run.hs             |  6 +--
 services/federator/src/Federator/Run.hs       | 18 ++++-----
 services/galley/src/Galley/App.hs             |  4 +-
 services/galley/src/Galley/Run.hs             | 13 ++++---
 services/gundeck/src/Gundeck/Run.hs           |  6 +--
 7 files changed, 50 insertions(+), 36 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 9dcbdf840f..d3f5e51acb 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -1,28 +1,37 @@
-module Wire.API.FederationUpdate where
+module Wire.API.FederationUpdate
+  ( FedUpdateCallback,
+    getAllowedDomainsInitial,
+    getAllowedDomainsLoop,
+    getAllowedDomainsLoop',
+  )
+where
 
 import Imports
+import Servant.Client (ClientEnv, ClientError, runClientM)
+import Servant.Client.Internal.HttpClient (ClientM)
+import qualified System.Logger as L
 import Wire.API.Routes.FederationDomainConfig (FederationDomainConfigs (updateInterval))
 import qualified Wire.API.Routes.Internal.Brig as IAPI
-import Servant.Client (runClientM, ClientError, ClientEnv)
 import Wire.API.Routes.Named (namedClient)
-import Servant.Client.Internal.HttpClient (ClientM)
 
 getFedRemotes :: ClientM FederationDomainConfigs
 getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
 
 -- Initial function for getting the set of domains from brig, and an update interval
-getAllowedDomainsInitial :: ClientEnv -> IO FederationDomainConfigs
-getAllowedDomainsInitial clientEnv =
+getAllowedDomainsInitial :: L.Logger -> ClientEnv -> IO FederationDomainConfigs
+getAllowedDomainsInitial logger clientEnv =
   let oneSec = 1000000 -- microsends
       go :: IO FederationDomainConfigs
       go = do
         getAllowedDomains clientEnv >>= \case
           Right s -> pure s
           Left e -> do
-            print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e -- TODO: log error or critical!
-            threadDelay oneSec
+            L.log logger L.Info $
+              L.msg (L.val "Could not retrieve an initial list of federation domains from Brig.")
+                L.~~ "error" L..= show e
+            threadDelay oneSec -- TODO: use retry instead.
             go
-  in go
+   in go
 
 getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
 getAllowedDomains = runClientM getFedRemotes
@@ -31,10 +40,13 @@ type FedUpdateCallback = FederationDomainConfigs -> FederationDomainConfigs -> I
 
 -- The callback takes the previous and the new values of the federation domain configs
 -- and runs a given action. This function is not called if a new config value cannot be fetched.
-getAllowedDomainsLoop :: ClientEnv -> IORef FederationDomainConfigs -> FedUpdateCallback -> IO ()
-getAllowedDomainsLoop clientEnv env callback = forever $ do
+getAllowedDomainsLoop :: L.Logger -> ClientEnv -> IORef FederationDomainConfigs -> FedUpdateCallback -> IO ()
+getAllowedDomainsLoop logger clientEnv env callback = forever $ do
   getAllowedDomains clientEnv >>= \case
-    Left e -> print $ "Could not retrieve the latest list of federation domains from Brig: " <> show e -- TODO: log error or critical!
+    Left e ->
+      L.log logger L.Fatal $
+        L.msg (L.val "Could not retrieve an updated list of federation domains from Brig; I'll keep trying!")
+          L.~~ "error" L..= show e
     Right cfg -> do
       old <- readIORef env
       callback old cfg
@@ -44,5 +56,5 @@ getAllowedDomainsLoop clientEnv env callback = forever $ do
 
 -- A version where the callback isn't needed. Most of the services don't care about
 -- when the list changes, just that they have the new list and can use it as-is
-getAllowedDomainsLoop' :: ClientEnv -> IORef FederationDomainConfigs -> IO ()
-getAllowedDomainsLoop' c r = getAllowedDomainsLoop c r $ \_ _ -> pure ()
\ No newline at end of file
+getAllowedDomainsLoop' :: L.Logger -> ClientEnv -> IORef FederationDomainConfigs -> IO ()
+getAllowedDomainsLoop' logger c r = getAllowedDomainsLoop logger c r $ \_ _ -> pure ()
diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal
index ddc2ba8c10..19b4b32962 100644
--- a/libs/wire-api/wire-api.cabal
+++ b/libs/wire-api/wire-api.cabal
@@ -289,6 +289,7 @@ library
     , tagged
     , text                       >=0.11
     , time                       >=1.4
+    , tinylog
     , transitive-anns
     , types-common               >=0.16
     , unordered-containers       >=0.2
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index 4ad246e7a3..fd52ddcc54 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -53,10 +53,10 @@ import System.Posix.Signals
 import qualified System.Posix.Signals as Signals
 import System.Random.MWC (createSystemRandom)
 import UnliftIO.Concurrent (myThreadId, throwTo)
+import Wire.API.FederationUpdate (getAllowedDomainsInitial, getAllowedDomainsLoop')
 import qualified Wire.API.Routes.Internal.Cannon as Internal
 import Wire.API.Routes.Public.Cannon
 import Wire.API.Routes.Version.Wai
-import Wire.API.FederationUpdate (getAllowedDomainsInitial, getAllowedDomainsLoop')
 
 type CombinedAPI = PublicAPI :<|> Internal.API
 
@@ -83,9 +83,9 @@ run o = do
   let Brig bh bp = o ^. brig
       baseUrl = BaseUrl Http (unpack bh) (fromIntegral bp) ""
       clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-  fedStrat <- getAllowedDomainsInitial clientEnv
+  fedStrat <- getAllowedDomainsInitial g clientEnv
   ioref <- newIORef fedStrat
-  updateDomainsThread <- Async.async $ getAllowedDomainsLoop' clientEnv ioref
+  updateDomainsThread <- Async.async $ getAllowedDomainsLoop' g clientEnv ioref
 
   let middleware :: Wai.Middleware
       middleware =
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index aa7cc82ab3..870b6d8099 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -50,13 +50,13 @@ import qualified Network.DNS as DNS
 import Network.HTTP.Client
 import qualified Network.HTTP.Client as HTTP
 import Servant.Client
-import qualified System.Logger.Class as Log
+import qualified System.Logger as Log
 import qualified System.Logger.Extended as LogExt
 import Util.Options
 import Wire.API.Federation.Component
+import Wire.API.FederationUpdate
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.Network.DNS.Helper as DNS
-import Wire.API.FederationUpdate
 
 ------------------------------------------------------------------------------
 -- run/app
@@ -65,17 +65,18 @@ import Wire.API.FederationUpdate
 run :: Opts -> IO ()
 run opts = do
   manager <- newManager defaultManagerSettings
+  logger <- LogExt.mkLogger (Opt.logLevel opts) (Opt.logNetStrings opts) (Opt.logFormat opts)
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
       Endpoint host port = brig opts
       baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-  okRemoteDomains <- getAllowedDomainsInitial clientEnv
+  okRemoteDomains <- getAllowedDomainsInitial logger clientEnv
   DNS.withCachingResolver resolvConf $ \res ->
-    bracket (newEnv opts res okRemoteDomains) closeEnv $ \env -> do
+    bracket (newEnv opts res logger okRemoteDomains) closeEnv $ \env -> do
       let externalServer = serveInward env portExternal
           internalServer = serveOutward env portInternal
-      withMonitor (env ^. applog) (onNewSSLContext env) (optSettings opts) $ do
-        updateAllowedDomainsThread <- async (getAllowedDomainsLoop' clientEnv $ view allowedRemoteDomains env)
+      withMonitor logger (onNewSSLContext env) (optSettings opts) $ do
+        updateAllowedDomainsThread <- async (getAllowedDomainsLoop' logger clientEnv $ view allowedRemoteDomains env)
         internalServerThread <- async internalServer
         externalServerThread <- async externalServer
         void $ waitAnyCancel [updateAllowedDomainsThread, internalServerThread, externalServerThread]
@@ -98,10 +99,9 @@ run opts = do
 -------------------------------------------------------------------------------
 -- Environment
 
-newEnv :: Opts -> DNS.Resolver -> FederationDomainConfigs -> IO Env
-newEnv o _dnsResolver okRemoteDomains = do
+newEnv :: Opts -> DNS.Resolver -> Log.Logger -> FederationDomainConfigs -> IO Env
+newEnv o _dnsResolver _applog okRemoteDomains = do
   _metrics <- Metrics.metrics
-  _applog <- LogExt.mkLogger (Opt.logLevel o) (Opt.logNetStrings o) (Opt.logFormat o)
   let _requestId = def
   let _runSettings = Opt.optSettings o
   let _service Brig = Opt.brig o
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index 7bf7520e05..f77dffcd5e 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -108,8 +108,8 @@ import qualified UnliftIO.Exception as UnliftIO
 import Util.Options
 import Wire.API.Error
 import Wire.API.Federation.Error
-import qualified Wire.Sem.Logger
 import Wire.API.FederationUpdate
+import qualified Wire.Sem.Logger
 
 -- Effects needed by the interpretation of other effects
 type GalleyEffects0 =
@@ -167,7 +167,7 @@ createEnv m o = do
       Endpoint h p = brigEndpoint
       baseUrl = SC.BaseUrl SC.Http (unpack h) (fromIntegral p) ""
       clientEnv = SC.ClientEnv mgr baseUrl Nothing SC.defaultMakeClientRequest
-  strat <- getAllowedDomainsInitial clientEnv
+  strat <- getAllowedDomainsInitial l clientEnv
   Env def m o l mgr h2mgr (o ^. optFederator) brigEndpoint cass
     <$> Q.new 16000
     <*> initExtEnv
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index b2aded9702..ae1d96fefb 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -69,11 +69,11 @@ import Servant.Client
   )
 import qualified System.Logger as Log
 import Util.Options
+import Wire.API.FederationUpdate
 import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
-import Wire.API.FederationUpdate
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -184,19 +184,20 @@ collectAuthMetrics m env = do
 updateFedDomains :: App ()
 updateFedDomains = do
   ioref <- view fedDomains
+  logger <- view applog
   manager' <- view manager
   Endpoint host port <- view brig
   let baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
-  
+
   liftIO $ do
-    okRemoteDomains <- getAllowedDomainsInitial clientEnv
+    okRemoteDomains <- getAllowedDomainsInitial logger clientEnv
     atomicWriteIORef ioref okRemoteDomains
     let domainListsEqual old new =
-            Set.fromList (domain <$> fromFederationDomainConfigs old)
-              == Set.fromList (domain <$> fromFederationDomainConfigs new)
+          Set.fromList (domain <$> fromFederationDomainConfigs old)
+            == Set.fromList (domain <$> fromFederationDomainConfigs new)
         callback old new = unless (domainListsEqual old new) $ do
           -- TODO: perform the database updates here
           -- This code will only run when there is a change in the domain lists
           pure ()
-    getAllowedDomainsLoop clientEnv ioref callback
+    getAllowedDomainsLoop logger clientEnv ioref callback
diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs
index d2104739a7..c9715adf96 100644
--- a/services/gundeck/src/Gundeck/Run.hs
+++ b/services/gundeck/src/Gundeck/Run.hs
@@ -54,9 +54,9 @@ import Servant.Client
 import qualified System.Logger as Log
 import qualified UnliftIO.Async as Async
 import Util.Options
+import Wire.API.FederationUpdate
 import Wire.API.Routes.Public.Gundeck (GundeckAPI)
 import Wire.API.Routes.Version.Wai
-import Wire.API.FederationUpdate
 
 run :: Opts -> IO ()
 run o = do
@@ -73,9 +73,9 @@ run o = do
   let Endpoint host port = o ^. optBrig
       baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
-  fedStrat <- getAllowedDomainsInitial clientEnv
+  fedStrat <- getAllowedDomainsInitial l clientEnv
   ioref <- newIORef fedStrat
-  updateDomainsThread <- Async.async $ getAllowedDomainsLoop' clientEnv ioref
+  updateDomainsThread <- Async.async $ getAllowedDomainsLoop' l clientEnv ioref
 
   lst <- Async.async $ Aws.execute (e ^. awsEnv) (Aws.listen throttleMillis (runDirect e . onEvent))
   wtbs <- forM (e ^. threadBudgetState) $ \tbs -> Async.async $ runDirect e $ watchThreadBudgetState m tbs 10

From 1a6a103e38205dc91c3041ab47351a648b302215 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 10 May 2023 14:31:35 +0200
Subject: [PATCH 046/220] sanitize-pr

---
 libs/wire-api/default.nix                                | 2 ++
 .../src/Wire/API/Routes/FederationDomainConfig.hs        | 6 +++---
 services/brig/src/Brig/API/Internal.hs                   | 9 +++++----
 services/federator/src/Federator/Options.hs              | 1 -
 services/federator/src/Federator/Response.hs             | 2 +-
 services/federator/src/Federator/Validation.hs           | 2 +-
 .../federator/test/unit/Test/Federator/ExternalServer.hs | 2 +-
 .../federator/test/unit/Test/Federator/InternalServer.hs | 2 +-
 .../federator/test/unit/Test/Federator/Validation.hs     | 4 ++--
 9 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/libs/wire-api/default.nix b/libs/wire-api/default.nix
index f8051e1799..96184c85b8 100644
--- a/libs/wire-api/default.nix
+++ b/libs/wire-api/default.nix
@@ -93,6 +93,7 @@
 , tasty-quickcheck
 , text
 , time
+, tinylog
 , transitive-anns
 , types-common
 , unliftio
@@ -190,6 +191,7 @@ mkDerivation {
     tagged
     text
     time
+    tinylog
     transitive-anns
     types-common
     unordered-containers
diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 00cfe35a62..725a5bd839 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -47,8 +47,8 @@ instance ToSchema FederationDomainConfig where
         <*> cfgSearchPolicy .= field "search_policy" schema
 
 data FederationDomainConfigs = FederationDomainConfigs
-  { fromFederationDomainConfigs :: [FederationDomainConfig]
-  , updateInterval :: Int
+  { fromFederationDomainConfigs :: [FederationDomainConfig],
+    updateInterval :: Int
   }
   deriving (Show, Generic, Eq)
   deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationDomainConfigs
@@ -58,4 +58,4 @@ instance ToSchema FederationDomainConfigs where
     object "FederationDomainConfigs" $
       FederationDomainConfigs
         <$> fromFederationDomainConfigs .= field "remotes" (array schema)
-        <*> updateInterval .= field "updateInterval" schema
\ No newline at end of file
+        <*> updateInterval .= field "updateInterval" schema
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 3d4673bddc..a0e8f418bc 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -200,10 +200,11 @@ getFederationRemotes = lift $ do
   -- more robust migration path.  See
   -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections,
   -- http://docs.wire.com/developer/developer/federation-design-aspects.html#configuring-remote-connections-dev-perspective
-  pure $ FederationDomainConfigs 
-    { fromFederationDomainConfigs = nub $ db <> cfg
-    , updateInterval = 1000000 -- TODO FIX ME!
-    }
+  pure $
+    FederationDomainConfigs
+      { fromFederationDomainConfigs = nub $ db <> cfg,
+        updateInterval = 1000000 -- TODO FIX ME!
+      }
 
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
diff --git a/services/federator/src/Federator/Options.hs b/services/federator/src/Federator/Options.hs
index f66bbd4392..87a503d801 100644
--- a/services/federator/src/Federator/Options.hs
+++ b/services/federator/src/Federator/Options.hs
@@ -1,4 +1,3 @@
-{-# LANGUAGE GeneralizedNewtypeDeriving #-}
 {-# LANGUAGE StrictData #-}
 
 -- This file is part of the Wire Server implementation.
diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index 0bc472b1c5..aaa238dd35 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -51,9 +51,9 @@ import Polysemy.Internal
 import Polysemy.TinyLog
 import Servant.Client.Core
 import Servant.Types.SourceT
+import Wire.API.Routes.FederationDomainConfig
 import Wire.Network.DNS.Effect
 import Wire.Sem.Logger.TinyLog
-import Wire.API.Routes.FederationDomainConfig
 
 defaultHeaders :: [HTTP.Header]
 defaultHeaders = [("Content-Type", "application/json")]
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index 114178c290..aaa58f5ac1 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -45,8 +45,8 @@ import qualified Network.Wai.Utilities.Error as Wai
 import Polysemy
 import Polysemy.Error
 import Polysemy.Input
-import Wire.Network.DNS.SRV (SrvTarget (..))
 import Wire.API.Routes.FederationDomainConfig
+import Wire.Network.DNS.SRV (SrvTarget (..))
 
 data ValidationError
   = NoClientCertificate
diff --git a/services/federator/test/unit/Test/Federator/ExternalServer.hs b/services/federator/test/unit/Test/Federator/ExternalServer.hs
index 6e0d9776e4..297f357803 100644
--- a/services/federator/test/unit/Test/Federator/ExternalServer.hs
+++ b/services/federator/test/unit/Test/Federator/ExternalServer.hs
@@ -45,8 +45,8 @@ import Test.Federator.Validation (mockDiscoveryTrivial)
 import Test.Tasty
 import Test.Tasty.HUnit
 import Wire.API.Federation.Component
-import Wire.Sem.Logger.TinyLog
 import Wire.API.Routes.FederationDomainConfig
+import Wire.Sem.Logger.TinyLog
 
 tests :: TestTree
 tests =
diff --git a/services/federator/test/unit/Test/Federator/InternalServer.hs b/services/federator/test/unit/Test/Federator/InternalServer.hs
index 2d56a2cd88..df8d82fdf3 100644
--- a/services/federator/test/unit/Test/Federator/InternalServer.hs
+++ b/services/federator/test/unit/Test/Federator/InternalServer.hs
@@ -43,9 +43,9 @@ import Test.Tasty
 import Test.Tasty.HUnit
 import Wire.API.Federation.Component
 import Wire.API.Federation.Domain
-import Wire.Sem.Logger.TinyLog
 import Wire.API.Routes.FederationDomainConfig
 import Wire.API.User.Search
+import Wire.Sem.Logger.TinyLog
 
 tests :: TestTree
 tests =
diff --git a/services/federator/test/unit/Test/Federator/Validation.hs b/services/federator/test/unit/Test/Federator/Validation.hs
index 57615d5da4..ec533b793b 100644
--- a/services/federator/test/unit/Test/Federator/Validation.hs
+++ b/services/federator/test/unit/Test/Federator/Validation.hs
@@ -37,9 +37,9 @@ import Test.Federator.Options (noClientCertSettings)
 import Test.Federator.Util
 import Test.Tasty
 import Test.Tasty.HUnit
-import Wire.Network.DNS.SRV (SrvTarget (..))
 import Wire.API.Routes.FederationDomainConfig
 import Wire.API.User.Search
+import Wire.Network.DNS.SRV (SrvTarget (..))
 
 mockDiscoveryTrivial :: Sem (DiscoverFederator ': r) x -> Sem r x
 mockDiscoveryTrivial = Polysemy.interpret $ \case
@@ -88,7 +88,7 @@ tests =
 federateWithAllowListSuccess :: TestTree
 federateWithAllowListSuccess =
   testCase "should give True when target domain is in the list" $ do
-    let settings = settingsWithAllowList 
+    let settings = settingsWithAllowList
     runM
       . assertNoError @ValidationError
       . runInputConst settings

From 950867e5439c94d8ef560f7495682bdfe206bc1f Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 10 May 2023 15:03:34 +0200
Subject: [PATCH 047/220] Use retry instead of threadDelay-loop.

---
 libs/wire-api/src/Wire/API/FederationUpdate.hs | 18 ++++++++++++------
 libs/wire-api/wire-api.cabal                   |  2 ++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index d3f5e51acb..67281c2bbf 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -6,6 +6,8 @@ module Wire.API.FederationUpdate
   )
 where
 
+import Control.Exception (ErrorCall (ErrorCall), throwIO)
+import qualified Control.Retry as R
 import Imports
 import Servant.Client (ClientEnv, ClientError, runClientM)
 import Servant.Client.Internal.HttpClient (ClientM)
@@ -20,18 +22,22 @@ getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
 -- Initial function for getting the set of domains from brig, and an update interval
 getAllowedDomainsInitial :: L.Logger -> ClientEnv -> IO FederationDomainConfigs
 getAllowedDomainsInitial logger clientEnv =
-  let oneSec = 1000000 -- microsends
-      go :: IO FederationDomainConfigs
+  let -- keep trying every 3s for one minute
+      policy :: R.RetryPolicy
+      policy = R.constantDelay 3_081_003 <> R.limitRetries 20
+
+      go :: IO (Maybe FederationDomainConfigs)
       go = do
         getAllowedDomains clientEnv >>= \case
-          Right s -> pure s
+          Right s -> pure $ Just s
           Left e -> do
             L.log logger L.Info $
               L.msg (L.val "Could not retrieve an initial list of federation domains from Brig.")
                 L.~~ "error" L..= show e
-            threadDelay oneSec -- TODO: use retry instead.
-            go
-   in go
+            pure Nothing
+   in R.retrying policy (const (pure . isNothing)) (const go) >>= \case
+        Just c -> pure c
+        Nothing -> throwIO $ ErrorCall "*** Failed to reach brig for federation setup, giving up!"
 
 getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
 getAllowedDomains = runClientM getFedRemotes
diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal
index 19b4b32962..870aa828ad 100644
--- a/libs/wire-api/wire-api.cabal
+++ b/libs/wire-api/wire-api.cabal
@@ -191,6 +191,7 @@ library
     MultiParamTypeClasses
     MultiWayIf
     NamedFieldPuns
+    NumericUnderscores
     OverloadedRecordDot
     OverloadedStrings
     PackageImports
@@ -269,6 +270,7 @@ library
     , quickcheck-instances       >=0.3.16
     , random                     >=1.2.0
     , resourcet
+    , retry
     , saml2-web-sso
     , schema-profunctor
     , scientific

From 676862a6fffe1173667dd2cf558f5f3ae81217ad Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 11 May 2023 10:40:57 +1000
Subject: [PATCH 048/220] FS-1115: Updating design docs on how update intervals
 are supplied

---
 docs/src/developer/developer/federation-design-aspects.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/src/developer/developer/federation-design-aspects.md b/docs/src/developer/developer/federation-design-aspects.md
index 54e50cd152..d41670aab5 100644
--- a/docs/src/developer/developer/federation-design-aspects.md
+++ b/docs/src/developer/developer/federation-design-aspects.md
@@ -20,7 +20,10 @@ brig itself for performance keeps a `TVar` that it updates at regular
 intervals.  Plus provides the contents of the `TVar` via an internal
 CRUD API (see {ref}`configuring-remote-connections` for the links).
 
-Update intervals are currently hard-wired into the code.
+Update intervals are currently supplied by Brig in same response that
+carries the federation domain lists. This allows for simplified control
+of the update times and minimises changes to both services and their
+configuration files.
 
 Introduced in
 [PR#3260](https://github.com/wireapp/wire-server/pull/3260).

From fc00935b2d28144887420261b070a097d8bfc276 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 11 May 2023 08:33:34 +0200
Subject: [PATCH 049/220] Clarify source comment.

---
 services/brig/src/Brig/API/Internal.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index a0e8f418bc..da77ed815f 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -196,8 +196,9 @@ getFederationRemotes :: ExceptT Brig.API.Error.Error (AppT r) FederationDomainCo
 getFederationRemotes = lift $ do
   db <- wrapClient Data.getFederationRemotes
   cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
-  -- FUTUREWORK: we should solely rely on `db` in the future; `cfg` is just for an easier,
-  -- more robust migration path.  See
+  -- FUTUREWORK: we should solely rely on `db` in the future for remote domains; merging
+  -- remote domains from `cfg` is just for providing an easier, more robust migration path.
+  -- See
   -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections,
   -- http://docs.wire.com/developer/developer/federation-design-aspects.html#configuring-remote-connections-dev-perspective
   pure $

From d39dd452d97a243e9ed733ace8ec49b9a878787b Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 11 May 2023 17:02:55 +1000
Subject: [PATCH 050/220] FS-1115: Cleaning up un-needed changes to options

---
 services/cannon/cannon.integration.yaml   | 1 -
 services/cannon/cannon2.integration.yaml  | 1 -
 services/cannon/src/Cannon/Options.hs     | 5 +----
 services/gundeck/gundeck.integration.yaml | 2 --
 services/gundeck/src/Gundeck/Options.hs   | 2 --
 5 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/services/cannon/cannon.integration.yaml b/services/cannon/cannon.integration.yaml
index e2d3e243c2..1886f69405 100644
--- a/services/cannon/cannon.integration.yaml
+++ b/services/cannon/cannon.integration.yaml
@@ -19,7 +19,6 @@ gundeck:
 brig:
   host: 0.0.0.0
   port: 8082
-domainUpdateInterval: 1000000
 
 drainOpts:
   gracePeriodSeconds: 1
diff --git a/services/cannon/cannon2.integration.yaml b/services/cannon/cannon2.integration.yaml
index 3baf6a8100..2aa003cfd6 100644
--- a/services/cannon/cannon2.integration.yaml
+++ b/services/cannon/cannon2.integration.yaml
@@ -19,7 +19,6 @@ gundeck:
 brig:
   host: 0.0.0.0
   port: 8082
-domainUpdateInterval: 1000000
 
 drainOpts:
   gracePeriodSeconds: 1
diff --git a/services/cannon/src/Cannon/Options.hs b/services/cannon/src/Cannon/Options.hs
index 0eab5e3908..8e59927ff4 100644
--- a/services/cannon/src/Cannon/Options.hs
+++ b/services/cannon/src/Cannon/Options.hs
@@ -37,7 +37,6 @@ module Cannon.Options
     millisecondsBetweenBatches,
     minBatchSize,
     disabledAPIVersions,
-    domainUpdateInterval,
     DrainOpts,
   )
 where
@@ -104,9 +103,7 @@ data Opts = Opts
     _optsLogNetStrings :: !(Maybe (Last Bool)),
     _optsLogFormat :: !(Maybe (Last LogFormat)),
     _optsDrainOpts :: DrainOpts,
-    _optsDisabledAPIVersions :: Maybe (Set Version),
-    -- | Domain update interval (microseconds)
-    _optsDomainUpdateInterval :: !Int
+    _optsDisabledAPIVersions :: Maybe (Set Version)
   }
   deriving (Eq, Show, Generic)
 
diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml
index dde2717184..b43f3520f4 100644
--- a/services/gundeck/gundeck.integration.yaml
+++ b/services/gundeck/gundeck.integration.yaml
@@ -41,7 +41,5 @@ settings:
     hard: 30  # more than this number of threads will not be allowed
     soft: 10  # more than this number of threads will be warned about
 
-domainUpdateInterval: 1000000
-
 logLevel: Warn
 logNetStrings: false
diff --git a/services/gundeck/src/Gundeck/Options.hs b/services/gundeck/src/Gundeck/Options.hs
index 8971f1c3b5..999a0cd088 100644
--- a/services/gundeck/src/Gundeck/Options.hs
+++ b/services/gundeck/src/Gundeck/Options.hs
@@ -123,8 +123,6 @@ data Opts = Opts
     _optAws :: !AWSOpts,
     _optDiscoUrl :: !(Maybe Text),
     _optSettings :: !Settings,
-    -- | Domain update interval (microseconds)
-    _optDomainUpdateInterval :: !Int,
     -- Logging
 
     -- | Log level (Debug, Info, etc)

From 6f5ee229348e99489dbe2226f74c7f43f23182d3 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 11 May 2023 11:07:13 +0200
Subject: [PATCH 051/220] update docs.wire.com

---
 .../developer/federation-design-aspects.md    | 25 +++++++++----------
 .../federation/backend-communication.md       | 12 +++++++++
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/docs/src/developer/developer/federation-design-aspects.md b/docs/src/developer/developer/federation-design-aspects.md
index d41670aab5..afd3711737 100644
--- a/docs/src/developer/developer/federation-design-aspects.md
+++ b/docs/src/developer/developer/federation-design-aspects.md
@@ -8,22 +8,21 @@ Federation can start and end.  These events need handlers to be called
 (like remove remote users from local conv), plus it is not convenient
 to edit and re-deploy config files every time that happens.  Hence
 remotes are stored in cassandra in brig, and every pod of every
-service keeps a cache in a `TVar` (this information is needed in many
-end-points).
+service keeps a cache in an `IORef` in its `Env` (this information is
+needed in many end-points, so it has to remain as fast as read access
+to `Env`).
 
-This secion elaborates on the implementation.  See
+This section elaborates on the implementation.  See
 {ref}`configuring-remote-connections` for the administrator's point of
-view.  Go read that section now!
+view.  If you haven't done so, go read that section now!
 
-The state is persistent in cassandra table `brig.federation_remotes`
-brig itself for performance keeps a `TVar` that it updates at regular
-intervals.  Plus provides the contents of the `TVar` via an internal
-CRUD API (see {ref}`configuring-remote-connections` for the links).
-
-Update intervals are currently supplied by Brig in same response that
-carries the federation domain lists. This allows for simplified control
-of the update times and minimises changes to both services and their
-configuration files.
+The state is persisted in cassandra table `brig.federation_remotes`.
+brig provides the contents via an internal CRUD API (see
+{ref}`configuring-remote-connections` for the links).  In the future,
+we may decide that brig needs to cache the table itself, but for now
+(`GET` is only used for the internal end-point to share it with other
+services) we hope to get away with the simple solution and always read
+from cassandra directly.
 
 Introduced in
 [PR#3260](https://github.com/wireapp/wire-server/pull/3260).
diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index 0b23610c72..cd314adc61 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -201,6 +201,18 @@ upgrade.
 
 -}
 
+TODO: explain how things need a while to stabilize (configurable), but
+that the other backend also needs to know us in order to be reachable.
+(how do we handle one backend being known to the other first for a few
+minutes / hours?)
+
+
+Update intervals are currently supplied by Brig in same response that
+carries the federation domain lists. This allows for simplified control
+of the update times and minimises changes to both services and their
+configuration files.
+
+
 See {ref}`configuring-remote-connections-dev-perspective` for the
 developer's point of view on this topic.
 

From db3970288f16c329fa2e70b7c40d49734d429858 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 11 May 2023 12:02:52 +0200
Subject: [PATCH 052/220] docs.

---
 .../src/developer/reference/config-options.md | 73 ++++----------
 .../federation/backend-communication.md       | 97 +++++++++++++++++++
 2 files changed, 114 insertions(+), 56 deletions(-)

diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md
index 5aa8a2b680..9df3b03391 100644
--- a/docs/src/developer/reference/config-options.md
+++ b/docs/src/developer/reference/config-options.md
@@ -393,44 +393,30 @@ settings:
 
 ### Federation allow list
 
-As of 2021-07, federation (whatever is implemented by the time you read this) is turned off by default by means of having an empty allow list:
-
-```yaml
-# federator.yaml
-optSettings:
-  federationStrategy:
-    allowedDomains: []
-```
-
-You can choose to federate with a specific list of allowed servers:
+As of the release containing [PR#3260](https://github.com/wireapp/wire-server/pull/3260), federator gets its configuration from brig (which has a cassandra instance and is thus better equipped to handle persistent dynamic data).  See {ref}`configuring-remote-connections` for the whole story.
 
+Federation is turned off by default, and you can turn it off
+explicitly switching to strategy "allowedDomains", and making sure
+that `brig.federation_remotes` (the response of the
+[`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
+request is empty).
 
 ```yaml
-# federator.yaml
+# brig.yaml
 optSettings:
-  federationStrategy:
-    allowedDomains:
-      - server1.example.com
-      - server2.example.com
+  federationStrategy: allowedDomains
 ```
 
-or, you can federate with everyone:
+If you want to federate with all domains that ask, change this to:
 
 ```yaml
-# federator.yaml
+# brig.yaml
 optSettings:
-  federationStrategy:
-    # note the 'empty' value after 'allowAll'
-    allowAll:
-
-# when configuring helm charts, this becomes (note 'true' after 'allowAll')
-# inside helm_vars/wire-server:
-federator:
-  optSettings:
-    federationStrategy:
-      allowAll: true
+  federationStrategy: allowAll
 ```
 
+If you want to federate selectively with a list of known peers, consult {ref}`configuring-remote-connections`.
+
 ### Federation TLS Config
 
 When a federator connects with another federator, it does so over HTTPS. There
@@ -612,36 +598,11 @@ any key package whose expiry date is set further than 15 days after upload time
 
 ### Federated domain specific configuration settings
 
-**This section is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
-https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
-for details.**
-
-#### Restrict user search
-
-TODO: deprecate this, also rename this section.  it's about federation now.
-
-TODO: should we consider the federation strategy from federator in the
-union returned by brig for a transition period as well?  (if not, we
-need to insist on updating brig's config before this upgrade.  no
-remote backend may be unlisted and use the search policy default.  we
-should also crash on startup when somebody tries that.)
-
-The lookup and search of users on a wire instance can be configured. This can be done per federated domain.
-
-```yaml
-# [brig.yaml]
-optSettings:
-  setFederationDomainConfigs:
-    - domain: example.com
-      search_policy: no_search
-```
-
-Valid values for `search_policy` are:
-- `no_search`: No users are returned by federated searches.
-- `exact_handle_search`: Only users where the handle exactly matches are returned.
-- `full_search`: Additionally to `exact_handle_search`, users are found by a freetext search on handle and display name.
+As of the release containing
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260), you can
+configure search policies on a peer-by-peer basis.  See
+{ref}`configuring-remote-connections` for how.
 
-If there is no configuration for a domain, it's defaulted to `no_search`.
 
 ### API Versioning
 
diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index cd314adc61..aa2f357dbb 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -246,3 +246,100 @@ config file will be ignored, and you should remove it at your
 convenience.  See
 [docs](https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
 for details."*)
+
+
+
+### noise!!!
+
+
+
+### Federation allow list
+
+As of 2021-07, federation (whatever is implemented by the time you read this) is turned off by default by means of having an empty allow list:
+
+```yaml
+# federator.yaml
+optSettings:
+  federationStrategy:
+    allowedDomains: []
+```
+
+You can choose to federate with a specific list of allowed servers:
+
+
+```yaml
+# federator.yaml
+optSettings:
+  federationStrategy:
+    allowedDomains:
+      - server1.example.com
+      - server2.example.com
+```
+
+or, you can federate with everyone:
+
+```yaml
+# federator.yaml
+optSettings:
+  federationStrategy:
+    # note the 'empty' value after 'allowAll'
+    allowAll:
+
+# when configuring helm charts, this becomes (note 'true' after 'allowAll')
+# inside helm_vars/wire-server:
+federator:
+  optSettings:
+    federationStrategy:
+      allowAll: true
+```
+
+
+
+
+this is deprecated:
+
+```
+  setFederationDomainConfigs:
+    - domain: example.com
+      search_policy: no_search
+
+```
+
+
+
+
+**This section is deprecated .  See
+https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
+for details.**
+
+#### Restrict user search
+
+TODO: deprecate this, also rename this section.  it's about federation now.
+
+TODO: should we consider the federation strategy from federator in the
+union returned by brig for a transition period as well?  (if not, we
+need to insist on updating brig's config before this upgrade.  no
+remote backend may be unlisted and use the search policy default.  we
+should also crash on startup when somebody tries that.)
+
+The lookup and search of users on a wire instance can be configured. This can be done per federated domain.
+
+```yaml
+# [brig.yaml]
+optSettings:
+  setFederationDomainConfigs:
+    - domain: example.com
+      search_policy: no_search
+```
+
+Valid values for `search_policy` are:
+- `no_search`: No users are returned by federated searches.
+- `exact_handle_search`: Only users where the handle exactly matches are returned.
+- `full_search`: Additionally to `exact_handle_search`, users are found by a freetext search on handle and display name.
+
+If there is no configuration for a domain, it's defaulted to `no_search`.
+
+
+
+
+does anybody know off the top of their heads: is [this section](https://wearezeta.atlassian.net/wiki/spaces/BAC/pages/288620677/Processes+shared+with+CS#Different-search-visibility-per-team) still up to date?  and is stern?  [this page](https://docs.wire.com/developer/reference/config-options.html#federated-domain-specific-configuration-settings) tells a different story...

From 3846ae3ac2fb1a1fcb7b3babc8bbcc4264eb69df Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 12 May 2023 10:48:06 +0200
Subject: [PATCH 053/220] docs.

---
 .../src/developer/reference/config-options.md | 19 +++++++++++--------
 .../federation/backend-communication.md       | 13 +++++++------
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md
index 9df3b03391..e2c8f2a339 100644
--- a/docs/src/developer/reference/config-options.md
+++ b/docs/src/developer/reference/config-options.md
@@ -391,13 +391,13 @@ settings:
   federationDomain: example.com
 ```
 
-### Federation allow list
+### Federation strategy: whom to federate with?
 
 As of the release containing [PR#3260](https://github.com/wireapp/wire-server/pull/3260), federator gets its configuration from brig (which has a cassandra instance and is thus better equipped to handle persistent dynamic data).  See {ref}`configuring-remote-connections` for the whole story.
 
-Federation is turned off by default, and you can turn it off
-explicitly switching to strategy "allowedDomains", and making sure
-that `brig.federation_remotes` (the response of the
+Federation is turned off by default.  You can also turn it off
+explicitly by setting federation strategy "allowedDomains", and making
+sure that `brig.federation_remotes` (the response of the
 [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
 request is empty).
 
@@ -598,10 +598,13 @@ any key package whose expiry date is set further than 15 days after upload time
 
 ### Federated domain specific configuration settings
 
-As of the release containing
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260), you can
-configure search policies on a peer-by-peer basis.  See
-{ref}`configuring-remote-connections` for how.
+#### Restrict user search
+
+You can configure search policies on a peer-by-peer basis, but using
+the config file for that is not supported any more since the release
+containing
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260).  See
+{ref}`configuring-remote-connections` for how to do this now.
 
 
 ### API Versioning
diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index aa2f357dbb..c8c6b3c126 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -160,13 +160,14 @@ search request from *Alice*, one of its clients.
 
 Up to the release containing
 [PR#3260](https://github.com/wireapp/wire-server/pull/3260), the
-config files of the individual services statically contained
-information about the remote connections.  Starting with this release,
-this information is stored in the database, and there is an internal
-REST API for adding and removing remotes:
+config files of the individual services statically contained the
+domains of remote connections.  Starting with this release, this and
+all information about remote connections is stored in the database,
+and there is an internal REST API for adding and removing remotes:
 
-* [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
 * [`POST`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/post_i_federation_remotes)
+* [`PUT`](TODO)
+* [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
 * [`DELETE`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/delete_i_federation_remotes__domain_)
 
 **WARNING:** If you delete a connection, all users from that remote
@@ -177,7 +178,7 @@ can be re-established should you decide to add the same backend later.
 
 Changing the configuration of existing edges via `PUT` is not
 implemented at the moment, if you need to do that, delete the
-connection and add it again.
+connection and add it again.  TODO: this is bullshit.  go implement it!
 
 {-
 TODO: this paragraph still annoys me.  move strategy to brig, too?  or

From 8b45f0f7cd5e0a8203fe2f7a2f34bbeca7b0522e Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Wed, 10 May 2023 18:05:15 +0200
Subject: [PATCH 054/220] Fix: `/i/user/meta-info` (stern) (#3281)

---
 changelog.d/3-bug-fixes/pr-3281     |  1 +
 tools/stern/src/Stern/Intra.hs      |  2 +-
 tools/stern/test/integration/API.hs | 10 +++++++++-
 3 files changed, 11 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/3-bug-fixes/pr-3281

diff --git a/changelog.d/3-bug-fixes/pr-3281 b/changelog.d/3-bug-fixes/pr-3281
new file mode 100644
index 0000000000..da3054c620
--- /dev/null
+++ b/changelog.d/3-bug-fixes/pr-3281
@@ -0,0 +1 @@
+Fixed `/i/user/meta-info` in backoffice/stern
diff --git a/tools/stern/src/Stern/Intra.hs b/tools/stern/src/Stern/Intra.hs
index 8a38eaf08e..5f311d1f93 100644
--- a/tools/stern/src/Stern/Intra.hs
+++ b/tools/stern/src/Stern/Intra.hs
@@ -813,7 +813,7 @@ getUserProperties uid = do
             b
             ( method GET
                 . header "Z-User" (toByteString' uid)
-                . versionedPaths ["/properties", toByteString' x]
+                . versionedPaths ["properties", toByteString' x]
                 . expect2xx
             )
       info $ msg ("Response" ++ show r)
diff --git a/tools/stern/test/integration/API.hs b/tools/stern/test/integration/API.hs
index 0ef9daa1d9..ca62050c10 100644
--- a/tools/stern/test/integration/API.hs
+++ b/tools/stern/test/integration/API.hs
@@ -27,7 +27,7 @@ import Bilge.Assert
 import Brig.Types.Intra
 import Control.Applicative
 import Control.Lens hiding ((.=))
-import Data.Aeson (ToJSON)
+import Data.Aeson (ToJSON, Value)
 import Data.ByteString.Conversion
 import Data.Handle
 import Data.Id
@@ -42,6 +42,7 @@ import Test.Tasty
 import Test.Tasty.HUnit
 import TestSetup
 import Util
+import Wire.API.Properties (PropertyKey)
 import Wire.API.Routes.Internal.Brig.Connection
 import qualified Wire.API.Routes.Internal.Brig.EJPD as EJPD
 import Wire.API.Routes.Internal.Galley.TeamsIntra (tdStatus)
@@ -116,6 +117,8 @@ testSearchVisibility = do
 testGetUserMetaInfo :: TestM ()
 testGetUserMetaInfo = do
   uid <- randomUser
+  let k = fromMaybe (error "invalid property key") $ fromByteString "WIRE_RECEIPT_MODE"
+  putUserProperty uid k "bar"
   -- Just make sure this returns a 200
   void $ getUserMetaInfo uid
 
@@ -615,3 +618,8 @@ unlockFeature ::
 unlockFeature tid = do
   g <- view tsGalley
   void $ put (g . paths ["i", "teams", toByteString' tid, "features", Public.featureNameBS @cfg, "unlocked"] . expect2xx)
+
+putUserProperty :: UserId -> PropertyKey -> Value -> TestM ()
+putUserProperty uid k v = do
+  b <- view tsBrig
+  void $ put (b . paths ["properties", toByteString' k] . json v . zUser uid . zConn "123" . expect2xx)

From 708bec255f6d8f2f4144b620e540f582d11c1dbc Mon Sep 17 00:00:00 2001
From: fisx <mf@zerobuzz.net>
Date: Thu, 11 May 2023 09:07:19 +0200
Subject: [PATCH 055/220] Handle race conditions in /integration (#3278)

---
 integration/test/Test/Demo.hs      | 10 ++++++++++
 integration/test/Testlib/App.hs    | 10 ++++++++++
 integration/test/Testlib/Cannon.hs | 13 ++++---------
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/integration/test/Test/Demo.hs b/integration/test/Test/Demo.hs
index f8a1bffa08..87202291f1 100644
--- a/integration/test/Test/Demo.hs
+++ b/integration/test/Test/Demo.hs
@@ -68,3 +68,13 @@ testMultipleBackends = do
   ownDomainRes `shouldMatch` ownDomain
   otherDomainRes `shouldMatch` otherDomain
   ownDomain `shouldNotMatch` otherDomain
+
+testUnrace :: App ()
+testUnrace = do
+  {-
+  -- the following would retry for ~30s and only then fail
+  unrace $ do
+    True `shouldMatch` True
+    True `shouldMatch` False
+  -}
+  unrace $ True `shouldMatch` True
diff --git a/integration/test/Testlib/App.hs b/integration/test/Testlib/App.hs
index ab88325489..b37469345b 100644
--- a/integration/test/Testlib/App.hs
+++ b/integration/test/Testlib/App.hs
@@ -1,6 +1,7 @@
 module Testlib.App where
 
 import Control.Monad.Reader
+import qualified Control.Retry as Retry
 import Data.Aeson hiding ((.=))
 import Data.IORef
 import qualified Data.Yaml as Yaml
@@ -49,3 +50,12 @@ ownDomain = asks (.domain1)
 
 otherDomain :: App String
 otherDomain = asks (.domain2)
+
+-- | Run an action, `recoverAll`ing with exponential backoff (min step 8ms, total timeout
+-- ~15s).  Search this package for examples how to use it.
+--
+-- Ideally, this will be the only thing you'll ever need from the retry package when writing
+-- integration tests.  If you are unhappy with it,, please consider fixing it so everybody can
+-- benefit.
+unrace :: App a -> App a
+unrace action = Retry.recoverAll (Retry.exponentialBackoff 8000 <> Retry.limitRetries 10) (const action)
diff --git a/integration/test/Testlib/Cannon.hs b/integration/test/Testlib/Cannon.hs
index caebbdeb54..94df997820 100644
--- a/integration/test/Testlib/Cannon.hs
+++ b/integration/test/Testlib/Cannon.hs
@@ -161,22 +161,17 @@ run wsConnect app = do
           )
           `onException` tryPutMVar latch ()
 
-  let waitForRegistry :: HasCallStack => Int -> App ()
-      waitForRegistry (0 :: Int) = failApp "Cannon: failed to register presence"
-      waitForRegistry n = do
+  let waitForRegistry :: HasCallStack => App ()
+      waitForRegistry = unrace $ do
         request <- baseRequest ownDomain Cannon Unversioned ("/i/presences/" <> wsConnect.user <> "/" <> connId)
         response <- submit "HEAD" request
-        unless (status response == 200) $ do
-          liftIO $ threadDelay $ 100 * 1000
-          waitForRegistry (n - 1)
+        status response `shouldMatchInt` 200
 
   liftIO $ takeMVar latch
   stat <- liftIO $ poll wsapp
   case stat of
     Just (Left ex) -> liftIO $ throwIO ex
-    _ -> waitForRegistry numRetries >> pure wsapp
-  where
-    numRetries = 30
+    _ -> waitForRegistry >> pure wsapp
 
 close :: MonadIO m => WebSocket -> m ()
 close ws = liftIO $ do

From e89b052b42f453c7676b99afba0250cdceb3a24c Mon Sep 17 00:00:00 2001
From: fisx <mf@zerobuzz.net>
Date: Thu, 11 May 2023 10:56:00 +0200
Subject: [PATCH 056/220] Update docs.wire.com (#3284)

* Deprecate start-services-only.sh

* Fix outdated docs on how to run swagger docs locally.
---
 README.md                                       |  4 ++--
 docs/src/developer/developer/how-to.md          | 17 +++++++----------
 .../api-client-perspective/swagger.md           |  2 ++
 services/spar/test-scim-suite/README.md         |  2 +-
 services/start-services-only.sh                 |  9 ++-------
 tools/stern/README.md                           |  2 +-
 6 files changed, 15 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index 9ac41294e7..fcfe538460 100644
--- a/README.md
+++ b/README.md
@@ -85,7 +85,7 @@ will, eventually, have built a range of docker images. Make sure to [give Docker
 
 See the `Makefile`s and `Dockerfile`s, as well as [build/ubuntu/README.md](build/ubuntu/README.md) for details.
 
-#### 2. Use nix-provided build environment 
+#### 2. Use nix-provided build environment
 
 This is suitable only for local development and testing. See [build instructions](./docs/src/developer/developer/building.md) in the developer documentation.
 
@@ -95,4 +95,4 @@ You have two options:
 
 * Option 1. (recommended) Install wire-server on kubernetes using the configuration and instructions provided in [wire-server-deploy](https://github.com/wireapp/wire-server-deploy). This is the best option to run it on a server and recommended if you want to self-host wire-server.
 
-* Option 2. Compile everything in this repo, then you can use the `services/start-services-only.sh`. This option is intended as a way to try out wire-server on your local development machine and not suited for production.
+* Option 2. Compile everything in this repo, then you can use the `services/run-services`. This option is intended as a way to try out wire-server on your local development machine and not suited for production.
diff --git a/docs/src/developer/developer/how-to.md b/docs/src/developer/developer/how-to.md
index 71d12c0851..6c7cc0e1cb 100644
--- a/docs/src/developer/developer/how-to.md
+++ b/docs/src/developer/developer/how-to.md
@@ -15,17 +15,14 @@ Terminal 1:
 * Set up backing services: `./deploy/dockerephemeral/run.sh`
 
 Terminal 2:
-* Compile all services: `make c`
-* Run services including nginz: `./services/start-services-only.sh`.
+* Build and start wire-server services: ` make c && ./services/run-services`
 
 Open your browser at:
-[http://localhost:8080/api/swagger-ui](http://localhost:8080/api/swagger-ui) for
-the Swagger 2.0 endpoints of the latest version. This endpoint is versioned;
-i.e. the Swagger docs refer to the API version. Refer to the [Swagger API
-documentation](../../understand/api-client-perspective/swagger.md) regarding
-Swagger and API versioning.
+[http://localhost:8080/api/swagger-ui](http://localhost:8080/api/swagger-ui) for a list of API verions.
 
-Swagger json is available under [http://localhost:8080/api/swagger.json](http://localhost:8080/api/swagger.json)
+Also check out the docs for swagger in our staging environment:
+{ref}`swagger-api-docs`.  Replace the staging domain by
+`localhost:8080` to get to your local build.
 
 
 ## How to run federation tests across two backends
@@ -211,6 +208,6 @@ Note: Simply deleting the namespaces is insufficient, because it leaves some res
 
 ## How to manage RabbitMQ
 
-We support two different ways of managing the docker-compose instance of rabbitmq: 
-* A web console interface is available [here](http://localhost:15672) 
+We support two different ways of managing the docker-compose instance of rabbitmq:
+* A web console interface is available [here](http://localhost:15672)
 * `rabbitmqadmin` CLI is made available in the dev environment
diff --git a/docs/src/understand/api-client-perspective/swagger.md b/docs/src/understand/api-client-perspective/swagger.md
index 3c9da8b96f..02d45a7ddb 100644
--- a/docs/src/understand/api-client-perspective/swagger.md
+++ b/docs/src/understand/api-client-perspective/swagger.md
@@ -1,3 +1,5 @@
+(swagger-api-docs)=
+
 # Swagger API documentation
 
 Our staging system provides [Swagger /
diff --git a/services/spar/test-scim-suite/README.md b/services/spar/test-scim-suite/README.md
index e7dfaaf85d..d2da32d467 100644
--- a/services/spar/test-scim-suite/README.md
+++ b/services/spar/test-scim-suite/README.md
@@ -4,7 +4,7 @@ The scripts in this directory allow to run the [SCIM Test Suite](https://github.
 
 How to run:
 ```sh
-./services/start-services-only.sh
+./services/run-services
 ./services/spar/test-scim-suite/runsuite.sh
 ```
 
diff --git a/services/start-services-only.sh b/services/start-services-only.sh
index 374e12f285..db5d23fd3b 100755
--- a/services/start-services-only.sh
+++ b/services/start-services-only.sh
@@ -1,10 +1,5 @@
 #!/usr/bin/env bash
 
-# Run all haskell services without immediately starting a test executable.
-# Can be useful for manually poking at the API.
-set -eo pipefail
-
 SERVICES_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# call run-services, show a message, then sleep (instead of executing a test executable)
-exec "$SERVICES_DIR/run-services"
+echo -e "\n\n\n*** $0 is deprecated.  please run '$SERVICES_DIR/run-services' instead.\n\n\n"
+exit 1
diff --git a/tools/stern/README.md b/tools/stern/README.md
index a5ad9a345b..884f486134 100644
--- a/tools/stern/README.md
+++ b/tools/stern/README.md
@@ -19,7 +19,7 @@ TODO: This section is under construction
 
 ## How to run stern locally
 
-Start local services via `services/start-services-only.sh`
+Start local services via `services/run-services`
 
 Open <http://localhost:8091> in a browser.
 

From 5638e5e4a9789bc15dd9d9a7f35f5b9f0b2de36c Mon Sep 17 00:00:00 2001
From: Stefan Matting <smatting@users.noreply.github.com>
Date: Thu, 11 May 2023 11:28:04 +0200
Subject: [PATCH 057/220] Restore deleted scripts (#3287)

---
 hack/bin/create_team.sh              |  67 +++++++++++++
 hack/bin/create_team_members.sh      | 101 +++++++++++++++++++
 hack/bin/create_team_request_code.sh |  46 +++++++++
 hack/bin/create_test_team_admins.sh  |  68 +++++++++++++
 hack/bin/create_test_team_members.sh | 139 +++++++++++++++++++++++++++
 hack/bin/create_test_user.sh         |  66 +++++++++++++
 6 files changed, 487 insertions(+)
 create mode 100755 hack/bin/create_team.sh
 create mode 100755 hack/bin/create_team_members.sh
 create mode 100755 hack/bin/create_team_request_code.sh
 create mode 100755 hack/bin/create_test_team_admins.sh
 create mode 100755 hack/bin/create_test_team_members.sh
 create mode 100755 hack/bin/create_test_user.sh

diff --git a/hack/bin/create_team.sh b/hack/bin/create_team.sh
new file mode 100755
index 0000000000..aa672a8594
--- /dev/null
+++ b/hack/bin/create_team.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# consider using create_team.py
+
+set -o pipefail
+set -o errexit
+
+BRIG_HOST="http://localhost:8080"
+OWNER_NAME="owner name n/a"
+OWNER_EMAIL="owner email n/a"
+OWNER_PASSWORD="owner pass n/a"
+EMAIL_CODE="email code n/a"
+TEAM_NAME="team name n/a"
+TEAM_CURRENCY="USD"
+
+USAGE="
+Request a code to create a team.  Call ./create_test_team_members.sh
+first, then use the code you will receive by email to call this script.
+
+USAGE: $0 -h <host> -o <owner_name> -e <owner_email> -p <owner_password> -v <email_code> -t <team_name> -c <team_currency>
+    -h <host>: Base URI of brig. default: ${BRIG_HOST}
+    -o <owner_name>: user display name of the owner of the team to be created.  default: ${OWNER_NAME}
+    -e <owner_email>: email address of the owner of the team to be created.  default: ${OWNER_EMAIL}
+    -p <owner_password>: owner password.  default: ${OWNER_PASSWORD}
+    -v <email_code>: validation code received by email.  default: ${EMAIL_CODE}
+    -t <team_name>: default: ${TEAM_NAME}
+    -c <team_currency>: default: ${TEAM_CURRENCY}
+
+"
+
+# Option parsing:
+# https://sookocheff.com/post/bash/parsing-bash-script-arguments-with-shopts/
+while getopts ":o:e:p:v:t:c:h:" opt; do
+  case ${opt} in
+    o ) OWNER_NAME="$OPTARG"
+      ;;
+    e ) OWNER_EMAIL="$OPTARG"
+      ;;
+    p ) OWNER_PASSWORD="$OPTARG"
+      ;;
+    v ) EMAIL_CODE="$OPTARG"
+      ;;
+    t ) TEAM_NAME="$OPTARG"
+      ;;
+    c ) TEAM_CURRENCY="$OPTARG"
+      ;;
+    h ) BRIG_HOST="$OPTARG"
+      ;;
+    : ) echo "-$OPTARG" requires an argument 1>&2
+        exit 1
+      ;;
+    \? ) echo "$USAGE" 1>&2
+         exit 1
+      ;;
+  esac
+done
+shift $((OPTIND -1))
+
+if [ "$#" -ne 0 ]; then
+  echo "$USAGE" 1>&2
+  exit 1
+fi
+
+curl -i -s --show-error \
+     -XPOST "$BRIG_HOST/register" \
+        -H'Content-type: application/json' \
+        -d'{"name":"'"$OWNER_NAME"'","email":"'"$OWNER_EMAIL"'","password":"'"$OWNER_PASSWORD"'","email_code":"'"$EMAIL_CODE"'","team":{"currency":"'"$TEAM_CURRENCY"'","icon":"default","name":"'"$TEAM_NAME"'"}}'
diff --git a/hack/bin/create_team_members.sh b/hack/bin/create_team_members.sh
new file mode 100755
index 0000000000..42740024dc
--- /dev/null
+++ b/hack/bin/create_team_members.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+#
+# consider using create_team.py
+
+set -e
+
+ADMIN_UUID="n/a"
+TEAM_UUID="n/a"
+BRIG_HOST="http://localhost:8080"
+CSV_FILE="n/a"
+
+USAGE="
+This bash script can be used to invite members to a given team.  Input
+is a csv file with email addresses and suggested user names.
+
+Note that this uses internal brig endpoints.  It is not exposed over
+nginz and can only be used if you have direct access to brig.
+
+USAGE: $0
+    -a <admin uuid>: User ID of the inviting admin.  default: ${ADMIN_UUID}
+    -t <team uuid>: ID of the inviting team.  default: ${TEAM_UUID}
+    -h <host>: Base URI of brig. default: ${BRIG_HOST}
+    -c <input file>: file containing info on the invitees in format 'Email,UserName,Role'.  default: ${CSV_FILE}
+
+If role is specified, it must be one of owner, admin, member, partner.
+If it is missing, default is member.
+
+If you tee(1) stdout, stderr of this script into a log file, you can
+grep that log file for errors like this:
+
+$ grep code out.log | grep email-exists  # the most common case
+$ grep code out.log | grep -v email-exists
+
+If you are in a hurry, you may want to change the sleep(1) at the end
+of the invite loop to less than a second.  If you want to give up on
+the first error, add an exit(1) where we check the $INVIDATION_ID.
+
+"
+
+# Option parsing:
+# https://sookocheff.com/post/bash/parsing-bash-script-arguments-with-shopts/
+while getopts ":a:t:h:c:" opt; do
+  case ${opt} in
+    a ) ADMIN_UUID="$OPTARG"
+      ;;
+    t ) TEAM_UUID="$OPTARG"
+      ;;
+    h ) BRIG_HOST="$OPTARG"
+      ;;
+    c ) CSV_FILE="$OPTARG"
+      ;;
+    : ) echo "-$OPTARG" requires an argument 1>&2
+        exit 1
+      ;;
+    \? ) echo "$USAGE" 1>&2
+         exit 1
+      ;;
+  esac
+done
+shift $((OPTIND -1))
+
+if [ "$#" -ne 0 ]; then
+  echo "$USAGE" 1>&2
+  exit 1
+fi
+
+if [ ! -e "$CSV_FILE" ]; then
+    echo -e "\n\n*** I need the name of an existing csv file.\n\n"
+    echo "$USAGE" 1>&2
+    exit 1
+fi
+
+# Generate users
+while IFS=, read -r EMAIL USER_NAME ROLE
+do
+    if ( echo "$ROLE" | grep -vq "\(owner\|admin\|member\|partner\)" ); then
+        export ROLE=member
+    fi
+
+    echo "inviting $USER_NAME <$EMAIL> with role $ROLE..." 1>&2
+
+    # Generate the invitation
+    CURL_OUT_INVITATION=$(curl -i -s --show-error \
+        -XPOST "$BRIG_HOST/teams/$TEAM_UUID/invitations" \
+        -H'Content-type: application/json' \
+        -H'Z-User: '"$ADMIN_UUID"'' \
+        -d'{"email":"'"$EMAIL"'","name":"'"$USER_NAME"'","role":"'"$ROLE"'"}')
+
+    INVITATION_ID=$(echo "$CURL_OUT_INVITATION" | tail -1 | sed 's/.*\"id\":\"\([a-z0-9-]*\)\".*/\1/')
+
+    echo "Created the invitation, sleeping 1 second..." 1>&2
+    sleep 1
+
+    if ( ( echo "$INVITATION_ID" | grep -q '"code"' ) &&
+         ( echo "$INVITATION_ID" | grep -q '"label"' ) ) ; then
+      echo "failed inviting $USER_NAME <$EMAIL>: $INVITATION_ID"
+    fi
+
+    echo "Sleeping 1 second..." 1>&2
+    sleep 1
+done < "$CSV_FILE"
diff --git a/hack/bin/create_team_request_code.sh b/hack/bin/create_team_request_code.sh
new file mode 100755
index 0000000000..6e6d85d1d1
--- /dev/null
+++ b/hack/bin/create_team_request_code.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+# consider using create_team.py
+
+set -o pipefail
+set -o errexit
+
+BRIG_HOST="http://localhost:8080"
+OWNER_EMAIL="owner email n/a"
+
+USAGE="
+Request a code to create a team.  Call this script first, then use the
+code you will receive by email to call ./create_team.sh
+
+USAGE: $0 -h <host> -e <email>
+    -h <host>: Base URI of brig. default: ${BRIG_HOST}
+    -e <email>: email address of the owner of the team to be created.  default: ${OWNER_EMAIL}
+
+"
+
+# Option parsing:
+while getopts ":e:h:" opt; do
+  case ${opt} in
+    e ) OWNER_EMAIL="$OPTARG"
+      ;;
+    h ) BRIG_HOST="$OPTARG"
+      ;;
+    : ) echo "-$OPTARG" requires an argument 1>&2
+        exit 1
+      ;;
+    \? ) echo "$USAGE" 1>&2
+         exit 1
+      ;;
+  esac
+done
+shift $((OPTIND -1))
+
+if [ "$#" -ne 0 ]; then
+  echo "$USAGE" 1>&2
+  exit 1
+fi
+
+curl -i -s --show-error \
+     -XPOST "$BRIG_HOST/activate/send" \
+        -H'Content-type: application/json' \
+        -d'{"email":"'"$OWNER_EMAIL"'"}'
diff --git a/hack/bin/create_test_team_admins.sh b/hack/bin/create_test_team_admins.sh
new file mode 100755
index 0000000000..e6af495131
--- /dev/null
+++ b/hack/bin/create_test_team_admins.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+#
+# consider using create_team.py
+
+set -e
+
+COUNT="1"
+BRIG_HOST="http://localhost:8082"
+CSV="false"
+
+USAGE="
+This bash script can be used to create active team admin users and
+their teams.
+
+Note that this uses an internal brig endpoint.  It is not exposed over
+nginz and can only be used if you have direct access to brig.
+
+USAGE: $0
+    -n <N>: Create <N> users. default: ${COUNT}
+    -h <host>: Base URI of brig. default: ${BRIG_HOST}
+    -c: Output as headerless CSV in format 'User-Id,Email,Password'. default: ${CSV}
+"
+
+# Option parsing:
+# https://sookocheff.com/post/bash/parsing-bash-script-arguments-with-shopts/
+while getopts ":n:h:c" opt; do
+  case ${opt} in
+    n ) COUNT="$OPTARG"
+      ;;
+    h ) BRIG_HOST="$OPTARG"
+      ;;
+    c ) CSV="true"
+      ;;
+    : ) echo "-$OPTARG" requires an argument 1>&2
+        exit 1
+      ;;
+    \? ) echo "$USAGE" 1>&2
+         exit 1
+      ;;
+  esac
+done
+shift $((OPTIND -1))
+
+if [ "$#" -ne 0 ]; then
+  echo "$USAGE" 1>&2
+  exit 1
+fi
+
+# Generate users
+
+for i in $(seq 1 "$COUNT")
+do
+    EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
+    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+
+    CURL_OUT=$(curl -i -s --show-error \
+        -XPOST "$BRIG_HOST/i/users" \
+        -H'Content-type: application/json' \
+        -d'{"email":"'"$EMAIL"'","password":"'"$PASSWORD"'","name":"demo","team":{"name":"Wire team","icon":"default"}}')
+
+    UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"id\":\"\([a-z0-9-]*\)\".*/\1/')
+    TEAM=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"team\":\"\([a-z0-9-]*\)\".*/\1/')
+
+    if [ "$CSV" == "false" ]
+        then echo -e "Succesfully created a team admin user: $UUID on team: $TEAM with email: $EMAIL and password: $PASSWORD"
+        else echo -e "$UUID,$EMAIL,$PASSWORD"
+    fi
+done
diff --git a/hack/bin/create_test_team_members.sh b/hack/bin/create_test_team_members.sh
new file mode 100755
index 0000000000..6a55f4a1b0
--- /dev/null
+++ b/hack/bin/create_test_team_members.sh
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+#
+# consider using create_team.py (you'll have to evolve it a little further to cover this use case, though)
+
+set -e
+
+ADMIN_UUID="a09e9521-e14e-4285-ad71-47caa97f4a16"
+TEAM_UUID="9e57a378-0dca-468f-9661-7872f5f1c910"
+BRIG_HOST="http://localhost:8082"
+START="1"
+COUNT="1"
+CSV="false"
+TARGET_EMAIL_DOMAIN=""
+
+USAGE="This bash script can be used to create active members in a
+given team.  Every member will have an email address of the form
+'w<number>@${TARGET_EMAIL_DOMAIN}', and will have to change that
+(after logging in with the password provided to the user from the
+output of this script).
+
+Note that this uses internal brig endpoints.  It is not exposed over
+nginz and can only be used if you have direct access to brig.
+
+USAGE: $0 -d <email domain> [OPTIONS...]
+    -d <email domain>: Domain part of the emails that the bogus
+                       invitations are sent to.  No default, you need
+                       to provide that.  Consider 'example.com', or an
+                       internal domain you control.
+
+                       WARNING: This may boost your reputation as a
+                       spammer.  Use with care!
+
+    -a <admin uuid>: User ID of the inviting admin.  default: ${ADMIN_UUID}
+    -t <team uuid>: ID of the inviting team.  default: ${TEAM_UUID}
+    -s <S>: Start at offset. default: ${START}
+    -n <N>: Create <N> users. default: ${COUNT}
+    -h <host>: Base URI of brig. default: ${BRIG_HOST}
+    -c: Output as headerless CSV in format 'User-Id,Email,Password'. default: ${CSV}
+"
+
+# Option parsing:
+# https://sookocheff.com/post/bash/parsing-bash-script-arguments-with-shopts/
+while getopts ":a:t:s:n:h:d:c" opt; do
+  case ${opt} in
+    a ) ADMIN_UUID="$OPTARG"
+      ;;
+    t ) TEAM_UUID="$OPTARG"
+      ;;
+    s ) START="$OPTARG"
+      ;;
+    n ) COUNT="$OPTARG"
+      ;;
+    h ) BRIG_HOST="$OPTARG"
+      ;;
+    d ) TARGET_EMAIL_DOMAIN="$OPTARG"
+      ;;
+    c ) CSV="true"
+      ;;
+    : ) echo "-$OPTARG" requires an argument 1>&2
+        exit 1
+      ;;
+    \? ) echo "$USAGE" 1>&2
+         exit 1
+      ;;
+  esac
+done
+shift $((OPTIND -1))
+
+if [ "$#" -ne 0 ]; then
+  echo "$USAGE" 1>&2
+  exit 1
+fi
+
+# Warn about sending emails
+
+if [ "$TARGET_EMAIL_DOMAIN" == "" ]; then
+    echo -e "\n\n*** Please provide an email domain if you want to run this script.\n\n"
+    echo "$USAGE" 1>&2
+    exit 1
+fi
+
+# Generate users
+END=$((COUNT + START - 1))
+for i in $(seq "$START" "$END")
+do
+    EMAIL='w'$(printf "%03d" "$i")"@$TARGET_EMAIL_DOMAIN"
+    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+
+    # Generate the invitation
+
+    CURL_OUT_INVITATION=$(curl -i -s --show-error \
+        -XPOST "$BRIG_HOST/teams/$TEAM_UUID/invitations" \
+        -H'Content-type: application/json' \
+        -H'Z-User: '"$ADMIN_UUID"'' \
+        -d'{"email":"'"$EMAIL"'","name":"Replace with name","inviter_name":"Team admin"}')
+
+    INVITATION_ID=$(echo "$CURL_OUT_INVITATION" | tail -1 | sed 's/.*\"id\":\"\([a-z0-9-]*\)\".*/\1/')
+
+    echo "Created the invitation, sleeping 1 second..." 1>&2
+    sleep 1
+
+    if ( ( echo "$INVITATION_ID" | grep -q '"code"' ) &&
+         ( echo "$INVITATION_ID" | grep -q '"label"' ) ) ; then
+      echo "Got an error while creating $EMAIL, aborting: $INVITATION_ID"
+      exit 1
+    fi
+
+    # Get the code
+    CURL_OUT_INVITATION_CODE=$(curl -i -s --show-error \
+        -XGET "$BRIG_HOST/i/teams/invitation-code?team=$TEAM_UUID&invitation_id=$INVITATION_ID")
+
+    INVITATION_CODE=$(echo "$CURL_OUT_INVITATION_CODE" | tail -1 | sed -n -e '/"code":/ s/^.*"\(.*\)".*/\1/p')
+
+    echo "Got the code, sleeping 1 second..." 1>&2
+    sleep 1
+
+    # Create the user using that code
+    CURL_OUT=$(curl -i -s --show-error \
+            -XPOST "$BRIG_HOST/i/users" \
+            -H'Content-type: application/json' \
+            -d'{"email":"'"$EMAIL"'","password":"'"$PASSWORD"'","name":"demo","team_code":"'"$INVITATION_CODE"'"}')
+
+    TEAM_MEMBER_UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"id\":\"\([a-z0-9-]*\)\".*/\1/')
+    TEAM=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"team\":\"\([a-z0-9-]*\)\".*/\1/')
+
+    if [ "$TEAM" != "$TEAM_UUID" ]; then
+        echo "unexpected error: user got assigned to no / the wrong team?!"
+        echo ${CURL_OUT}
+        exit 1
+    fi
+
+    if [ "$CSV" == "false" ]
+        then echo -e "Succesfully created a team member: $TEAM_MEMBER_UUID on team: $TEAM_UUID with email: $EMAIL and password: $PASSWORD"
+        else echo -e "$UUID,$EMAIL,$PASSWORD"
+    fi
+
+    echo "Sleeping 1 second..." 1>&2
+    sleep 1
+done
diff --git a/hack/bin/create_test_user.sh b/hack/bin/create_test_user.sh
new file mode 100755
index 0000000000..18d3435ad1
--- /dev/null
+++ b/hack/bin/create_test_user.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+# consider using create_team.py
+
+set -e
+
+#
+# This bash script can be used to create an active user by using an internal
+# brig endpoint. Note that this is not exposed over nginz and can only be used
+# if you have direct access to brig
+#
+
+USAGE="USAGE: $0
+    -n <N>:  Create <N> users. default: 1
+    -h <host>: Base URI of brig. default: http://localhost:8082
+    -c: Output as headerless CSV in format 'User-Id,Email,Password'. default: false
+"
+
+BRIG_HOST="http://localhost:8082"
+COUNT="1"
+CSV="false"
+
+# Option parsing:
+# https://sookocheff.com/post/bash/parsing-bash-script-arguments-with-shopts/
+while getopts ":n:h:c" opt; do
+  case ${opt} in
+    n ) COUNT="$OPTARG"
+      ;;
+    h ) BRIG_HOST="$OPTARG"
+      ;;
+    c ) CSV="true"
+      ;;
+    : ) echo "-$OPTARG" requires an argument 1>&2
+        exit 1
+      ;;
+    \? ) echo "$USAGE" 1>&2
+         exit 1
+      ;;
+  esac
+done
+shift $((OPTIND -1))
+
+if [ "$#" -ne 0 ]; then
+  echo "$USAGE" 1>&2
+  exit 1
+fi;
+
+# Generate users
+
+for i in `seq 1 $COUNT`
+do
+    EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
+    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+
+    CURL_OUT=$(curl -i -s --show-error \
+        -XPOST "$BRIG_HOST/i/users" \
+        -H'Content-type: application/json' \
+        -d'{"email":"'$EMAIL'","password":"'$PASSWORD'","name":"demo"}')
+
+    UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"id\":\"\([a-z0-9-]*\)\".*/\1/')
+
+    if [ "$CSV" == "false" ]
+        then echo -e "Succesfully created a user with email: "$EMAIL" and password: "$PASSWORD
+        else echo -e $UUID","$EMAIL","$PASSWORD
+    fi
+done

From 417e368e19f0cfdbafbce1a31bb3d1d730f45dc0 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 12 May 2023 11:14:53 +0200
Subject: [PATCH 058/220] Refactor federation domain configuration.

- move strategy config from federator to brig.
- remove domain list from strategy type.
- add 'none' strategy (more explicit that 'list' with empty list, but
  both still work)
---
 .../Wire/API/Routes/FederationDomainConfig.hs | 42 +++++++++++++++++--
 .../unit/Test/Wire/API/Roundtrip/Aeson.hs     |  3 ++
 services/brig/brig.cabal                      |  1 +
 services/brig/src/Brig/API/Internal.hs        | 34 +++++++++++----
 services/brig/src/Brig/Options.hs             | 25 +++++++----
 .../federator/src/Federator/ExternalServer.hs |  2 -
 .../federator/src/Federator/InternalServer.hs |  2 -
 services/federator/src/Federator/Options.hs   | 30 -------------
 .../federator/src/Federator/Validation.hs     |  9 ++--
 .../unit/Test/Federator/ExternalServer.hs     | 19 +++++----
 .../test/unit/Test/Federator/Options.hs       | 32 +-------------
 11 files changed, 102 insertions(+), 97 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 725a5bd839..026bee2d0b 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -18,6 +18,8 @@
 module Wire.API.Routes.FederationDomainConfig
   ( FederationDomainConfig (..),
     FederationDomainConfigs (..),
+    FederationStrategy (..),
+    defFederationDomainConfigs,
   )
 where
 
@@ -28,6 +30,7 @@ import qualified Data.Swagger as S
 import GHC.Generics
 import Imports
 import Wire.API.User.Search (FederatedUserSearchPolicy)
+import Wire.Arbitrary (Arbitrary, GenericUniform (..))
 
 -- | Everything we need to know about a remote instance in order to federate with it.  Comes
 -- in `AllowedDomains` if `AllowStrategy` is `AllowList`.  If `AllowAll`, we still use this
@@ -38,6 +41,7 @@ data FederationDomainConfig = FederationDomainConfig
   }
   deriving (Eq, Ord, Show, Generic)
   deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationDomainConfig
+  deriving (Arbitrary) via (GenericUniform FederationDomainConfig)
 
 instance ToSchema FederationDomainConfig where
   schema =
@@ -47,15 +51,47 @@ instance ToSchema FederationDomainConfig where
         <*> cfgSearchPolicy .= field "search_policy" schema
 
 data FederationDomainConfigs = FederationDomainConfigs
-  { fromFederationDomainConfigs :: [FederationDomainConfig],
+  { strategy :: FederationStrategy,
+    fromFederationDomainConfigs :: [FederationDomainConfig], -- TODO: rename to `remotes`
     updateInterval :: Int
   }
   deriving (Show, Generic, Eq)
   deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationDomainConfigs
+  deriving (Arbitrary) via (GenericUniform FederationDomainConfigs)
+
+defFederationDomainConfigs :: FederationDomainConfigs
+defFederationDomainConfigs =
+  FederationDomainConfigs
+    { strategy = AllowNone,
+      fromFederationDomainConfigs = [],
+      updateInterval = 10
+    }
 
 instance ToSchema FederationDomainConfigs where
   schema =
     object "FederationDomainConfigs" $
       FederationDomainConfigs
-        <$> fromFederationDomainConfigs .= field "remotes" (array schema)
-        <*> updateInterval .= field "updateInterval" schema
+        <$> strategy .= field "strategy" schema
+        <*> fromFederationDomainConfigs .= field "remotes" (array schema)
+        <*> updateInterval .= field "updateInterval (seconds)" schema
+
+data FederationStrategy
+  = -- | Disable federation.
+    AllowNone
+  | -- | Allow any backend that asks.
+    AllowAll
+  | -- | Any backend explicitly configured in table `brig.federation_remotes` (if that table
+    -- is empty, this is the same as `AllowNone`).
+    AllowList
+  deriving (Eq, Show, Generic)
+  deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationStrategy
+  deriving (Arbitrary) via (GenericUniform FederationStrategy)
+
+instance ToSchema FederationStrategy where
+  schema =
+    enum @Text "FederationStrategy" $
+      mconcat
+        [ element "allowNone" AllowNone,
+          element "allowAll" AllowAll,
+          element "allowList" AllowList
+        ]
diff --git a/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs b/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs
index 23b7ebdb7d..6af6e6dc3d 100644
--- a/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs
+++ b/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs
@@ -47,6 +47,7 @@ import qualified Wire.API.Provider.External as Provider.External
 import qualified Wire.API.Provider.Service as Provider.Service
 import qualified Wire.API.Provider.Service.Tag as Provider.Service.Tag
 import qualified Wire.API.Push.Token as Push.Token
+import qualified Wire.API.Routes.FederationDomainConfig as FederationDomainConfig
 import qualified Wire.API.Routes.Internal.Galley.TeamsIntra as TeamsIntra
 import qualified Wire.API.Routes.Version as Routes.Version
 import qualified Wire.API.SystemSettings as SystemSettings
@@ -143,6 +144,8 @@ tests =
       testRoundTrip @Event.Conversation.OtrMessage,
       testRoundTrip @Event.Team.Event,
       testRoundTrip @Event.Team.EventType,
+      testRoundTrip @FederationDomainConfig.FederationDomainConfigs,
+      testRoundTrip @FederationDomainConfig.FederationStrategy,
       testRoundTrip @Message.Priority,
       testRoundTrip @Message.OtrRecipients,
       testRoundTrip @Message.NewOtrMessage,
diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 69d02377fa..5e63f987c9 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -260,6 +260,7 @@ library
     , polysemy-plugin
     , polysemy-wire-zoo
     , proto-lens                 >=0.1
+    , random
     , random-shuffle             >=0.0.3
     , resource-pool              >=0.2
     , resourcet                  >=1.1
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index da77ed815f..57c202662d 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -1,3 +1,5 @@
+{-# LANGUAGE NumericUnderscores #-}
+
 -- This file is part of the Wire Server implementation.
 --
 -- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
@@ -62,7 +64,7 @@ import qualified Brig.User.API.Search as Search
 import qualified Brig.User.EJPD
 import qualified Brig.User.Search.Index as Index
 import Control.Error hiding (bool)
-import Control.Lens (view)
+import Control.Lens (to, view, (^.))
 import Data.Aeson hiding (json)
 import Data.ByteString.Conversion
 import qualified Data.ByteString.Conversion as List
@@ -83,6 +85,7 @@ import Polysemy
 import Servant hiding (Handler, JSON, addHeader, respond)
 import Servant.Swagger.Internal.Orphans ()
 import qualified System.Logger.Class as Log
+import System.Random (randomRIO)
 import UnliftIO.Async
 import Wire.API.Connection
 import Wire.API.Error
@@ -194,18 +197,33 @@ addFederationRemote fedDomConf = do
 
 getFederationRemotes :: ExceptT Brig.API.Error.Error (AppT r) FederationDomainConfigs
 getFederationRemotes = lift $ do
-  db <- wrapClient Data.getFederationRemotes
-  cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
   -- FUTUREWORK: we should solely rely on `db` in the future for remote domains; merging
   -- remote domains from `cfg` is just for providing an easier, more robust migration path.
   -- See
   -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections,
   -- http://docs.wire.com/developer/developer/federation-design-aspects.html#configuring-remote-connections-dev-perspective
-  pure $
-    FederationDomainConfigs
-      { fromFederationDomainConfigs = nub $ db <> cfg,
-        updateInterval = 1000000 -- TODO FIX ME!
-      }
+  db <- wrapClient Data.getFederationRemotes
+  (ms :: Maybe FederationStrategy, mf :: Maybe [FederationDomainConfig], mu :: Maybe Int) <- do
+    cfg :: Env <- ask
+    pure
+      ( setFederationStrategy (cfg ^. settings),
+        cfg ^. settings . to setFederationDomainConfigs,
+        setFederationDomainConfigsUpdateFreq (cfg ^. settings)
+      )
+
+  -- update frequency settings of <= 0 are ignored.  only warn about this every now and
+  -- then, that'll be noise enough for the logs given the traffic on this end-point.
+  unless (maybe True (> 0) mu) $
+    randomRIO (0 :: Int, 1000)
+      >>= \case
+        0 -> Log.warn (Log.msg (Log.val "Invalid brig configuration: setFederationDomainConfigsUpdateFreq must be > 0, using default 10 seconds."))
+        _ -> pure ()
+
+  defFederationDomainConfigs
+    & maybe id (\v cfg -> cfg {strategy = v}) ms
+    & (\cfg -> cfg {fromFederationDomainConfigs = nub $ db <> fromMaybe mempty mf})
+    & maybe id (\v cfg -> cfg {updateInterval = min 10 v}) mu
+    & pure
 
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index ae69755429..d46bb6109d 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -539,27 +539,38 @@ data Settings = Settings
     -- returns users from the same team
     setSearchSameTeamOnly :: !(Maybe Bool),
     -- | FederationDomain is required, even when not wanting to federate with other backends
-    -- (in that case the 'allowedDomains' can be set to empty in Federator)
+    -- (in that case the 'setFederationStrategy' can be set to `allowNone` below, or to
+    -- `allowList` while keeping the list of allowed domains empty, see
+    -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
     -- Federation domain is used to qualify local IDs and handles,
     -- e.g. 0c4d8944-70fa-480e-a8b7-9d929862d18c@wire.com and somehandle@wire.com.
     -- It should also match the SRV DNS records under which other wire-server installations can find this backend:
-    --    _wire-server-federator._tcp.<federationDomain>
+    -- >>>   _wire-server-federator._tcp.<federationDomain>
     -- Once set, DO NOT change it: if you do, existing users may have a broken experience and/or stop working
     -- Remember to keep it the same in all services.
     -- Example:
-    --   allowedDomains:
-    --     - wire.com
-    --     - example.com
+    -- >>>  allowedDomains:
+    -- >>>    - wire.com
+    -- >>>    - example.com
     setFederationDomain :: !Domain,
-    -- | 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
+    -- | See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
+    -- default: AllowNone
+    setFederationStrategy :: !(Maybe FederationStrategy),
+    -- | 'setFederationDomainConfigs' is introduced in
+    -- https://github.com/wireapp/wire-server/pull/3260 for the sole purpose of transitioning
+    -- to dynamic federation remote configuration.  See
     -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
     -- for details.
+    -- default: []
     setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]),
+    -- | In seconds.  Values <=0 are ignored.  Default: 10 seconds.  See
+    -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
+    setFederationDomainConfigsUpdateFreq :: !(Maybe Int),
     -- | The amount of time in milliseconds to wait after reading from an SQS queue
     -- returns no message, before asking for messages from SQS again.
     -- defaults to 'defSqsThrottleMillis'.
     -- When using real SQS from AWS, throttling isn't needed as much, since using
-    --   SQS.rmWaitTimeSeconds (Just 20) in Brig.AWS.listen
+    -- >>> SQS.rmWaitTimeSeconds (Just 20) in Brig.AWS.listen
     -- ensures that there is only one request every 20 seconds.
     -- However, that parameter is not honoured when using fake-sqs
     -- (where throttling can thus make sense)
diff --git a/services/federator/src/Federator/ExternalServer.hs b/services/federator/src/Federator/ExternalServer.hs
index 22fabfa279..d146e7c518 100644
--- a/services/federator/src/Federator/ExternalServer.hs
+++ b/services/federator/src/Federator/ExternalServer.hs
@@ -25,7 +25,6 @@ import qualified Data.Text as Text
 import Federator.Discovery
 import Federator.Env
 import Federator.Error.ServerError
-import Federator.Options (RunSettings)
 import Federator.Response
 import Federator.Service
 import Federator.Validation
@@ -52,7 +51,6 @@ callInward ::
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
     Member (Error ServerError) r,
-    Member (Input RunSettings) r,
     Member (Input FederationDomainConfigs) r
   ) =>
   Wai.Request ->
diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs
index 1e82471a33..b739fc9892 100644
--- a/services/federator/src/Federator/InternalServer.hs
+++ b/services/federator/src/Federator/InternalServer.hs
@@ -25,7 +25,6 @@ import qualified Data.ByteString as BS
 import qualified Data.Text as Text
 import Federator.Env
 import Federator.Error.ServerError
-import Federator.Options (RunSettings)
 import Federator.Remote
 import Federator.Response
 import Federator.Validation
@@ -83,7 +82,6 @@ callOutward ::
     Member (Embed IO) r,
     Member (Error ValidationError) r,
     Member (Error ServerError) r,
-    Member (Input RunSettings) r,
     Member (Input FederationDomainConfigs) r
   ) =>
   Wai.Request ->
diff --git a/services/federator/src/Federator/Options.hs b/services/federator/src/Federator/Options.hs
index 87a503d801..d5bc71da53 100644
--- a/services/federator/src/Federator/Options.hs
+++ b/services/federator/src/Federator/Options.hs
@@ -24,39 +24,9 @@ import Imports
 import System.Logger.Extended (Level, LogFormat)
 import Util.Options
 
-data FederationStrategy
-  = -- | This backend allows federating with any other Wire-Server backend
-    AllowAll
-  | -- | Any backend explicitly configured in table `brig.federation_remotes`.
-    AllowList
-  deriving (Eq, Show, Generic)
-
-instance ToJSON FederationStrategy where
-  toJSON AllowAll =
-    object
-      [ "allowAll" .= object []
-      ]
-  toJSON AllowList =
-    object
-      [ "allowedDomains" .= object []
-      ]
-
--- | This parser is a bit odd: for historical reasons, we support a list of sub-items (for
--- allowlist), but we don't keep that any more.
-instance FromJSON FederationStrategy where
-  parseJSON = withObject "FederationStrategy" $ \o -> do
-    -- Only inspect field content once we committed to one, for better error messages.
-    allowAll :: Maybe Value <- o .:! "allowAll"
-    allowList :: Maybe Value <- o .:! "allowedDomains"
-    case (allowAll, allowList) of
-      (Just _, Nothing) -> pure AllowAll
-      (Nothing, Just _) -> pure AllowList
-      _ -> fail "invalid FederationStrategy: expected either allowAll or allowedDomains"
-
 -- | Options that persist as runtime settings.
 data RunSettings = RunSettings
   { -- | Would you like to federate with everyone or only with a select set of other wire-server installations?
-    federationStrategy :: FederationStrategy,
     useSystemCAStore :: Bool,
     remoteCAStore :: Maybe FilePath,
     clientCertificate :: FilePath,
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index aaa58f5ac1..d5fab49748 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -38,7 +38,6 @@ import qualified Data.X509 as X509
 import qualified Data.X509.Validation as X509
 import Federator.Discovery
 import Federator.Error
-import Federator.Options
 import Imports
 import qualified Network.HTTP.Types as HTTP
 import qualified Network.Wai.Utilities.Error as Wai
@@ -94,17 +93,16 @@ validationErrorStatus _ = HTTP.status403
 -- `brig.federation_remotes`, cached in `Env`).
 ensureCanFederateWith ::
   ( Member (Input FederationDomainConfigs) r,
-    Member (Input RunSettings) r,
     Member (Error ValidationError) r
   ) =>
   Domain ->
   Sem r ()
 ensureCanFederateWith targetDomain = do
-  strategy <- inputs federationStrategy
+  FederationDomainConfigs strategy domains _ <- input
   case strategy of
+    AllowNone -> throw (FederationDenied targetDomain)
     AllowAll -> pure ()
     AllowList -> do
-      FederationDomainConfigs domains _ <- input
       unless (targetDomain `elem` fmap domain domains) $
         throw (FederationDenied targetDomain)
 
@@ -142,8 +140,7 @@ parseDomainText domain =
 -- federator startup configuration and checks that it matches the names reported
 -- by the client certificate
 validateDomain ::
-  ( Member (Input RunSettings) r,
-    Member (Input FederationDomainConfigs) r,
+  ( Member (Input FederationDomainConfigs) r,
     Member (Error ValidationError) r,
     Member (Error DiscoveryFailure) r,
     Member DiscoverFederator r
diff --git a/services/federator/test/unit/Test/Federator/ExternalServer.hs b/services/federator/test/unit/Test/Federator/ExternalServer.hs
index 297f357803..3748e9e70d 100644
--- a/services/federator/test/unit/Test/Federator/ExternalServer.hs
+++ b/services/federator/test/unit/Test/Federator/ExternalServer.hs
@@ -114,7 +114,7 @@ requestBrigSuccess =
         . discardTinyLogs
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ callInward request
     let expectedCall = Call Brig "/federation/get-user-by-handle" "\"foo\"" aValidDomain
     assertEqual "one call to brig should be made" [expectedCall] actualCalls
@@ -140,7 +140,7 @@ requestBrigFailure =
         . discardTinyLogs
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ callInward request
 
     let expectedCall = Call Brig "/federation/get-user-by-handle" "\"foo\"" aValidDomain
@@ -167,7 +167,7 @@ requestGalleySuccess =
           . discardTinyLogs
           . mockDiscoveryTrivial
           . runInputConst noClientCertSettings
-          . runInputConst (FederationDomainConfigs [] 0)
+          . runInputConst scaffoldingFederationDomainConfigs
           $ callInward request
       let expectedCall = Call Galley "/federation/get-conversations" "\"foo\"" aValidDomain
       embed $ assertEqual "one call to galley should be made" [expectedCall] actualCalls
@@ -196,7 +196,7 @@ requestNoDomain =
           . discardTinyLogs
           . mockDiscoveryTrivial
           . runInputConst noClientCertSettings
-          . runInputConst (FederationDomainConfigs [] 0)
+          . runInputConst scaffoldingFederationDomainConfigs
           $ callInward request
 
       embed $ assertEqual "no calls to services should be made" [] actualCalls
@@ -222,7 +222,7 @@ requestNoCertificate =
         . discardTinyLogs
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ callInward request
 
     assertEqual "no calls to services should be made" [] actualCalls
@@ -274,7 +274,7 @@ testInvalidPaths = do
           . discardTinyLogs
           . mockDiscoveryTrivial
           . runInputConst noClientCertSettings
-          . runInputConst (FederationDomainConfigs [] 0)
+          . runInputConst scaffoldingFederationDomainConfigs
           $ callInward request
 
       assertEqual ("Expected request with path \"" <> cs invalidPath <> "\" to fail") (Left InvalidRoute) (void res)
@@ -298,7 +298,7 @@ testInvalidComponent =
         . discardTinyLogs
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ callInward request
 
     void res @?= Left (UnknownComponent "mast")
@@ -327,7 +327,7 @@ testMethod =
           . discardTinyLogs
           . mockDiscoveryTrivial
           . runInputConst noClientCertSettings
-          . runInputConst (FederationDomainConfigs [] 0)
+          . runInputConst scaffoldingFederationDomainConfigs
           $ callInward request
       void res @?= Left InvalidRoute
 
@@ -336,3 +336,6 @@ exampleDomain = "localhost.example.com"
 
 aValidDomain :: Domain
 aValidDomain = Domain exampleDomain
+
+scaffoldingFederationDomainConfigs :: FederationDomainConfigs
+scaffoldingFederationDomainConfigs = defFederationDomainConfigs & strategy .~ AllowAll
diff --git a/services/federator/test/unit/Test/Federator/Options.hs b/services/federator/test/unit/Test/Federator/Options.hs
index b197c310a8..64e442f37d 100644
--- a/services/federator/test/unit/Test/Federator/Options.hs
+++ b/services/federator/test/unit/Test/Federator/Options.hs
@@ -51,39 +51,9 @@ tests :: TestTree
 tests =
   testGroup
     "Options"
-    [ -- parseFederationStrategy,
-      testSettings
+    [ testSettings
     ]
 
-{- TODO fixme
-parseFederationStrategy :: TestTree
-parseFederationStrategy =
-  testCase "parse FederationStrategy examples" $ do
-    assertParsesAs AllowAll $
-      "allowAll: null"
-    assertParsesAs (withAllowList []) $
-      "allowedDomains: []"
-    assertParsesAs (withAllowList ["test.org"]) . B8.pack $
-      [QQ.i|
-      allowedDomains:
-        - test.org|]
-    assertParsesAs (withAllowList ["example.com", "wire.com"]) . B8.pack $
-      [QQ.i|
-      allowedDomains:
-        - example.com
-        - wire.com|]
-    -- manual roundtrip example AllowAll
-    let allowA = toStrict $ Aeson.encode AllowAll
-    assertParsesAs AllowAll $ allowA
-    -- manual roundtrip example AllowList
-    let allowWire = withAllowList ["wire.com"]
-    let allowedDom = toStrict $ Aeson.encode allowWire
-    assertParsesAs allowWire $ allowedDom
-  where
-    withAllowList =
-      AllowedDomains . map (either error id . mkDomain)
--}
-
 testSettings :: TestTree
 testSettings =
   testGroup

From 5bed5644d37f9a815dd93999e34fe7e50d92f4c9 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 12 May 2023 12:43:50 +0200
Subject: [PATCH 059/220] Fixup

---
 .../Wire/API/Routes/FederationDomainConfig.hs |  2 +-
 .../unit/Test/Federator/ExternalServer.hs     |  2 +-
 .../unit/Test/Federator/InternalServer.hs     | 11 ++--
 .../test/unit/Test/Federator/Options.hs       |  6 +--
 .../test/unit/Test/Federator/Validation.hs    | 51 ++++++++++---------
 5 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 026bee2d0b..9b51550bf4 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -73,7 +73,7 @@ instance ToSchema FederationDomainConfigs where
       FederationDomainConfigs
         <$> strategy .= field "strategy" schema
         <*> fromFederationDomainConfigs .= field "remotes" (array schema)
-        <*> updateInterval .= field "updateInterval (seconds)" schema
+        <*> updateInterval .= field "update_interval" schema
 
 data FederationStrategy
   = -- | Disable federation.
diff --git a/services/federator/test/unit/Test/Federator/ExternalServer.hs b/services/federator/test/unit/Test/Federator/ExternalServer.hs
index 3748e9e70d..01812f1bab 100644
--- a/services/federator/test/unit/Test/Federator/ExternalServer.hs
+++ b/services/federator/test/unit/Test/Federator/ExternalServer.hs
@@ -338,4 +338,4 @@ aValidDomain :: Domain
 aValidDomain = Domain exampleDomain
 
 scaffoldingFederationDomainConfigs :: FederationDomainConfigs
-scaffoldingFederationDomainConfigs = defFederationDomainConfigs & strategy .~ AllowAll
+scaffoldingFederationDomainConfigs = defFederationDomainConfigs {strategy = AllowAll}
diff --git a/services/federator/test/unit/Test/Federator/InternalServer.hs b/services/federator/test/unit/Test/Federator/InternalServer.hs
index df8d82fdf3..c2d4d82653 100644
--- a/services/federator/test/unit/Test/Federator/InternalServer.hs
+++ b/services/federator/test/unit/Test/Federator/InternalServer.hs
@@ -25,7 +25,6 @@ import Data.Default
 import Data.Domain
 import Federator.Error.ServerError
 import Federator.InternalServer (callOutward)
-import Federator.Options (FederationStrategy (..), RunSettings (..))
 import Federator.Remote
 import Federator.Validation
 import Imports
@@ -58,10 +57,6 @@ tests =
         ]
     ]
 
-settingsWithAllowList :: RunSettings
-settingsWithAllowList =
-  noClientCertSettings {federationStrategy = AllowList}
-
 federatedRequestSuccess :: TestTree
 federatedRequestSuccess =
   testCase "should successfully return success response" $ do
@@ -97,7 +92,7 @@ federatedRequestSuccess =
         . assertNoError @ServerError
         . discardTinyLogs
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "target.example.com") FullSearch] 10)
         $ callOutward request
     Wai.responseStatus res @?= HTTP.status200
     body <- Wai.lazyResponseBody res
@@ -109,7 +104,7 @@ federatedRequestSuccess =
 federatedRequestFailureAllowList :: TestTree
 federatedRequestFailureAllowList =
   testCase "should not make a call when target domain not in the allowList" $ do
-    let settings = settingsWithAllowList
+    let settings = noClientCertSettings
     let targetDomain = Domain "target.example.com"
         headers = [(originDomainHeaderName, "origin.example.com")]
     request <-
@@ -139,7 +134,7 @@ federatedRequestFailureAllowList =
         . assertNoError @ServerError
         . discardTinyLogs
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "hello.world") FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "hello.world") FullSearch] 10)
         $ callOutward request
     eith @?= Left (FederationDenied targetDomain)
 
diff --git a/services/federator/test/unit/Test/Federator/Options.hs b/services/federator/test/unit/Test/Federator/Options.hs
index 64e442f37d..ce8af8375b 100644
--- a/services/federator/test/unit/Test/Federator/Options.hs
+++ b/services/federator/test/unit/Test/Federator/Options.hs
@@ -35,8 +35,7 @@ import Test.Tasty.HUnit
 defRunSettings :: FilePath -> FilePath -> RunSettings
 defRunSettings client key =
   RunSettings
-    { federationStrategy = AllowAll,
-      useSystemCAStore = True,
+    { useSystemCAStore = True,
       remoteCAStore = Nothing,
       clientCertificate = client,
       clientPrivateKey = key,
@@ -72,8 +71,7 @@ testSettings =
       testCase "parse configuration example (closed federation)" $ do
         let settings =
               (defRunSettings "client.pem" "client-key.pem")
-                { federationStrategy = AllowList,
-                  useSystemCAStore = False
+                { useSystemCAStore = False
                 }
         assertParsesAs settings . B8.pack $
           [QQ.i|
diff --git a/services/federator/test/unit/Test/Federator/Validation.hs b/services/federator/test/unit/Test/Federator/Validation.hs
index ec533b793b..e122957447 100644
--- a/services/federator/test/unit/Test/Federator/Validation.hs
+++ b/services/federator/test/unit/Test/Federator/Validation.hs
@@ -26,7 +26,6 @@ import Data.List.NonEmpty (NonEmpty (..))
 import qualified Data.Text.Encoding as Text
 import qualified Data.X509.Validation as X509
 import Federator.Discovery
-import Federator.Options
 import Federator.Validation
 import Imports
 import Polysemy
@@ -60,6 +59,16 @@ mockDiscoveryFailure = Polysemy.interpret $ \case
   DiscoverFederator _ -> error "Not mocked"
   DiscoverAllFederators _ -> pure . Left $ DiscoveryFailureDNSError "mock DNS error"
 
+scaffoldingFederationDomainConfigs :: FederationDomainConfigs
+scaffoldingFederationDomainConfigs =
+  FederationDomainConfigs
+    AllowList
+    [ FederationDomainConfig (Domain "foo.example.com") FullSearch,
+      FederationDomainConfig (Domain "example.com") FullSearch,
+      FederationDomainConfig (Domain "federator.example.com") FullSearch
+    ]
+    10
+
 tests :: TestTree
 tests =
   testGroup
@@ -88,22 +97,22 @@ tests =
 federateWithAllowListSuccess :: TestTree
 federateWithAllowListSuccess =
   testCase "should give True when target domain is in the list" $ do
-    let settings = settingsWithAllowList
+    let settings = noClientCertSettings
     runM
       . assertNoError @ValidationError
       . runInputConst settings
-      . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "hello.world") FullSearch] 0)
+      . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "hello.world") FullSearch] 0)
       $ ensureCanFederateWith (Domain "hello.world")
 
 federateWithAllowListFail :: TestTree
 federateWithAllowListFail =
   testCase "should give False when target domain is not in the list" $ do
-    let settings = settingsWithAllowList
+    let settings = noClientCertSettings
     eith :: Either ValidationError () <-
       runM
         . runError @ValidationError
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ ensureCanFederateWith (Domain "hello.world")
     assertBool "federating should not be allowed" (isLeft eith)
 
@@ -111,14 +120,14 @@ validateDomainAllowListFailSemantic :: TestTree
 validateDomainAllowListFailSemantic =
   testCase "semantic validation" $ do
     exampleCert <- BS.readFile "test/resources/unit/localhost.pem"
-    let settings = settingsWithAllowList
+    let settings = noClientCertSettings
     res <-
       runM
         . runError
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ validateDomain (Just exampleCert) "invalid//.><-semantic-&@-domain"
     res @?= Left (DomainParseError "invalid//.><-semantic-&@-domain")
 
@@ -129,14 +138,14 @@ validateDomainAllowListFail :: TestTree
 validateDomainAllowListFail =
   testCase "allow list validation" $ do
     exampleCert <- BS.readFile "test/resources/unit/localhost.example.com.pem"
-    let settings = settingsWithAllowList
+    let settings = noClientCertSettings
     res <-
       runM
         . runError
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ validateDomain (Just exampleCert) "localhost.example.com"
     res @?= Left (FederationDenied (Domain "localhost.example.com"))
 
@@ -147,14 +156,14 @@ validateDomainAllowListSuccess =
   testCase "should give parsed domain if in the allow list" $ do
     exampleCert <- BS.readFile "test/resources/unit/localhost.example.com.pem"
     let domain = Domain "localhost.example.com"
-        settings = settingsWithAllowList
+        settings = noClientCertSettings
     res <-
       runM
         . assertNoError @ValidationError
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs [FederationDomainConfig domain FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig domain FullSearch] 0)
         $ validateDomain (Just exampleCert) (toByteString' domain)
     assertEqual "validateDomain should give 'localhost.example.com' as domain" domain res
 
@@ -167,7 +176,7 @@ validateDomainCertMissing =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst defFederationDomainConfigs
         $ validateDomain Nothing "foo.example.com"
     res @?= Left NoClientCertificate
 
@@ -182,7 +191,7 @@ validateDomainCertInvalid =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ validateDomain (Just "not a certificate") "foo.example.com"
     res @?= Left (CertificateParseError "no certificate found")
 
@@ -202,7 +211,7 @@ validateDomainCertWrongDomain =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ validateDomain (Just exampleCert) "foo.example.com"
     res @?= Left (AuthenticationFailure (pure [X509.NameMismatch "foo.example.com"]))
 
@@ -219,7 +228,7 @@ validateDomainCertCN =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ validateDomain (Just exampleCert) (toByteString' domain)
     res @?= domain
 
@@ -234,7 +243,7 @@ validateDomainCertSAN =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ validateDomain (Just exampleCert) (toByteString' domain)
     res @?= domain
 
@@ -249,7 +258,7 @@ validateDomainMultipleFederators =
             . assertNoError @DiscoveryFailure
             . mockDiscoveryMapping domain ("localhost.example.com" :| ["second-federator.example.com"])
             . runInputConst noClientCertSettings
-            . runInputConst (FederationDomainConfigs [] 0)
+            . runInputConst scaffoldingFederationDomainConfigs
         domain = Domain "foo.example.com"
     resFirst <-
       runValidation $
@@ -271,7 +280,7 @@ validateDomainDiscoveryFailed =
         . assertNoError @ValidationError
         . mockDiscoveryFailure
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ validateDomain (Just exampleCert) "example.com"
     res @?= Left (DiscoveryFailureDNSError "mock DNS error")
 
@@ -286,10 +295,6 @@ validateDomainNonIdentitySRV =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryMapping domain ("localhost.example.com" :| [])
         . runInputConst noClientCertSettings
-        . runInputConst (FederationDomainConfigs [] 0)
+        . runInputConst scaffoldingFederationDomainConfigs
         $ validateDomain (Just exampleCert) (toByteString' domain)
     res @?= domain
-
-settingsWithAllowList :: RunSettings
-settingsWithAllowList =
-  noClientCertSettings {federationStrategy = AllowList}

From 4199dc5fd2893e35441b77bc19ab0b3c20e073dc Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 12 May 2023 13:05:16 +0200
Subject: [PATCH 060/220] Complete CRUD api for federator remotes (Update is
 missing) [WIP].

---
 .../federation/backend-communication.md         |  2 +-
 .../src/Wire/API/Routes/Internal/Brig.hs        | 17 +++++++++++++----
 services/brig/src/Brig/API/Internal.hs          |  8 ++++++--
 .../brig/test/integration/API/Federation.hs     |  9 +++++++++
 4 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index c8c6b3c126..de0bbe0331 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -166,8 +166,8 @@ all information about remote connections is stored in the database,
 and there is an internal REST API for adding and removing remotes:
 
 * [`POST`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/post_i_federation_remotes)
-* [`PUT`](TODO)
 * [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
+* [`PUT`](TODO)
 * [`DELETE`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/delete_i_federation_remotes__domain_)
 
 **WARNING:** If you delete a connection, all users from that remote
diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index 07cabed987..64c22e4bca 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -406,19 +406,28 @@ type AuthAPI =
 -- of changes via rabbitmq, we argue it's "fine" for federators to ask brig once on startup.
 type FederationRemotesAPI =
   Named
-    "get-federation-remotes"
+    "add-federation-remotes"
     ( Description FederationRemotesAPIDescription
         :> "federation"
         :> "remotes"
-        :> Get '[JSON] FederationDomainConfigs
+        :> ReqBody '[JSON] FederationDomainConfig
+        :> Post '[JSON] ()
     )
     :<|> Named
-           "add-federation-remotes"
+           "get-federation-remotes"
            ( Description FederationRemotesAPIDescription
                :> "federation"
                :> "remotes"
+               :> Get '[JSON] FederationDomainConfigs
+           )
+    :<|> Named
+           "update-federation-remotes"
+           ( Description FederationRemotesAPIDescription
+               :> "federation"
+               :> "remotes"
+               :> Capture "domain" Domain
                :> ReqBody '[JSON] FederationDomainConfig
-               :> Post '[JSON] ()
+               :> Put '[JSON] ()
            )
     :<|> Named
            "delete-federation-remotes"
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 57c202662d..d020e59bef 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -181,8 +181,9 @@ authAPI =
 
 federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
 federationRemotesAPI =
-  Named @"get-federation-remotes" getFederationRemotes
-    :<|> Named @"add-federation-remotes" addFederationRemote
+  Named @"add-federation-remotes" addFederationRemote
+    :<|> Named @"get-federation-remotes" getFederationRemotes
+    :<|> Named @"update-federation-remotes" updateFederationRemotes
     :<|> Named @"delete-federation-remotes" deleteFederationRemotes
 
 addFederationRemote :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
@@ -225,6 +226,9 @@ getFederationRemotes = lift $ do
     & maybe id (\v cfg -> cfg {updateInterval = min 10 v}) mu
     & pure
 
+updateFederationRemotes :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
+updateFederationRemotes = _
+
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
   cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index ebc3b1d128..c2f9897d40 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -492,6 +492,15 @@ crudFederationRemotes opts brig = do
   -- deleting from the config file triggers an error
   deleteFederationRemote' id brig (domain $ head $ cfgRemotes) !!! const 533 === statusCode
 
+  -- updating search strategy works
+  _
+
+  -- updating from config file fails
+  _
+
+  -- updating domain fails
+  _
+
   -- TODO: how do we test that the TVar is updated in all services?  some fancy unit test?
   -- duplicate internal end-point to all services, and implement the hanlers in a library?
   pure ()

From 728567a10c9f8410b22c92d6270f9af8615cb685 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 12 May 2023 17:32:39 +0200
Subject: [PATCH 061/220] Complete CRUD api for federator remotes (Update is
 missing): tests.

---
 .../understand/federation/backend-communication.md |  4 ----
 services/brig/test/integration/API/Federation.hs   | 14 +++++++++-----
 services/brig/test/integration/Util.hs             |  8 ++++++++
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index de0bbe0331..5036a284a6 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -176,10 +176,6 @@ by that remote will be removed from the local backend.  Connections
 between local and remote users that are removed will be archived, and
 can be re-established should you decide to add the same backend later.
 
-Changing the configuration of existing edges via `PUT` is not
-implemented at the moment, if you need to do that, delete the
-connection and add it again.  TODO: this is bullshit.  go implement it!
-
 {-
 TODO: this paragraph still annoys me.  move strategy to brig, too?  or
 at least to a different syntax, and force admin to use both old and
diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index c2f9897d40..e8bce7e87d 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -473,12 +473,12 @@ crudFederationRemotes opts brig = do
   let remote1 = FederationDomainConfig (Domain "good.example.com") NoSearch
   addFederationRemote brig remote1
   res2 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and good.example.com" (nub $ sort $ cfgRemotes <> [remote1]) (sort res2)
+  liftIO $ assertEqual "should return config values and good.example.com" (sort $ remote1 : cfgRemotes) (sort res2)
 
   -- idempotency
   addFederationRemote brig remote1
   res2' <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and good.example.com" (nub $ sort $ cfgRemotes <> [remote1]) (sort res2')
+  liftIO $ assertEqual "should return config values and good.example.com" (sort $ remote1 : cfgRemotes) (sort res2')
 
   let remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
   addFederationRemote brig remote2
@@ -493,13 +493,17 @@ crudFederationRemotes opts brig = do
   deleteFederationRemote' id brig (domain $ head $ cfgRemotes) !!! const 533 === statusCode
 
   -- updating search strategy works
-  _
+  let remote2' = remote2 {cfgSearchPolicy = NoSearch}
+  () <- updateFederationRemote brig (domain remote2) remote2'
+  res5 <- getFederationRemotes brig
+  liftIO $ assertEqual "should be NoSearch" (nub $ sort $ cfgRemotes <> [remote1, remote2']) (sort res5)
 
   -- updating from config file fails
-  _
+  updateFederationRemote' id brig (domain $ head $ cfgRemotes) (head $ cfgRemotes) !!! const 533 === statusCode
 
   -- updating domain fails
-  _
+  let remote2'' = remote2' {domain = Domain "broken.example.com"}
+  updateFederationRemote' id brig (domain remote2) remote2'' !!! const 533 === statusCode
 
   -- TODO: how do we test that the TVar is updated in all services?  some fancy unit test?
   -- duplicate internal end-point to all services, and implement the hanlers in a library?
diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs
index 4ea86efa23..262bac3f17 100644
--- a/services/brig/test/integration/Util.hs
+++ b/services/brig/test/integration/Util.hs
@@ -1085,6 +1085,14 @@ addFederationRemote :: Brig -> FederationDomainConfig -> Http ()
 addFederationRemote brig remote =
   void $ post (brig . paths ["i", "federation", "remotes"] . contentJson . json remote . expect2xx)
 
+updateFederationRemote :: Brig -> Domain -> FederationDomainConfig -> Http ()
+updateFederationRemote brig rdom remote =
+  void $ updateFederationRemote' expect2xx brig rdom remote
+
+updateFederationRemote' :: (Request -> Request) -> Brig -> Domain -> FederationDomainConfig -> Http ResponseLBS
+updateFederationRemote' mods brig rdom remote =
+  put (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . json remote . mods)
+
 deleteFederationRemote :: Brig -> Domain -> Http ()
 deleteFederationRemote brig rdom =
   void $ deleteFederationRemote' expect2xx brig rdom

From f5c1b3e6854b4ee704c9496cd66e9fc8df334134 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 12 May 2023 17:40:36 +0200
Subject: [PATCH 062/220] docs.

---
 docs/src/understand/federation/backend-communication.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index 5036a284a6..9c92d48139 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -215,6 +215,8 @@ developer's point of view on this topic.
 
 ### Transitioning from config file to database state
 
+transitioning is only necessary if you (1) upgrade and not install fresh; and (2) already have federation enabled before the upgrade.
+
 TODO: you need to update config files!
   - complete list of search policies, no more defaults
   - new fed strategy syntax (keep the old, just copy)

From a6e92554410430ac7a74e708284350d23e998cc2 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 12 May 2023 17:44:43 +0200
Subject: [PATCH 063/220] Cleanup

---
 libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs | 6 +++---
 services/brig/src/Brig/API/Internal.hs                      | 4 ++--
 services/brig/test/integration/Util.hs                      | 2 +-
 services/galley/src/Galley/Run.hs                           | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 9b51550bf4..0acc9d0f40 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -52,7 +52,7 @@ instance ToSchema FederationDomainConfig where
 
 data FederationDomainConfigs = FederationDomainConfigs
   { strategy :: FederationStrategy,
-    fromFederationDomainConfigs :: [FederationDomainConfig], -- TODO: rename to `remotes`
+    remotes :: [FederationDomainConfig],
     updateInterval :: Int
   }
   deriving (Show, Generic, Eq)
@@ -63,7 +63,7 @@ defFederationDomainConfigs :: FederationDomainConfigs
 defFederationDomainConfigs =
   FederationDomainConfigs
     { strategy = AllowNone,
-      fromFederationDomainConfigs = [],
+      remotes = [],
       updateInterval = 10
     }
 
@@ -72,7 +72,7 @@ instance ToSchema FederationDomainConfigs where
     object "FederationDomainConfigs" $
       FederationDomainConfigs
         <$> strategy .= field "strategy" schema
-        <*> fromFederationDomainConfigs .= field "remotes" (array schema)
+        <*> remotes .= field "remotes" (array schema)
         <*> updateInterval .= field "update_interval" schema
 
 data FederationStrategy
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index d020e59bef..42cc4d9344 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -222,12 +222,12 @@ getFederationRemotes = lift $ do
 
   defFederationDomainConfigs
     & maybe id (\v cfg -> cfg {strategy = v}) ms
-    & (\cfg -> cfg {fromFederationDomainConfigs = nub $ db <> fromMaybe mempty mf})
+    & (\cfg -> cfg {remotes = nub $ db <> fromMaybe mempty mf})
     & maybe id (\v cfg -> cfg {updateInterval = min 10 v}) mu
     & pure
 
 updateFederationRemotes :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
-updateFederationRemotes = _
+updateFederationRemotes = undefined
 
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs
index 262bac3f17..a30c99f3d2 100644
--- a/services/brig/test/integration/Util.hs
+++ b/services/brig/test/integration/Util.hs
@@ -1078,7 +1078,7 @@ withDomainsBlockedForRegistration opts domains sess = do
 
 getFederationRemotes :: Brig -> Http [FederationDomainConfig]
 getFederationRemotes brig =
-  fromFederationDomainConfigs . responseJsonUnsafe <$> do
+  remotes . responseJsonUnsafe <$> do
     get (brig . paths ["i", "federation", "remotes"] . contentJson . expect2xx)
 
 addFederationRemote :: Brig -> FederationDomainConfig -> Http ()
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index ae1d96fefb..ab4b4b83ce 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -194,8 +194,8 @@ updateFedDomains = do
     okRemoteDomains <- getAllowedDomainsInitial logger clientEnv
     atomicWriteIORef ioref okRemoteDomains
     let domainListsEqual old new =
-          Set.fromList (domain <$> fromFederationDomainConfigs old)
-            == Set.fromList (domain <$> fromFederationDomainConfigs new)
+          Set.fromList (domain <$> remotes old)
+            == Set.fromList (domain <$> remotes new)
         callback old new = unless (domainListsEqual old new) $ do
           -- TODO: perform the database updates here
           -- This code will only run when there is a change in the domain lists

From 64b2420993ecc3f207916981032cddd69fd5a5ab Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 15 May 2023 09:43:52 +1000
Subject: [PATCH 064/220] FS-1179: Setting up a location in galley for
 processing deleted domains

NOTE: Broken! Currently not compiling

Adding a new query to cassandra for getting remote members and
conversation IDs that they are in, setting up loops for removing these
members and potentially conversations using existing APIs.
---
 .../src/Galley/Cassandra/Conversation.hs      |  2 +-
 .../Galley/Cassandra/Conversation/Members.hs  |  7 ++
 .../galley/src/Galley/Cassandra/Queries.hs    |  4 +
 .../galley/src/Galley/Effects/MemberStore.hs  |  3 +
 services/galley/src/Galley/Run.hs             | 78 +++++++++++++++----
 5 files changed, 77 insertions(+), 17 deletions(-)

diff --git a/services/galley/src/Galley/Cassandra/Conversation.hs b/services/galley/src/Galley/Cassandra/Conversation.hs
index d4d82fcd45..05bcd01b8f 100644
--- a/services/galley/src/Galley/Cassandra/Conversation.hs
+++ b/services/galley/src/Galley/Cassandra/Conversation.hs
@@ -405,4 +405,4 @@ interpretConversationStoreToCassandra = interpret $ \case
   SetGroupId gId cid -> embedClient $ mapGroupId gId cid
   SetPublicGroupState cid gib -> embedClient $ setPublicGroupState cid gib
   AcquireCommitLock gId epoch ttl -> embedClient $ acquireCommitLock gId epoch ttl
-  ReleaseCommitLock gId epoch -> embedClient $ releaseCommitLock gId epoch
+  ReleaseCommitLock gId epoch -> embedClient $ releaseCommitLock gId epoch
\ No newline at end of file
diff --git a/services/galley/src/Galley/Cassandra/Conversation/Members.hs b/services/galley/src/Galley/Cassandra/Conversation/Members.hs
index 8590b2496e..1b34d39943 100644
--- a/services/galley/src/Galley/Cassandra/Conversation/Members.hs
+++ b/services/galley/src/Galley/Cassandra/Conversation/Members.hs
@@ -199,6 +199,12 @@ lookupRemoteMembers conv = do
           rmConvRoleName = role
         }
 
+lookupRemoteMembersByDomain :: Domain -> Client [(ConvId, RemoteMember)]
+lookupRemoteMembersByDomain dom = do
+  fmap (fmap mkConvMem) . retry x1 $ query Cql.selectRemoteMembersByDomain (params LocalQuorum (Identity dom))
+  where
+    mkConvMem (convId, usr, role) = (convId, RemoteMember (toRemoteUnsafe dom usr) role)
+
 member ::
   ConvId ->
   UserId ->
@@ -390,3 +396,4 @@ interpretMemberStoreToCassandra = interpret $ \case
   AddMLSClients lcnv quid cs -> embedClient $ addMLSClients lcnv quid cs
   RemoveMLSClients lcnv quid cs -> embedClient $ removeMLSClients lcnv quid cs
   LookupMLSClients lcnv -> embedClient $ lookupMLSClients lcnv
+  GetRemoteMembersByDomain dom -> embedClient $ lookupRemoteMembersByDomain dom
diff --git a/services/galley/src/Galley/Cassandra/Queries.hs b/services/galley/src/Galley/Cassandra/Queries.hs
index db3b5b91a8..9e14bb4f74 100644
--- a/services/galley/src/Galley/Cassandra/Queries.hs
+++ b/services/galley/src/Galley/Cassandra/Queries.hs
@@ -369,6 +369,10 @@ selectRemoteMembers = "select user_remote_domain, user_remote_id, conversation_r
 updateRemoteMemberConvRoleName :: PrepQuery W (RoleName, ConvId, Domain, UserId) ()
 updateRemoteMemberConvRoleName = "update member_remote_user set conversation_role = ? where conv = ? and user_remote_domain = ? and user_remote_id = ?"
 
+-- Used when removing a federation domain, so that we can quickly list all of the affected remote users and conversations
+selectRemoteMembersByDomain :: PrepQuery R (Identity Domain) (ConvId, UserId, RoleName)
+selectRemoteMembersByDomain = "select conv, user_remote_id, conversation_role from member_remote_user where user_remote_domain = ?"
+
 -- local user with remote conversations
 
 insertUserRemoteConv :: PrepQuery W (UserId, Domain, ConvId) ()
diff --git a/services/galley/src/Galley/Effects/MemberStore.hs b/services/galley/src/Galley/Effects/MemberStore.hs
index f9f5b57d50..b5a96a6dfe 100644
--- a/services/galley/src/Galley/Effects/MemberStore.hs
+++ b/services/galley/src/Galley/Effects/MemberStore.hs
@@ -34,6 +34,7 @@ module Galley.Effects.MemberStore
     getRemoteMembers,
     checkLocalMemberRemoteConv,
     selectRemoteMembers,
+    getRemoteMembersByDomain,
 
     -- * Update members
     setSelfMember,
@@ -60,6 +61,7 @@ import Wire.API.Conversation.Member hiding (Member)
 import Wire.API.MLS.Group
 import Wire.API.MLS.KeyPackage
 import Wire.API.Provider.Service
+import Data.Domain
 
 data MemberStore m a where
   CreateMembers :: ToUserRole u => ConvId -> UserList u -> MemberStore m ([LocalMember], [RemoteMember])
@@ -80,6 +82,7 @@ data MemberStore m a where
   LookupMLSClients ::
     GroupId ->
     MemberStore m (Map (Qualified UserId) (Set (ClientId, KeyPackageRef)))
+  GetRemoteMembersByDomain :: Domain -> MemberStore m [(ConvId, RemoteMember)]
 
 makeSem ''MemberStore
 
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index ab4b4b83ce..337eab329d 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -34,6 +34,7 @@ import Control.Monad.Codensity
 import qualified Data.Aeson as Aeson
 import Data.Default
 import Data.Id
+import qualified Data.Map as Map
 import Data.Metrics (Metrics)
 import Data.Metrics.AWS (gaugeTokenRemaing)
 import qualified Data.Metrics.Middleware as M
@@ -74,6 +75,16 @@ import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
+import qualified Galley.Effects.MemberStore as E
+import qualified Data.List.NonEmpty as N
+import Galley.API.Action
+import Galley.Types.Conversations.Members
+import Data.Qualified
+import Wire.API.Error.Galley
+import Wire.API.Conversation.Role
+import qualified Wire.API.Federation.API.Galley as F
+import Wire.API.Error
+import Polysemy.Error
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -183,21 +194,56 @@ collectAuthMetrics m env = do
 
 updateFedDomains :: App ()
 updateFedDomains = do
-  ioref <- view fedDomains
-  logger <- view applog
-  manager' <- view manager
-  Endpoint host port <- view brig
-  let baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
+  env <- ask
+  let ioref              = env ^. fedDomains
+      logger             = env ^. applog
+      manager'           = env ^. manager
+      Endpoint host port = env ^. brig
+      baseUrl   = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
-
-  liftIO $ do
-    okRemoteDomains <- getAllowedDomainsInitial logger clientEnv
-    atomicWriteIORef ioref okRemoteDomains
-    let domainListsEqual old new =
-          Set.fromList (domain <$> remotes old)
-            == Set.fromList (domain <$> remotes new)
-        callback old new = unless (domainListsEqual old new) $ do
-          -- TODO: perform the database updates here
-          -- This code will only run when there is a change in the domain lists
+  liftIO $ getAllowedDomainsLoop logger clientEnv ioref $ callback env
+  where
+    callback env old new = do
+      -- TODO: perform the database updates here
+      -- This code will only run when there is a change in the domain lists
+      let fromFedList = Set.fromList . fromFederationDomainConfigs
+          prevDoms = fromFedList old
+          currDoms = fromFedList new
+      unless (prevDoms == currDoms) $ do
+        -- Perform updates before rewriting the tvar
+        -- This means that if the update fails on a
+        -- particular invocation, it can be run again
+        -- on the next firing as it isn't likely that
+        -- the domain list is changing frequently.
+        -- FS-1179 is handling this part.
+        let deletedDomains = Set.difference prevDoms currDoms
+            addedDomains = Set.difference currDoms prevDoms
+        for_ deletedDomains $ \fedDomCfg -> do
+          -- https://wearezeta.atlassian.net/browse/FS-1179
+          -- TODO
+          -- * Remove remote users for the given domain from all conversations owned by the current host
+          -- * Remove all local users from remote conversations owned by the given domain.
+          --   NOTE: This is NOT sent to other backends, as this information is not authoratative, but is
+          --   good enough to tell local users about the federation connection being removed.
+          -- * Delete all connections from local users to users for the remote domain.
+          -- Get all remote users for the given domain, along with conversation IDs that they are in
+          remoteUsers <- liftIO $ evalGalleyToIO env $ E.getRemoteMembersByDomain $ domain fedDomCfg
+          let cnvMap :: Map ConvId (N.NonEmpty RemoteMember)
+              cnvMap = foldr insertIntoMap mempty remoteUsers
+              -- Build the map, keyed by conversations to the list of remote members
+              insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
+          for_ (Map.toList cnvMap) $ \(cnv, rUsers) -> do
+            res <- liftIO $ evalGalleyToIO env $ runError 
+              . mapToRuntimeError @'ConvNotFound F.RemoveFromConversationErrorNotFound
+              . mapToRuntimeError @('ActionDenied 'LeaveConversation) F.RemoveFromConversationErrorRemovalNotAllowed
+              . mapToRuntimeError @'InvalidOperation F.RemoveFromConversationErrorRemovalNotAllowed
+              . mapError @NoChanges (const F.RemoveFromConversationErrorUnchanged) $
+              updateLocalConversation
+              @'ConversationRemoveMembersTag
+              (toLocalUnsafe (domain fedDomCfg) cnv)
+              undefined
+              Nothing $
+              tUntagged . rmId <$> rUsers
+            pure ()
+        for_ addedDomains $ \_domain -> do
           pure ()
-    getAllowedDomainsLoop logger clientEnv ioref callback

From 8f348c84113490ed3fa9973168cf0332da0db269 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 11 May 2023 20:20:05 +1000
Subject: [PATCH 065/220] FS-1179: Fixing the polysemy error.

Picking an error type and documenting what might need to be done for
better error mapping.
---
 services/galley/src/Galley/Run.hs | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 337eab329d..d20511a21b 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -85,6 +85,7 @@ import Wire.API.Conversation.Role
 import qualified Wire.API.Federation.API.Galley as F
 import Wire.API.Error
 import Polysemy.Error
+import Galley.API.Error
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -233,17 +234,25 @@ updateFedDomains = do
               -- Build the map, keyed by conversations to the list of remote members
               insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
           for_ (Map.toList cnvMap) $ \(cnv, rUsers) -> do
-            res <- liftIO $ evalGalleyToIO env $ runError 
-              . mapToRuntimeError @'ConvNotFound F.RemoveFromConversationErrorNotFound
-              . mapToRuntimeError @('ActionDenied 'LeaveConversation) F.RemoveFromConversationErrorRemovalNotAllowed
-              . mapToRuntimeError @'InvalidOperation F.RemoveFromConversationErrorRemovalNotAllowed
-              . mapError @NoChanges (const F.RemoveFromConversationErrorUnchanged) $
-              updateLocalConversation
-              @'ConversationRemoveMembersTag
-              (toLocalUnsafe (domain fedDomCfg) cnv)
-              undefined
-              Nothing $
-              tUntagged . rmId <$> rUsers
+            -- This value contains an event that we might need to
+            -- send out to all of the local clients that are a party
+            -- to the conversation. However we also don't want to DOS
+            -- clients. Maybe suppress and send out a bulk version?
+            _res <- liftIO $ evalGalleyToIO env
+              -- TODO: Are these the right error types we should be using?
+              -- TODO: We are restricted to the errors listed in GalleyEffects,
+              -- TODO: and none of those seem like a great fit.
+              $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Foo")
+              . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Bar")
+              . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Baz")
+              . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription  "Qux")
+              . mapError @NoChanges (const (InternalErrorWithDescription  "Qwe"))
+              $ updateLocalConversation
+                @'ConversationRemoveMembersTag
+                (toLocalUnsafe (domain fedDomCfg) cnv)
+                undefined
+                Nothing $
+                tUntagged . rmId <$> rUsers
             pure ()
         for_ addedDomains $ \_domain -> do
           pure ()

From 1e086c277b6eb7c8fd0803f721adebc1d1c0bc46 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 15 May 2023 09:37:42 +1000
Subject: [PATCH 066/220] FS-1179: Initial processing for deleting remote
 domains.

Setting up Galley to remove remote users from local conversations, local
users from remote conversations, and to delete all user connections for
the federation domain that is being dropped.
---
 .../src/Wire/API/Federation/API/Brig.hs       |  2 +
 services/brig/src/Brig/API/Federation.hs      |  5 ++
 services/brig/src/Brig/API/Internal.hs        |  5 +-
 services/brig/src/Brig/Data/Connection.hs     | 12 +++
 services/galley/src/Galley/API/Update.hs      |  1 -
 .../Galley/Cassandra/Conversation/Members.hs  |  5 ++
 .../galley/src/Galley/Cassandra/Queries.hs    |  6 ++
 .../galley/src/Galley/Effects/MemberStore.hs  |  2 +
 services/galley/src/Galley/Run.hs             | 85 ++++++++++++++-----
 9 files changed, 102 insertions(+), 21 deletions(-)

diff --git a/libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs b/libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs
index f9c36367bd..2235844af0 100644
--- a/libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs
+++ b/libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs
@@ -36,6 +36,7 @@ import Wire.API.User.Search
 import Wire.API.UserMap (UserMap)
 import Wire.API.Util.Aeson (CustomEncoded (..))
 import Wire.Arbitrary (GenericUniform (..))
+import Data.Domain (Domain)
 
 newtype SearchRequest = SearchRequest {term :: Text}
   deriving (Show, Eq, Generic, Typeable)
@@ -70,6 +71,7 @@ type BrigApi =
     :<|> FedEndpoint "get-mls-clients" MLSClientsRequest (Set ClientInfo)
     :<|> FedEndpoint "send-connection-action" NewConnectionRequest NewConnectionResponse
     :<|> FedEndpoint "on-user-deleted-connections" UserDeletedConnectionsNotification EmptyResponse
+    :<|> FedEndpoint "on-domain-unfederated" Domain EmptyResponse
     :<|> FedEndpoint "claim-key-packages" ClaimKeyPackageRequest (Maybe KeyPackageBundle)
 
 newtype GetUserClients = GetUserClients
diff --git a/services/brig/src/Brig/API/Federation.hs b/services/brig/src/Brig/API/Federation.hs
index 61edef6ae9..5bdad5cd1c 100644
--- a/services/brig/src/Brig/API/Federation.hs
+++ b/services/brig/src/Brig/API/Federation.hs
@@ -86,6 +86,7 @@ federationSitemap =
     :<|> Named @"get-mls-clients" getMLSClients
     :<|> Named @"send-connection-action" sendConnectionAction
     :<|> Named @"on-user-deleted-connections" onUserDeleted
+    :<|> Named @"on-domain-unfederated" (\d _ -> onDomainUnfederated d)
     :<|> Named @"claim-key-packages" fedClaimKeyPackages
 
 sendConnectionAction :: Domain -> NewConnectionRequest -> Handler r NewConnectionResponse
@@ -220,3 +221,7 @@ onUserDeleted origDomain udcn = lift $ do
       notify event (tUnqualified deletedUser) Push.RouteDirect Nothing (pure recipients)
   wrapClient $ Data.deleteRemoteConnections deletedUser connections
   pure EmptyResponse
+
+onDomainUnfederated :: Domain -> Handler r EmptyResponse
+onDomainUnfederated fedDomain = lift $
+  EmptyResponse <$ wrapClient (Data.deleteRemoteConnectionsByDomain fedDomain)
\ No newline at end of file
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 42cc4d9344..a8165a2e30 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -217,7 +217,10 @@ getFederationRemotes = lift $ do
   unless (maybe True (> 0) mu) $
     randomRIO (0 :: Int, 1000)
       >>= \case
-        0 -> Log.warn (Log.msg (Log.val "Invalid brig configuration: setFederationDomainConfigsUpdateFreq must be > 0, using default 10 seconds."))
+        0 -> do
+          let n = updateInterval defFederationDomainConfigs
+          -- Use the value from defFederationDomainConfig rather than hard coding it.
+          Log.warn $ Log.msg $ Log.val $ fromString $ "Invalid brig configuration: setFederationDomainConfigsUpdateFreq must be > 0, using default " <> show n <> " seconds."
         _ -> pure ()
 
   defFederationDomainConfigs
diff --git a/services/brig/src/Brig/Data/Connection.hs b/services/brig/src/Brig/Data/Connection.hs
index 21f2969da4..c326dfdd09 100644
--- a/services/brig/src/Brig/Data/Connection.hs
+++ b/services/brig/src/Brig/Data/Connection.hs
@@ -37,10 +37,12 @@ module Brig.Data.Connection
     countConnections,
     deleteConnections,
     deleteRemoteConnections,
+    deleteRemoteConnectionsByDomain, 
     remoteConnectionInsert,
     remoteConnectionSelect,
     remoteConnectionSelectFrom,
     remoteConnectionDelete,
+    remoteConnectionsDeleteByDomain,
     remoteConnectionClear,
 
     -- * Re-exports
@@ -323,6 +325,13 @@ deleteRemoteConnections (tUntagged -> Qualified remoteUser remoteDomain) (fromRa
   pooledForConcurrentlyN_ 16 locals $ \u ->
     write remoteConnectionDelete $ params LocalQuorum (u, remoteDomain, remoteUser)
 
+deleteRemoteConnectionsByDomain
+  :: MonadClient m
+  => Domain
+  -> m ()
+deleteRemoteConnectionsByDomain domain =
+  retry x1 . write remoteConnectionsDeleteByDomain $ params LocalQuorum $ pure domain
+
 -- Queries
 
 connectionInsert :: PrepQuery W (UserId, UserId, RelationWithHistory, UTCTimeMillis, ConvId) ()
@@ -385,6 +394,9 @@ remoteConnectionUpdate = "UPDATE connection_remote set status = ?, last_update =
 remoteConnectionDelete :: PrepQuery W (UserId, Domain, UserId) ()
 remoteConnectionDelete = "DELETE FROM connection_remote where left = ? AND right_domain = ? AND right_user = ?"
 
+remoteConnectionsDeleteByDomain :: PrepQuery W (Identity Domain) ()
+remoteConnectionsDeleteByDomain = "DELETE FROM connection_remote where right_domain = ?"
+
 remoteConnectionClear :: PrepQuery W (Identity UserId) ()
 remoteConnectionClear = "DELETE FROM connection_remote where left = ?"
 
diff --git a/services/galley/src/Galley/API/Update.hs b/services/galley/src/Galley/API/Update.hs
index c09a74bba7..ecc3f5ebf2 100644
--- a/services/galley/src/Galley/API/Update.hs
+++ b/services/galley/src/Galley/API/Update.hs
@@ -371,7 +371,6 @@ updateRemoteConversation rcnv lusr conn action = getUpdateResult $ do
     ConversationUpdateResponseNoChanges -> throw NoChanges
     ConversationUpdateResponseError err' -> rethrowErrors @(HasConversationActionGalleyErrors tag) err'
     ConversationUpdateResponseUpdate convUpdate -> pure convUpdate
-
   onConversationUpdated (tDomain rcnv) convUpdate
   notifyRemoteConversationAction lusr (qualifyAs rcnv convUpdate) (Just conn)
 
diff --git a/services/galley/src/Galley/Cassandra/Conversation/Members.hs b/services/galley/src/Galley/Cassandra/Conversation/Members.hs
index 1b34d39943..39d8f95e05 100644
--- a/services/galley/src/Galley/Cassandra/Conversation/Members.hs
+++ b/services/galley/src/Galley/Cassandra/Conversation/Members.hs
@@ -205,6 +205,10 @@ lookupRemoteMembersByDomain dom = do
   where
     mkConvMem (convId, usr, role) = (convId, RemoteMember (toRemoteUnsafe dom usr) role)
 
+lookupLocalMembersByDomain :: Domain -> Client [(ConvId, UserId)]
+lookupLocalMembersByDomain dom = do
+  retry x1 $ query Cql.selectLocalMembersByDomain (params LocalQuorum (Identity dom))
+
 member ::
   ConvId ->
   UserId ->
@@ -397,3 +401,4 @@ interpretMemberStoreToCassandra = interpret $ \case
   RemoveMLSClients lcnv quid cs -> embedClient $ removeMLSClients lcnv quid cs
   LookupMLSClients lcnv -> embedClient $ lookupMLSClients lcnv
   GetRemoteMembersByDomain dom -> embedClient $ lookupRemoteMembersByDomain dom
+  GetLocalMembersByDomain dom -> embedClient $ lookupLocalMembersByDomain dom
diff --git a/services/galley/src/Galley/Cassandra/Queries.hs b/services/galley/src/Galley/Cassandra/Queries.hs
index 9e14bb4f74..8d53bf938d 100644
--- a/services/galley/src/Galley/Cassandra/Queries.hs
+++ b/services/galley/src/Galley/Cassandra/Queries.hs
@@ -370,6 +370,7 @@ updateRemoteMemberConvRoleName :: PrepQuery W (RoleName, ConvId, Domain, UserId)
 updateRemoteMemberConvRoleName = "update member_remote_user set conversation_role = ? where conv = ? and user_remote_domain = ? and user_remote_id = ?"
 
 -- Used when removing a federation domain, so that we can quickly list all of the affected remote users and conversations
+-- This returns local conversation IDs and remote users
 selectRemoteMembersByDomain :: PrepQuery R (Identity Domain) (ConvId, UserId, RoleName)
 selectRemoteMembersByDomain = "select conv, user_remote_id, conversation_role from member_remote_user where user_remote_domain = ?"
 
@@ -390,6 +391,11 @@ selectRemoteConvMembers = "select user from user_remote_conv where user = ? and
 deleteUserRemoteConv :: PrepQuery W (UserId, Domain, ConvId) ()
 deleteUserRemoteConv = "delete from user_remote_conv where user = ? and conv_remote_domain = ? and conv_remote_id = ?"
 
+-- Used when removing a federation domain, so that we can quickly list all of the affected local users and conversations
+-- This returns remote conversation IDs and local users
+selectLocalMembersByDomain :: PrepQuery R (Identity Domain) (ConvId, UserId)
+selectLocalMembersByDomain = "select conv_remote_id, user from user_remote_conv where conv_remote_domain = ?"
+
 -- remote conversation status for local user
 
 updateRemoteOtrMemberMutedStatus :: PrepQuery W (MutedStatus, Maybe Text, Domain, ConvId, UserId) ()
diff --git a/services/galley/src/Galley/Effects/MemberStore.hs b/services/galley/src/Galley/Effects/MemberStore.hs
index b5a96a6dfe..a62d3f653b 100644
--- a/services/galley/src/Galley/Effects/MemberStore.hs
+++ b/services/galley/src/Galley/Effects/MemberStore.hs
@@ -35,6 +35,7 @@ module Galley.Effects.MemberStore
     checkLocalMemberRemoteConv,
     selectRemoteMembers,
     getRemoteMembersByDomain,
+    getLocalMembersByDomain,
 
     -- * Update members
     setSelfMember,
@@ -83,6 +84,7 @@ data MemberStore m a where
     GroupId ->
     MemberStore m (Map (Qualified UserId) (Set (ClientId, KeyPackageRef)))
   GetRemoteMembersByDomain :: Domain -> MemberStore m [(ConvId, RemoteMember)]
+  GetLocalMembersByDomain :: Domain -> MemberStore m [(ConvId, UserId)]
 
 makeSem ''MemberStore
 
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index d20511a21b..64f6cb1c89 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -44,7 +44,6 @@ import qualified Data.Set as Set
 import Data.String.Conversions (cs)
 import Data.Text (unpack)
 import qualified Galley.API as API
-import Galley.API.Federation (FederationAPI, federationSitemap)
 import Galley.API.Internal
 import Galley.App
 import qualified Galley.App as App
@@ -67,6 +66,7 @@ import Servant.Client
     ClientEnv (ClientEnv),
     Scheme (Http),
     defaultMakeClientRequest,
+    runClientM
   )
 import qualified System.Logger as Log
 import Util.Options
@@ -75,7 +75,6 @@ import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
-import qualified Galley.Effects.MemberStore as E
 import qualified Data.List.NonEmpty as N
 import Galley.API.Action
 import Galley.Types.Conversations.Members
@@ -86,6 +85,14 @@ import qualified Wire.API.Federation.API.Galley as F
 import Wire.API.Error
 import Polysemy.Error
 import Galley.API.Error
+import qualified Galley.Effects.MemberStore as E
+import Data.Time (getCurrentTime)
+import Wire.API.Conversation.Action
+import Galley.API.Federation
+import Data.Singletons
+import qualified Wire.API.Federation.API.Brig as Brig
+import Wire.API.Routes.Named (namedClient)
+import qualified System.Logger as L
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -202,14 +209,17 @@ updateFedDomains = do
       Endpoint host port = env ^. brig
       baseUrl   = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
-  liftIO $ getAllowedDomainsLoop logger clientEnv ioref $ callback env
+  liftIO $ getAllowedDomainsLoop logger clientEnv ioref $ callback env clientEnv
   where
-    callback env old new = do
+    -- Build the map, keyed by conversations to the list of members
+    insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
+    callback env clientEnv old new = do
       -- TODO: perform the database updates here
       -- This code will only run when there is a change in the domain lists
-      let fromFedList = Set.fromList . fromFederationDomainConfigs
+      let fromFedList = Set.fromList . remotes
           prevDoms = fromFedList old
           currDoms = fromFedList new
+          localDomain = env ^. options . optSettings . setFederationDomain
       unless (prevDoms == currDoms) $ do
         -- Perform updates before rewriting the tvar
         -- This means that if the update fails on a
@@ -218,7 +228,7 @@ updateFedDomains = do
         -- the domain list is changing frequently.
         -- FS-1179 is handling this part.
         let deletedDomains = Set.difference prevDoms currDoms
-            addedDomains = Set.difference currDoms prevDoms
+            -- addedDomains = Set.difference currDoms prevDoms
         for_ deletedDomains $ \fedDomCfg -> do
           -- https://wearezeta.atlassian.net/browse/FS-1179
           -- TODO
@@ -227,13 +237,12 @@ updateFedDomains = do
           --   NOTE: This is NOT sent to other backends, as this information is not authoratative, but is
           --   good enough to tell local users about the federation connection being removed.
           -- * Delete all connections from local users to users for the remote domain.
+
           -- Get all remote users for the given domain, along with conversation IDs that they are in
-          remoteUsers <- liftIO $ evalGalleyToIO env $ E.getRemoteMembersByDomain $ domain fedDomCfg
-          let cnvMap :: Map ConvId (N.NonEmpty RemoteMember)
-              cnvMap = foldr insertIntoMap mempty remoteUsers
-              -- Build the map, keyed by conversations to the list of remote members
-              insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
-          for_ (Map.toList cnvMap) $ \(cnv, rUsers) -> do
+          let dom = domain fedDomCfg
+          remoteUsers <- liftIO $ evalGalleyToIO env $ E.getRemoteMembersByDomain dom
+          let lCnvMap = foldr insertIntoMap mempty remoteUsers
+          for_ (Map.toList lCnvMap) $ \(cnv, rUsers) -> do
             -- This value contains an event that we might need to
             -- send out to all of the local clients that are a party
             -- to the conversation. However we also don't want to DOS
@@ -242,11 +251,12 @@ updateFedDomains = do
               -- TODO: Are these the right error types we should be using?
               -- TODO: We are restricted to the errors listed in GalleyEffects,
               -- TODO: and none of those seem like a great fit.
-              $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Foo")
-              . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Bar")
-              . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Baz")
-              . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription  "Qux")
-              . mapError @NoChanges (const (InternalErrorWithDescription  "Qwe"))
+              $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Remove From Conversation Error")
+              . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Conv Not Found")
+              . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Action Denied: Remove Conversation Member")
+              . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription  "Invalid Operation")
+              . mapError @NoChanges (const (InternalErrorWithDescription  "No Changes"))
+              -- This is allowed to send notifications to _local_ clients.
               $ updateLocalConversation
                 @'ConversationRemoveMembersTag
                 (toLocalUnsafe (domain fedDomCfg) cnv)
@@ -254,5 +264,42 @@ updateFedDomains = do
                 Nothing $
                 tUntagged . rmId <$> rUsers
             pure ()
-        for_ addedDomains $ \_domain -> do
-          pure ()
+
+          -- Get all local users for the given domain, along with remote conversation IDs that they are in
+          localUsers <- liftIO $ evalGalleyToIO env $ E.getLocalMembersByDomain dom
+          -- As above, build the map so we can get all local users per conversation
+          let rCnvMap = foldr insertIntoMap mempty localUsers
+          -- Process each user.
+          for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
+            _res <- liftIO $ evalGalleyToIO env
+              $ mapError @NoChanges (const (InternalErrorWithDescription  "No Changes: Could not remove a local member from a remote conversation."))
+              $ do
+                now <- liftIO $ getCurrentTime
+                for_ lUsers $ \user -> do
+                  let lUser = toLocalUnsafe localDomain user
+                      convUpdate = F.ConversationUpdate
+                        { cuTime = now
+                        , cuOrigUserId = tUntagged lUser
+                        , cuConvId = cnv
+                        , cuAlreadyPresentUsers = mempty
+                        , cuAction = SomeConversationAction (sing @'ConversationDeleteTag) ()
+                        }
+                  -- These functions are used directly rather than as part of a larger conversation
+                  -- delete function, as we don't have an originating user, and we can't send data
+                  -- to the remote backend.
+                  onConversationUpdated dom convUpdate
+                  let rcnv = toRemoteUnsafe dom cnv
+                  notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
+            pure ()
+
+          -- Remove the remote one-on-one conversations between local members and remote members for the given domain.
+          -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
+          let delFedDomain = namedClient @Brig.BrigApi @"on-domain-unfederated"
+          -- deleteRemoteConnectionsByDomain dom
+          liftIO (runClientM (delFedDomain dom) clientEnv) >>= \case
+            Right _ -> pure ()
+            Left e -> L.log (env ^. applog) L.Info $
+              L.msg (L.val "Could not delete remote user connections in Brig")
+                L.~~ "error" L..= show e
+        -- for_ addedDomains $ \_domain -> do
+        --   pure ()

From d7f60e3e499fa6f7aa4a37847bc7fcb167406cfa Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 16 May 2023 16:05:19 +1000
Subject: [PATCH 067/220] FS-1179: Fixing an error http reponse code.

---
 services/federator/src/Federator/Validation.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index d5fab49748..6bae69730c 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -86,7 +86,7 @@ validationErrorStatus :: ValidationError -> HTTP.Status
 -- the FederationDenied case is handled differently, because it may be caused
 -- by wrong input in the original request, so we let this error propagate to the
 -- client
-validationErrorStatus (FederationDenied _) = HTTP.status400
+validationErrorStatus (FederationDenied _) = HTTP.status422
 validationErrorStatus _ = HTTP.status403
 
 -- | Validates an already-parsed domain against the allowList (stored in

From 6a583bfb86eba6ccb9cc0ac720c55c1e5b5a291f Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 18 May 2023 13:04:43 +1000
Subject: [PATCH 068/220] wip

---
 .../src/Wire/API/Federation/API/Brig.hs       |  2 -
 services/brig/src/Brig/API/Federation.hs      |  7 +-
 services/brig/src/Brig/API/Internal.hs        |  2 +
 services/galley/src/Galley/API/Action.hs      | 22 +++++++
 services/galley/src/Galley/Run.hs             | 66 ++++++++++++-------
 5 files changed, 67 insertions(+), 32 deletions(-)

diff --git a/libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs b/libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs
index 2235844af0..f9c36367bd 100644
--- a/libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs
+++ b/libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs
@@ -36,7 +36,6 @@ import Wire.API.User.Search
 import Wire.API.UserMap (UserMap)
 import Wire.API.Util.Aeson (CustomEncoded (..))
 import Wire.Arbitrary (GenericUniform (..))
-import Data.Domain (Domain)
 
 newtype SearchRequest = SearchRequest {term :: Text}
   deriving (Show, Eq, Generic, Typeable)
@@ -71,7 +70,6 @@ type BrigApi =
     :<|> FedEndpoint "get-mls-clients" MLSClientsRequest (Set ClientInfo)
     :<|> FedEndpoint "send-connection-action" NewConnectionRequest NewConnectionResponse
     :<|> FedEndpoint "on-user-deleted-connections" UserDeletedConnectionsNotification EmptyResponse
-    :<|> FedEndpoint "on-domain-unfederated" Domain EmptyResponse
     :<|> FedEndpoint "claim-key-packages" ClaimKeyPackageRequest (Maybe KeyPackageBundle)
 
 newtype GetUserClients = GetUserClients
diff --git a/services/brig/src/Brig/API/Federation.hs b/services/brig/src/Brig/API/Federation.hs
index 5bdad5cd1c..fd00e6136d 100644
--- a/services/brig/src/Brig/API/Federation.hs
+++ b/services/brig/src/Brig/API/Federation.hs
@@ -86,7 +86,6 @@ federationSitemap =
     :<|> Named @"get-mls-clients" getMLSClients
     :<|> Named @"send-connection-action" sendConnectionAction
     :<|> Named @"on-user-deleted-connections" onUserDeleted
-    :<|> Named @"on-domain-unfederated" (\d _ -> onDomainUnfederated d)
     :<|> Named @"claim-key-packages" fedClaimKeyPackages
 
 sendConnectionAction :: Domain -> NewConnectionRequest -> Handler r NewConnectionResponse
@@ -220,8 +219,4 @@ onUserDeleted origDomain udcn = lift $ do
     pooledForConcurrentlyN_ 16 (nonEmpty acceptedLocals) $ \(List1 -> recipients) ->
       notify event (tUnqualified deletedUser) Push.RouteDirect Nothing (pure recipients)
   wrapClient $ Data.deleteRemoteConnections deletedUser connections
-  pure EmptyResponse
-
-onDomainUnfederated :: Domain -> Handler r EmptyResponse
-onDomainUnfederated fedDomain = lift $
-  EmptyResponse <$ wrapClient (Data.deleteRemoteConnectionsByDomain fedDomain)
\ No newline at end of file
+  pure EmptyResponse
\ No newline at end of file
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index a8165a2e30..0b088e97c1 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -241,6 +241,8 @@ deleteFederationRemotes dom = do
       \do that, removing items listed in the config file is not allowed."
   -- FUTUREWORK: see 'getFederationRemotes'.
   lift . wrapClient . Data.deleteFederationRemote $ dom
+  -- Also drop connections to the remote domain
+  lift . wrapClient . Data.deleteRemoteConnectionsByDomain $ dom
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)
diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs
index 95cae36a04..89dc3a82a7 100644
--- a/services/galley/src/Galley/API/Action.hs
+++ b/services/galley/src/Galley/API/Action.hs
@@ -26,6 +26,7 @@ module Galley.API.Action
     -- * Performing actions
     updateLocalConversation,
     updateLocalConversationUnchecked,
+    updateLocalConversationUserUnchecked,
     NoChanges (..),
     LocalConversationUpdate (..),
     notifyTypingIndicator,
@@ -652,6 +653,27 @@ updateLocalConversationUnchecked lconv qusr con action = do
     (convBotsAndMembers conv <> extraTargets)
     action'
 
+
+-- | Similar to 'updateLocalConversationUnchecked', but skips performing
+-- user authorisation checks. This is written for use in de-federation code
+-- where conversations for many users will be torn down at once and must work.
+--
+-- Additionally, this function doesn't make notification calls to clients.
+updateLocalConversationUserUnchecked ::
+  forall tag r.
+  ( SingI tag,
+    HasConversationActionEffects tag r
+  ) =>
+  Local Conversation ->
+  Qualified UserId ->
+  ConversationAction tag ->
+  Sem r ()
+updateLocalConversationUserUnchecked lconv qusr action = do
+  let tag = sing @tag
+
+  -- perform action
+  void $ performAction tag qusr lconv action
+
 -- --------------------------------------------------------------------------------
 -- -- Utilities
 
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 64f6cb1c89..a0beab5fa2 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -90,9 +90,12 @@ import Data.Time (getCurrentTime)
 import Wire.API.Conversation.Action
 import Galley.API.Federation
 import Data.Singletons
-import qualified Wire.API.Federation.API.Brig as Brig
+import qualified Wire.API.Routes.Internal.Brig as Brig
 import Wire.API.Routes.Named (namedClient)
 import qualified System.Logger as L
+import Wire.API.Conversation (ConvType(One2OneConv), cnvmType, ConvType (ConnectConv, One2OneConv))
+import Galley.Data.Conversation.Types (convMetadata)
+import Galley.API.Util (getConversationWithError)
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -214,7 +217,6 @@ updateFedDomains = do
     -- Build the map, keyed by conversations to the list of members
     insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
     callback env clientEnv old new = do
-      -- TODO: perform the database updates here
       -- This code will only run when there is a change in the domain lists
       let fromFedList = Set.fromList . remotes
           prevDoms = fromFedList old
@@ -228,10 +230,8 @@ updateFedDomains = do
         -- the domain list is changing frequently.
         -- FS-1179 is handling this part.
         let deletedDomains = Set.difference prevDoms currDoms
-            -- addedDomains = Set.difference currDoms prevDoms
         for_ deletedDomains $ \fedDomCfg -> do
           -- https://wearezeta.atlassian.net/browse/FS-1179
-          -- TODO
           -- * Remove remote users for the given domain from all conversations owned by the current host
           -- * Remove all local users from remote conversations owned by the given domain.
           --   NOTE: This is NOT sent to other backends, as this information is not authoratative, but is
@@ -242,27 +242,44 @@ updateFedDomains = do
           let dom = domain fedDomCfg
           remoteUsers <- liftIO $ evalGalleyToIO env $ E.getRemoteMembersByDomain dom
           let lCnvMap = foldr insertIntoMap mempty remoteUsers
-          for_ (Map.toList lCnvMap) $ \(cnv, rUsers) -> do
+          for_ (Map.toList lCnvMap) $ \(cnvId, rUsers) -> do
+            let lCnvId = toLocalUnsafe dom cnvId
             -- This value contains an event that we might need to
             -- send out to all of the local clients that are a party
             -- to the conversation. However we also don't want to DOS
             -- clients. Maybe suppress and send out a bulk version?
-            _res <- liftIO $ evalGalleyToIO env
-              -- TODO: Are these the right error types we should be using?
-              -- TODO: We are restricted to the errors listed in GalleyEffects,
-              -- TODO: and none of those seem like a great fit.
-              $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Remove From Conversation Error")
-              . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Conv Not Found")
-              . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Action Denied: Remove Conversation Member")
-              . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription  "Invalid Operation")
-              . mapError @NoChanges (const (InternalErrorWithDescription  "No Changes"))
+            liftIO $ evalGalleyToIO env
+              $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Federation domain removal: Remove from conversation error")
+              . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
+              . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Federation domain removal: Action denied, remove conversation member")
+              . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription  "Federation domain removal: Invalid operation")
+              . mapToRuntimeError @'NotATeamMember (InternalErrorWithDescription  "Federation domain removal: Not a team member")
+              . mapError @NoChanges (const (InternalErrorWithDescription  "Federation domain removal: No changes"))
               -- This is allowed to send notifications to _local_ clients.
-              $ updateLocalConversation
-                @'ConversationRemoveMembersTag
-                (toLocalUnsafe (domain fedDomCfg) cnv)
-                undefined
-                Nothing $
-                tUntagged . rmId <$> rUsers
+              -- But we are suppressing those events as we don't want to
+              -- DOS our users if a large and deeply interconnected federation
+              -- member is removed. Sending out hundreds or thousands of events
+              -- to each client isn't something we want to be doing.
+              $ do
+                conv <- getConversationWithError lCnvId
+                let lConv = toLocalUnsafe dom conv
+                updateLocalConversationUserUnchecked
+                  @'ConversationRemoveMembersTag
+                  lConv
+                  undefined $ -- This field can be undefined as the path for ConversationRemoveMembersTag doens't use it
+                  tUntagged . rmId <$> rUsers
+                -- Check if the conversation if type 2 or 3, one-on-one conversations.
+                -- If it is, then we need to remove the entire conversation as users
+                -- aren't able to delete those types of conversations themselves.
+                -- Check that we are in a type 2 or a type 3 conversation
+                when (cnvmType (convMetadata conv) `elem` [One2OneConv, ConnectConv]) $
+                  -- If we are, delete it.
+                  updateLocalConversationUserUnchecked
+                    @'ConversationDeleteTag
+                    lConv
+                    undefined
+                    ()
+                  
             pure ()
 
           -- Get all local users for the given domain, along with remote conversation IDs that they are in
@@ -287,6 +304,9 @@ updateFedDomains = do
                   -- These functions are used directly rather than as part of a larger conversation
                   -- delete function, as we don't have an originating user, and we can't send data
                   -- to the remote backend.
+                  -- We don't need to check the conversation type here, as we can't tell the
+                  -- remote federation server to delete the conversation. They will have to do a
+                  -- similar processing run for removing the local domain from their federation list.
                   onConversationUpdated dom convUpdate
                   let rcnv = toRemoteUnsafe dom cnv
                   notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
@@ -294,12 +314,10 @@ updateFedDomains = do
 
           -- Remove the remote one-on-one conversations between local members and remote members for the given domain.
           -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
-          let delFedDomain = namedClient @Brig.BrigApi @"on-domain-unfederated"
-          -- deleteRemoteConnectionsByDomain dom
+          let delFedDomain = namedClient @Brig.FederationRemotesAPI @"delete-federation-remotes"
           liftIO (runClientM (delFedDomain dom) clientEnv) >>= \case
             Right _ -> pure ()
             Left e -> L.log (env ^. applog) L.Info $
               L.msg (L.val "Could not delete remote user connections in Brig")
                 L.~~ "error" L..= show e
-        -- for_ addedDomains $ \_domain -> do
-        --   pure ()
+

From c0df89aafa7064484c571ae406dce1dc36747cbb Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 11:01:56 +0200
Subject: [PATCH 069/220] Remove dead code.

---
 libs/types-common/src/Data/MessageQueue.hs | 21 ---------------------
 libs/types-common/types-common.cabal       |  1 -
 services/brig/src/Brig/Options.hs          |  5 +----
 3 files changed, 1 insertion(+), 26 deletions(-)
 delete mode 100644 libs/types-common/src/Data/MessageQueue.hs

diff --git a/libs/types-common/src/Data/MessageQueue.hs b/libs/types-common/src/Data/MessageQueue.hs
deleted file mode 100644
index eccc6b77d0..0000000000
--- a/libs/types-common/src/Data/MessageQueue.hs
+++ /dev/null
@@ -1,21 +0,0 @@
-module Data.MessageQueue
-  ( MessageQueueSettings (..),
-  )
-where
-
-import Data.Aeson (FromJSON)
-import Data.Text (Text)
-import GHC.Generics (Generic)
-import Prelude (Show, String)
-
--- | Options for connecting to the message queue system
-data MessageQueueSettings = MessageQueueSettings
-  { host :: String,
-    vHost :: Text,
-    user :: Text,
-    pass :: Text,
-    queue :: Text
-  }
-  deriving (Show, Generic)
-
-instance FromJSON MessageQueueSettings
diff --git a/libs/types-common/types-common.cabal b/libs/types-common/types-common.cabal
index f2f1f6b525..00d9e81282 100644
--- a/libs/types-common/types-common.cabal
+++ b/libs/types-common/types-common.cabal
@@ -22,7 +22,6 @@ library
     Data.Json.Util
     Data.LegalHold
     Data.List1
-    Data.MessageQueue
     Data.Misc
     Data.Nonce
     Data.PEMKeys
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index d46bb6109d..c8c2984dfb 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -39,7 +39,6 @@ import qualified Data.Code as Code
 import Data.Domain (Domain (..))
 import Data.Id
 import Data.LanguageCodes (ISO639_1 (EN))
-import Data.MessageQueue
 import Data.Misc (HttpsUrl)
 import Data.Nonce
 import Data.Range
@@ -459,9 +458,7 @@ data Opts = Opts
     -- | SFT Settings
     sft :: !(Maybe SFTOptions),
     -- | Runtime settings
-    optSettings :: !Settings,
-    -- | Message Queue settings
-    mqSettings :: !(Maybe MessageQueueSettings)
+    optSettings :: !Settings
   }
   deriving (Show, Generic)
 

From ea18bcb116befa7d97318705da40dbdcc59e6c03 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 11:09:07 +0200
Subject: [PATCH 070/220] nit-pick.

---
 libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index 64c22e4bca..05435a1e05 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -431,7 +431,8 @@ type FederationRemotesAPI =
            )
     :<|> Named
            "delete-federation-remotes"
-           ( Description FederationRemotesAPIDeleteDescription
+           ( Description FederationRemotesAPIDescription
+               :> Description FederationRemotesAPIDeleteDescription
                :> "federation"
                :> "remotes"
                :> Capture "domain" Domain
@@ -439,12 +440,11 @@ type FederationRemotesAPI =
            )
 
 type FederationRemotesAPIDescription =
-  "See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections for background."
+  "See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections for background. "
 
 type FederationRemotesAPIDeleteDescription =
-  "WARNING!  If you remove a remote connection, all users from that remote will be removed from local conversations, and all \
-  \group conversations hosted by that remote will be removed from the local backend.  This cannot be reverted!  See \
-  \https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections for background."
+  "**WARNING!** If you remove a remote connection, all users from that remote will be removed from local conversations, and all \
+  \group conversations hosted by that remote will be removed from the local backend. This cannot be reverted! "
 
 swaggerDoc :: Swagger
 swaggerDoc =

From 4b79d4ba7d6ac861d93095c4236dcba63f080fa7 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 11:21:50 +0200
Subject: [PATCH 071/220] Typo.

---
 services/cannon/src/Cannon/Run.hs   | 2 +-
 services/gundeck/src/Gundeck/Run.hs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index fd52ddcc54..f707708dfc 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -78,7 +78,7 @@ run o = do
   refreshMetricsThread <- Async.async $ runCannon' e refreshMetrics
   s <- newSettings $ Server (o ^. cannon . host) (o ^. cannon . port) (applog e) m (Just idleTimeout)
 
-  -- Get the federaion domain list from Brig and start the updater loop
+  -- Get the federation domain list from Brig and start the updater loop
   manager <- newManager defaultManagerSettings
   let Brig bh bp = o ^. brig
       baseUrl = BaseUrl Http (unpack bh) (fromIntegral bp) ""
diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs
index c9715adf96..34a7888908 100644
--- a/services/gundeck/src/Gundeck/Run.hs
+++ b/services/gundeck/src/Gundeck/Run.hs
@@ -68,7 +68,7 @@ run o = do
   s <- newSettings $ defaultServer (unpack $ o ^. optGundeck . epHost) (o ^. optGundeck . epPort) l m
   let throttleMillis = fromMaybe defSqsThrottleMillis $ o ^. (optSettings . setSqsThrottleMillis)
 
-  -- Get the federaion domain list from Brig and start the updater loop
+  -- Get the federation domain list from Brig and start the updater loop
   mgr <- newManager defaultManagerSettings
   let Endpoint host port = o ^. optBrig
       baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""

From 2f3ec7bd7c550e1e3646e3437ffc97482b5d4fa8 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 11:29:48 +0200
Subject: [PATCH 072/220] Fine-tune logging.

---
 libs/wire-api/src/Wire/API/FederationUpdate.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 67281c2bbf..3c99ff5f5d 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -31,7 +31,7 @@ getAllowedDomainsInitial logger clientEnv =
         getAllowedDomains clientEnv >>= \case
           Right s -> pure $ Just s
           Left e -> do
-            L.log logger L.Info $
+            L.log logger L.Debug $
               L.msg (L.val "Could not retrieve an initial list of federation domains from Brig.")
                 L.~~ "error" L..= show e
             pure Nothing

From 84e78f0a276ca3095285a7643d8fe07d341f528b Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 12:20:10 +0200
Subject: [PATCH 073/220] Fix haddocks.

---
 services/brig/src/Brig/Options.hs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index c8c2984dfb..6ba4bc6499 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -543,12 +543,8 @@ data Settings = Settings
     -- e.g. 0c4d8944-70fa-480e-a8b7-9d929862d18c@wire.com and somehandle@wire.com.
     -- It should also match the SRV DNS records under which other wire-server installations can find this backend:
     -- >>>   _wire-server-federator._tcp.<federationDomain>
-    -- Once set, DO NOT change it: if you do, existing users may have a broken experience and/or stop working
+    -- Once set, DO NOT change it: if you do, existing users may have a broken experience and/or stop working.
     -- Remember to keep it the same in all services.
-    -- Example:
-    -- >>>  allowedDomains:
-    -- >>>    - wire.com
-    -- >>>    - example.com
     setFederationDomain :: !Domain,
     -- | See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
     -- default: AllowNone

From 6d1f7191a5af5ca7bae0a6101164a02431236d96 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 12:30:38 +0200
Subject: [PATCH 074/220] Implement put.

---
 services/brig/test/integration/API/Federation.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index e8bce7e87d..c465585f41 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -494,9 +494,10 @@ crudFederationRemotes opts brig = do
 
   -- updating search strategy works
   let remote2' = remote2 {cfgSearchPolicy = NoSearch}
-  () <- updateFederationRemote brig (domain remote2) remote2'
+  updateFederationRemote brig (domain remote2) remote2'
   res5 <- getFederationRemotes brig
-  liftIO $ assertEqual "should be NoSearch" (nub $ sort $ cfgRemotes <> [remote1, remote2']) (sort res5)
+  -- (move the dynamic remotes to the beginning here to make sure we look for `remote2'`, not `remote`.)
+  liftIO $ assertEqual "should be NoSearch" (nub $ sort $ [remote1, remote2'] <> cfgRemotes) (sort res5)
 
   -- updating from config file fails
   updateFederationRemote' id brig (domain $ head $ cfgRemotes) (head $ cfgRemotes) !!! const 533 === statusCode

From e85c97d013a4e8bbb389aa1d6b143875aea9afe2 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 15:27:54 +0200
Subject: [PATCH 075/220] Clariy updateFrequency everywhere.

---
 docs/src/understand/federation/backend-communication.md    | 3 +++
 .../wire-api/src/Wire/API/Routes/FederationDomainConfig.hs | 7 +++++--
 services/brig/src/Brig/API/Internal.hs                     | 4 ++--
 services/brig/src/Brig/Options.hs                          | 2 +-
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index 9c92d48139..299b34be21 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -342,3 +342,6 @@ If there is no configuration for a domain, it's defaulted to `no_search`.
 
 
 does anybody know off the top of their heads: is [this section](https://wearezeta.atlassian.net/wiki/spaces/BAC/pages/288620677/Processes+shared+with+CS#Different-search-visibility-per-team) still up to date?  and is stern?  [this page](https://docs.wire.com/developer/reference/config-options.html#federated-domain-specific-configuration-settings) tells a different story...
+
+
+TODO: explain setFederationDomainConfigsUpdateFreq
diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 0acc9d0f40..70f4c1fd6f 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -23,6 +23,7 @@ module Wire.API.Routes.FederationDomainConfig
   )
 where
 
+import Control.Lens ((?~))
 import Data.Aeson (FromJSON, ToJSON)
 import Data.Domain (Domain)
 import Data.Schema
@@ -69,8 +70,10 @@ defFederationDomainConfigs =
 
 instance ToSchema FederationDomainConfigs where
   schema =
-    object "FederationDomainConfigs" $
-      FederationDomainConfigs
+    objectWithDocModifier
+      "FederationDomainConfigs"
+      (description ?~ "See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections.")
+      $ FederationDomainConfigs
         <$> strategy .= field "strategy" schema
         <*> remotes .= field "remotes" (array schema)
         <*> updateInterval .= field "update_interval" schema
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 42cc4d9344..58b5e3a285 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -212,7 +212,7 @@ getFederationRemotes = lift $ do
         setFederationDomainConfigsUpdateFreq (cfg ^. settings)
       )
 
-  -- update frequency settings of <= 0 are ignored.  only warn about this every now and
+  -- update frequency settings of `<1` are interpreted as `1 second`.  only warn about this every now and
   -- then, that'll be noise enough for the logs given the traffic on this end-point.
   unless (maybe True (> 0) mu) $
     randomRIO (0 :: Int, 1000)
@@ -223,7 +223,7 @@ getFederationRemotes = lift $ do
   defFederationDomainConfigs
     & maybe id (\v cfg -> cfg {strategy = v}) ms
     & (\cfg -> cfg {remotes = nub $ db <> fromMaybe mempty mf})
-    & maybe id (\v cfg -> cfg {updateInterval = min 10 v}) mu
+    & maybe id (\v cfg -> cfg {updateInterval = min 1 v}) mu
     & pure
 
 updateFederationRemotes :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index 6ba4bc6499..17dbd386a0 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -556,7 +556,7 @@ data Settings = Settings
     -- for details.
     -- default: []
     setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]),
-    -- | In seconds.  Values <=0 are ignored.  Default: 10 seconds.  See
+    -- | In seconds.  Default: 10 seconds.  Values <1 are silently replaced by 1.  See
     -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
     setFederationDomainConfigsUpdateFreq :: !(Maybe Int),
     -- | The amount of time in milliseconds to wait after reading from an SQS queue

From 6327defae922801e88ea3aa8f10fdce5773686a9 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 15:32:35 +0200
Subject: [PATCH 076/220] nit-pick.

---
 services/brig/src/Brig/API/Internal.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 58b5e3a285..e9330544a5 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -233,10 +233,10 @@ deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
   cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
   when (dom `elem` (domain <$> cfg)) $ do
+    -- FUTUREWORK: see 'getFederationRemotes'.
     throwError . fedError . FederationUnexpectedError $
       "keeping track of remote domains in the brig config file is deprecated, but as long as we \
       \do that, removing items listed in the config file is not allowed."
-  -- FUTUREWORK: see 'getFederationRemotes'.
   lift . wrapClient . Data.deleteFederationRemote $ dom
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.

From 5915c8228ce17004be89a1c21c412be11b736503 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 16:23:45 +0200
Subject: [PATCH 077/220] Fixup

---
 libs/wire-api/src/Wire/API/FederationUpdate.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 3c99ff5f5d..8c56daa52a 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -58,7 +58,7 @@ getAllowedDomainsLoop logger clientEnv env callback = forever $ do
       callback old cfg
       atomicWriteIORef env cfg
   delay <- updateInterval <$> readIORef env
-  threadDelay delay
+  threadDelay (delay * 1_000_000)
 
 -- A version where the callback isn't needed. Most of the services don't care about
 -- when the list changes, just that they have the new list and can use it as-is

From f2e998d54436b281e5659e9c36338dced4467e2a Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 16:32:57 +0200
Subject: [PATCH 078/220] Cleanup

---
 services/brig/src/Brig/API/Internal.hs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index e9330544a5..1697870833 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -229,14 +229,18 @@ getFederationRemotes = lift $ do
 updateFederationRemotes :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 updateFederationRemotes = undefined
 
-deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
-deleteFederationRemotes dom = do
+-- | FUTUREWORK: should go away in the future; see 'getFederationRemotes'.
+assertNoDomainsFromConfigFiles :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
+assertNoDomainsFromConfigFiles dom = do
   cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
   when (dom `elem` (domain <$> cfg)) $ do
-    -- FUTUREWORK: see 'getFederationRemotes'.
     throwError . fedError . FederationUnexpectedError $
       "keeping track of remote domains in the brig config file is deprecated, but as long as we \
-      \do that, removing items listed in the config file is not allowed."
+      \do that, removing or updating items listed in the config file is not allowed."
+
+deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
+deleteFederationRemotes dom = do
+  assertNoDomainsFromConfigFiles dom
   lift . wrapClient . Data.deleteFederationRemote $ dom
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.

From 012f39db3ad9d0aa3e8bd715170e996f287b18a4 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 16:53:25 +0200
Subject: [PATCH 079/220] Implement put.  (For real this time.)

---
 services/brig/src/Brig/API/Internal.hs    | 12 +++++++++++-
 services/brig/src/Brig/Data/Federation.hs |  8 ++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 1697870833..c9ff060625 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -74,6 +74,7 @@ import Data.Id as Id
 import qualified Data.Map.Strict as Map
 import Data.Qualified
 import qualified Data.Set as Set
+import Data.String.Conversions (cs)
 import Imports hiding (head)
 import Network.HTTP.Types.Status
 import Network.Wai (Response)
@@ -227,7 +228,16 @@ getFederationRemotes = lift $ do
     & pure
 
 updateFederationRemotes :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
-updateFederationRemotes = undefined
+updateFederationRemotes dom fedcfg = do
+  assertDomainIsNotUpdated dom fedcfg
+  assertNoDomainsFromConfigFiles dom
+  lift . wrapClient . Data.updateFederationRemote $ fedcfg
+
+assertDomainIsNotUpdated :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
+assertDomainIsNotUpdated dom fedcfg = do
+  when (dom /= domain fedcfg) $
+    throwError . fedError . FederationUnexpectedError . cs $
+      "federation domain of a given peer cannot be changed from " <> show (domain fedcfg) <> " to " <> show dom <> "."
 
 -- | FUTUREWORK: should go away in the future; see 'getFederationRemotes'.
 assertNoDomainsFromConfigFiles :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
diff --git a/services/brig/src/Brig/Data/Federation.hs b/services/brig/src/Brig/Data/Federation.hs
index ce2deb5da7..ee06bbedcf 100644
--- a/services/brig/src/Brig/Data/Federation.hs
+++ b/services/brig/src/Brig/Data/Federation.hs
@@ -18,6 +18,7 @@
 module Brig.Data.Federation
   ( getFederationRemotes,
     addFederationRemote,
+    updateFederationRemote,
     deleteFederationRemote,
     AddFederationRemoteResult (..),
   )
@@ -54,6 +55,13 @@ addFederationRemote (FederationDomainConfig rdom searchpolicy) = do
     add :: PrepQuery W (Domain, FederatedUserSearchPolicy) ()
     add = "INSERT INTO federation_remotes (domain, search_policy) VALUES (?, ?)"
 
+updateFederationRemote :: MonadClient m => FederationDomainConfig -> m ()
+updateFederationRemote (FederationDomainConfig rdom spol) = do
+  retry x1 $ write upd (params LocalQuorum (spol, rdom))
+  where
+    upd :: PrepQuery W (FederatedUserSearchPolicy, Domain) ()
+    upd = "UPDATE federation_remotes SET search_policy = ? WHERE domain = ? IF EXISTS"
+
 deleteFederationRemote :: MonadClient m => Domain -> m ()
 deleteFederationRemote rdom =
   retry x1 $ write delete (params LocalQuorum (Identity rdom))

From aa52e918fdf402da29a97de3f8f61cf957fab769 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 19 May 2023 16:54:08 +0200
Subject: [PATCH 080/220] Move integration tests to /integration.

---
 integration/test/API/BrigInternal.hs          | 54 ++++++++++++++++++
 integration/test/Test/Brig.hs                 | 52 ++++++++++++++++++
 .../brig/test/integration/API/Federation.hs   | 55 +------------------
 services/brig/test/integration/Util.hs        | 37 -------------
 4 files changed, 107 insertions(+), 91 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 1e73f4366c..050c9e7d7b 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -43,3 +43,57 @@ createUser domain cu = do
                  | cu.team
                ]
         )
+
+data FedConn = FedConn
+  { domain :: Maybe String,
+    searchStrategy :: Maybe String
+  }
+
+instance Default FedConn where
+  def =
+    FedConn
+      { domain = Nothing,
+        searchStrategy = Nothing
+      }
+
+createFedConn :: HasCallStack => FedConn -> App Response
+createFedConn = undefined
+
+{-
+  post (brig . paths ["i", "federation", "remotes"] . contentJson . json remote . expect2xx)
+-}
+
+readFedConn :: HasCallStack => App Response
+readFedConn = undefined
+
+{-
+  remotes . responseJsonUnsafe <$> do
+    get (brig . paths ["i", "federation", "remotes"] . contentJson . expect2xx)
+-}
+
+updateFedConn :: HasCallStack => FedConn -> App Response
+updateFedConn = undefined
+
+deleteFedConn :: HasCallStack => String -> App Response
+deleteFedConn = undefined
+
+{-
+updateFederationRemote' :: (Request -> Request) -> Brig -> Domain -> FederationDomainConfig -> Http ResponseLBS
+updateFederationRemote' mods brig rdom remote =
+  put (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . json remote . mods)
+
+deleteFederationRemote' :: (Request -> Request) -> Brig -> Domain -> Http ResponseLBS
+deleteFederationRemote' mods brig rdom =
+  delete (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . mods)
+
+-- this one needs to go elsewhere
+resetFederationRemotes :: Opts -> Brig -> Http ()
+resetFederationRemotes opts brig = do
+  rs <- getFederationRemotes brig
+  -- Filter out domains that are in the config file.
+  -- These values can't be deleted yet, so don't even try.
+  forM_ (notCfgRemotes rs) $ \(FederationDomainConfig rdom _) -> deleteFederationRemote brig rdom
+  where
+    cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
+    notCfgRemotes = filter (`notElem` cfgRemotes)
+-}
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 170017c8f8..93d2754439 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -18,3 +18,55 @@ testSearchContactForExternalUsers = do
 
   bindResponse (Public.searchContacts partner (owner %. "name")) $ \resp ->
     resp.status `shouldMatchInt` 403
+
+testCrudFederationRemotes :: HasCallStack => App ()
+testCrudFederationRemotes = do
+  -- Delete the remotes from the database
+  -- This doesn't do anything with the remotes
+  -- defined in config files.
+  resetFederationRemotes opts brig
+
+  res1 <- getFederationRemotes brig
+  liftIO $ assertEqual "should return config values" cfgRemotes res1
+
+  let remote1 = FederationDomainConfig (Domain "good.example.com") NoSearch
+  addFederationRemote brig remote1
+  res2 <- getFederationRemotes brig
+  liftIO $ assertEqual "should return config values and good.example.com" (sort $ remote1 : cfgRemotes) (sort res2)
+
+  -- idempotency
+  addFederationRemote brig remote1
+  res2' <- getFederationRemotes brig
+  liftIO $ assertEqual "should return config values and good.example.com" (sort $ remote1 : cfgRemotes) (sort res2')
+
+  let remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
+  addFederationRemote brig remote2
+  res3 <- getFederationRemotes brig
+  liftIO $ assertEqual "should return config values and {good,evil}.example.com" (nub $ sort $ cfgRemotes <> [remote1, remote2]) (sort res3)
+
+  deleteFederationRemote brig (domain remote1)
+  res4 <- getFederationRemotes brig
+  liftIO $ assertEqual "should return config values and evil.example.com" (nub $ sort $ cfgRemotes <> [remote2]) (sort res4)
+
+  -- deleting from the config file triggers an error
+  deleteFederationRemote' id brig (domain $ head $ cfgRemotes) !!! const 533 === statusCode
+
+  -- updating search strategy works
+  let remote2' = remote2 {cfgSearchPolicy = NoSearch}
+  updateFederationRemote brig (domain remote2) remote2'
+  res5 <- getFederationRemotes brig
+  -- (move the dynamic remotes to the beginning here to make sure we look for `remote2'`, not `remote`.)
+  liftIO $ assertEqual "should be NoSearch" (nub $ sort $ [remote1, remote2'] <> cfgRemotes) (sort res5)
+
+  -- updating from config file fails
+  updateFederationRemote' id brig (domain $ head $ cfgRemotes) (head $ cfgRemotes) !!! const 533 === statusCode
+
+  -- updating domain fails
+  let remote2'' = remote2' {domain = Domain "broken.example.com"}
+  updateFederationRemote' id brig (domain remote2) remote2'' !!! const 533 === statusCode
+
+  -- TODO: how do we test that the TVar is updated in all services?  some fancy unit test?
+  -- duplicate internal end-point to all services, and implement the hanlers in a library?
+  pure ()
+  where
+    cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index c465585f41..27f01e2bf0 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -89,8 +89,7 @@ tests m opts brig cannon fedBrigClient =
         test m "POST /federation/on-user-deleted-connections : 200" (testRemoteUserGetsDeleted opts brig cannon fedBrigClient),
         test m "POST /federation/api-version : 200" (testAPIVersion brig fedBrigClient),
         test m "POST /federation/claim-key-packages : 200" (testClaimKeyPackages brig fedBrigClient),
-        test m "POST /federation/claim-key-packages (MLS disabled) : 200" (testClaimKeyPackagesMLSDisabled opts brig),
-        test m "CRUD /i/federation/remotes" (crudFederationRemotes opts brig)
+        test m "POST /federation/claim-key-packages (MLS disabled) : 200" (testClaimKeyPackagesMLSDisabled opts brig)
       ]
 
 allowFullSearch :: Domain -> Opt.Opts -> Opt.Opts
@@ -459,55 +458,3 @@ testClaimKeyPackagesMLSDisabled opts brig = do
           ClaimKeyPackageRequest (qUnqualified alice) (qUnqualified bob)
 
   liftIO $ mbundle @?= Nothing
-
-crudFederationRemotes :: HasCallStack => Opt.Opts -> Brig -> Http ()
-crudFederationRemotes opts brig = do
-  -- Delete the remotes from the database
-  -- This doesn't do anything with the remotes
-  -- defined in config files.
-  resetFederationRemotes opts brig
-
-  res1 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values" cfgRemotes res1
-
-  let remote1 = FederationDomainConfig (Domain "good.example.com") NoSearch
-  addFederationRemote brig remote1
-  res2 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and good.example.com" (sort $ remote1 : cfgRemotes) (sort res2)
-
-  -- idempotency
-  addFederationRemote brig remote1
-  res2' <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and good.example.com" (sort $ remote1 : cfgRemotes) (sort res2')
-
-  let remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
-  addFederationRemote brig remote2
-  res3 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and {good,evil}.example.com" (nub $ sort $ cfgRemotes <> [remote1, remote2]) (sort res3)
-
-  deleteFederationRemote brig (domain remote1)
-  res4 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and evil.example.com" (nub $ sort $ cfgRemotes <> [remote2]) (sort res4)
-
-  -- deleting from the config file triggers an error
-  deleteFederationRemote' id brig (domain $ head $ cfgRemotes) !!! const 533 === statusCode
-
-  -- updating search strategy works
-  let remote2' = remote2 {cfgSearchPolicy = NoSearch}
-  updateFederationRemote brig (domain remote2) remote2'
-  res5 <- getFederationRemotes brig
-  -- (move the dynamic remotes to the beginning here to make sure we look for `remote2'`, not `remote`.)
-  liftIO $ assertEqual "should be NoSearch" (nub $ sort $ [remote1, remote2'] <> cfgRemotes) (sort res5)
-
-  -- updating from config file fails
-  updateFederationRemote' id brig (domain $ head $ cfgRemotes) (head $ cfgRemotes) !!! const 533 === statusCode
-
-  -- updating domain fails
-  let remote2'' = remote2' {domain = Domain "broken.example.com"}
-  updateFederationRemote' id brig (domain remote2) remote2'' !!! const 533 === statusCode
-
-  -- TODO: how do we test that the TVar is updated in all services?  some fancy unit test?
-  -- duplicate internal end-point to all services, and implement the hanlers in a library?
-  pure ()
-  where
-    cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs
index a30c99f3d2..dd8368dba2 100644
--- a/services/brig/test/integration/Util.hs
+++ b/services/brig/test/integration/Util.hs
@@ -29,7 +29,6 @@ import Brig.AWS.Types
 import Brig.App (applog, fsWatcher, sftEnv, turnEnv)
 import Brig.Calling as Calling
 import qualified Brig.Code as Code
-import Brig.Options (Opts)
 import qualified Brig.Options as Opt
 import qualified Brig.Options as Opts
 import qualified Brig.Run as Run
@@ -112,7 +111,6 @@ import Wire.API.Conversation.Role (roleNameWireAdmin)
 import Wire.API.Federation.API
 import Wire.API.Federation.Domain
 import Wire.API.Internal.Notification
-import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.MultiTablePaging
 import Wire.API.Team.Member hiding (userId)
 import Wire.API.User
@@ -1076,41 +1074,6 @@ withDomainsBlockedForRegistration opts domains sess = do
       unsafeMkDomain = either error id . mkDomain
   withSettingsOverrides opts' sess
 
-getFederationRemotes :: Brig -> Http [FederationDomainConfig]
-getFederationRemotes brig =
-  remotes . responseJsonUnsafe <$> do
-    get (brig . paths ["i", "federation", "remotes"] . contentJson . expect2xx)
-
-addFederationRemote :: Brig -> FederationDomainConfig -> Http ()
-addFederationRemote brig remote =
-  void $ post (brig . paths ["i", "federation", "remotes"] . contentJson . json remote . expect2xx)
-
-updateFederationRemote :: Brig -> Domain -> FederationDomainConfig -> Http ()
-updateFederationRemote brig rdom remote =
-  void $ updateFederationRemote' expect2xx brig rdom remote
-
-updateFederationRemote' :: (Request -> Request) -> Brig -> Domain -> FederationDomainConfig -> Http ResponseLBS
-updateFederationRemote' mods brig rdom remote =
-  put (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . json remote . mods)
-
-deleteFederationRemote :: Brig -> Domain -> Http ()
-deleteFederationRemote brig rdom =
-  void $ deleteFederationRemote' expect2xx brig rdom
-
-deleteFederationRemote' :: (Request -> Request) -> Brig -> Domain -> Http ResponseLBS
-deleteFederationRemote' mods brig rdom =
-  delete (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . mods)
-
-resetFederationRemotes :: Opts -> Brig -> Http ()
-resetFederationRemotes opts brig = do
-  rs <- getFederationRemotes brig
-  -- Filter out domains that are in the config file.
-  -- These values can't be deleted yet, so don't even try.
-  forM_ (notCfgRemotes rs) $ \(FederationDomainConfig rdom _) -> deleteFederationRemote brig rdom
-  where
-    cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
-    notCfgRemotes = filter (`notElem` cfgRemotes)
-
 -- | Run a probe several times, until a "good" value materializes or until patience runs out
 aFewTimes ::
   (HasCallStack, MonadIO m) =>

From 9c17a695000b5bd67bc8d91fc56abbe658838e79 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 22 May 2023 17:21:49 +1000
Subject: [PATCH 081/220] FS-1179: Work in progress. Tests are broken due to
 timing issues

---
 services/brig/src/Brig/API/Internal.hs        |   2 -
 services/brig/src/Brig/Data/Connection.hs     |  12 -
 services/galley/galley.cabal                  |   2 +
 services/galley/src/Galley/API/Action.hs      |  10 +
 services/galley/src/Galley/API/Util.hs        |   7 +
 .../galley/src/Galley/Cassandra/Connection.hs |  20 ++
 .../galley/src/Galley/Cassandra/Queries.hs    |   4 +-
 services/galley/src/Galley/Run.hs             | 237 +++++++++---------
 .../galley/test/integration/Federation.hs     | 126 ++++++++++
 services/galley/test/integration/Main.hs      |   5 +-
 10 files changed, 294 insertions(+), 131 deletions(-)
 create mode 100644 services/galley/src/Galley/Cassandra/Connection.hs
 create mode 100644 services/galley/test/integration/Federation.hs

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 0b088e97c1..a8165a2e30 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -241,8 +241,6 @@ deleteFederationRemotes dom = do
       \do that, removing items listed in the config file is not allowed."
   -- FUTUREWORK: see 'getFederationRemotes'.
   lift . wrapClient . Data.deleteFederationRemote $ dom
-  -- Also drop connections to the remote domain
-  lift . wrapClient . Data.deleteRemoteConnectionsByDomain $ dom
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)
diff --git a/services/brig/src/Brig/Data/Connection.hs b/services/brig/src/Brig/Data/Connection.hs
index c326dfdd09..21f2969da4 100644
--- a/services/brig/src/Brig/Data/Connection.hs
+++ b/services/brig/src/Brig/Data/Connection.hs
@@ -37,12 +37,10 @@ module Brig.Data.Connection
     countConnections,
     deleteConnections,
     deleteRemoteConnections,
-    deleteRemoteConnectionsByDomain, 
     remoteConnectionInsert,
     remoteConnectionSelect,
     remoteConnectionSelectFrom,
     remoteConnectionDelete,
-    remoteConnectionsDeleteByDomain,
     remoteConnectionClear,
 
     -- * Re-exports
@@ -325,13 +323,6 @@ deleteRemoteConnections (tUntagged -> Qualified remoteUser remoteDomain) (fromRa
   pooledForConcurrentlyN_ 16 locals $ \u ->
     write remoteConnectionDelete $ params LocalQuorum (u, remoteDomain, remoteUser)
 
-deleteRemoteConnectionsByDomain
-  :: MonadClient m
-  => Domain
-  -> m ()
-deleteRemoteConnectionsByDomain domain =
-  retry x1 . write remoteConnectionsDeleteByDomain $ params LocalQuorum $ pure domain
-
 -- Queries
 
 connectionInsert :: PrepQuery W (UserId, UserId, RelationWithHistory, UTCTimeMillis, ConvId) ()
@@ -394,9 +385,6 @@ remoteConnectionUpdate = "UPDATE connection_remote set status = ?, last_update =
 remoteConnectionDelete :: PrepQuery W (UserId, Domain, UserId) ()
 remoteConnectionDelete = "DELETE FROM connection_remote where left = ? AND right_domain = ? AND right_user = ?"
 
-remoteConnectionsDeleteByDomain :: PrepQuery W (Identity Domain) ()
-remoteConnectionsDeleteByDomain = "DELETE FROM connection_remote where right_domain = ?"
-
 remoteConnectionClear :: PrepQuery W (Identity UserId) ()
 remoteConnectionClear = "DELETE FROM connection_remote where left = ?"
 
diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index 17ad027d81..e64470665f 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -68,6 +68,7 @@ library
     Galley.Cassandra.Access
     Galley.Cassandra.Client
     Galley.Cassandra.Code
+    Galley.Cassandra.Connection
     Galley.Cassandra.Conversation
     Galley.Cassandra.Conversation.Members
     Galley.Cassandra.Conversation.MLS
@@ -364,6 +365,7 @@ executable galley-integration
     API.Teams.LegalHold.Util
     API.Util
     API.Util.TeamFeature
+    Federation
     Main
     TestHelpers
     TestSetup
diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs
index 89dc3a82a7..4d8057d0d1 100644
--- a/services/galley/src/Galley/API/Action.hs
+++ b/services/galley/src/Galley/API/Action.hs
@@ -100,6 +100,7 @@ import Wire.API.Federation.Error
 import Wire.API.Team.LegalHold
 import Wire.API.Team.Member
 import qualified Wire.API.User as User
+import qualified Polysemy.TinyLog as TinyLog
 
 data NoChanges = NoChanges
 
@@ -145,6 +146,7 @@ type family HasConversationActionEffects (tag :: ConversationActionTag) r :: Con
     )
   HasConversationActionEffects 'ConversationRemoveMembersTag r =
     ( Member MemberStore r,
+      Member TinyLog r,
       Member (Error NoChanges) r
     )
   HasConversationActionEffects 'ConversationMemberUpdateTag r =
@@ -330,6 +332,14 @@ performAction tag origUser lconv action = do
       pure (mempty, action)
     SConversationRemoveMembersTag -> do
       let presentVictims = filter (isConvMemberL lconv) (toList action)
+      _ <- error $
+        "-----------------------------\n\n\n" <>
+        "lconv = " <> show lconv <> "\n\n\n" <>
+        "action = " <> show action <> "\n\n\n" <>
+        "presentVictims = " <> show presentVictims <> "\n\n\n" <>
+        "-----------------------------"
+      TinyLog.err $ Log.msg ("action" :: String) . Log.field "values" (show action)
+      TinyLog.err $ Log.msg ("presentVictims" :: String) . Log.field "values" (show presentVictims)
       when (null presentVictims) noChanges
       E.deleteMembers (tUnqualified lcnv) (toUserList lconv presentVictims)
       pure (mempty, action) -- FUTUREWORK: should we return the filtered action here?
diff --git a/services/galley/src/Galley/API/Util.hs b/services/galley/src/Galley/API/Util.hs
index 040b327a17..a93c4b17f2 100644
--- a/services/galley/src/Galley/API/Util.hs
+++ b/services/galley/src/Galley/API/Util.hs
@@ -375,6 +375,13 @@ instance IsConvMemberId (Local UserId) LocalMember where
 
 instance IsConvMemberId (Remote UserId) RemoteMember where
   getConvMember _ conv u = find ((u ==) . rmId) (Data.convRemoteMembers conv)
+    -- error $
+    --   "----------------" <>
+    --   "\n\n\nconv = " <> show conv <>
+    --   "\n\n\nu = " <> show u <>
+    --   "\n\n\nresult = " <> show (find ((u ==) . rmId) (Data.convRemoteMembers conv)) <>
+    --   "\n\n\n" <>
+    --   "----------------"
 
 instance IsConvMemberId (Qualified UserId) (Either LocalMember RemoteMember) where
   getConvMember loc conv =
diff --git a/services/galley/src/Galley/Cassandra/Connection.hs b/services/galley/src/Galley/Cassandra/Connection.hs
new file mode 100644
index 0000000000..10a4f786aa
--- /dev/null
+++ b/services/galley/src/Galley/Cassandra/Connection.hs
@@ -0,0 +1,20 @@
+module Galley.Cassandra.Connection where
+
+import Cassandra (MonadClient, PrepQuery, W, params, Consistency (LocalQuorum), retry, x1, write)
+import Imports
+import Data.Domain
+import Galley.Cassandra.Instances ()
+
+-- Queries targeting this table are usually in Brig, but I've put this one
+-- here so that we don't have yet another network call to Brig when most
+-- everything is already happening in galley
+
+deleteRemoteConnectionsByDomain
+  :: MonadClient m
+  => Domain
+  -> m ()
+deleteRemoteConnectionsByDomain domain =
+  retry x1 . write remoteConnectionsDeleteByDomain $ params LocalQuorum $ pure domain
+
+remoteConnectionsDeleteByDomain :: PrepQuery W (Identity Domain) ()
+remoteConnectionsDeleteByDomain = "DELETE FROM connection_remote where right_domain = ?"
\ No newline at end of file
diff --git a/services/galley/src/Galley/Cassandra/Queries.hs b/services/galley/src/Galley/Cassandra/Queries.hs
index 8d53bf938d..da287e71b2 100644
--- a/services/galley/src/Galley/Cassandra/Queries.hs
+++ b/services/galley/src/Galley/Cassandra/Queries.hs
@@ -372,7 +372,7 @@ updateRemoteMemberConvRoleName = "update member_remote_user set conversation_rol
 -- Used when removing a federation domain, so that we can quickly list all of the affected remote users and conversations
 -- This returns local conversation IDs and remote users
 selectRemoteMembersByDomain :: PrepQuery R (Identity Domain) (ConvId, UserId, RoleName)
-selectRemoteMembersByDomain = "select conv, user_remote_id, conversation_role from member_remote_user where user_remote_domain = ?"
+selectRemoteMembersByDomain = "select conv, user_remote_id, conversation_role from member_remote_user where user_remote_domain = ? ALLOW FILTERING"
 
 -- local user with remote conversations
 
@@ -394,7 +394,7 @@ deleteUserRemoteConv = "delete from user_remote_conv where user = ? and conv_rem
 -- Used when removing a federation domain, so that we can quickly list all of the affected local users and conversations
 -- This returns remote conversation IDs and local users
 selectLocalMembersByDomain :: PrepQuery R (Identity Domain) (ConvId, UserId)
-selectLocalMembersByDomain = "select conv_remote_id, user from user_remote_conv where conv_remote_domain = ?"
+selectLocalMembersByDomain = "select conv_remote_id, user from user_remote_conv where conv_remote_domain = ? ALLOW FILTERING"
 
 -- remote conversation status for local user
 
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index a0beab5fa2..9e62a5d2ea 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -19,6 +19,7 @@
 module Galley.Run
   ( run,
     mkApp,
+    updateFedDomainsCallback
   )
 where
 
@@ -65,8 +66,7 @@ import Servant.Client
   ( BaseUrl (BaseUrl),
     ClientEnv (ClientEnv),
     Scheme (Http),
-    defaultMakeClientRequest,
-    runClientM
+    defaultMakeClientRequest
   )
 import qualified System.Logger as Log
 import Util.Options
@@ -90,12 +90,12 @@ import Data.Time (getCurrentTime)
 import Wire.API.Conversation.Action
 import Galley.API.Federation
 import Data.Singletons
-import qualified Wire.API.Routes.Internal.Brig as Brig
-import Wire.API.Routes.Named (namedClient)
-import qualified System.Logger as L
 import Wire.API.Conversation (ConvType(One2OneConv), cnvmType, ConvType (ConnectConv, One2OneConv))
 import Galley.Data.Conversation.Types (convMetadata)
 import Galley.API.Util (getConversationWithError)
+import Data.Domain (Domain)
+import Galley.Cassandra.Connection
+import UnliftIO.Retry
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -110,10 +110,14 @@ run opts = lowerCodensity $ do
           (env ^. monitor)
 
   forM_ (env ^. aEnv) $ \aws ->
-    void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
-  void $ Codensity $ Async.withAsync $ runApp env updateFedDomains
-  void $ Codensity $ Async.withAsync $ runApp env deleteLoop
-  void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
+    void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. moni  where
+    rtree = compile API.sitemap
+    runGalley e r k = evalGalleyToIO e (route rtree r k)
+    -- the servant API wraps the one defined using wai-routing
+    servantApp e0 r =
+      let e = reqId .~ lookupReqId r $ e0
+       in Servant.serveWithContext
+            (Proxy @CombinedAPI)
   void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
@@ -212,112 +216,117 @@ updateFedDomains = do
       Endpoint host port = env ^. brig
       baseUrl   = BaseUrl Http (unpack host) (fromIntegral port) ""
       clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
-  liftIO $ getAllowedDomainsLoop logger clientEnv ioref $ callback env clientEnv
-  where
-    -- Build the map, keyed by conversations to the list of members
-    insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
-    callback env clientEnv old new = do
-      -- This code will only run when there is a change in the domain lists
-      let fromFedList = Set.fromList . remotes
-          prevDoms = fromFedList old
-          currDoms = fromFedList new
-          localDomain = env ^. options . optSettings . setFederationDomain
-      unless (prevDoms == currDoms) $ do
-        -- Perform updates before rewriting the tvar
-        -- This means that if the update fails on a
-        -- particular invocation, it can be run again
-        -- on the next firing as it isn't likely that
-        -- the domain list is changing frequently.
-        -- FS-1179 is handling this part.
-        let deletedDomains = Set.difference prevDoms currDoms
-        for_ deletedDomains $ \fedDomCfg -> do
-          -- https://wearezeta.atlassian.net/browse/FS-1179
-          -- * Remove remote users for the given domain from all conversations owned by the current host
-          -- * Remove all local users from remote conversations owned by the given domain.
-          --   NOTE: This is NOT sent to other backends, as this information is not authoratative, but is
-          --   good enough to tell local users about the federation connection being removed.
-          -- * Delete all connections from local users to users for the remote domain.
+  liftIO $ getAllowedDomainsLoop logger clientEnv ioref $ updateFedDomainsCallback env
 
-          -- Get all remote users for the given domain, along with conversation IDs that they are in
-          let dom = domain fedDomCfg
-          remoteUsers <- liftIO $ evalGalleyToIO env $ E.getRemoteMembersByDomain dom
-          let lCnvMap = foldr insertIntoMap mempty remoteUsers
-          for_ (Map.toList lCnvMap) $ \(cnvId, rUsers) -> do
-            let lCnvId = toLocalUnsafe dom cnvId
-            -- This value contains an event that we might need to
-            -- send out to all of the local clients that are a party
-            -- to the conversation. However we also don't want to DOS
-            -- clients. Maybe suppress and send out a bulk version?
-            liftIO $ evalGalleyToIO env
-              $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Federation domain removal: Remove from conversation error")
-              . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
-              . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Federation domain removal: Action denied, remove conversation member")
-              . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription  "Federation domain removal: Invalid operation")
-              . mapToRuntimeError @'NotATeamMember (InternalErrorWithDescription  "Federation domain removal: Not a team member")
-              . mapError @NoChanges (const (InternalErrorWithDescription  "Federation domain removal: No changes"))
-              -- This is allowed to send notifications to _local_ clients.
-              -- But we are suppressing those events as we don't want to
-              -- DOS our users if a large and deeply interconnected federation
-              -- member is removed. Sending out hundreds or thousands of events
-              -- to each client isn't something we want to be doing.
-              $ do
-                conv <- getConversationWithError lCnvId
-                let lConv = toLocalUnsafe dom conv
-                updateLocalConversationUserUnchecked
-                  @'ConversationRemoveMembersTag
-                  lConv
-                  undefined $ -- This field can be undefined as the path for ConversationRemoveMembersTag doens't use it
-                  tUntagged . rmId <$> rUsers
-                -- Check if the conversation if type 2 or 3, one-on-one conversations.
-                -- If it is, then we need to remove the entire conversation as users
-                -- aren't able to delete those types of conversations themselves.
-                -- Check that we are in a type 2 or a type 3 conversation
-                when (cnvmType (convMetadata conv) `elem` [One2OneConv, ConnectConv]) $
-                  -- If we are, delete it.
-                  updateLocalConversationUserUnchecked
-                    @'ConversationDeleteTag
-                    lConv
-                    undefined
-                    ()
-                  
-            pure ()
+-- Build the map, keyed by conversations to the list of members
+insertIntoMap :: (ConvId, a) -> Map ConvId (N.NonEmpty a) -> Map ConvId (N.NonEmpty a)
+insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
 
-          -- Get all local users for the given domain, along with remote conversation IDs that they are in
-          localUsers <- liftIO $ evalGalleyToIO env $ E.getLocalMembersByDomain dom
-          -- As above, build the map so we can get all local users per conversation
-          let rCnvMap = foldr insertIntoMap mempty localUsers
-          -- Process each user.
-          for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
-            _res <- liftIO $ evalGalleyToIO env
-              $ mapError @NoChanges (const (InternalErrorWithDescription  "No Changes: Could not remove a local member from a remote conversation."))
-              $ do
-                now <- liftIO $ getCurrentTime
-                for_ lUsers $ \user -> do
-                  let lUser = toLocalUnsafe localDomain user
-                      convUpdate = F.ConversationUpdate
-                        { cuTime = now
-                        , cuOrigUserId = tUntagged lUser
-                        , cuConvId = cnv
-                        , cuAlreadyPresentUsers = mempty
-                        , cuAction = SomeConversationAction (sing @'ConversationDeleteTag) ()
-                        }
-                  -- These functions are used directly rather than as part of a larger conversation
-                  -- delete function, as we don't have an originating user, and we can't send data
-                  -- to the remote backend.
-                  -- We don't need to check the conversation type here, as we can't tell the
-                  -- remote federation server to delete the conversation. They will have to do a
-                  -- similar processing run for removing the local domain from their federation list.
-                  onConversationUpdated dom convUpdate
-                  let rcnv = toRemoteUnsafe dom cnv
-                  notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
-            pure ()
+deleteFederationDomainRemote :: Env -> Domain -> IO ()
+deleteFederationDomainRemote env dom = do
+  remoteUsers <- evalGalleyToIO env $ E.getRemoteMembersByDomain dom
+  let lCnvMap = foldr insertIntoMap mempty remoteUsers
+  for_ (Map.toList lCnvMap) $ \(cnvId, rUsers) -> do
+    let lCnvId = toLocalUnsafe dom cnvId
+    -- This value contains an event that we might need to
+    -- send out to all of the local clients that are a party
+    -- to the conversation. However we also don't want to DOS
+    -- clients. Maybe suppress and send out a bulk version?
+    evalGalleyToIO env
+      $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Federation domain removal: Remove from conversation error")
+      . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
+      . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Federation domain removal: Action denied, remove conversation member")
+      . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription  "Federation domain removal: Invalid operation")
+      . mapToRuntimeError @'NotATeamMember (InternalErrorWithDescription  "Federation domain removal: Not a team member")
+      . mapError @NoChanges (const (InternalErrorWithDescription  "Federation domain removal: No changes"))
+      -- This is allowed to send notifications to _local_ clients.
+      -- But we are suppressing those events as we don't want to
+      -- DOS our users if a large and deeply interconnected federation
+      -- member is removed. Sending out hundreds or thousands of events
+      -- to each client isn't something we want to be doing.
+      $ do
+        conv <- getConversationWithError lCnvId
+        let lConv = toLocalUnsafe dom conv
+        updateLocalConversationUserUnchecked
+          @'ConversationRemoveMembersTag
+          lConv
+          undefined $ -- This field can be undefined as the path for ConversationRemoveMembersTag doens't use it
+          tUntagged . rmId <$> rUsers
+        -- Check if the conversation if type 2 or 3, one-on-one conversations.
+        -- If it is, then we need to remove the entire conversation as users
+        -- aren't able to delete those types of conversations themselves.
+        -- Check that we are in a type 2 or a type 3 conversation
+        when (cnvmType (convMetadata conv) `elem` [One2OneConv, ConnectConv]) $
+          -- If we are, delete it.
+          updateLocalConversationUserUnchecked
+            @'ConversationDeleteTag
+            lConv
+            undefined
+            ()
+
+deleteFederationDomainLocal :: Env -> Domain -> IO ()
+deleteFederationDomainLocal env dom = do
+  localUsers <- evalGalleyToIO env $ E.getLocalMembersByDomain dom
+  -- As above, build the map so we can get all local users per conversation
+  let rCnvMap = foldr insertIntoMap mempty localUsers
+  -- Process each user.
+  for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
+    evalGalleyToIO env
+      $ mapError @NoChanges (const (InternalErrorWithDescription  "No Changes: Could not remove a local member from a remote conversation."))
+      $ do
+        now <- liftIO $ getCurrentTime
+        for_ lUsers $ \user -> do
+          let lUser = toLocalUnsafe localDomain user
+              convUpdate = F.ConversationUpdate
+                { cuTime = now
+                , cuOrigUserId = tUntagged lUser
+                , cuConvId = cnv
+                , cuAlreadyPresentUsers = mempty
+                , cuAction = SomeConversationAction (sing @'ConversationDeleteTag) ()
+                }
+          -- These functions are used directly rather than as part of a larger conversation
+          -- delete function, as we don't have an originating user, and we can't send data
+          -- to the remote backend.
+          -- We don't need to check the conversation type here, as we can't tell the
+          -- remote federation server to delete the conversation. They will have to do a
+          -- similar processing run for removing the local domain from their federation list.
+          onConversationUpdated dom convUpdate
+          -- let rcnv = toRemoteUnsafe dom cnv
+          -- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
+  where
+    localDomain = env ^. options . optSettings . setFederationDomain
 
-          -- Remove the remote one-on-one conversations between local members and remote members for the given domain.
-          -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
-          let delFedDomain = namedClient @Brig.FederationRemotesAPI @"delete-federation-remotes"
-          liftIO (runClientM (delFedDomain dom) clientEnv) >>= \case
-            Right _ -> pure ()
-            Left e -> L.log (env ^. applog) L.Info $
-              L.msg (L.val "Could not delete remote user connections in Brig")
-                L.~~ "error" L..= show e
+deleteFederationDomain :: Env -> Set FederationDomainConfig -> IO ()
+deleteFederationDomain env deletedDomains = do
+  for_ deletedDomains $ \fedDomCfg -> do
+    -- https://wearezeta.atlassian.net/browse/FS-1179
+    -- * Remove remote users for the given domain from all conversations owned by the current host
+    -- * Remove all local users from remote conversations owned by the given domain.
+    --   NOTE: This is NOT sent to other backends, as this information is not authoratative, but is
+    --   good enough to tell local users about the federation connection being removed.
+    -- * Delete all connections from local users to users for the remote domain
+    -- Get all remote users for the given domain, along with conversation IDs that they are in
+    let dom = domain fedDomCfg
+    deleteFederationDomainRemote env dom
+    -- Get all local users for the given domain, along with remote conversation IDs that they are in
+    deleteFederationDomainLocal env dom
+    -- Remove the remote one-on-one conversations between local members and remote members for the given domain.
+    -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
+    runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
+    
+updateFedDomainsCallback :: Env -> FederationDomainConfigs -> FederationDomainConfigs -> IO ()
+updateFedDomainsCallback env old new = do
+  -- This code will only run when there is a change in the domain lists
+  let fromFedList = Set.fromList . remotes
+      prevDoms = fromFedList old
+      currDoms = fromFedList new
+  unless (prevDoms == currDoms) $ do
+    -- Perform updates before rewriting the tvar
+    -- This means that if the update fails on a
+    -- particular invocation, it can be run again
+    -- on the next firing as it isn't likely that
+    -- the domain list is changing frequently.
+    -- FS-1179 is handling this part.
+    let deletedDomains = Set.difference prevDoms currDoms
+    deleteFederationDomain env deletedDomains
 
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
new file mode 100644
index 0000000000..f0ae7d0122
--- /dev/null
+++ b/services/galley/test/integration/Federation.hs
@@ -0,0 +1,126 @@
+module Federation where
+import TestSetup
+import Control.Lens ((^.))
+import Imports
+import Galley.Run
+import Control.Monad.Codensity (lowerCodensity)
+import Wire.API.Routes.FederationDomainConfig
+import Data.Domain
+import Wire.API.User.Search
+import API.Util
+import Data.Id
+import Data.List.NonEmpty
+import Data.Qualified
+import Wire.API.Conversation
+import Bilge.Assert
+import Bilge.Response
+import Galley.Options (optSettings, setFederationDomain)
+import Galley.Env
+
+updateFedDomainsTest :: TestM ()
+updateFedDomainsTest = do
+  s <- ask
+  let opts = s ^. tsGConf
+  -- Don't need the actual server, and we certainly don't want it running.
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  -- Common variables.
+  let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain  = Domain "far-away.example.com"
+      remoteDomain2 = Domain "far-away-two.example.com"
+
+  -- Setup a conversation for a known remote domain.
+  -- Include that domain in the old and new lists so
+  -- if the function is acting up we know it will be
+  -- working on the domain.
+  -- updateFedDomainsTestNoop env remoteDomain interval
+
+  -- Adding a new federation domain, this too should be a no-op
+  -- updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval  
+
+  -- Removing a single domain
+  updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval  
+
+  -- Removing multiple domains
+  -- liftIO $ updateFedDomainsCallback env old new
+
+updateFedDomainRemoveRemoteFromLocal :: Env -> Domain -> Domain -> Int -> TestM ()
+updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = recovering x3 [const . Handler $ pure . _] $ const $ do
+  s <- ask
+  let opts = s ^. tsGConf
+      localDomain = opts ^. optSettings . setFederationDomain
+      old = FederationDomainConfigs AllowList [FederationDomainConfig remoteDomain FullSearch, FederationDomainConfig remoteDomain2 FullSearch] interval
+      new = old { remotes = [FederationDomainConfig remoteDomain2 FullSearch] }
+  qalice <- randomQualifiedUser
+  bobId <- randomId
+  charlieId <- randomId
+  let alice = qUnqualified qalice
+      remoteBob = Qualified bobId remoteDomain
+      remoteCharlie = Qualified charlieId remoteDomain2
+  -- Create a conversation
+  convId <- decodeConvId <$> postConv alice [] (Just "remote gossip") [] Nothing Nothing
+  connectWithRemoteUser alice remoteBob
+  connectWithRemoteUser alice remoteCharlie
+  _ <- postQualifiedMembers alice (remoteCharlie <| remoteBob :| []) convId
+  liftIO $ threadDelay $ 10000
+  -- Remove the remote user from the local domain
+  liftIO $ updateFedDomainsCallback env old new
+  -- Check that the conversation still exists.
+  getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
+    const 200 === statusCode
+    let findRemote :: Qualified UserId -> Conversation -> Maybe (Qualified UserId)
+        findRemote u = find (== u) . fmap omQualifiedId . cmOthers . cnvMembers
+    -- Check that only one remote user was removed.
+    const (Right Nothing) === (fmap (findRemote remoteBob) <$> responseJsonEither)
+    const (Right $ pure remoteCharlie) === (fmap (findRemote remoteCharlie) <$> responseJsonEither)
+    const (Right qalice) === (fmap (memId . cmSelf . cnvMembers) <$> responseJsonEither)
+
+updateFedDomainsAddRemote :: Env -> Domain -> Domain -> Int -> TestM ()
+updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval = do
+  s <- ask
+  let opts = s ^. tsGConf
+      localDomain = opts ^. optSettings . setFederationDomain
+      old = FederationDomainConfigs AllowList [FederationDomainConfig remoteDomain FullSearch] interval
+      new = old { remotes = FederationDomainConfig remoteDomain2 FullSearch : remotes old }
+  qalice <- randomQualifiedUser
+  bobId <- randomId
+  let alice = qUnqualified qalice
+      remoteBob = Qualified bobId remoteDomain
+  -- Create a conversation
+  convId <- decodeConvId <$> postConv alice [] (Just "remote gossip") [] Nothing Nothing
+  connectWithRemoteUser alice remoteBob
+  _ <- postQualifiedMembers alice (remoteBob :| []) convId
+
+  -- No-op
+  liftIO $ updateFedDomainsCallback env old new
+  -- Check that the conversation still exists.
+  getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
+    const 200 === statusCode
+    let findRemote :: Conversation -> Maybe (Qualified UserId)
+        findRemote = find (== remoteBob) . fmap omQualifiedId . cmOthers . cnvMembers
+    const (Right $ pure remoteBob) === (fmap findRemote <$> responseJsonEither)
+    const (Right qalice) === (fmap (memId . cmSelf . cnvMembers) <$> responseJsonEither)
+
+updateFedDomainsTestNoop :: Env -> Domain -> Int -> TestM ()
+updateFedDomainsTestNoop env remoteDomain interval = do
+  s <- ask
+  let opts = s ^. tsGConf
+      localDomain = opts ^. optSettings . setFederationDomain
+      old = FederationDomainConfigs AllowList [FederationDomainConfig remoteDomain FullSearch] interval
+      new = old
+  qalice <- randomQualifiedUser
+  bobId <- randomId
+  let alice = qUnqualified qalice
+      remoteBob = Qualified bobId remoteDomain
+  -- Create a conversation
+  convId <- decodeConvId <$> postConv alice [] (Just "remote gossip") [] Nothing Nothing
+  connectWithRemoteUser alice remoteBob
+  _ <- postQualifiedMembers alice (remoteBob :| []) convId
+  -- No-op
+  liftIO $ updateFedDomainsCallback env old new
+  -- Check that the conversation still exists.
+  getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
+    const 200 === statusCode
+    let findRemote :: Conversation -> Maybe (Qualified UserId)
+        findRemote = find (== remoteBob) . fmap omQualifiedId . cmOthers . cnvMembers
+    const (Right $ pure remoteBob) === (fmap findRemote <$> responseJsonEither)
+    const (Right qalice) === (fmap (memId . cmSelf . cnvMembers) <$> responseJsonEither)
\ No newline at end of file
diff --git a/services/galley/test/integration/Main.hs b/services/galley/test/integration/Main.hs
index c67d355dfa..8351db5040 100644
--- a/services/galley/test/integration/Main.hs
+++ b/services/galley/test/integration/Main.hs
@@ -52,6 +52,8 @@ import Util.Options
 import Util.Options.Common
 import Util.Test
 import qualified Util.Test.SQS as SQS
+import Federation
+import TestHelpers (test)
 
 newtype ServiceConfigFile = ServiceConfigFile String
   deriving (Eq, Ord, Typeable)
@@ -93,7 +95,8 @@ main = withOpenSSL $ runTests go
               "inconsistent sitemap"
               mempty
               (pathsConsistencyCheck . treeToPaths . compile $ Galley.API.sitemap),
-          API.tests setup
+          API.tests setup,
+          test setup "Federation Domains" updateFedDomainsTest
         ]
     getOpts gFile iFile = do
       m <- newManager tlsManagerSettings {managerResponseTimeout = responseTimeoutMicro 300000000}

From 1d388037e506199863ab89b33da55d28ef4d7b97 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 22 May 2023 17:26:54 +1000
Subject: [PATCH 082/220] FS-1179: Fixing an errant delete

---
 services/galley/src/Galley/Run.hs | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 9e62a5d2ea..b76a702e46 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -110,14 +110,10 @@ run opts = lowerCodensity $ do
           (env ^. monitor)
 
   forM_ (env ^. aEnv) $ \aws ->
-    void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. moni  where
-    rtree = compile API.sitemap
-    runGalley e r k = evalGalleyToIO e (route rtree r k)
-    -- the servant API wraps the one defined using wai-routing
-    servantApp e0 r =
-      let e = reqId .~ lookupReqId r $ e0
-       in Servant.serveWithContext
-            (Proxy @CombinedAPI)
+    void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
+  void $ Codensity $ Async.withAsync $ runApp env updateFedDomains
+  void $ Codensity $ Async.withAsync $ runApp env deleteLoop
+  void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
   void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 

From e2015dcf247a7b73104bee8220582beaeecc8dc1 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 22 May 2023 17:41:51 +1000
Subject: [PATCH 083/220] FS-1179: Commiting and switching to FS-1115

---
 services/galley/src/Galley/Run.hs              |  1 -
 services/galley/test/integration/Federation.hs | 13 +++++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index b76a702e46..bb7bf33d0e 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -95,7 +95,6 @@ import Galley.Data.Conversation.Types (convMetadata)
 import Galley.API.Util (getConversationWithError)
 import Data.Domain (Domain)
 import Galley.Cassandra.Connection
-import UnliftIO.Retry
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index f0ae7d0122..f3924872a4 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -1,4 +1,5 @@
 module Federation where
+
 import TestSetup
 import Control.Lens ((^.))
 import Imports
@@ -16,6 +17,11 @@ import Bilge.Assert
 import Bilge.Response
 import Galley.Options (optSettings, setFederationDomain)
 import Galley.Env
+import UnliftIO.Retry
+import Control.Monad.Catch
+
+x3 :: RetryPolicy
+x3 = limitRetries 3 <> exponentialBackoff 100000
 
 updateFedDomainsTest :: TestM ()
 updateFedDomainsTest = do
@@ -43,8 +49,11 @@ updateFedDomainsTest = do
   -- Removing multiple domains
   -- liftIO $ updateFedDomainsCallback env old new
 
+constHandlers :: MonadIO m => [RetryStatus -> Handler m Bool]
+constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
+
 updateFedDomainRemoveRemoteFromLocal :: Env -> Domain -> Domain -> Int -> TestM ()
-updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = recovering x3 [const . Handler $ pure . _] $ const $ do
+updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = recovering x3 constHandlers $ const $ do
   s <- ask
   let opts = s ^. tsGConf
       localDomain = opts ^. optSettings . setFederationDomain
@@ -61,7 +70,7 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
   connectWithRemoteUser alice remoteBob
   connectWithRemoteUser alice remoteCharlie
   _ <- postQualifiedMembers alice (remoteCharlie <| remoteBob :| []) convId
-  liftIO $ threadDelay $ 10000
+  liftIO $ threadDelay $ 3  * 1000000
   -- Remove the remote user from the local domain
   liftIO $ updateFedDomainsCallback env old new
   -- Check that the conversation still exists.

From 3374dd658a1fb2d2de416cfca07f2fb48c8a745a Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 09:43:07 +0200
Subject: [PATCH 084/220] ...

---
 integration/test/API/BrigInternal.hs   |  7 ++-
 integration/test/Test/Brig.hs          | 78 ++++++++++++++++++--------
 services/brig/test/integration/Util.hs |  9 ++-
 services/cannon/src/Cannon/Run.hs      |  1 +
 4 files changed, 66 insertions(+), 29 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 050c9e7d7b..45f845037b 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -63,8 +63,8 @@ createFedConn = undefined
   post (brig . paths ["i", "federation", "remotes"] . contentJson . json remote . expect2xx)
 -}
 
-readFedConn :: HasCallStack => App Response
-readFedConn = undefined
+readFedConns :: HasCallStack => App Response
+readFedConns = undefined
 
 {-
   remotes . responseJsonUnsafe <$> do
@@ -77,6 +77,9 @@ updateFedConn = undefined
 deleteFedConn :: HasCallStack => String -> App Response
 deleteFedConn = undefined
 
+resetFedConns :: HasCallStack => App Response
+resetFedConns = undefined
+
 {-
 updateFederationRemote' :: (Request -> Request) -> Brig -> Domain -> FederationDomainConfig -> Http ResponseLBS
 updateFederationRemote' mods brig rdom remote =
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 93d2754439..d4f08f46f4 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -21,38 +21,71 @@ testSearchContactForExternalUsers = do
 
 testCrudFederationRemotes :: HasCallStack => App ()
 testCrudFederationRemotes = do
+  let remote1 = FederationDomainConfig (Domain "good.example.com") NoSearch
+      remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
+      remote2' = remote2 {cfgSearchPolicy = NoSearch}
+
+      parseFedConns :: HasCallStack => Response -> App [FederationDomainConfig]
+      parseFedConns = undefined
+
+      shouldMatchFedConns :: HasCallStack => [FederationDomainConfig] -> [FederationDomainConfig] -> App ()
+      shouldMatchFedConns _ _ = do
+        liftIO $ assertEqual "should return config values and good.example.com" (sort $ rem : cfgRemotes) (sort res2)
+        undefined
+
+      addOnce :: HasCallStack => FederationDomainConfig -> App ()
+      addOnce rem = do
+        createFedConn rem
+        res <- parseFedConns =<< readFedConns
+        res `shouldMatchFedConns` (sort $ rem : cfgRemotes)
+
+      deleteOnce :: HasCallStack => Domain -> App ()
+      deleteOnce = undefined
+
+      deleteFail :: HasCallStack => Domain -> App ()
+      deleteFail = undefined
+
+      updateOnce :: HasCallStack => Domain -> FederationDomainConfig -> App ()
+      updateOnce = undefined
+
+      updateFail :: HasCallStack => Domain -> App ()
+      updateFail = undefined
+
   -- Delete the remotes from the database
   -- This doesn't do anything with the remotes
   -- defined in config files.
-  resetFederationRemotes opts brig
+  resetFedConns
+  cfgRemotes <- parseFedConns =<< readFedConns
+  cfgRemotes `shouldMatchFedConns` [remote2]
+  deleteFail (domain $ head $ cfgRemotes)
 
-  res1 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values" cfgRemotes res1
+  addOnce remote1
+  readFedConns `shouldContainFedConns` remote1
 
-  let remote1 = FederationDomainConfig (Domain "good.example.com") NoSearch
-  addFederationRemote brig remote1
-  res2 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and good.example.com" (sort $ remote1 : cfgRemotes) (sort res2)
+  addOnce remote1 -- idempotency
+  readFedConns `shouldContainFedConns` remote1
 
-  -- idempotency
-  addFederationRemote brig remote1
-  res2' <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and good.example.com" (sort $ remote1 : cfgRemotes) (sort res2')
+  deleteOnce (domain remote1)
+  readFedConns `shouldNotContainFedConns` remote1
 
-  let remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
-  addFederationRemote brig remote2
-  res3 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and {good,evil}.example.com" (nub $ sort $ cfgRemotes <> [remote1, remote2]) (sort res3)
+  deleteOnce (domain remote1) -- idempotency
+  readFedConns `shouldNotContainFedConns` remote1
 
-  deleteFederationRemote brig (domain remote1)
-  res4 <- getFederationRemotes brig
-  liftIO $ assertEqual "should return config values and evil.example.com" (nub $ sort $ cfgRemotes <> [remote2]) (sort res4)
+  addOnce remote2
+  deleteFail (domain remote2) -- removing from cfg file doesn't work whether it's in the database or not
+  readFedConns `shouldContainFedConns` remote2
 
-  -- deleting from the config file triggers an error
-  deleteFederationRemote' id brig (domain $ head $ cfgRemotes) !!! const 533 === statusCode
+  updateOnce (domain remote2) remote2')
+  readFedConns `shouldNotContainFedConns` remote2
+  readFedConns `shouldContainFedConns` remote2'
+
+  updateOnce (domain remote2) remote2') -- idempotency
+  readFedConns `shouldNotContainFedConns` remote2
+  readFedConns `shouldContainFedConns` remote2'
+
+{-
 
   -- updating search strategy works
-  let remote2' = remote2 {cfgSearchPolicy = NoSearch}
   updateFederationRemote brig (domain remote2) remote2'
   res5 <- getFederationRemotes brig
   -- (move the dynamic remotes to the beginning here to make sure we look for `remote2'`, not `remote`.)
@@ -69,4 +102,5 @@ testCrudFederationRemotes = do
   -- duplicate internal end-point to all services, and implement the hanlers in a library?
   pure ()
   where
-    cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
+
+-}
diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs
index dd8368dba2..29534bcdbf 100644
--- a/services/brig/test/integration/Util.hs
+++ b/services/brig/test/integration/Util.hs
@@ -30,7 +30,6 @@ import Brig.App (applog, fsWatcher, sftEnv, turnEnv)
 import Brig.Calling as Calling
 import qualified Brig.Code as Code
 import qualified Brig.Options as Opt
-import qualified Brig.Options as Opts
 import qualified Brig.Run as Run
 import Brig.Types.Activation
 import Brig.Types.Intra
@@ -1053,7 +1052,7 @@ circumventSettingsOverride = runHttpT
 --
 --   Beware: (1) Not all async parts of brig are running in this.  (2) other services will
 --   see the old, unaltered brig.
-withSettingsOverrides :: MonadIO m => Opts.Opts -> WaiTest.Session a -> m a
+withSettingsOverrides :: MonadIO m => Opt.Opts -> WaiTest.Session a -> m a
 withSettingsOverrides opts action = liftIO $ do
   (brigApp, env) <- Run.mkApp opts
   sftDiscovery <-
@@ -1067,10 +1066,10 @@ withSettingsOverrides opts action = liftIO $ do
 
 -- | When we remove the customer-specific extension of domain blocking, this test will fail to
 -- compile.
-withDomainsBlockedForRegistration :: (MonadIO m) => Opts.Opts -> [Text] -> WaiTest.Session a -> m a
+withDomainsBlockedForRegistration :: (MonadIO m) => Opt.Opts -> [Text] -> WaiTest.Session a -> m a
 withDomainsBlockedForRegistration opts domains sess = do
-  let opts' = opts {Opts.optSettings = (Opts.optSettings opts) {Opts.setCustomerExtensions = Just blocked}}
-      blocked = Opts.CustomerExtensions (Opts.DomainsBlockedForRegistration (unsafeMkDomain <$> domains))
+  let opts' = opts {Opt.optSettings = (Opt.optSettings opts) {Opt.setCustomerExtensions = Just blocked}}
+      blocked = Opt.CustomerExtensions (Opt.DomainsBlockedForRegistration (unsafeMkDomain <$> domains))
       unsafeMkDomain = either error id . mkDomain
   withSettingsOverrides opts' sess
 
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index f707708dfc..ea1cce9b24 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -79,6 +79,7 @@ run o = do
   s <- newSettings $ Server (o ^. cannon . host) (o ^. cannon . port) (applog e) m (Just idleTimeout)
 
   -- Get the federation domain list from Brig and start the updater loop
+  -- TODO: move this block into a function in libs/wire-api/src/Wire/API/FederationUpdate.hs; check all services for the same block and use the function.
   manager <- newManager defaultManagerSettings
   let Brig bh bp = o ^. brig
       baseUrl = BaseUrl Http (unpack bh) (fromIntegral bp) ""

From 411f03d23435d5be9749874db4bce8c04a67fda4 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 22 May 2023 18:01:18 +1000
Subject: [PATCH 085/220] wip

---
 integration/integration.cabal          |  2 ++
 integration/test/Test/Brig.hs          | 20 +++++++++++++-------
 services/brig/src/Brig/API/Internal.hs |  4 ++--
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/integration/integration.cabal b/integration/integration.cabal
index 1c33462eb6..916604845a 100644
--- a/integration/integration.cabal
+++ b/integration/integration.cabal
@@ -142,8 +142,10 @@ library
     , text
     , time
     , transformers
+    , types-common
     , unix
     , unliftio
     , uuid
     , websockets
+    , wire-api
     , yaml
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index d4f08f46f4..04cf87ee1e 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -7,6 +7,12 @@ import qualified API.GalleyInternal as Internal
 import GHC.Stack
 import SetupHelpers
 import Testlib.Prelude
+import Wire.API.Routes.FederationDomainConfig
+import qualified Data.Domain as D
+import Wire.API.User.Search
+import Data.String.Conversions
+import Control.Monad.IO.Class
+import TestLib.Assertions
 
 testSearchContactForExternalUsers :: HasCallStack => App ()
 testSearchContactForExternalUsers = do
@@ -21,8 +27,8 @@ testSearchContactForExternalUsers = do
 
 testCrudFederationRemotes :: HasCallStack => App ()
 testCrudFederationRemotes = do
-  let remote1 = FederationDomainConfig (Domain "good.example.com") NoSearch
-      remote2 = FederationDomainConfig (Domain "evil.example.com") ExactHandleSearch
+  let remote1 = FederationDomainConfig (D.Domain $ cs "good.example.com") NoSearch
+      remote2 = FederationDomainConfig (D.Domain $ cs "evil.example.com") ExactHandleSearch
       remote2' = remote2 {cfgSearchPolicy = NoSearch}
 
       parseFedConns :: HasCallStack => Response -> App [FederationDomainConfig]
@@ -30,7 +36,7 @@ testCrudFederationRemotes = do
 
       shouldMatchFedConns :: HasCallStack => [FederationDomainConfig] -> [FederationDomainConfig] -> App ()
       shouldMatchFedConns _ _ = do
-        liftIO $ assertEqual "should return config values and good.example.com" (sort $ rem : cfgRemotes) (sort res2)
+        liftIO $ shouldMatch _ _ --  "should return config values and good.example.com" (sort $ rem : cfgRemotes) (sort res2)
         undefined
 
       addOnce :: HasCallStack => FederationDomainConfig -> App ()
@@ -45,10 +51,10 @@ testCrudFederationRemotes = do
       deleteFail :: HasCallStack => Domain -> App ()
       deleteFail = undefined
 
-      updateOnce :: HasCallStack => Domain -> FederationDomainConfig -> App ()
+      updateOnce :: HasCallStack => D.Domain -> FederationDomainConfig -> App ()
       updateOnce = undefined
 
-      updateFail :: HasCallStack => Domain -> App ()
+      updateFail :: HasCallStack => D.Domain -> App ()
       updateFail = undefined
 
   -- Delete the remotes from the database
@@ -75,11 +81,11 @@ testCrudFederationRemotes = do
   deleteFail (domain remote2) -- removing from cfg file doesn't work whether it's in the database or not
   readFedConns `shouldContainFedConns` remote2
 
-  updateOnce (domain remote2) remote2')
+  updateOnce (domain remote2) remote2'
   readFedConns `shouldNotContainFedConns` remote2
   readFedConns `shouldContainFedConns` remote2'
 
-  updateOnce (domain remote2) remote2') -- idempotency
+  updateOnce (domain remote2) remote2' -- idempotency
   readFedConns `shouldNotContainFedConns` remote2
   readFedConns `shouldContainFedConns` remote2'
 
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index c9ff060625..3cad4424f7 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -328,8 +328,8 @@ getMLSClients usr _ss = do
   pure . Set.fromList . map (uncurry ClientInfo) $ clientInfo
   where
     getResult [] = pure mempty
-    getResult ((u, cs) : rs)
-      | u == usr = pure cs
+    getResult ((u, cs') : rs)
+      | u == usr = pure cs'
       | otherwise = getResult rs
 
     getValidity lusr cid =

From 96521f5b87767746d731b2d839c804dac98b1ed6 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 10:35:47 +0200
Subject: [PATCH 086/220] ...

---
 integration/integration.cabal        |   1 -
 integration/test/API/BrigInternal.hs |  12 +--
 integration/test/Test/Brig.hs        | 114 +++++++++------------------
 3 files changed, 39 insertions(+), 88 deletions(-)

diff --git a/integration/integration.cabal b/integration/integration.cabal
index 916604845a..43b7ec5ce7 100644
--- a/integration/integration.cabal
+++ b/integration/integration.cabal
@@ -147,5 +147,4 @@ library
     , unliftio
     , uuid
     , websockets
-    , wire-api
     , yaml
diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 45f845037b..a231e3c969 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -45,16 +45,10 @@ createUser domain cu = do
         )
 
 data FedConn = FedConn
-  { domain :: Maybe String,
-    searchStrategy :: Maybe String
+  { domain :: String,
+    searchStrategy :: String
   }
-
-instance Default FedConn where
-  def =
-    FedConn
-      { domain = Nothing,
-        searchStrategy = Nothing
-      }
+  deriving (Eq, Ord)
 
 createFedConn :: HasCallStack => FedConn -> App Response
 createFedConn = undefined
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 04cf87ee1e..c0caba7e9e 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -4,15 +4,12 @@ import qualified API.Brig as Public
 import qualified API.BrigInternal as Internal
 import qualified API.Common as API
 import qualified API.GalleyInternal as Internal
+import Control.Monad.IO.Class
+import Data.String.Conversions
 import GHC.Stack
 import SetupHelpers
+import Testlib.Assertions
 import Testlib.Prelude
-import Wire.API.Routes.FederationDomainConfig
-import qualified Data.Domain as D
-import Wire.API.User.Search
-import Data.String.Conversions
-import Control.Monad.IO.Class
-import TestLib.Assertions
 
 testSearchContactForExternalUsers :: HasCallStack => App ()
 testSearchContactForExternalUsers = do
@@ -27,86 +24,47 @@ testSearchContactForExternalUsers = do
 
 testCrudFederationRemotes :: HasCallStack => App ()
 testCrudFederationRemotes = do
-  let remote1 = FederationDomainConfig (D.Domain $ cs "good.example.com") NoSearch
-      remote2 = FederationDomainConfig (D.Domain $ cs "evil.example.com") ExactHandleSearch
-      remote2' = remote2 {cfgSearchPolicy = NoSearch}
-
-      parseFedConns :: HasCallStack => Response -> App [FederationDomainConfig]
+  let parseFedConns :: HasCallStack => Response -> App [Internal.FedConn]
       parseFedConns = undefined
 
-      shouldMatchFedConns :: HasCallStack => [FederationDomainConfig] -> [FederationDomainConfig] -> App ()
-      shouldMatchFedConns _ _ = do
-        liftIO $ shouldMatch _ _ --  "should return config values and good.example.com" (sort $ rem : cfgRemotes) (sort res2)
-        undefined
+      addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
+      addOnce rem want = do
+        Internal.createFedConn rem
+        res <- parseFedConns =<< Internal.readFedConns
+        sort res `shouldMatch` sort want
 
-      addOnce :: HasCallStack => FederationDomainConfig -> App ()
-      addOnce rem = do
-        createFedConn rem
-        res <- parseFedConns =<< readFedConns
-        res `shouldMatchFedConns` (sort $ rem : cfgRemotes)
-
-      deleteOnce :: HasCallStack => Domain -> App ()
+      deleteOnce :: HasCallStack => String -> [Internal.FedConn] -> App ()
       deleteOnce = undefined
 
-      deleteFail :: HasCallStack => Domain -> App ()
+      deleteFail :: HasCallStack => String -> App ()
       deleteFail = undefined
 
-      updateOnce :: HasCallStack => D.Domain -> FederationDomainConfig -> App ()
+      updateOnce :: HasCallStack => String -> Internal.FedConn -> [Internal.FedConn] -> App ()
       updateOnce = undefined
 
-      updateFail :: HasCallStack => D.Domain -> App ()
+      updateFail :: HasCallStack => String -> Internal.FedConn -> App ()
       updateFail = undefined
 
-  -- Delete the remotes from the database
-  -- This doesn't do anything with the remotes
-  -- defined in config files.
-  resetFedConns
-  cfgRemotes <- parseFedConns =<< readFedConns
-  cfgRemotes `shouldMatchFedConns` [remote2]
-  deleteFail (domain $ head $ cfgRemotes)
-
-  addOnce remote1
-  readFedConns `shouldContainFedConns` remote1
-
-  addOnce remote1 -- idempotency
-  readFedConns `shouldContainFedConns` remote1
-
-  deleteOnce (domain remote1)
-  readFedConns `shouldNotContainFedConns` remote1
-
-  deleteOnce (domain remote1) -- idempotency
-  readFedConns `shouldNotContainFedConns` remote1
-
-  addOnce remote2
-  deleteFail (domain remote2) -- removing from cfg file doesn't work whether it's in the database or not
-  readFedConns `shouldContainFedConns` remote2
-
-  updateOnce (domain remote2) remote2'
-  readFedConns `shouldNotContainFedConns` remote2
-  readFedConns `shouldContainFedConns` remote2'
-
-  updateOnce (domain remote2) remote2' -- idempotency
-  readFedConns `shouldNotContainFedConns` remote2
-  readFedConns `shouldContainFedConns` remote2'
-
-{-
-
-  -- updating search strategy works
-  updateFederationRemote brig (domain remote2) remote2'
-  res5 <- getFederationRemotes brig
-  -- (move the dynamic remotes to the beginning here to make sure we look for `remote2'`, not `remote`.)
-  liftIO $ assertEqual "should be NoSearch" (nub $ sort $ [remote1, remote2'] <> cfgRemotes) (sort res5)
-
-  -- updating from config file fails
-  updateFederationRemote' id brig (domain $ head $ cfgRemotes) (head $ cfgRemotes) !!! const 533 === statusCode
-
-  -- updating domain fails
-  let remote2'' = remote2' {domain = Domain "broken.example.com"}
-  updateFederationRemote' id brig (domain remote2) remote2'' !!! const 533 === statusCode
-
-  -- TODO: how do we test that the TVar is updated in all services?  some fancy unit test?
-  -- duplicate internal end-point to all services, and implement the hanlers in a library?
-  pure ()
-  where
-
--}
+  Internal.resetFedConns
+  cfgRemotes <- parseFedConns =<< Internal.readFedConns
+
+  let remote1, remote1', remote1'' :: Internal.FedConn
+      remote1 = Internal.FedConn (cs "good.example.com") "no_search"
+      remote1' = remote1 {Internal.searchStrategy = "full_search"}
+      remote1'' = remote1 {Internal.domain = "meh.example.com"}
+
+      remote2 = Internal.FedConn (cs "evil.example.com") "exact_handle_search"
+      remote2' = remote2 {Internal.searchStrategy = "no_search"}
+
+  cfgRemotes `shouldMatch` [remote2]
+  deleteFail (Internal.domain remote2)
+
+  addOnce remote1 [remote1, remote2]
+  addOnce remote1 [remote1, remote2] -- idempotency
+  updateOnce (Internal.domain remote1) remote1' [remote1', remote2]
+  updateFail (Internal.domain remote1) remote1''
+  deleteOnce (Internal.domain remote1) cfgRemotes
+  deleteOnce (Internal.domain remote1) cfgRemotes -- idempotency
+  addOnce remote2 cfgRemotes
+  deleteFail (Internal.domain remote2) -- removing from cfg file doesn't work whether it's in the database or not
+  updateFail (Internal.domain remote2) remote2'

From e474e0bbd9e450b460066f82dd6aa14f63c6e4c5 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 22 May 2023 19:50:36 +1000
Subject: [PATCH 087/220] FS-1115: Setting up calls to Brig from integration
 tests.

JSON, calling, etc.
---
 integration/test/API/BrigInternal.hs   | 30 +++++++++++++++++++++-----
 integration/test/Test/Brig.hs          | 25 ++++++++++++++-------
 integration/test/Testlib/Assertions.hs | 17 +++++++++++++++
 3 files changed, 59 insertions(+), 13 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index a231e3c969..9b03b19495 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -4,6 +4,7 @@ import API.Common
 import Data.Function
 import Data.Maybe
 import Testlib.Prelude
+import qualified Data.Aeson as Aeson
 
 data CreateUser = CreateUser
   { email :: Maybe String,
@@ -50,26 +51,45 @@ data FedConn = FedConn
   }
   deriving (Eq, Ord)
 
+instance ToJSON FedConn where
+  toJSON (FedConn d s) =
+    Aeson.object
+      [ "domain" .= d
+      , "search_policy" .= s
+      ]
+
 createFedConn :: HasCallStack => FedConn -> App Response
-createFedConn = undefined
+createFedConn fedConn = do
+  dom <- ownDomain
+  req <- rawBaseRequest dom Brig Versioned "/i/federation/remotes"
+  res <- submit "POST" $ req & addJSON fedConn
+  res.status `shouldMatchRange` (200, 299)
+  pure res
 
 {-
   post (brig . paths ["i", "federation", "remotes"] . contentJson . json remote . expect2xx)
 -}
 
 readFedConns :: HasCallStack => App Response
-readFedConns = undefined
+readFedConns = do
+  dom <- ownDomain
+  req <- rawBaseRequest dom Brig Versioned "/i/federation/remotes"
+  res <- submit "GET" req
+  res.status `shouldMatchRange` (200, 299)
+  pure re
 
 {-
   remotes . responseJsonUnsafe <$> do
     get (brig . paths ["i", "federation", "remotes"] . contentJson . expect2xx)
 -}
 
-updateFedConn :: HasCallStack => FedConn -> App Response
-updateFedConn = undefined
+updateFedConn :: HasCallStack => String -> FedConn -> App Response
+updateFedConn _dom _fedConn = do
+  dom <- ownDomain
+  req <- rawBaseRequest
 
 deleteFedConn :: HasCallStack => String -> App Response
-deleteFedConn = undefined
+deleteFedConn _dom = undefined
 
 resetFedConns :: HasCallStack => App Response
 resetFedConns = undefined
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index c0caba7e9e..5ee27184b4 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -4,7 +4,6 @@ import qualified API.Brig as Public
 import qualified API.BrigInternal as Internal
 import qualified API.Common as API
 import qualified API.GalleyInternal as Internal
-import Control.Monad.IO.Class
 import Data.String.Conversions
 import GHC.Stack
 import SetupHelpers
@@ -28,24 +27,34 @@ testCrudFederationRemotes = do
       parseFedConns = undefined
 
       addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
-      addOnce rem want = do
-        Internal.createFedConn rem
+      addOnce fedConn want = do
+        _res <- Internal.createFedConn fedConn
         res <- parseFedConns =<< Internal.readFedConns
         sort res `shouldMatch` sort want
 
       deleteOnce :: HasCallStack => String -> [Internal.FedConn] -> App ()
-      deleteOnce = undefined
+      deleteOnce domain want = do
+        _res <- Internal.deleteFedConn domain
+        res <- parseFedConns =<< Internal.readFedConns
+        sort res `shouldMatch` sort want
 
       deleteFail :: HasCallStack => String -> App ()
-      deleteFail = undefined
+      deleteFail del = do
+        res <- Internal.deleteFedConn del
+        res.status `shouldMatchInt` 400
 
       updateOnce :: HasCallStack => String -> Internal.FedConn -> [Internal.FedConn] -> App ()
-      updateOnce = undefined
+      updateOnce domain fedConn want = do
+        _res <- Internal.updateFedConn domain fedConn
+        res <- parseFedConns =<< Internal.readFedConns
+        sort res `shouldMatch` sort want
 
       updateFail :: HasCallStack => String -> Internal.FedConn -> App ()
-      updateFail = undefined
+      updateFail domain fedConn = do
+        res <- Internal.updateFedConn domain fedConn
+        res.status `shouldMatchInt` 400
 
-  Internal.resetFedConns
+  _res <- Internal.resetFedConns
   cfgRemotes <- parseFedConns =<< Internal.readFedConns
 
   let remote1, remote1', remote1'' :: Internal.FedConn
diff --git a/integration/test/Testlib/Assertions.hs b/integration/test/Testlib/Assertions.hs
index c056fe1c8d..7e5a6704d1 100644
--- a/integration/test/Testlib/Assertions.hs
+++ b/integration/test/Testlib/Assertions.hs
@@ -84,6 +84,23 @@ shouldMatchInt ::
   App ()
 shouldMatchInt = shouldMatch
 
+shouldMatchRange ::
+  (MakesValue a, HasCallStack) =>
+  -- | The actual value
+  a ->
+  -- | The expected range, inclusive both sides
+  (Int, Int) ->
+  App ()
+shouldMatchRange a (lower, upper) = do
+  xa <- make a
+  xl <- make lower
+  xu <- make upper
+  unless (xa < xl || xa > xu) $ do
+    pa <- prettyJSON xa
+    pu <- prettyJSON xu
+    pl <- prettyJSON xl
+    assertFailure $ "Actual:\n" <> pa <> "\nExpected:\n(" <> pl <> "," <> pu <> ")"
+
 liftP2 ::
   (MakesValue a, MakesValue b, HasCallStack) =>
   (Value -> Value -> c) ->

From af21829ba8fb3eca64e1c4e8d4fd5f9727ddfc36 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 14:22:22 +0200
Subject: [PATCH 088/220] ...

---
 integration/test/API/BrigInternal.hs | 99 ++++++++++++++++------------
 1 file changed, 56 insertions(+), 43 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 9b03b19495..2809eaa248 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -1,10 +1,10 @@
 module API.BrigInternal where
 
 import API.Common
+import qualified Data.Aeson as Aeson
 import Data.Function
 import Data.Maybe
 import Testlib.Prelude
-import qualified Data.Aeson as Aeson
 
 data CreateUser = CreateUser
   { email :: Maybe String,
@@ -54,63 +54,76 @@ data FedConn = FedConn
 instance ToJSON FedConn where
   toJSON (FedConn d s) =
     Aeson.object
-      [ "domain" .= d
-      , "search_policy" .= s
+      [ "domain" .= d,
+        "search_policy" .= s
       ]
 
+instance FromJSON FedConn where
+  parseJSON = withObject "FedConn" $ \obj -> do
+    FedConn
+      <$> obj .: fromString "domain"
+      <*> obj .: fromString "search_policy"
+
 createFedConn :: HasCallStack => FedConn -> App Response
 createFedConn fedConn = do
-  dom <- ownDomain
-  req <- rawBaseRequest dom Brig Versioned "/i/federation/remotes"
-  res <- submit "POST" $ req & addJSON fedConn
+  res <- createFedConn' fedConn
   res.status `shouldMatchRange` (200, 299)
   pure res
 
-{-
-  post (brig . paths ["i", "federation", "remotes"] . contentJson . json remote . expect2xx)
--}
+createFedConn' :: HasCallStack => FedConn -> App Response
+createFedConn' fedConn = do
+  owndom <- ownDomain
+  req <- rawBaseRequest owndom Brig Versioned "/i/federation/remotes"
+  submit "POST" $ req & addJSON fedConn
 
 readFedConns :: HasCallStack => App Response
 readFedConns = do
-  dom <- ownDomain
-  req <- rawBaseRequest dom Brig Versioned "/i/federation/remotes"
-  res <- submit "GET" req
+  res <- readFedConns'
   res.status `shouldMatchRange` (200, 299)
-  pure re
+  pure res
 
-{-
-  remotes . responseJsonUnsafe <$> do
-    get (brig . paths ["i", "federation", "remotes"] . contentJson . expect2xx)
--}
+readFedConns' :: HasCallStack => App Response
+readFedConns' = do
+  owndom <- ownDomain
+  req <- rawBaseRequest owndom Brig Versioned "/i/federation/remotes"
+  submit "GET" req
 
 updateFedConn :: HasCallStack => String -> FedConn -> App Response
-updateFedConn _dom _fedConn = do
-  dom <- ownDomain
-  req <- rawBaseRequest
+updateFedConn dom fedConn = do
+  res <- updateFedConn' dom fedConn
+  res.status `shouldMatchRange` (200, 299)
+  pure res
+
+updateFedConn' :: HasCallStack => String -> FedConn -> App Response
+updateFedConn' dom fedConn = do
+  owndom <- ownDomain
+  req <- rawBaseRequest owndom Brig Versioned ("/i/federation/remotes" <> dom)
+  submit "PUT" (fedConn `addJSON` req)
 
 deleteFedConn :: HasCallStack => String -> App Response
-deleteFedConn _dom = undefined
+deleteFedConn dom = do
+  res <- deleteFedConn' dom
+  res.status `shouldMatchRange` (200, 299)
+  pure res
+
+deleteFedConn' :: HasCallStack => String -> App Response
+deleteFedConn' dom = do
+  owndom <- ownDomain
+  req <- rawBaseRequest owndom Brig Versioned ("/i/federation/remotes" <> dom)
+  submit "DELETE" req
 
 resetFedConns :: HasCallStack => App Response
-resetFedConns = undefined
-
-{-
-updateFederationRemote' :: (Request -> Request) -> Brig -> Domain -> FederationDomainConfig -> Http ResponseLBS
-updateFederationRemote' mods brig rdom remote =
-  put (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . json remote . mods)
-
-deleteFederationRemote' :: (Request -> Request) -> Brig -> Domain -> Http ResponseLBS
-deleteFederationRemote' mods brig rdom =
-  delete (brig . paths ["i", "federation", "remotes", toByteString' rdom] . contentJson . mods)
-
--- this one needs to go elsewhere
-resetFederationRemotes :: Opts -> Brig -> Http ()
-resetFederationRemotes opts brig = do
-  rs <- getFederationRemotes brig
-  -- Filter out domains that are in the config file.
-  -- These values can't be deleted yet, so don't even try.
-  forM_ (notCfgRemotes rs) $ \(FederationDomainConfig rdom _) -> deleteFederationRemote brig rdom
-  where
-    cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
-    notCfgRemotes = filter (`notElem` cfgRemotes)
--}
+resetFedConns = do
+  {-
+  -- this one needs to go elsewhere
+  resetFederationRemotes :: Opts -> Brig -> Http ()
+  resetFederationRemotes opts brig = do
+    rs <- getFederationRemotes brig
+    -- Filter out domains that are in the config file.
+    -- These values can't be deleted yet, so don't even try.
+    forM_ (notCfgRemotes rs) $ \(FederationDomainConfig rdom _) -> deleteFederationRemote brig rdom
+    where
+      cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
+      notCfgRemotes = filter (`notElem` cfgRemotes)
+  -}
+  pure undefined

From 3287003efd813ffeaf5e0d386340fb1bf1fb1457 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 14:33:52 +0200
Subject: [PATCH 089/220] ...

---
 integration/test/API/BrigInternal.hs | 8 ++++----
 integration/test/Test/Brig.hs        | 7 ++++---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 2809eaa248..2927940110 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -73,7 +73,7 @@ createFedConn fedConn = do
 createFedConn' :: HasCallStack => FedConn -> App Response
 createFedConn' fedConn = do
   owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Versioned "/i/federation/remotes"
+  req <- rawBaseRequest owndom Brig Unversioned "/i/federation/remotes"
   submit "POST" $ req & addJSON fedConn
 
 readFedConns :: HasCallStack => App Response
@@ -85,7 +85,7 @@ readFedConns = do
 readFedConns' :: HasCallStack => App Response
 readFedConns' = do
   owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Versioned "/i/federation/remotes"
+  req <- rawBaseRequest owndom Brig Unversioned "/i/federation/remotes"
   submit "GET" req
 
 updateFedConn :: HasCallStack => String -> FedConn -> App Response
@@ -97,7 +97,7 @@ updateFedConn dom fedConn = do
 updateFedConn' :: HasCallStack => String -> FedConn -> App Response
 updateFedConn' dom fedConn = do
   owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Versioned ("/i/federation/remotes" <> dom)
+  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes" <> dom)
   submit "PUT" (fedConn `addJSON` req)
 
 deleteFedConn :: HasCallStack => String -> App Response
@@ -109,7 +109,7 @@ deleteFedConn dom = do
 deleteFedConn' :: HasCallStack => String -> App Response
 deleteFedConn' dom = do
   owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Versioned ("/i/federation/remotes" <> dom)
+  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes" <> dom)
   submit "DELETE" req
 
 resetFedConns :: HasCallStack => App Response
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 5ee27184b4..6ecd5c8ef8 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -4,6 +4,7 @@ import qualified API.Brig as Public
 import qualified API.BrigInternal as Internal
 import qualified API.Common as API
 import qualified API.GalleyInternal as Internal
+import qualified Data.Aeson as Aeson
 import Data.String.Conversions
 import GHC.Stack
 import SetupHelpers
@@ -24,7 +25,7 @@ testSearchContactForExternalUsers = do
 testCrudFederationRemotes :: HasCallStack => App ()
 testCrudFederationRemotes = do
   let parseFedConns :: HasCallStack => Response -> App [Internal.FedConn]
-      parseFedConns = undefined
+      parseFedConns resp = fromJust . Aeson.decode . Aeson.encode . fromJust <$> ((`lookupField` "remotes") =<< getJSON 200 resp)
 
       addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
       addOnce fedConn want = do
@@ -40,7 +41,7 @@ testCrudFederationRemotes = do
 
       deleteFail :: HasCallStack => String -> App ()
       deleteFail del = do
-        res <- Internal.deleteFedConn del
+        res <- Internal.deleteFedConn' del
         res.status `shouldMatchInt` 400
 
       updateOnce :: HasCallStack => String -> Internal.FedConn -> [Internal.FedConn] -> App ()
@@ -51,7 +52,7 @@ testCrudFederationRemotes = do
 
       updateFail :: HasCallStack => String -> Internal.FedConn -> App ()
       updateFail domain fedConn = do
-        res <- Internal.updateFedConn domain fedConn
+        res <- Internal.updateFedConn' domain fedConn
         res.status `shouldMatchInt` 400
 
   _res <- Internal.resetFedConns

From 166e2ba6cf5169b855e040bfc39d037d391544fd Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 15:31:10 +0200
Subject: [PATCH 090/220] ...

---
 integration/test/Test/Brig.hs          | 13 +++++++++----
 integration/test/Testlib/Assertions.hs |  2 +-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 6ecd5c8ef8..cb62f328e6 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -63,12 +63,17 @@ testCrudFederationRemotes = do
       remote1' = remote1 {Internal.searchStrategy = "full_search"}
       remote1'' = remote1 {Internal.domain = "meh.example.com"}
 
-      remote2 = Internal.FedConn (cs "evil.example.com") "exact_handle_search"
-      remote2' = remote2 {Internal.searchStrategy = "no_search"}
+      remote2 = Internal.FedConn (cs "evil.example.com") "no_search"
+      remote2' = remote2 {Internal.searchStrategy = "exact_handle_search"}
 
-  cfgRemotes `shouldMatch` [remote2]
-  deleteFail (Internal.domain remote2)
+      cfgRemotesExpect :: [Internal.FedConn]
+      cfgRemotesExpect =
+        [ remote2,
+          Internal.FedConn (cs "example.com") "full_search"
+        ]
 
+  sort cfgRemotes `shouldMatch` sort cfgRemotesExpect
+  deleteFail (Internal.domain remote2)
   addOnce remote1 [remote1, remote2]
   addOnce remote1 [remote1, remote2] -- idempotency
   updateOnce (Internal.domain remote1) remote1' [remote1', remote2]
diff --git a/integration/test/Testlib/Assertions.hs b/integration/test/Testlib/Assertions.hs
index 7e5a6704d1..5c2c746d9a 100644
--- a/integration/test/Testlib/Assertions.hs
+++ b/integration/test/Testlib/Assertions.hs
@@ -95,7 +95,7 @@ shouldMatchRange a (lower, upper) = do
   xa <- make a
   xl <- make lower
   xu <- make upper
-  unless (xa < xl || xa > xu) $ do
+  when (xa < xl || xa > xu) $ do
     pa <- prettyJSON xa
     pu <- prettyJSON xu
     pl <- prettyJSON xl

From bad86a19523a7393b876f974ef3ab52241ffd33e Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 15:36:47 +0200
Subject: [PATCH 091/220] ...

---
 integration/test/API/BrigInternal.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 2927940110..f8f1193a35 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -97,7 +97,7 @@ updateFedConn dom fedConn = do
 updateFedConn' :: HasCallStack => String -> FedConn -> App Response
 updateFedConn' dom fedConn = do
   owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes" <> dom)
+  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "PUT" (fedConn `addJSON` req)
 
 deleteFedConn :: HasCallStack => String -> App Response
@@ -109,7 +109,7 @@ deleteFedConn dom = do
 deleteFedConn' :: HasCallStack => String -> App Response
 deleteFedConn' dom = do
   owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes" <> dom)
+  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "DELETE" req
 
 resetFedConns :: HasCallStack => App Response

From 63906749b6813baca8cf8bd4dc6a36d6e0e02491 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 15:55:36 +0200
Subject: [PATCH 092/220] ...

---
 integration/test/API/BrigInternal.hs | 16 -----------
 integration/test/SetupHelpers.hs     |  8 ++++++
 integration/test/Test/Brig.hs        | 41 ++++++++++++++--------------
 3 files changed, 29 insertions(+), 36 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index f8f1193a35..911e9bc812 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -111,19 +111,3 @@ deleteFedConn' dom = do
   owndom <- ownDomain
   req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "DELETE" req
-
-resetFedConns :: HasCallStack => App Response
-resetFedConns = do
-  {-
-  -- this one needs to go elsewhere
-  resetFederationRemotes :: Opts -> Brig -> Http ()
-  resetFederationRemotes opts brig = do
-    rs <- getFederationRemotes brig
-    -- Filter out domains that are in the config file.
-    -- These values can't be deleted yet, so don't even try.
-    forM_ (notCfgRemotes rs) $ \(FederationDomainConfig rdom _) -> deleteFederationRemote brig rdom
-    where
-      cfgRemotes = fromMaybe [] . Opt.setFederationDomainConfigs $ Opt.optSettings opts
-      notCfgRemotes = filter (`notElem` cfgRemotes)
-  -}
-  pure undefined
diff --git a/integration/test/SetupHelpers.hs b/integration/test/SetupHelpers.hs
index a46e8e8cc5..24b016b381 100644
--- a/integration/test/SetupHelpers.hs
+++ b/integration/test/SetupHelpers.hs
@@ -58,3 +58,11 @@ getAllConvs u = do
     resp.status `shouldMatchInt` 200
     resp.json
   result %. "found" & asList
+
+resetFedConns :: HasCallStack => App ()
+resetFedConns = do
+  bindResponse Internal.readFedConns $ \resp -> do
+    rdoms :: [String] <- do
+      rawlist <- resp.json %. "remotes" & asList
+      (asString . (%. "domain")) `mapM` rawlist
+    Internal.deleteFedConn' `mapM_` rdoms
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index cb62f328e6..a28dbe995b 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -42,7 +42,7 @@ testCrudFederationRemotes = do
       deleteFail :: HasCallStack => String -> App ()
       deleteFail del = do
         res <- Internal.deleteFedConn' del
-        res.status `shouldMatchInt` 400
+        res.status `shouldMatchInt` 533
 
       updateOnce :: HasCallStack => String -> Internal.FedConn -> [Internal.FedConn] -> App ()
       updateOnce domain fedConn want = do
@@ -53,33 +53,34 @@ testCrudFederationRemotes = do
       updateFail :: HasCallStack => String -> Internal.FedConn -> App ()
       updateFail domain fedConn = do
         res <- Internal.updateFedConn' domain fedConn
-        res.status `shouldMatchInt` 400
-
-  _res <- Internal.resetFedConns
-  cfgRemotes <- parseFedConns =<< Internal.readFedConns
+        res.status `shouldMatchInt` 533
 
   let remote1, remote1', remote1'' :: Internal.FedConn
       remote1 = Internal.FedConn (cs "good.example.com") "no_search"
       remote1' = remote1 {Internal.searchStrategy = "full_search"}
       remote1'' = remote1 {Internal.domain = "meh.example.com"}
 
-      remote2 = Internal.FedConn (cs "evil.example.com") "no_search"
-      remote2' = remote2 {Internal.searchStrategy = "exact_handle_search"}
+      cfgRemotesExpect :: Internal.FedConn
+      cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
+
+  resetFedConns
+  cfgRemotes <- parseFedConns =<< Internal.readFedConns
+  cfgRemotes `shouldMatch` [cfgRemotesExpect]
+
+  -- entries present in the config file can be idempotently added, but cannot be deleted or
+  -- updated.
+  addOnce cfgRemotesExpect [cfgRemotesExpect]
+  deleteFail (Internal.domain cfgRemotesExpect)
+  updateFail (Internal.domain cfgRemotesExpect) cfgRemotesExpect {Internal.searchStrategy = "no_search"}
 
-      cfgRemotesExpect :: [Internal.FedConn]
-      cfgRemotesExpect =
-        [ remote2,
-          Internal.FedConn (cs "example.com") "full_search"
-        ]
+  -- create
+  addOnce remote1 (remote1 : cfgRemotes)
+  addOnce remote1 (remote1 : cfgRemotes) -- idempotency
 
-  sort cfgRemotes `shouldMatch` sort cfgRemotesExpect
-  deleteFail (Internal.domain remote2)
-  addOnce remote1 [remote1, remote2]
-  addOnce remote1 [remote1, remote2] -- idempotency
-  updateOnce (Internal.domain remote1) remote1' [remote1', remote2]
+  -- update
+  updateOnce (Internal.domain remote1) remote1' (remote1' : cfgRemotes)
   updateFail (Internal.domain remote1) remote1''
+
+  -- delete
   deleteOnce (Internal.domain remote1) cfgRemotes
   deleteOnce (Internal.domain remote1) cfgRemotes -- idempotency
-  addOnce remote2 cfgRemotes
-  deleteFail (Internal.domain remote2) -- removing from cfg file doesn't work whether it's in the database or not
-  updateFail (Internal.domain remote2) remote2'

From b49a74292072e4db7b7a9196c49186603d8ade26 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 16:04:48 +0200
Subject: [PATCH 093/220] ...

---
 integration/test/Test/Brig.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index a28dbe995b..89b0eab29a 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -65,7 +65,7 @@ testCrudFederationRemotes = do
 
   resetFedConns
   cfgRemotes <- parseFedConns =<< Internal.readFedConns
-  cfgRemotes `shouldMatch` [cfgRemotesExpect]
+  cfgRemotes `shouldMatch` [cfgRemotesExpect] -- this fails and returns two entries for example.com.  maybe resetFedConns or createFedConn is broken in brig?
 
   -- entries present in the config file can be idempotently added, but cannot be deleted or
   -- updated.

From f02819f565201350e9f6a8f58389482b105c67a7 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 16:29:25 +0200
Subject: [PATCH 094/220] Fix

---
 services/brig/src/Brig/API/Internal.hs | 29 ++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 3cad4424f7..ff27afd305 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -189,6 +189,7 @@ federationRemotesAPI =
 
 addFederationRemote :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 addFederationRemote fedDomConf = do
+  assertNoDivergingDomainInConfigFiles fedDomConf
   result <- lift . wrapClient $ Data.addFederationRemote fedDomConf
   case result of
     Data.AddFederationRemoteSuccess -> pure ()
@@ -197,6 +198,34 @@ addFederationRemote fedDomConf = do
         "Maximum number of remote backends reached.  If you need to create more connections, \
         \please contact wire.com."
 
+-- | If remote domain is registered in config file, the version that can be added to the
+-- database must be the same.
+assertNoDivergingDomainInConfigFiles :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
+assertNoDivergingDomainInConfigFiles fedComConf = do
+  cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
+  let dict = Map.fromListWith merge keyvals
+        where
+          merge c c' =
+            if c == c'
+              then c
+              else error $ "error in config file: conflicting parameters on domain: " <> show (c, c')
+
+          keyvals = [(domain cnf, cnf) | cnf <- cfg]
+  let diverges = case Map.lookup (domain fedComConf) dict of
+        Nothing -> False
+        Just fedComConf' -> fedComConf' /= fedComConf
+  when diverges $ do
+    throwError . fedError . FederationUnexpectedError $
+      "keeping track of remote domains in the brig config file is deprecated, but as long as we \
+      \do that, adding a domain with different settings than in the config file is nto allowed.  want "
+        <> ( "Just "
+               <> cs (show fedComConf)
+               <> "or Nothing, "
+           )
+        <> ( "got "
+               <> cs (show (Map.lookup (domain fedComConf) dict))
+           )
+
 getFederationRemotes :: ExceptT Brig.API.Error.Error (AppT r) FederationDomainConfigs
 getFederationRemotes = lift $ do
   -- FUTUREWORK: we should solely rely on `db` in the future for remote domains; merging

From 15ec903bd1741f34f1269092de14f0aaf34ebd36 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 22 May 2023 17:21:32 +0200
Subject: [PATCH 095/220] ...

---
 integration/test/Test/Brig.hs          | 40 +++++++++++++++++---------
 services/brig/src/Brig/API/Internal.hs | 38 +++++++++++++-----------
 2 files changed, 48 insertions(+), 30 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 89b0eab29a..a0c94b0261 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -25,19 +25,29 @@ testSearchContactForExternalUsers = do
 testCrudFederationRemotes :: HasCallStack => App ()
 testCrudFederationRemotes = do
   let parseFedConns :: HasCallStack => Response -> App [Internal.FedConn]
-      parseFedConns resp = fromJust . Aeson.decode . Aeson.encode . fromJust <$> ((`lookupField` "remotes") =<< getJSON 200 resp)
+      parseFedConns resp = do
+        -- TODO: not idiomatic!  try `getJSON 200 resp %. "remotes" & asList & mapM asObjOrSomething`
+        -- Some ideas: There is asList to assert that a Value is actually a an array of Values. Then you can sort that to have a defined order.
+        fromJust . Aeson.decode . Aeson.encode . fromJust <$> ((`lookupField` "remotes") =<< getJSON 200 resp)
 
       addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
       addOnce fedConn want = do
-        _res <- Internal.createFedConn fedConn
-        res <- parseFedConns =<< Internal.readFedConns
-        sort res `shouldMatch` sort want
+        res <- Internal.createFedConn fedConn
+        res.status `shouldMatchInt` 200
+        res2 <- parseFedConns =<< Internal.readFedConns
+        sort res2 `shouldMatch` sort want
+
+      addFail :: HasCallStack => Internal.FedConn -> App ()
+      addFail fedConn = do
+        res <- Internal.createFedConn' fedConn
+        res.status `shouldMatchInt` 533
 
       deleteOnce :: HasCallStack => String -> [Internal.FedConn] -> App ()
       deleteOnce domain want = do
-        _res <- Internal.deleteFedConn domain
-        res <- parseFedConns =<< Internal.readFedConns
-        sort res `shouldMatch` sort want
+        res <- Internal.deleteFedConn domain
+        res.status `shouldMatchInt` 200
+        res2 <- parseFedConns =<< Internal.readFedConns
+        sort res2 `shouldMatch` sort want
 
       deleteFail :: HasCallStack => String -> App ()
       deleteFail del = do
@@ -46,9 +56,10 @@ testCrudFederationRemotes = do
 
       updateOnce :: HasCallStack => String -> Internal.FedConn -> [Internal.FedConn] -> App ()
       updateOnce domain fedConn want = do
-        _res <- Internal.updateFedConn domain fedConn
-        res <- parseFedConns =<< Internal.readFedConns
-        sort res `shouldMatch` sort want
+        res <- Internal.updateFedConn domain fedConn
+        res.status `shouldMatchInt` 200
+        res2 <- parseFedConns =<< Internal.readFedConns
+        sort res2 `shouldMatch` sort want
 
       updateFail :: HasCallStack => String -> Internal.FedConn -> App ()
       updateFail domain fedConn = do
@@ -63,15 +74,16 @@ testCrudFederationRemotes = do
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
 
-  resetFedConns
+  resetFedConns -- TODO: if you `make cqlsh and look at the table, you'll find this doesn't delete anything
   cfgRemotes <- parseFedConns =<< Internal.readFedConns
   cfgRemotes `shouldMatch` [cfgRemotesExpect] -- this fails and returns two entries for example.com.  maybe resetFedConns or createFedConn is broken in brig?
 
-  -- entries present in the config file can be idempotently added, but cannot be deleted or
-  -- updated.
+  -- entries present in the config file can be idempotently added if identical, but cannot be
+  -- updated, deleted or updated.
   addOnce cfgRemotesExpect [cfgRemotesExpect]
+  addFail (cfgRemotesExpect {Internal.searchStrategy = "no_search"})
   deleteFail (Internal.domain cfgRemotesExpect)
-  updateFail (Internal.domain cfgRemotesExpect) cfgRemotesExpect {Internal.searchStrategy = "no_search"}
+  updateFail (Internal.domain cfgRemotesExpect) (cfgRemotesExpect {Internal.searchStrategy = "no_search"})
 
   -- create
   addOnce remote1 (remote1 : cfgRemotes)
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index ff27afd305..09256e5604 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -64,7 +64,7 @@ import qualified Brig.User.API.Search as Search
 import qualified Brig.User.EJPD
 import qualified Brig.User.Search.Index as Index
 import Control.Error hiding (bool)
-import Control.Lens (to, view, (^.))
+import Control.Lens (view, (^.))
 import Data.Aeson hiding (json)
 import Data.ByteString.Conversion
 import qualified Data.ByteString.Conversion as List
@@ -198,20 +198,25 @@ addFederationRemote fedDomConf = do
         "Maximum number of remote backends reached.  If you need to create more connections, \
         \please contact wire.com."
 
+remotesMapFromCfgFile :: AppT r (Map Domain FederationDomainConfig)
+remotesMapFromCfgFile = do
+  cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
+  let dict = [(domain cnf, cnf) | cnf <- cfg]
+      merge c c' =
+        if c == c'
+          then c
+          else error $ "error in config file: conflicting parameters on domain: " <> show (c, c')
+  pure $ Map.fromListWith merge dict
+
+remotesListFromCfgFile :: AppT r [FederationDomainConfig]
+remotesListFromCfgFile = Map.elems <$> remotesMapFromCfgFile
+
 -- | If remote domain is registered in config file, the version that can be added to the
 -- database must be the same.
 assertNoDivergingDomainInConfigFiles :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 assertNoDivergingDomainInConfigFiles fedComConf = do
-  cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
-  let dict = Map.fromListWith merge keyvals
-        where
-          merge c c' =
-            if c == c'
-              then c
-              else error $ "error in config file: conflicting parameters on domain: " <> show (c, c')
-
-          keyvals = [(domain cnf, cnf) | cnf <- cfg]
-  let diverges = case Map.lookup (domain fedComConf) dict of
+  cfg <- lift remotesMapFromCfgFile
+  let diverges = case Map.lookup (domain fedComConf) cfg of
         Nothing -> False
         Just fedComConf' -> fedComConf' /= fedComConf
   when diverges $ do
@@ -223,7 +228,7 @@ assertNoDivergingDomainInConfigFiles fedComConf = do
                <> "or Nothing, "
            )
         <> ( "got "
-               <> cs (show (Map.lookup (domain fedComConf) dict))
+               <> cs (show (Map.lookup (domain fedComConf) cfg))
            )
 
 getFederationRemotes :: ExceptT Brig.API.Error.Error (AppT r) FederationDomainConfigs
@@ -234,11 +239,12 @@ getFederationRemotes = lift $ do
   -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections,
   -- http://docs.wire.com/developer/developer/federation-design-aspects.html#configuring-remote-connections-dev-perspective
   db <- wrapClient Data.getFederationRemotes
-  (ms :: Maybe FederationStrategy, mf :: Maybe [FederationDomainConfig], mu :: Maybe Int) <- do
-    cfg :: Env <- ask
+  (ms :: Maybe FederationStrategy, mf :: [FederationDomainConfig], mu :: Maybe Int) <- do
+    cfg <- ask
+    domcfgs <- remotesListFromCfgFile -- (it's not very elegant to prove the env twice here, but this code is transitory.)
     pure
       ( setFederationStrategy (cfg ^. settings),
-        cfg ^. settings . to setFederationDomainConfigs,
+        domcfgs,
         setFederationDomainConfigsUpdateFreq (cfg ^. settings)
       )
 
@@ -252,7 +258,7 @@ getFederationRemotes = lift $ do
 
   defFederationDomainConfigs
     & maybe id (\v cfg -> cfg {strategy = v}) ms
-    & (\cfg -> cfg {remotes = nub $ db <> fromMaybe mempty mf})
+    & (\cfg -> cfg {remotes = nub $ db <> mf})
     & maybe id (\v cfg -> cfg {updateInterval = min 1 v}) mu
     & pure
 

From dd061ae109ccb4fef1d75f4403bada1f8f8218d2 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 23 May 2023 17:00:07 +1000
Subject: [PATCH 096/220] WIP: Chasing down cache issues between some bug fixes

---
 integration/test/API/BrigInternal.hs   |  2 +-
 integration/test/SetupHelpers.hs       | 10 ++++++----
 integration/test/Test/Brig.hs          | 21 +++++++++------------
 integration/test/Testlib/Types.hs      |  2 +-
 services/brig/src/Brig/API/Internal.hs |  7 ++++++-
 5 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 911e9bc812..2a4a032d16 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -49,7 +49,7 @@ data FedConn = FedConn
   { domain :: String,
     searchStrategy :: String
   }
-  deriving (Eq, Ord)
+  deriving (Eq, Ord, Show)
 
 instance ToJSON FedConn where
   toJSON (FedConn d s) =
diff --git a/integration/test/SetupHelpers.hs b/integration/test/SetupHelpers.hs
index 24b016b381..ff2521c581 100644
--- a/integration/test/SetupHelpers.hs
+++ b/integration/test/SetupHelpers.hs
@@ -8,6 +8,7 @@ import Data.Default
 import Data.Function
 import GHC.Stack
 import Testlib.Prelude
+import API.BrigInternal
 
 randomUser :: (HasCallStack, MakesValue domain) => domain -> Internal.CreateUser -> App Value
 randomUser domain cu = bindResponse (Internal.createUser domain cu) $ \resp -> do
@@ -59,10 +60,11 @@ getAllConvs u = do
     resp.json
   result %. "found" & asList
 
-resetFedConns :: HasCallStack => App ()
-resetFedConns = do
+resetFedConns :: HasCallStack => [String] -> App ()
+resetFedConns cfgRemotes= do
   bindResponse Internal.readFedConns $ \resp -> do
-    rdoms :: [String] <- do
+    rdoms' :: [String] <- do
       rawlist <- resp.json %. "remotes" & asList
       (asString . (%. "domain")) `mapM` rawlist
-    Internal.deleteFedConn' `mapM_` rdoms
+    let rdoms = filter (`notElem` cfgRemotes) rdoms'
+    Internal.deleteFedConn' `mapM_` rdoms
\ No newline at end of file
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index a0c94b0261..d23126cf83 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -33,38 +33,38 @@ testCrudFederationRemotes = do
       addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
       addOnce fedConn want = do
         res <- Internal.createFedConn fedConn
-        res.status `shouldMatchInt` 200
+        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns
         sort res2 `shouldMatch` sort want
 
       addFail :: HasCallStack => Internal.FedConn -> App ()
       addFail fedConn = do
         res <- Internal.createFedConn' fedConn
-        res.status `shouldMatchInt` 533
+        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
       deleteOnce :: HasCallStack => String -> [Internal.FedConn] -> App ()
       deleteOnce domain want = do
         res <- Internal.deleteFedConn domain
-        res.status `shouldMatchInt` 200
+        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns
         sort res2 `shouldMatch` sort want
 
       deleteFail :: HasCallStack => String -> App ()
       deleteFail del = do
         res <- Internal.deleteFedConn' del
-        res.status `shouldMatchInt` 533
+        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
       updateOnce :: HasCallStack => String -> Internal.FedConn -> [Internal.FedConn] -> App ()
       updateOnce domain fedConn want = do
         res <- Internal.updateFedConn domain fedConn
-        res.status `shouldMatchInt` 200
+        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns
         sort res2 `shouldMatch` sort want
 
       updateFail :: HasCallStack => String -> Internal.FedConn -> App ()
       updateFail domain fedConn = do
         res <- Internal.updateFedConn' domain fedConn
-        res.status `shouldMatchInt` 533
+        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
   let remote1, remote1', remote1'' :: Internal.FedConn
       remote1 = Internal.FedConn (cs "good.example.com") "no_search"
@@ -74,25 +74,22 @@ testCrudFederationRemotes = do
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
 
-  resetFedConns -- TODO: if you `make cqlsh and look at the table, you'll find this doesn't delete anything
+  -- Pass in a list of domains to _not_ delete
+  resetFedConns $ Internal.domain <$> [cfgRemotesExpect] -- TODO: if you `make cqlsh and look at the table, you'll find this doesn't delete anything
   cfgRemotes <- parseFedConns =<< Internal.readFedConns
   cfgRemotes `shouldMatch` [cfgRemotesExpect] -- this fails and returns two entries for example.com.  maybe resetFedConns or createFedConn is broken in brig?
-
   -- entries present in the config file can be idempotently added if identical, but cannot be
   -- updated, deleted or updated.
   addOnce cfgRemotesExpect [cfgRemotesExpect]
   addFail (cfgRemotesExpect {Internal.searchStrategy = "no_search"})
   deleteFail (Internal.domain cfgRemotesExpect)
   updateFail (Internal.domain cfgRemotesExpect) (cfgRemotesExpect {Internal.searchStrategy = "no_search"})
-
   -- create
   addOnce remote1 (remote1 : cfgRemotes)
   addOnce remote1 (remote1 : cfgRemotes) -- idempotency
-
   -- update
-  updateOnce (Internal.domain remote1) remote1' (remote1' : cfgRemotes)
+  updateOnce (Internal.domain remote1) remote1' (remote1' : cfgRemotes) -- This fails
   updateFail (Internal.domain remote1) remote1''
-
   -- delete
   deleteOnce (Internal.domain remote1) cfgRemotes
   deleteOnce (Internal.domain remote1) cfgRemotes -- idempotency
diff --git a/integration/test/Testlib/Types.hs b/integration/test/Testlib/Types.hs
index d0d88935d9..2e5ac154c9 100644
--- a/integration/test/Testlib/Types.hs
+++ b/integration/test/Testlib/Types.hs
@@ -32,7 +32,7 @@ data Response = Response
     status :: Int,
     headers :: [HTTP.Header],
     request :: HTTP.Request
-  }
+  } deriving Show
 
 instance HasField "json" Response (App Aeson.Value) where
   getField response = maybe (assertFailure "Response has no json body") pure response.jsonBody
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 09256e5604..999de1d396 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -216,12 +216,17 @@ remotesListFromCfgFile = Map.elems <$> remotesMapFromCfgFile
 assertNoDivergingDomainInConfigFiles :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 assertNoDivergingDomainInConfigFiles fedComConf = do
   cfg <- lift remotesMapFromCfgFile
+  -- _ <- error $
+  --   "-------------------\n\n\n" <>
+  --   "cfg = " <> show cfg <> "\n\n\n" <>
+  --   "fedComConf = " <> show fedComConf <> "\n\n\n" <>
+  --   "-------------------\n\n\n"
   let diverges = case Map.lookup (domain fedComConf) cfg of
         Nothing -> False
         Just fedComConf' -> fedComConf' /= fedComConf
   when diverges $ do
     throwError . fedError . FederationUnexpectedError $
-      "keeping track of remote domains in the brig config file is deprecated, but as long as we \
+      "foooooooooooooooo keeping track of remote domains in the brig config file is deprecated, but as long as we \
       \do that, adding a domain with different settings than in the config file is nto allowed.  want "
         <> ( "Just "
                <> cs (show fedComConf)

From a6bab561984cafc3d1453596d6896f2020315fd5 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 23 May 2023 09:01:04 +0200
Subject: [PATCH 097/220] ...

---
 integration/integration.cabal | 1 -
 1 file changed, 1 deletion(-)

diff --git a/integration/integration.cabal b/integration/integration.cabal
index 43b7ec5ce7..1c33462eb6 100644
--- a/integration/integration.cabal
+++ b/integration/integration.cabal
@@ -142,7 +142,6 @@ library
     , text
     , time
     , transformers
-    , types-common
     , unix
     , unliftio
     , uuid

From 8f553bd30810c697397864ec7b9be4ccd9ca5053 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 23 May 2023 09:07:14 +0200
Subject: [PATCH 098/220] ...

---
 integration/test/API/BrigInternal.hs | 12 ++++--------
 integration/test/SetupHelpers.hs     |  5 ++---
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 2a4a032d16..ed24f7f177 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -72,8 +72,7 @@ createFedConn fedConn = do
 
 createFedConn' :: HasCallStack => FedConn -> App Response
 createFedConn' fedConn = do
-  owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Unversioned "/i/federation/remotes"
+  req <- rawBaseRequest ownDomain Brig Unversioned "/i/federation/remotes"
   submit "POST" $ req & addJSON fedConn
 
 readFedConns :: HasCallStack => App Response
@@ -84,8 +83,7 @@ readFedConns = do
 
 readFedConns' :: HasCallStack => App Response
 readFedConns' = do
-  owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Unversioned "/i/federation/remotes"
+  req <- rawBaseRequest ownDomain Brig Unversioned "/i/federation/remotes"
   submit "GET" req
 
 updateFedConn :: HasCallStack => String -> FedConn -> App Response
@@ -96,8 +94,7 @@ updateFedConn dom fedConn = do
 
 updateFedConn' :: HasCallStack => String -> FedConn -> App Response
 updateFedConn' dom fedConn = do
-  owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
+  req <- rawBaseRequest ownDomain Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "PUT" (fedConn `addJSON` req)
 
 deleteFedConn :: HasCallStack => String -> App Response
@@ -108,6 +105,5 @@ deleteFedConn dom = do
 
 deleteFedConn' :: HasCallStack => String -> App Response
 deleteFedConn' dom = do
-  owndom <- ownDomain
-  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
+  req <- rawBaseRequest ownDomain Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "DELETE" req
diff --git a/integration/test/SetupHelpers.hs b/integration/test/SetupHelpers.hs
index ff2521c581..4e435d9db3 100644
--- a/integration/test/SetupHelpers.hs
+++ b/integration/test/SetupHelpers.hs
@@ -8,7 +8,6 @@ import Data.Default
 import Data.Function
 import GHC.Stack
 import Testlib.Prelude
-import API.BrigInternal
 
 randomUser :: (HasCallStack, MakesValue domain) => domain -> Internal.CreateUser -> App Value
 randomUser domain cu = bindResponse (Internal.createUser domain cu) $ \resp -> do
@@ -61,10 +60,10 @@ getAllConvs u = do
   result %. "found" & asList
 
 resetFedConns :: HasCallStack => [String] -> App ()
-resetFedConns cfgRemotes= do
+resetFedConns cfgRemotes = do
   bindResponse Internal.readFedConns $ \resp -> do
     rdoms' :: [String] <- do
       rawlist <- resp.json %. "remotes" & asList
       (asString . (%. "domain")) `mapM` rawlist
     let rdoms = filter (`notElem` cfgRemotes) rdoms'
-    Internal.deleteFedConn' `mapM_` rdoms
\ No newline at end of file
+    Internal.deleteFedConn' `mapM_` rdoms

From e44cfe91cd3626cc373d92ba7f37a9494383d781 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 23 May 2023 09:17:53 +0200
Subject: [PATCH 099/220] ...

---
 integration/test/SetupHelpers.hs       | 7 +++----
 integration/test/Test/Brig.hs          | 5 ++---
 services/brig/src/Brig/API/Internal.hs | 7 +------
 3 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/integration/test/SetupHelpers.hs b/integration/test/SetupHelpers.hs
index 4e435d9db3..24b016b381 100644
--- a/integration/test/SetupHelpers.hs
+++ b/integration/test/SetupHelpers.hs
@@ -59,11 +59,10 @@ getAllConvs u = do
     resp.json
   result %. "found" & asList
 
-resetFedConns :: HasCallStack => [String] -> App ()
-resetFedConns cfgRemotes = do
+resetFedConns :: HasCallStack => App ()
+resetFedConns = do
   bindResponse Internal.readFedConns $ \resp -> do
-    rdoms' :: [String] <- do
+    rdoms :: [String] <- do
       rawlist <- resp.json %. "remotes" & asList
       (asString . (%. "domain")) `mapM` rawlist
-    let rdoms = filter (`notElem` cfgRemotes) rdoms'
     Internal.deleteFedConn' `mapM_` rdoms
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index d23126cf83..293766ae3d 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -74,10 +74,9 @@ testCrudFederationRemotes = do
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
 
-  -- Pass in a list of domains to _not_ delete
-  resetFedConns $ Internal.domain <$> [cfgRemotesExpect] -- TODO: if you `make cqlsh and look at the table, you'll find this doesn't delete anything
+  resetFedConns
   cfgRemotes <- parseFedConns =<< Internal.readFedConns
-  cfgRemotes `shouldMatch` [cfgRemotesExpect] -- this fails and returns two entries for example.com.  maybe resetFedConns or createFedConn is broken in brig?
+  cfgRemotes `shouldMatch` [cfgRemotesExpect]
   -- entries present in the config file can be idempotently added if identical, but cannot be
   -- updated, deleted or updated.
   addOnce cfgRemotesExpect [cfgRemotesExpect]
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 999de1d396..09256e5604 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -216,17 +216,12 @@ remotesListFromCfgFile = Map.elems <$> remotesMapFromCfgFile
 assertNoDivergingDomainInConfigFiles :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 assertNoDivergingDomainInConfigFiles fedComConf = do
   cfg <- lift remotesMapFromCfgFile
-  -- _ <- error $
-  --   "-------------------\n\n\n" <>
-  --   "cfg = " <> show cfg <> "\n\n\n" <>
-  --   "fedComConf = " <> show fedComConf <> "\n\n\n" <>
-  --   "-------------------\n\n\n"
   let diverges = case Map.lookup (domain fedComConf) cfg of
         Nothing -> False
         Just fedComConf' -> fedComConf' /= fedComConf
   when diverges $ do
     throwError . fedError . FederationUnexpectedError $
-      "foooooooooooooooo keeping track of remote domains in the brig config file is deprecated, but as long as we \
+      "keeping track of remote domains in the brig config file is deprecated, but as long as we \
       \do that, adding a domain with different settings than in the config file is nto allowed.  want "
         <> ( "Just "
                <> cs (show fedComConf)

From 19f273282d263a26189bd2b7ffd89fc31dac6181 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 23 May 2023 09:26:50 +0200
Subject: [PATCH 100/220] ...

---
 services/brig/src/Brig/API/Internal.hs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 09256e5604..e4319cde5e 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -198,6 +198,8 @@ addFederationRemote fedDomConf = do
         "Maximum number of remote backends reached.  If you need to create more connections, \
         \please contact wire.com."
 
+-- | Compile config file list into a map indexed by domains.  Use this to make sure the config
+-- file is consistent (ie., no two entries for the same domain).
 remotesMapFromCfgFile :: AppT r (Map Domain FederationDomainConfig)
 remotesMapFromCfgFile = do
   cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings)
@@ -208,6 +210,8 @@ remotesMapFromCfgFile = do
           else error $ "error in config file: conflicting parameters on domain: " <> show (c, c')
   pure $ Map.fromListWith merge dict
 
+-- | Return the config file list.  Use this to make sure the config file is consistent (ie.,
+-- no two entries for the same domain).  Based on `remotesMapFromCfgFile`.
 remotesListFromCfgFile :: AppT r [FederationDomainConfig]
 remotesListFromCfgFile = Map.elems <$> remotesMapFromCfgFile
 
@@ -283,10 +287,13 @@ assertNoDomainsFromConfigFiles dom = do
       "keeping track of remote domains in the brig config file is deprecated, but as long as we \
       \do that, removing or updating items listed in the config file is not allowed."
 
+-- | Remove the entry from the database if present (or do nothing if not).  This responds with
+-- 533 if the entry was also present in the config file, but only *after* it has removed the
+-- entry from cassandra.
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
-  assertNoDomainsFromConfigFiles dom
   lift . wrapClient . Data.deleteFederationRemote $ dom
+  assertNoDomainsFromConfigFiles dom
 
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)

From d662c85db864521588067ef26cddbc3c94146094 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 23 May 2023 09:47:05 +0200
Subject: [PATCH 101/220] Tests are passing!

---
 integration/test/Test/Brig.hs             |  2 +-
 services/brig/brig.cabal                  |  1 +
 services/brig/src/Brig/API/Internal.hs    |  6 +++++-
 services/brig/src/Brig/Data/Federation.hs | 12 +++++++++---
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 293766ae3d..f49ee2322a 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -87,7 +87,7 @@ testCrudFederationRemotes = do
   addOnce remote1 (remote1 : cfgRemotes)
   addOnce remote1 (remote1 : cfgRemotes) -- idempotency
   -- update
-  updateOnce (Internal.domain remote1) remote1' (remote1' : cfgRemotes) -- This fails
+  updateOnce (Internal.domain remote1) remote1' (remote1' : cfgRemotes)
   updateFail (Internal.domain remote1) remote1''
   -- delete
   deleteOnce (Internal.domain remote1) cfgRemotes
diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 5e63f987c9..bcd9b3f4f9 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -208,6 +208,7 @@ library
     , conduit                    >=1.2.8
     , containers                 >=0.5
     , cookie                     >=0.4
+    , cql
     , cryptobox-haskell          >=0.1.1
     , currency-codes             >=2.0
     , data-default               >=0.5
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index e4319cde5e..6811fac465 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -270,7 +270,11 @@ updateFederationRemotes :: Domain -> FederationDomainConfig -> ExceptT Brig.API.
 updateFederationRemotes dom fedcfg = do
   assertDomainIsNotUpdated dom fedcfg
   assertNoDomainsFromConfigFiles dom
-  lift . wrapClient . Data.updateFederationRemote $ fedcfg
+  (lift . wrapClient . Data.updateFederationRemote $ fedcfg) >>= \case
+    True -> pure ()
+    False ->
+      throwError . fedError . FederationUnexpectedError . cs $
+        "federation domain does not exist and cannot be updated: " <> show (dom, fedcfg)
 
 assertDomainIsNotUpdated :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 assertDomainIsNotUpdated dom fedcfg = do
diff --git a/services/brig/src/Brig/Data/Federation.hs b/services/brig/src/Brig/Data/Federation.hs
index ee06bbedcf..a8f3b88df3 100644
--- a/services/brig/src/Brig/Data/Federation.hs
+++ b/services/brig/src/Brig/Data/Federation.hs
@@ -26,7 +26,10 @@ where
 
 import Brig.Data.Instances ()
 import Cassandra
+import Control.Exception (ErrorCall (ErrorCall))
+import Control.Monad.Catch (throwM)
 import Data.Domain
+import Database.CQL.Protocol (SerialConsistency (LocalSerialConsistency), serialConsistency)
 import Imports
 import Wire.API.Routes.FederationDomainConfig
 import Wire.API.User.Search
@@ -55,11 +58,14 @@ addFederationRemote (FederationDomainConfig rdom searchpolicy) = do
     add :: PrepQuery W (Domain, FederatedUserSearchPolicy) ()
     add = "INSERT INTO federation_remotes (domain, search_policy) VALUES (?, ?)"
 
-updateFederationRemote :: MonadClient m => FederationDomainConfig -> m ()
+updateFederationRemote :: MonadClient m => FederationDomainConfig -> m Bool
 updateFederationRemote (FederationDomainConfig rdom spol) = do
-  retry x1 $ write upd (params LocalQuorum (spol, rdom))
+  (retry x1 $ trans upd (params LocalQuorum (spol, rdom)) {serialConsistency = Just LocalSerialConsistency}) >>= \case
+    [] -> pure False
+    [_] -> pure True
+    _ -> throwM $ ErrorCall "Primary key violation detected federation_remotes"
   where
-    upd :: PrepQuery W (FederatedUserSearchPolicy, Domain) ()
+    upd :: PrepQuery W (FederatedUserSearchPolicy, Domain) x
     upd = "UPDATE federation_remotes SET search_policy = ? WHERE domain = ? IF EXISTS"
 
 deleteFederationRemote :: MonadClient m => Domain -> m ()

From f6ae0c7aab7a885d25ca0366804cb89d7f3cdfa0 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 23 May 2023 11:01:56 +0200
Subject: [PATCH 102/220] process leif's feedback.

---
 charts/brig/templates/configmap.yaml          |  6 ++++++
 charts/federator/templates/configmap.yaml     | 13 -------------
 charts/federator/values.yaml                  |  2 --
 hack/helm_vars/wire-server/values.yaml.gotmpl |  2 --
 services/brig/brig.integration.yaml           |  4 +++-
 services/federator/federator.integration.yaml | 16 +---------------
 6 files changed, 10 insertions(+), 33 deletions(-)

diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml
index 60cffe82ad..116141ffd2 100644
--- a/charts/brig/templates/configmap.yaml
+++ b/charts/brig/templates/configmap.yaml
@@ -247,6 +247,12 @@ data:
       setPropertyMaxValueLen: {{ .setPropertyMaxValueLen }}
       setDeleteThrottleMillis: {{ .setDeleteThrottleMillis }}
       setFederationDomain: {{ .setFederationDomain }}
+      {{- if .setFederationStrategy }}
+      setFederationStrategy: {{ toYaml .setFederationStrategy | nindent 8 }}
+      {{- end }}
+      {{- if .setFederationDomainConfigsUpdateFreq }}
+      setFederationDomainConfigsUpdateFreq: {{ toYaml .setFederationDomainConfigsUpdateFreq | nindent 8 }}
+      {{- end }}
       {{- if .setFederationDomainConfigs }}
       # 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
       # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
diff --git a/charts/federator/templates/configmap.yaml b/charts/federator/templates/configmap.yaml
index cbfd53e9e7..44de627107 100644
--- a/charts/federator/templates/configmap.yaml
+++ b/charts/federator/templates/configmap.yaml
@@ -51,18 +51,5 @@ data:
       clientCertificate: "/etc/wire/federator/secrets/tls.crt"
       clientPrivateKey: "/etc/wire/federator/secrets/tls.key"
       useSystemCAStore: {{ .useSystemCAStore }}
-      federationStrategy:
-        {{- if .federationStrategy.allowAll }}
-        allowAll:
-        {{- else if .federationStrategy.allowedDomains }}
-        allowedDomains:
-        {{- range $domain := .federationStrategy.allowedDomains }}
-          - {{ $domain | quote }}
-        {{- end }}
-        {{- else }}
-        # In gotemplate there is no way to distinguish between empty list and no
-        # list, we assume empty list when there is no list
-        allowedDomains: []
-        {{- end}}
     {{- end }}
     {{- end }}
diff --git a/charts/federator/values.yaml b/charts/federator/values.yaml
index 8e8a777e6a..e8bfac0280 100644
--- a/charts/federator/values.yaml
+++ b/charts/federator/values.yaml
@@ -41,5 +41,3 @@ config:
     # A client certificate and corresponding private key can be specified
     # similarly to a custom CA store.
     useSystemCAStore: true
-    federationStrategy:
-      allowedDomains: []
diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl
index e260b2f254..2b02af44d7 100644
--- a/hack/helm_vars/wire-server/values.yaml.gotmpl
+++ b/hack/helm_vars/wire-server/values.yaml.gotmpl
@@ -299,6 +299,4 @@ federator:
   imagePullPolicy: {{ .Values.imagePullPolicy }}
   config:
     optSettings:
-      federationStrategy:
-        allowAll: true
       useSystemCAStore: false
diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml
index 253fe9ff84..4c1395e593 100644
--- a/services/brig/brig.integration.yaml
+++ b/services/brig/brig.integration.yaml
@@ -189,6 +189,8 @@ optSettings:
   #   Remember to keep it the same in Galley.
   setFederationDomain: example.com
   setFeatureFlags:  # see #RefConfigOptions in `/docs/reference`
+  setFederationDomainConfigsUpdateFreq: 1
+  setFederationStrategy: allowList
   setFederationDomainConfigs:
     - domain: example.com
       search_policy: full_search
@@ -213,4 +215,4 @@ optSettings:
   setOAuthMaxActiveRefreshTokens: 10
 
 logLevel: Warn
-logNetStrings: false
\ No newline at end of file
+logNetStrings: false
diff --git a/services/federator/federator.integration.yaml b/services/federator/federator.integration.yaml
index 06a341d4f0..e0d8ec2355 100644
--- a/services/federator/federator.integration.yaml
+++ b/services/federator/federator.integration.yaml
@@ -21,22 +21,8 @@ optSettings:
   # Filepath to one or more PEM-encoded server certificates to use as a trust
   # store when making requests to remote backends
   remoteCAStore: "test/resources/integration-ca.pem"
-
-  # Would you like to federate with every wire-server installation ?
-  #
-  federationStrategy:
-    allowAll:
-  #
-  # or only with a select set of other wire-server installations?
-  #
-  # federationStrategy:
-  #   allowedDomains:
-  #     - wire.com
-  #     - example.com
-
   useSystemCAStore: false
-
   clientCertificate: "test/resources/integration-leaf.pem"
   clientPrivateKey: "test/resources/integration-leaf-key.pem"
   dnsHost: "127.0.0.1"
-  dnsPort: 9053
\ No newline at end of file
+  dnsPort: 9053

From f0bfa62ab1f2688291478394437675a2ba407c83 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 23 May 2023 17:16:35 +0200
Subject: [PATCH 103/220] ...

---
 integration/test/API/BrigInternal.hs | 48 ++++++++++++++--------------
 integration/test/SetupHelpers.hs     |  8 ++---
 integration/test/Test/Brig.hs        | 22 ++++++-------
 3 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 40f7bc07a2..12ba3c5810 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -64,48 +64,48 @@ instance FromJSON FedConn where
       <$> obj .: fromString "domain"
       <*> obj .: fromString "search_policy"
 
-createFedConn :: HasCallStack => FedConn -> App Response
-createFedConn fedConn = do
-  res <- createFedConn' fedConn
+createFedConn :: (HasCallStack, MakesValue dom) => dom -> FedConn -> App Response
+createFedConn dom fedConn = do
+  res <- createFedConn' dom fedConn
   res.status `shouldMatchRange` (200, 299)
   pure res
 
-createFedConn' :: HasCallStack => FedConn -> App Response
-createFedConn' fedConn = do
-  req <- rawBaseRequest ownDomain Brig Unversioned "/i/federation/remotes"
+createFedConn' :: (HasCallStack, MakesValue dom) => dom -> FedConn -> App Response
+createFedConn' dom fedConn = do
+  req <- rawBaseRequest dom Brig Unversioned "/i/federation/remotes"
   submit "POST" $ req & addJSON fedConn
 
-readFedConns :: HasCallStack => App Response
-readFedConns = do
-  res <- readFedConns'
+readFedConns :: (HasCallStack, MakesValue dom) => dom -> App Response
+readFedConns dom = do
+  res <- readFedConns' dom
   res.status `shouldMatchRange` (200, 299)
   pure res
 
-readFedConns' :: HasCallStack => App Response
-readFedConns' = do
-  req <- rawBaseRequest ownDomain Brig Unversioned "/i/federation/remotes"
+readFedConns' :: (HasCallStack, MakesValue dom) => dom -> App Response
+readFedConns' dom = do
+  req <- rawBaseRequest dom Brig Unversioned "/i/federation/remotes"
   submit "GET" req
 
-updateFedConn :: HasCallStack => String -> FedConn -> App Response
-updateFedConn dom fedConn = do
-  res <- updateFedConn' dom fedConn
+updateFedConn :: (HasCallStack, MakesValue owndom) => owndom -> String -> FedConn -> App Response
+updateFedConn owndom dom fedConn = do
+  res <- updateFedConn' owndom dom fedConn
   res.status `shouldMatchRange` (200, 299)
   pure res
 
-updateFedConn' :: HasCallStack => String -> FedConn -> App Response
-updateFedConn' dom fedConn = do
-  req <- rawBaseRequest ownDomain Brig Unversioned ("/i/federation/remotes/" <> dom)
+updateFedConn' :: (HasCallStack, MakesValue owndom) => owndom -> String -> FedConn -> App Response
+updateFedConn' owndom dom fedConn = do
+  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "PUT" (fedConn `addJSON` req)
 
-deleteFedConn :: HasCallStack => String -> App Response
-deleteFedConn dom = do
-  res <- deleteFedConn' dom
+deleteFedConn :: (HasCallStack, MakesValue owndom) => owndom -> String -> App Response
+deleteFedConn owndom dom = do
+  res <- deleteFedConn' owndom dom
   res.status `shouldMatchRange` (200, 299)
   pure res
 
-deleteFedConn' :: HasCallStack => String -> App Response
-deleteFedConn' dom = do
-  req <- rawBaseRequest ownDomain Brig Unversioned ("/i/federation/remotes/" <> dom)
+deleteFedConn' :: (HasCallStack, MakesValue owndom) => owndom -> String -> App Response
+deleteFedConn' owndom dom = do
+  req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "DELETE" req
 
 registerOAuthClient :: (HasCallStack, MakesValue user, MakesValue name, MakesValue url) => user -> name -> url -> App Response
diff --git a/integration/test/SetupHelpers.hs b/integration/test/SetupHelpers.hs
index 24b016b381..ac028577ae 100644
--- a/integration/test/SetupHelpers.hs
+++ b/integration/test/SetupHelpers.hs
@@ -59,10 +59,10 @@ getAllConvs u = do
     resp.json
   result %. "found" & asList
 
-resetFedConns :: HasCallStack => App ()
-resetFedConns = do
-  bindResponse Internal.readFedConns $ \resp -> do
+resetFedConns :: (HasCallStack, MakesValue owndom) => owndom -> App ()
+resetFedConns owndom = do
+  bindResponse (Internal.readFedConns owndom) $ \resp -> do
     rdoms :: [String] <- do
       rawlist <- resp.json %. "remotes" & asList
       (asString . (%. "domain")) `mapM` rawlist
-    Internal.deleteFedConn' `mapM_` rdoms
+    Internal.deleteFedConn' owndom `mapM_` rdoms
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 7efd9dfe4a..c47e4100bc 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -32,38 +32,38 @@ testCrudFederationRemotes = do
 
       addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
       addOnce fedConn want = do
-        res <- Internal.createFedConn fedConn
+        res <- Internal.createFedConn OwnDomain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
-        res2 <- parseFedConns =<< Internal.readFedConns
+        res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
         sort res2 `shouldMatch` sort want
 
       addFail :: HasCallStack => Internal.FedConn -> App ()
       addFail fedConn = do
-        res <- Internal.createFedConn' fedConn
+        res <- Internal.createFedConn' OwnDomain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
       deleteOnce :: HasCallStack => String -> [Internal.FedConn] -> App ()
       deleteOnce domain want = do
-        res <- Internal.deleteFedConn domain
+        res <- Internal.deleteFedConn OwnDomain domain
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
-        res2 <- parseFedConns =<< Internal.readFedConns
+        res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
         sort res2 `shouldMatch` sort want
 
       deleteFail :: HasCallStack => String -> App ()
       deleteFail del = do
-        res <- Internal.deleteFedConn' del
+        res <- Internal.deleteFedConn' OwnDomain del
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
       updateOnce :: HasCallStack => String -> Internal.FedConn -> [Internal.FedConn] -> App ()
       updateOnce domain fedConn want = do
-        res <- Internal.updateFedConn domain fedConn
+        res <- Internal.updateFedConn OwnDomain domain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
-        res2 <- parseFedConns =<< Internal.readFedConns
+        res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
         sort res2 `shouldMatch` sort want
 
       updateFail :: HasCallStack => String -> Internal.FedConn -> App ()
       updateFail domain fedConn = do
-        res <- Internal.updateFedConn' domain fedConn
+        res <- Internal.updateFedConn' OwnDomain domain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
   let remote1, remote1', remote1'' :: Internal.FedConn
@@ -74,8 +74,8 @@ testCrudFederationRemotes = do
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
 
-  resetFedConns
-  cfgRemotes <- parseFedConns =<< Internal.readFedConns
+  resetFedConns OwnDomain
+  cfgRemotes <- parseFedConns =<< Internal.readFedConns OwnDomain
   cfgRemotes `shouldMatch` [cfgRemotesExpect]
   -- entries present in the config file can be idempotently added if identical, but cannot be
   -- updated, deleted or updated.

From 2e959b059056e03c3b8c2ef75ca25b28a9c61cbb Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 24 May 2023 13:24:00 +1000
Subject: [PATCH 104/220] Fixing compile issues after a merge

---
 integration/test/API/BrigInternal.hs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 40f7bc07a2..18b421398b 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -5,6 +5,7 @@ import qualified Data.Aeson as Aeson
 import Data.Function
 import Data.Maybe
 import Testlib.Prelude
+import Control.Monad.Reader (asks)
 
 data CreateUser = CreateUser
   { email :: Maybe String,
@@ -72,6 +73,7 @@ createFedConn fedConn = do
 
 createFedConn' :: HasCallStack => FedConn -> App Response
 createFedConn' fedConn = do
+  ownDomain <- asks domain1
   req <- rawBaseRequest ownDomain Brig Unversioned "/i/federation/remotes"
   submit "POST" $ req & addJSON fedConn
 
@@ -83,6 +85,7 @@ readFedConns = do
 
 readFedConns' :: HasCallStack => App Response
 readFedConns' = do
+  ownDomain <- asks domain1
   req <- rawBaseRequest ownDomain Brig Unversioned "/i/federation/remotes"
   submit "GET" req
 
@@ -94,6 +97,7 @@ updateFedConn dom fedConn = do
 
 updateFedConn' :: HasCallStack => String -> FedConn -> App Response
 updateFedConn' dom fedConn = do
+  ownDomain <- asks domain1
   req <- rawBaseRequest ownDomain Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "PUT" (fedConn `addJSON` req)
 
@@ -105,6 +109,7 @@ deleteFedConn dom = do
 
 deleteFedConn' :: HasCallStack => String -> App Response
 deleteFedConn' dom = do
+  ownDomain <- asks domain1
   req <- rawBaseRequest ownDomain Brig Unversioned ("/i/federation/remotes/" <> dom)
   submit "DELETE" req
 

From 37b4f094563b58c6ad98eb941df5904c945f2e3d Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 24 May 2023 13:59:42 +1000
Subject: [PATCH 105/220] Updating templates

---
 hack/helm_vars/wire-server/values.yaml.gotmpl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl
index 2b02af44d7..cede80f96e 100644
--- a/hack/helm_vars/wire-server/values.yaml.gotmpl
+++ b/hack/helm_vars/wire-server/values.yaml.gotmpl
@@ -85,6 +85,8 @@ brig:
           search_policy: full_search
         - domain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
           search_policy: full_search
+      setFederationStrategy: allowList
+      setFederationDomainConfigsUpdateFreq: 10
       set2FACodeGenerationDelaySecs: 5
       setNonceTtlSecs: 300
       setDpopMaxSkewSecs: 1

From 8f8af7cb683b48fdd3bc127b683398306fb38cb8 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 24 May 2023 13:59:47 +1000
Subject: [PATCH 106/220] Removing a JSON roundtrip, better using aeson.

---
 integration/test/Test/Brig.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 7efd9dfe4a..8b95bb0c53 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -4,12 +4,12 @@ import qualified API.Brig as Public
 import qualified API.BrigInternal as Internal
 import qualified API.Common as API
 import qualified API.GalleyInternal as Internal
-import qualified Data.Aeson as Aeson
 import Data.String.Conversions
 import GHC.Stack
 import SetupHelpers
 import Testlib.Assertions
 import Testlib.Prelude
+import Data.Aeson.Types (parseMaybe)
 
 testSearchContactForExternalUsers :: HasCallStack => App ()
 testSearchContactForExternalUsers = do
@@ -28,7 +28,7 @@ testCrudFederationRemotes = do
       parseFedConns resp = do
         -- TODO: not idiomatic!  try `getJSON 200 resp %. "remotes" & asList & mapM asObjOrSomething`
         -- Some ideas: There is asList to assert that a Value is actually a an array of Values. Then you can sort that to have a defined order.
-        fromJust . Aeson.decode . Aeson.encode . fromJust <$> ((`lookupField` "remotes") =<< getJSON 200 resp)
+        fromJust . parseMaybe parseJSON . fromJust <$> ((`lookupField` "remotes") =<< getJSON 200 resp)
 
       addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
       addOnce fedConn want = do

From 236f4bb93e47ffbc38195546b39e2e6e774dcb11 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 24 May 2023 16:18:16 +1000
Subject: [PATCH 107/220] Moving more of the federation domain update code into
 wire-api.

Tests are broken with these changes.
---
 .../wire-api/src/Wire/API/FederationUpdate.hs | 29 +++++++----
 libs/wire-api/wire-api.cabal                  |  2 +
 services/cannon/cannon.cabal                  |  1 -
 services/cannon/src/Cannon/Run.hs             | 17 +++---
 services/federator/federator.cabal            |  1 -
 services/federator/src/Federator/Run.hs       | 25 +++------
 services/galley/src/Galley/App.hs             | 16 ++----
 services/galley/src/Galley/Run.hs             | 52 +++++++------------
 services/galley/test/integration/API/Util.hs  |  4 +-
 services/gundeck/gundeck.cabal                |  1 -
 services/gundeck/src/Gundeck/Run.hs           | 10 +---
 11 files changed, 62 insertions(+), 96 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 8c56daa52a..dd3a182ee5 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -1,20 +1,22 @@
 module Wire.API.FederationUpdate
   ( FedUpdateCallback,
-    getAllowedDomainsInitial,
-    getAllowedDomainsLoop,
-    getAllowedDomainsLoop',
+    updateFedDomains
   )
 where
 
 import Control.Exception (ErrorCall (ErrorCall), throwIO)
 import qualified Control.Retry as R
 import Imports
-import Servant.Client (ClientEnv, ClientError, runClientM)
-import Servant.Client.Internal.HttpClient (ClientM)
+import Servant.Client (ClientEnv (ClientEnv), ClientError, runClientM, BaseUrl (BaseUrl), Scheme (Http))
+import Servant.Client.Internal.HttpClient (ClientM, defaultMakeClientRequest)
 import qualified System.Logger as L
 import Wire.API.Routes.FederationDomainConfig (FederationDomainConfigs (updateInterval))
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
+import Util.Options (Endpoint (..))
+import Control.Concurrent.Async
+import Network.HTTP.Client (newManager, defaultManagerSettings)
+import Data.Text (unpack)
 
 getFedRemotes :: ClientM FederationDomainConfigs
 getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
@@ -42,12 +44,13 @@ getAllowedDomainsInitial logger clientEnv =
 getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
 getAllowedDomains = runClientM getFedRemotes
 
+-- Old value -> new value -> action
 type FedUpdateCallback = FederationDomainConfigs -> FederationDomainConfigs -> IO ()
 
 -- The callback takes the previous and the new values of the federation domain configs
 -- and runs a given action. This function is not called if a new config value cannot be fetched.
-getAllowedDomainsLoop :: L.Logger -> ClientEnv -> IORef FederationDomainConfigs -> FedUpdateCallback -> IO ()
-getAllowedDomainsLoop logger clientEnv env callback = forever $ do
+getAllowedDomainsLoop :: L.Logger -> ClientEnv -> FedUpdateCallback -> IORef FederationDomainConfigs -> IO ()
+getAllowedDomainsLoop logger clientEnv callback env = forever $ do
   getAllowedDomains clientEnv >>= \case
     Left e ->
       L.log logger L.Fatal $
@@ -60,7 +63,11 @@ getAllowedDomainsLoop logger clientEnv env callback = forever $ do
   delay <- updateInterval <$> readIORef env
   threadDelay (delay * 1_000_000)
 
--- A version where the callback isn't needed. Most of the services don't care about
--- when the list changes, just that they have the new list and can use it as-is
-getAllowedDomainsLoop' :: L.Logger -> ClientEnv -> IORef FederationDomainConfigs -> IO ()
-getAllowedDomainsLoop' logger c r = getAllowedDomainsLoop logger c r $ \_ _ -> pure ()
+updateFedDomains :: Endpoint -> L.Logger -> FedUpdateCallback -> IO (IORef FederationDomainConfigs, Async ())
+updateFedDomains (Endpoint h p) log' cb = do
+  clientEnv <- newManager defaultManagerSettings <&> \mgr -> ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
+  ioref <- newIORef =<< getAllowedDomainsInitial log' clientEnv
+  updateDomainsThread <- async $ getAllowedDomainsLoop log' clientEnv cb ioref
+  pure (ioref, updateDomainsThread)
+  where
+    baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
\ No newline at end of file
diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal
index 0b767123f9..010c0850dc 100644
--- a/libs/wire-api/wire-api.cabal
+++ b/libs/wire-api/wire-api.cabal
@@ -217,6 +217,7 @@ library
 
   build-depends:
       aeson                      >=2.0.1.0
+    , async
     , attoparsec                 >=0.10
     , base                       >=4       && <5
     , base64-bytestring          >=1.0
@@ -250,6 +251,7 @@ library
     , hscim
     , HsOpenSSL
     , http-api-data
+    , http-client
     , http-media
     , http-types
     , imports
diff --git a/services/cannon/cannon.cabal b/services/cannon/cannon.cabal
index 2be10d9594..75c1e6188c 100644
--- a/services/cannon/cannon.cabal
+++ b/services/cannon/cannon.cabal
@@ -100,7 +100,6 @@ library
     , mwc-random             >=0.13
     , retry                  >=0.7
     , safe-exceptions
-    , servant-client
     , servant-conduit
     , servant-server
     , strict                 >=0.3.2
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index ea1cce9b24..4fa3f9c33d 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -37,7 +37,7 @@ import Data.Metrics.Middleware (gaugeSet, path)
 import qualified Data.Metrics.Middleware as Middleware
 import Data.Metrics.Servant
 import Data.Proxy
-import Data.Text (pack, strip, unpack)
+import Data.Text (pack, strip)
 import Data.Text.Encoding (encodeUtf8)
 import Imports hiding (head)
 import qualified Network.Wai as Wai
@@ -45,7 +45,6 @@ import Network.Wai.Handler.Warp hiding (run)
 import qualified Network.Wai.Middleware.Gzip as Gzip
 import Network.Wai.Utilities.Server
 import Servant
-import Servant.Client
 import qualified System.IO.Strict as Strict
 import qualified System.Logger.Class as LC
 import qualified System.Logger.Extended as L
@@ -53,10 +52,11 @@ import System.Posix.Signals
 import qualified System.Posix.Signals as Signals
 import System.Random.MWC (createSystemRandom)
 import UnliftIO.Concurrent (myThreadId, throwTo)
-import Wire.API.FederationUpdate (getAllowedDomainsInitial, getAllowedDomainsLoop')
+import Wire.API.FederationUpdate
 import qualified Wire.API.Routes.Internal.Cannon as Internal
 import Wire.API.Routes.Public.Cannon
 import Wire.API.Routes.Version.Wai
+import Util.Options (Endpoint(..))
 
 type CombinedAPI = PublicAPI :<|> Internal.API
 
@@ -79,14 +79,9 @@ run o = do
   s <- newSettings $ Server (o ^. cannon . host) (o ^. cannon . port) (applog e) m (Just idleTimeout)
 
   -- Get the federation domain list from Brig and start the updater loop
-  -- TODO: move this block into a function in libs/wire-api/src/Wire/API/FederationUpdate.hs; check all services for the same block and use the function.
-  manager <- newManager defaultManagerSettings
-  let Brig bh bp = o ^. brig
-      baseUrl = BaseUrl Http (unpack bh) (fromIntegral bp) ""
-      clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-  fedStrat <- getAllowedDomainsInitial g clientEnv
-  ioref <- newIORef fedStrat
-  updateDomainsThread <- Async.async $ getAllowedDomainsLoop' g clientEnv ioref
+  let brigEndpoint = Endpoint bh bp
+      Brig bh bp = o ^. brig
+  (_, updateDomainsThread) <- updateFedDomains brigEndpoint g (\_ _ -> pure ())
 
   let middleware :: Wai.Middleware
       middleware =
diff --git a/services/federator/federator.cabal b/services/federator/federator.cabal
index 995d3d0c92..24ab6552e0 100644
--- a/services/federator/federator.cabal
+++ b/services/federator/federator.cabal
@@ -132,7 +132,6 @@ library
     , polysemy
     , polysemy-wire-zoo
     , servant
-    , servant-client
     , servant-client-core
     , string-conversions
     , text
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 870b6d8099..9b2e6a390a 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -36,10 +36,9 @@ where
 
 import Control.Concurrent.Async
 import Control.Exception (bracket)
-import Control.Lens (view, (^.))
+import Control.Lens ((^.))
 import Data.Default (def)
 import qualified Data.Metrics.Middleware as Metrics
-import Data.Text
 import Federator.Env
 import Federator.ExternalServer (serveInward)
 import Federator.InternalServer (serveOutward)
@@ -47,9 +46,7 @@ import Federator.Monitor
 import Federator.Options as Opt
 import Imports
 import qualified Network.DNS as DNS
-import Network.HTTP.Client
 import qualified Network.HTTP.Client as HTTP
-import Servant.Client
 import qualified System.Logger as Log
 import qualified System.Logger.Extended as LogExt
 import Util.Options
@@ -64,19 +61,14 @@ import qualified Wire.Network.DNS.Helper as DNS
 -- FUTUREWORK(federation): Add metrics and status endpoints
 run :: Opts -> IO ()
 run opts = do
-  manager <- newManager defaultManagerSettings
   logger <- LogExt.mkLogger (Opt.logLevel opts) (Opt.logNetStrings opts) (Opt.logFormat opts)
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
-      Endpoint host port = brig opts
-      baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
-      clientEnv = ClientEnv manager baseUrl Nothing defaultMakeClientRequest
-  okRemoteDomains <- getAllowedDomainsInitial logger clientEnv
-  DNS.withCachingResolver resolvConf $ \res ->
-    bracket (newEnv opts res logger okRemoteDomains) closeEnv $ \env -> do
+  DNS.withCachingResolver resolvConf $ \res -> do
+    (ioref, updateAllowedDomainsThread) <- updateFedDomains (brig opts) logger (\_ _ -> pure ())
+    bracket (newEnv opts res logger ioref) closeEnv $ \env -> do
       let externalServer = serveInward env portExternal
           internalServer = serveOutward env portInternal
       withMonitor logger (onNewSSLContext env) (optSettings opts) $ do
-        updateAllowedDomainsThread <- async (getAllowedDomainsLoop' logger clientEnv $ view allowedRemoteDomains env)
         internalServerThread <- async internalServer
         externalServerThread <- async externalServer
         void $ waitAnyCancel [updateAllowedDomainsThread, internalServerThread, externalServerThread]
@@ -99,16 +91,15 @@ run opts = do
 -------------------------------------------------------------------------------
 -- Environment
 
-newEnv :: Opts -> DNS.Resolver -> Log.Logger -> FederationDomainConfigs -> IO Env
-newEnv o _dnsResolver _applog okRemoteDomains = do
+newEnv :: Opts -> DNS.Resolver -> Log.Logger -> IORef FederationDomainConfigs -> IO Env
+newEnv o _dnsResolver _applog _allowedRemoteDomains = do
   _metrics <- Metrics.metrics
   let _requestId = def
-  let _runSettings = Opt.optSettings o
-  let _service Brig = Opt.brig o
+      _runSettings = Opt.optSettings o
+      _service Brig = Opt.brig o
       _service Galley = Opt.galley o
       _service Cargohold = Opt.cargohold o
   _httpManager <- initHttpManager
-  _allowedRemoteDomains <- newIORef okRemoteDomains
   sslContext <- mkTLSSettingsOrThrow _runSettings
   _http2Manager <- newIORef =<< mkHttp2Manager sslContext
   pure Env {..}
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index f77dffcd5e..c77ba37eea 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -99,7 +99,6 @@ import Polysemy.Internal (Append)
 import Polysemy.Resource
 import qualified Polysemy.TinyLog as P
 import qualified Servant
-import qualified Servant.Client as SC
 import Ssl.Util
 import qualified System.Logger as Log
 import System.Logger.Class
@@ -108,7 +107,7 @@ import qualified UnliftIO.Exception as UnliftIO
 import Util.Options
 import Wire.API.Error
 import Wire.API.Federation.Error
-import Wire.API.FederationUpdate
+import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.Sem.Logger
 
 -- Effects needed by the interpretation of other effects
@@ -152,28 +151,21 @@ validateOptions l o = do
   when (settings ^. setMaxTeamSize < optFanoutLimit) $
     error "setMaxTeamSize cannot be < setTruncationLimit"
 
-createEnv :: Metrics -> Opts -> IO Env
-createEnv m o = do
+createEnv :: Metrics -> Opts -> IORef FederationDomainConfigs -> IO Env
+createEnv m o r = do
   l <- Logger.mkLogger (o ^. optLogLevel) (o ^. optLogNetStrings) (o ^. optLogFormat)
   cass <- initCassandra o l
   mgr <- initHttpManager o
   h2mgr <- initHttp2Manager
   validateOptions l o
 
-  -- Fetch the initial federation domain list so we always start with
-  -- a known update to date dataset.
-
   let brigEndpoint = o ^. optBrig
-      Endpoint h p = brigEndpoint
-      baseUrl = SC.BaseUrl SC.Http (unpack h) (fromIntegral p) ""
-      clientEnv = SC.ClientEnv mgr baseUrl Nothing SC.defaultMakeClientRequest
-  strat <- getAllowedDomainsInitial l clientEnv
   Env def m o l mgr h2mgr (o ^. optFederator) brigEndpoint cass
     <$> Q.new 16000
     <*> initExtEnv
     <*> maybe (pure Nothing) (fmap Just . Aws.mkEnv l mgr) (o ^. optJournal)
     <*> loadAllMLSKeys (fold (o ^. optSettings . setMlsPrivateKeyPaths))
-    <*> newIORef strat
+    <*> pure r
 
 initCassandra :: Opts -> Logger -> IO ClientState
 initCassandra o l = do
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index ab4b4b83ce..f1285a5901 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -49,7 +49,6 @@ import Galley.App
 import qualified Galley.App as App
 import Galley.Aws (awsEnv)
 import Galley.Cassandra
-import Galley.Env (fedDomains)
 import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
@@ -61,12 +60,6 @@ import qualified Network.Wai.Middleware.Gunzip as GZip
 import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
 import Servant hiding (route)
-import Servant.Client
-  ( BaseUrl (BaseUrl),
-    ClientEnv (ClientEnv),
-    Scheme (Http),
-    defaultMakeClientRequest,
-  )
 import qualified System.Logger as Log
 import Util.Options
 import Wire.API.FederationUpdate
@@ -74,10 +67,17 @@ import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
+import System.Logger.Extended (mkLogger)
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
-  (app, env) <- mkApp opts
+  (ioref, _) <- lift $ do
+    -- Duplicating the logger from App.createEnv so that we don't have to deal
+    -- with recursive monadic actions to get the logger before we have the initial
+    -- IORef of values from brig. It's just easier this way.
+    l <- mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
+    updateFedDomains (opts ^. optBrig) l callback
+  (app, env) <- mkApp opts ioref
   settings <-
     lift $
       newSettings $
@@ -89,17 +89,16 @@ run opts = lowerCodensity $ do
 
   forM_ (env ^. aEnv) $ \aws ->
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
-  void $ Codensity $ Async.withAsync $ runApp env updateFedDomains
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
   void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
-mkApp :: Opts -> Codensity IO (Application, Env)
-mkApp opts =
+mkApp :: Opts -> IORef FederationDomainConfigs -> Codensity IO (Application, Env)
+mkApp opts fedDoms =
   do
     metrics <- lift $ M.metrics
-    env <- lift $ App.createEnv metrics opts
+    env <- lift $ App.createEnv metrics opts fedDoms
     lift $ runClient (env ^. cstate) $ versionCheck schemaVersion
 
     let logger = env ^. App.applog
@@ -181,23 +180,12 @@ collectAuthMetrics m env = do
       gaugeTokenRemaing m mbRemaining
       threadDelay 1_000_000
 
-updateFedDomains :: App ()
-updateFedDomains = do
-  ioref <- view fedDomains
-  logger <- view applog
-  manager' <- view manager
-  Endpoint host port <- view brig
-  let baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
-      clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
-
-  liftIO $ do
-    okRemoteDomains <- getAllowedDomainsInitial logger clientEnv
-    atomicWriteIORef ioref okRemoteDomains
-    let domainListsEqual old new =
-          Set.fromList (domain <$> remotes old)
-            == Set.fromList (domain <$> remotes new)
-        callback old new = unless (domainListsEqual old new) $ do
-          -- TODO: perform the database updates here
-          -- This code will only run when there is a change in the domain lists
-          pure ()
-    getAllowedDomainsLoop logger clientEnv ioref callback
+callback :: FedUpdateCallback
+callback old new = unless (domainListsEqual old new) $ do
+  -- TODO: perform the database updates here
+  -- This code will only run when there is a change in the domain lists
+  pure ()
+  where
+    domainListsEqual o n =
+      Set.fromList (domain <$> remotes o) ==
+      Set.fromList (domain <$> remotes n)
\ No newline at end of file
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index 6341179f8f..1be79e7e3f 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -144,6 +144,7 @@ import Wire.API.User.Auth hiding (Access)
 import Wire.API.User.Client
 import qualified Wire.API.User.Client as Client
 import Wire.API.User.Client.Prekey
+import Wire.API.Routes.FederationDomainConfig
 
 -------------------------------------------------------------------------------
 -- API Operations
@@ -2456,7 +2457,8 @@ instance HasSettingsOverrides TestM where
     ts :: TestSetup <- ask
     let opts = f (ts ^. tsGConf)
     liftIO . lowerCodensity $ do
-      (galleyApp, _env) <- Run.mkApp opts
+      ioref <- newIORef defFederationDomainConfigs 
+      (galleyApp, _env) <- Run.mkApp opts ioref
       port' <- withMockServer galleyApp
       liftIO $
         runReaderT
diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal
index a7465feca1..8f586043cd 100644
--- a/services/gundeck/gundeck.cabal
+++ b/services/gundeck/gundeck.cabal
@@ -133,7 +133,6 @@ library
     , resourcet              >=1.1
     , retry                  >=0.5
     , safe-exceptions
-    , servant-client
     , servant-server
     , text                   >=1.1
     , time                   >=1.4
diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs
index 34a7888908..e83b11a614 100644
--- a/services/gundeck/src/Gundeck/Run.hs
+++ b/services/gundeck/src/Gundeck/Run.hs
@@ -43,14 +43,12 @@ import Gundeck.Options
 import Gundeck.React
 import Gundeck.ThreadBudget
 import Imports hiding (head)
-import Network.HTTP.Client (defaultManagerSettings, newManager)
 import Network.Wai as Wai
 import qualified Network.Wai.Middleware.Gunzip as GZip
 import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server hiding (serverPort)
 import Servant (Handler (Handler), (:<|>) (..))
 import qualified Servant
-import Servant.Client
 import qualified System.Logger as Log
 import qualified UnliftIO.Async as Async
 import Util.Options
@@ -69,13 +67,7 @@ run o = do
   let throttleMillis = fromMaybe defSqsThrottleMillis $ o ^. (optSettings . setSqsThrottleMillis)
 
   -- Get the federation domain list from Brig and start the updater loop
-  mgr <- newManager defaultManagerSettings
-  let Endpoint host port = o ^. optBrig
-      baseUrl = BaseUrl Http (unpack host) (fromIntegral port) ""
-      clientEnv = ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
-  fedStrat <- getAllowedDomainsInitial l clientEnv
-  ioref <- newIORef fedStrat
-  updateDomainsThread <- Async.async $ getAllowedDomainsLoop' l clientEnv ioref
+  (_, updateDomainsThread) <- updateFedDomains (o ^. optBrig) l (\_ _ -> pure ())
 
   lst <- Async.async $ Aws.execute (e ^. awsEnv) (Aws.listen throttleMillis (runDirect e . onEvent))
   wtbs <- forM (e ^. threadBudgetState) $ \tbs -> Async.async $ runDirect e $ watchThreadBudgetState m tbs 10

From fff988390f3ce7d8d292c72bfb39ca9861493b20 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 24 May 2023 17:24:00 +1000
Subject: [PATCH 108/220] wip

---
 services/galley/test/integration/API/Util.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index 1be79e7e3f..2dc1555eb6 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -2457,7 +2457,7 @@ instance HasSettingsOverrides TestM where
     ts :: TestSetup <- ask
     let opts = f (ts ^. tsGConf)
     liftIO . lowerCodensity $ do
-      ioref <- newIORef defFederationDomainConfigs 
+      ioref <- newIORef $ FederationDomainConfigs AllowAll [] $ updateInterval defFederationDomainConfigs
       (galleyApp, _env) <- Run.mkApp opts ioref
       port' <- withMockServer galleyApp
       liftIO $

From 4efa58ee684e820add9dba068f3688a5bf67f12d Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 24 May 2023 10:11:24 +0200
Subject: [PATCH 109/220] sanitize-pr

---
 integration/test/API/BrigInternal.hs           |  1 -
 integration/test/Test/Brig.hs                  |  2 +-
 integration/test/Testlib/Types.hs              |  3 ++-
 libs/wire-api/default.nix                      |  5 +++++
 libs/wire-api/src/Wire/API/FederationUpdate.hs | 14 +++++++-------
 services/brig/default.nix                      |  3 +++
 services/brig/src/Brig/API/Internal.hs         |  2 --
 services/brig/src/Brig/Data/Federation.hs      |  2 +-
 services/cannon/default.nix                    |  2 --
 services/cannon/src/Cannon/Run.hs              |  2 +-
 services/federator/default.nix                 |  1 -
 services/galley/src/Galley/Run.hs              |  6 +++---
 services/galley/test/integration/API/Util.hs   |  4 ++--
 services/gundeck/default.nix                   |  2 --
 14 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 6a1307c25b..12ba3c5810 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -1,7 +1,6 @@
 module API.BrigInternal where
 
 import API.Common
-import Control.Monad.Reader (asks)
 import qualified Data.Aeson as Aeson
 import Data.Function
 import Data.Maybe
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index b32f611fb7..ae21bff68f 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -4,12 +4,12 @@ import qualified API.Brig as Public
 import qualified API.BrigInternal as Internal
 import qualified API.Common as API
 import qualified API.GalleyInternal as Internal
+import Data.Aeson.Types (parseMaybe)
 import Data.String.Conversions
 import GHC.Stack
 import SetupHelpers
 import Testlib.Assertions
 import Testlib.Prelude
-import Data.Aeson.Types (parseMaybe)
 
 testSearchContactForExternalUsers :: HasCallStack => App ()
 testSearchContactForExternalUsers = do
diff --git a/integration/test/Testlib/Types.hs b/integration/test/Testlib/Types.hs
index 2e5ac154c9..6dd90efaf1 100644
--- a/integration/test/Testlib/Types.hs
+++ b/integration/test/Testlib/Types.hs
@@ -32,7 +32,8 @@ data Response = Response
     status :: Int,
     headers :: [HTTP.Header],
     request :: HTTP.Request
-  } deriving Show
+  }
+  deriving (Show)
 
 instance HasField "json" Response (App Aeson.Value) where
   getField response = maybe (assertFailure "Response has no json body") pure response.jsonBody
diff --git a/libs/wire-api/default.nix b/libs/wire-api/default.nix
index 96184c85b8..c31ab27d3c 100644
--- a/libs/wire-api/default.nix
+++ b/libs/wire-api/default.nix
@@ -45,6 +45,7 @@
 , hspec
 , hspec-wai
 , http-api-data
+, http-client
 , http-media
 , http-types
 , imports
@@ -69,6 +70,7 @@
 , quickcheck-instances
 , random
 , resourcet
+, retry
 , saml2-web-sso
 , schema-profunctor
 , scientific
@@ -117,6 +119,7 @@ mkDerivation {
   src = gitignoreSource ./.;
   libraryHaskellDepends = [
     aeson
+    async
     attoparsec
     base
     base64-bytestring
@@ -150,6 +153,7 @@ mkDerivation {
     hscim
     HsOpenSSL
     http-api-data
+    http-client
     http-media
     http-types
     imports
@@ -171,6 +175,7 @@ mkDerivation {
     quickcheck-instances
     random
     resourcet
+    retry
     saml2-web-sso
     schema-profunctor
     scientific
diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index dd3a182ee5..af7d24cb47 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -1,22 +1,22 @@
 module Wire.API.FederationUpdate
   ( FedUpdateCallback,
-    updateFedDomains
+    updateFedDomains,
   )
 where
 
+import Control.Concurrent.Async
 import Control.Exception (ErrorCall (ErrorCall), throwIO)
 import qualified Control.Retry as R
+import Data.Text (unpack)
 import Imports
-import Servant.Client (ClientEnv (ClientEnv), ClientError, runClientM, BaseUrl (BaseUrl), Scheme (Http))
+import Network.HTTP.Client (defaultManagerSettings, newManager)
+import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), ClientError, Scheme (Http), runClientM)
 import Servant.Client.Internal.HttpClient (ClientM, defaultMakeClientRequest)
 import qualified System.Logger as L
+import Util.Options (Endpoint (..))
 import Wire.API.Routes.FederationDomainConfig (FederationDomainConfigs (updateInterval))
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
-import Util.Options (Endpoint (..))
-import Control.Concurrent.Async
-import Network.HTTP.Client (newManager, defaultManagerSettings)
-import Data.Text (unpack)
 
 getFedRemotes :: ClientM FederationDomainConfigs
 getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
@@ -70,4 +70,4 @@ updateFedDomains (Endpoint h p) log' cb = do
   updateDomainsThread <- async $ getAllowedDomainsLoop log' clientEnv cb ioref
   pure (ioref, updateDomainsThread)
   where
-    baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
\ No newline at end of file
+    baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
diff --git a/services/brig/default.nix b/services/brig/default.nix
index bad459003c..0e23334dba 100644
--- a/services/brig/default.nix
+++ b/services/brig/default.nix
@@ -28,6 +28,7 @@
 , conduit
 , containers
 , cookie
+, cql
 , cryptobox-haskell
 , currency-codes
 , data-default
@@ -187,6 +188,7 @@ mkDerivation {
     conduit
     containers
     cookie
+    cql
     cryptobox-haskell
     currency-codes
     data-default
@@ -239,6 +241,7 @@ mkDerivation {
     polysemy-plugin
     polysemy-wire-zoo
     proto-lens
+    random
     random-shuffle
     resource-pool
     resourcet
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 6811fac465..3eeb17c65d 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -1,5 +1,3 @@
-{-# LANGUAGE NumericUnderscores #-}
-
 -- This file is part of the Wire Server implementation.
 --
 -- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
diff --git a/services/brig/src/Brig/Data/Federation.hs b/services/brig/src/Brig/Data/Federation.hs
index a8f3b88df3..3a1ec6cda8 100644
--- a/services/brig/src/Brig/Data/Federation.hs
+++ b/services/brig/src/Brig/Data/Federation.hs
@@ -60,7 +60,7 @@ addFederationRemote (FederationDomainConfig rdom searchpolicy) = do
 
 updateFederationRemote :: MonadClient m => FederationDomainConfig -> m Bool
 updateFederationRemote (FederationDomainConfig rdom spol) = do
-  (retry x1 $ trans upd (params LocalQuorum (spol, rdom)) {serialConsistency = Just LocalSerialConsistency}) >>= \case
+  retry x1 (trans upd (params LocalQuorum (spol, rdom)) {serialConsistency = Just LocalSerialConsistency}) >>= \case
     [] -> pure False
     [_] -> pure True
     _ -> throwM $ ErrorCall "Primary key violation detected federation_remotes"
diff --git a/services/cannon/default.nix b/services/cannon/default.nix
index 746e0d09e0..1032b92eb1 100644
--- a/services/cannon/default.nix
+++ b/services/cannon/default.nix
@@ -31,7 +31,6 @@
 , random
 , retry
 , safe-exceptions
-, servant-client
 , servant-conduit
 , servant-server
 , strict
@@ -82,7 +81,6 @@ mkDerivation {
     mwc-random
     retry
     safe-exceptions
-    servant-client
     servant-conduit
     servant-server
     strict
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index 4fa3f9c33d..74fc58ca05 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -52,11 +52,11 @@ import System.Posix.Signals
 import qualified System.Posix.Signals as Signals
 import System.Random.MWC (createSystemRandom)
 import UnliftIO.Concurrent (myThreadId, throwTo)
+import Util.Options (Endpoint (..))
 import Wire.API.FederationUpdate
 import qualified Wire.API.Routes.Internal.Cannon as Internal
 import Wire.API.Routes.Public.Cannon
 import Wire.API.Routes.Version.Wai
-import Util.Options (Endpoint(..))
 
 type CombinedAPI = PublicAPI :<|> Internal.API
 
diff --git a/services/federator/default.nix b/services/federator/default.nix
index ccdb55ccf6..8a9d1e9a98 100644
--- a/services/federator/default.nix
+++ b/services/federator/default.nix
@@ -106,7 +106,6 @@ mkDerivation {
     polysemy
     polysemy-wire-zoo
     servant
-    servant-client
     servant-client-core
     string-conversions
     text
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index f1285a5901..ea3be6ad48 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -61,13 +61,13 @@ import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
 import Servant hiding (route)
 import qualified System.Logger as Log
+import System.Logger.Extended (mkLogger)
 import Util.Options
 import Wire.API.FederationUpdate
 import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
-import System.Logger.Extended (mkLogger)
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -187,5 +187,5 @@ callback old new = unless (domainListsEqual old new) $ do
   pure ()
   where
     domainListsEqual o n =
-      Set.fromList (domain <$> remotes o) ==
-      Set.fromList (domain <$> remotes n)
\ No newline at end of file
+      Set.fromList (domain <$> remotes o)
+        == Set.fromList (domain <$> remotes n)
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index 1be79e7e3f..82893604a2 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -126,6 +126,7 @@ import Wire.API.MLS.Proposal
 import Wire.API.MLS.Serialisation
 import Wire.API.Message
 import qualified Wire.API.Message.Proto as Proto
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.Internal.Brig.Connection
 import Wire.API.Routes.Internal.Galley.ConversationsIntra
 import qualified Wire.API.Routes.Internal.Galley.TeamFeatureNoConfigMulti as Multi
@@ -144,7 +145,6 @@ import Wire.API.User.Auth hiding (Access)
 import Wire.API.User.Client
 import qualified Wire.API.User.Client as Client
 import Wire.API.User.Client.Prekey
-import Wire.API.Routes.FederationDomainConfig
 
 -------------------------------------------------------------------------------
 -- API Operations
@@ -2457,7 +2457,7 @@ instance HasSettingsOverrides TestM where
     ts :: TestSetup <- ask
     let opts = f (ts ^. tsGConf)
     liftIO . lowerCodensity $ do
-      ioref <- newIORef defFederationDomainConfigs 
+      ioref <- newIORef defFederationDomainConfigs
       (galleyApp, _env) <- Run.mkApp opts ioref
       port' <- withMockServer galleyApp
       liftIO $
diff --git a/services/gundeck/default.nix b/services/gundeck/default.nix
index 2d76650bff..8c74c72e95 100644
--- a/services/gundeck/default.nix
+++ b/services/gundeck/default.nix
@@ -56,7 +56,6 @@
 , safe
 , safe-exceptions
 , scientific
-, servant-client
 , servant-server
 , string-conversions
 , tagged
@@ -124,7 +123,6 @@ mkDerivation {
     resourcet
     retry
     safe-exceptions
-    servant-client
     servant-server
     text
     time

From 9c68bd8f9f5719726aee775bbf75ec33a805c522 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 24 May 2023 19:21:56 +1000
Subject: [PATCH 110/220] FS-1115: Updating brig integration config to help
 tests

---
 services/brig/brig.integration.yaml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml
index ba73ab49a5..d9547a4ab6 100644
--- a/services/brig/brig.integration.yaml
+++ b/services/brig/brig.integration.yaml
@@ -190,16 +190,10 @@ optSettings:
   setFederationDomain: example.com
   setFeatureFlags:  # see #RefConfigOptions in `/docs/reference`
   setFederationDomainConfigsUpdateFreq: 1
-  setFederationStrategy: allowList
+  setFederationStrategy: allowAll
   setFederationDomainConfigs:
     - domain: example.com
       search_policy: full_search
-    - domain: b.example.com
-      search_policy: full_search
-    - domain: invalid.example.com
-      search_policy: full_search
-    - domain: far-away.example.com
-      search_policy: full_search
   set2FACodeGenerationDelaySecs: 5
   setNonceTtlSecs: 5
   setDpopMaxSkewSecs: 1

From c8cac98725bd7965f5c25250c0824d7f2b4b6306 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 24 May 2023 14:17:20 +0200
Subject: [PATCH 111/220] s/AllowList/AllowDynamic/g

---
 .../src/Wire/API/Routes/FederationDomainConfig.hs  |  6 +++---
 services/brig/src/Brig/Options.hs                  |  2 +-
 services/federator/src/Federator/Validation.hs     |  6 +++---
 .../test/unit/Test/Federator/InternalServer.hs     |  8 ++++----
 .../test/unit/Test/Federator/Validation.hs         | 14 +++++++-------
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 70f4c1fd6f..6891d5a466 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -34,7 +34,7 @@ import Wire.API.User.Search (FederatedUserSearchPolicy)
 import Wire.Arbitrary (Arbitrary, GenericUniform (..))
 
 -- | Everything we need to know about a remote instance in order to federate with it.  Comes
--- in `AllowedDomains` if `AllowStrategy` is `AllowList`.  If `AllowAll`, we still use this
+-- in `AllowedDomains` if `AllowStrategy` is `AllowDynamic`.  If `AllowAll`, we still use this
 -- information for search policy.
 data FederationDomainConfig = FederationDomainConfig
   { domain :: Domain,
@@ -85,7 +85,7 @@ data FederationStrategy
     AllowAll
   | -- | Any backend explicitly configured in table `brig.federation_remotes` (if that table
     -- is empty, this is the same as `AllowNone`).
-    AllowList
+    AllowDynamic
   deriving (Eq, Show, Generic)
   deriving (ToJSON, FromJSON, S.ToSchema) via Schema FederationStrategy
   deriving (Arbitrary) via (GenericUniform FederationStrategy)
@@ -96,5 +96,5 @@ instance ToSchema FederationStrategy where
       mconcat
         [ element "allowNone" AllowNone,
           element "allowAll" AllowAll,
-          element "allowList" AllowList
+          element "allowDynamic" AllowDynamic
         ]
diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs
index 17dbd386a0..87450d6e59 100644
--- a/services/brig/src/Brig/Options.hs
+++ b/services/brig/src/Brig/Options.hs
@@ -537,7 +537,7 @@ data Settings = Settings
     setSearchSameTeamOnly :: !(Maybe Bool),
     -- | FederationDomain is required, even when not wanting to federate with other backends
     -- (in that case the 'setFederationStrategy' can be set to `allowNone` below, or to
-    -- `allowList` while keeping the list of allowed domains empty, see
+    -- `allowDynamic` while keeping the list of allowed domains empty, see
     -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
     -- Federation domain is used to qualify local IDs and handles,
     -- e.g. 0c4d8944-70fa-480e-a8b7-9d929862d18c@wire.com and somehandle@wire.com.
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index d5fab49748..172619b27b 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -89,7 +89,7 @@ validationErrorStatus :: ValidationError -> HTTP.Status
 validationErrorStatus (FederationDenied _) = HTTP.status400
 validationErrorStatus _ = HTTP.status403
 
--- | Validates an already-parsed domain against the allowList (stored in
+-- | Validates an already-parsed domain against the allow list (stored in
 -- `brig.federation_remotes`, cached in `Env`).
 ensureCanFederateWith ::
   ( Member (Input FederationDomainConfigs) r,
@@ -102,7 +102,7 @@ ensureCanFederateWith targetDomain = do
   case strategy of
     AllowNone -> throw (FederationDenied targetDomain)
     AllowAll -> pure ()
-    AllowList -> do
+    AllowDynamic -> do
       unless (targetDomain `elem` fmap domain domains) $
         throw (FederationDenied targetDomain)
 
@@ -136,7 +136,7 @@ parseDomainText domain =
     . mkDomain
     $ domain
 
--- | Validates an unknown domain string against the allowList using the
+-- | Validates an unknown domain string against the allow list using the
 -- federator startup configuration and checks that it matches the names reported
 -- by the client certificate
 validateDomain ::
diff --git a/services/federator/test/unit/Test/Federator/InternalServer.hs b/services/federator/test/unit/Test/Federator/InternalServer.hs
index c2d4d82653..ee9861b50d 100644
--- a/services/federator/test/unit/Test/Federator/InternalServer.hs
+++ b/services/federator/test/unit/Test/Federator/InternalServer.hs
@@ -92,7 +92,7 @@ federatedRequestSuccess =
         . assertNoError @ServerError
         . discardTinyLogs
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "target.example.com") FullSearch] 10)
+        . runInputConst (FederationDomainConfigs AllowDynamic [FederationDomainConfig (Domain "target.example.com") FullSearch] 10)
         $ callOutward request
     Wai.responseStatus res @?= HTTP.status200
     body <- Wai.lazyResponseBody res
@@ -100,10 +100,10 @@ federatedRequestSuccess =
 
 -- @SF.Federation @TSFI.Federate @TSFI.DNS @S2 @S3 @S7
 --
--- Refuse to send outgoing request to non-included domain when allowlist is configured.
+-- Refuse to send outgoing request to non-included domain when AllowDynamic is configured.
 federatedRequestFailureAllowList :: TestTree
 federatedRequestFailureAllowList =
-  testCase "should not make a call when target domain not in the allowList" $ do
+  testCase "should not make a call when target domain not in the allow list" $ do
     let settings = noClientCertSettings
     let targetDomain = Domain "target.example.com"
         headers = [(originDomainHeaderName, "origin.example.com")]
@@ -134,7 +134,7 @@ federatedRequestFailureAllowList =
         . assertNoError @ServerError
         . discardTinyLogs
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "hello.world") FullSearch] 10)
+        . runInputConst (FederationDomainConfigs AllowDynamic [FederationDomainConfig (Domain "hello.world") FullSearch] 10)
         $ callOutward request
     eith @?= Left (FederationDenied targetDomain)
 
diff --git a/services/federator/test/unit/Test/Federator/Validation.hs b/services/federator/test/unit/Test/Federator/Validation.hs
index e122957447..6c1d7ebc95 100644
--- a/services/federator/test/unit/Test/Federator/Validation.hs
+++ b/services/federator/test/unit/Test/Federator/Validation.hs
@@ -62,7 +62,7 @@ mockDiscoveryFailure = Polysemy.interpret $ \case
 scaffoldingFederationDomainConfigs :: FederationDomainConfigs
 scaffoldingFederationDomainConfigs =
   FederationDomainConfigs
-    AllowList
+    AllowDynamic
     [ FederationDomainConfig (Domain "foo.example.com") FullSearch,
       FederationDomainConfig (Domain "example.com") FullSearch,
       FederationDomainConfig (Domain "federator.example.com") FullSearch
@@ -101,7 +101,7 @@ federateWithAllowListSuccess =
     runM
       . assertNoError @ValidationError
       . runInputConst settings
-      . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "hello.world") FullSearch] 0)
+      . runInputConst (FederationDomainConfigs AllowDynamic [FederationDomainConfig (Domain "hello.world") FullSearch] 0)
       $ ensureCanFederateWith (Domain "hello.world")
 
 federateWithAllowListFail :: TestTree
@@ -112,7 +112,7 @@ federateWithAllowListFail =
       runM
         . runError @ValidationError
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowDynamic [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ ensureCanFederateWith (Domain "hello.world")
     assertBool "federating should not be allowed" (isLeft eith)
 
@@ -127,13 +127,13 @@ validateDomainAllowListFailSemantic =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowDynamic [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ validateDomain (Just exampleCert) "invalid//.><-semantic-&@-domain"
     res @?= Left (DomainParseError "invalid//.><-semantic-&@-domain")
 
 -- @SF.Federation @TSFI.Federate @TSFI.DNS @S2 @S3 @S7
 --
--- Refuse to send outgoing request to non-included domain when allowlist is configured.
+-- Refuse to send outgoing request to non-included domain when AllowDynamic is configured.
 validateDomainAllowListFail :: TestTree
 validateDomainAllowListFail =
   testCase "allow list validation" $ do
@@ -145,7 +145,7 @@ validateDomainAllowListFail =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowDynamic [FederationDomainConfig (Domain "only.other.domain") FullSearch] 0)
         $ validateDomain (Just exampleCert) "localhost.example.com"
     res @?= Left (FederationDenied (Domain "localhost.example.com"))
 
@@ -163,7 +163,7 @@ validateDomainAllowListSuccess =
         . assertNoError @DiscoveryFailure
         . mockDiscoveryTrivial
         . runInputConst settings
-        . runInputConst (FederationDomainConfigs AllowList [FederationDomainConfig domain FullSearch] 0)
+        . runInputConst (FederationDomainConfigs AllowDynamic [FederationDomainConfig domain FullSearch] 0)
         $ validateDomain (Just exampleCert) (toByteString' domain)
     assertEqual "validateDomain should give 'localhost.example.com' as domain" domain res
 

From ca572dda8cca404f4cbb6595edae3db55718ac7b Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 24 May 2023 14:40:01 +0200
Subject: [PATCH 112/220] rm trailing whitespace.

---
 docs/src/understand/configure-federation.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index fd092ad20f..e87feb9caa 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -62,7 +62,7 @@ backend.
     domain. Your user known to you as Alice, and known on your
     server with ID `ac41a202-2555-11ec-9341-00163e5e6c00` will
     become known for other servers you federate with as
-    
+
     ``` json
     {
       "user": {
@@ -73,7 +73,7 @@ backend.
     ```
 
 -   This domain is shown in the User Interface
-    alongside user information. 
+    alongside user information.
 
     Example: Using the same example as above, for backends you
     federate with, Alice would be displayed with the
@@ -138,7 +138,7 @@ The SRV record would look as follows:
 _wire-server-federator._tcp.example.com. 600 IN SRV 0        10     443  federator.wire.example.org.
 ```
 
-### DNS A record for the federator 
+### DNS A record for the federator
 
 Background: `federator` is the server component responsible for incoming
 and outgoing requests to other backend; but it is proxied on the
@@ -151,7 +151,7 @@ also needs to point to the IP of your ingress, i.e. the IP you want to
 provide services on.
 
 (federation-certificate-setup)=
-## Generate and configure TLS server and client certificates 
+## Generate and configure TLS server and client certificates
 
 Are your servers on the public internet? Then you have the option of
 using TLS certificates from [Let\'s encrypt](https://letsencrypt.org/).
@@ -196,7 +196,7 @@ FS-33 and FS-49 (tickets only visible to Wire employees).
 ```
 
 
-### (A) Let\'s encrypt TLS server and client certificate generation and renewal 
+### (A) Let\'s encrypt TLS server and client certificate generation and renewal
 
 The following will make use of [Let\'s
 encrypt](https://letsencrypt.org/) for both server certificates (used
@@ -395,7 +395,7 @@ cargohold:
       federationDomain: example.com # your chosen "backend domain"
 ```
 
-### Configure federator process to run and allow incoming traffic 
+### Configure federator process to run and allow incoming traffic
 
 For federation to work, the `federator` subchart of wire-server has to
 be enabled:
@@ -422,7 +422,7 @@ config:
     federator: federator.wire.example.org # set this to your "infra" domain
 ```
 
-### Configure the validation depth when handling client certificates 
+### Configure the validation depth when handling client certificates
 
 By default, `verify_depth` is `1`, meaning that in order to validate an
 incoming request from another backend, this backend needs to have a
@@ -482,14 +482,14 @@ federator:
         allowAll: true
 ```
 
-## Applying all configuration changes 
+## Applying all configuration changes
 
 Depending on your installation method and time you initially installed
 your first version of wire-server, commands to run to apply all of the
 above configrations may vary. You want to ensure that you upgrade the
 `nginx-ingress-services` and `wire-server` helm charts at a minimum.
 
-## Manually test that your configurations work as expected 
+## Manually test that your configurations work as expected
 
 ### Manually test DNS
 
@@ -518,7 +518,7 @@ DOMAIN to your
 {ref}`federation infrastructure domain <glossary_infra_domain>`. They should include your domain as part of the SAN (Subject
 Alternative Names) and not have expired.
 
-### Manually test that federation works 
+### Manually test that federation works
 
 Prerequisites:
 

From d3997dc53a8c45a80406dba2b3083edbb13f1d8a Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Wed, 24 May 2023 15:01:58 +0200
Subject: [PATCH 113/220] docs

---
 .../developer/federation-design-aspects.md    |  5 +-
 .../src/developer/reference/config-options.md | 32 +-------
 docs/src/understand/configure-federation.md   | 24 ++++++
 .../federation/backend-communication.md       | 77 +++++++++++++------
 4 files changed, 82 insertions(+), 56 deletions(-)

diff --git a/docs/src/developer/developer/federation-design-aspects.md b/docs/src/developer/developer/federation-design-aspects.md
index afd3711737..c2bccf47c0 100644
--- a/docs/src/developer/developer/federation-design-aspects.md
+++ b/docs/src/developer/developer/federation-design-aspects.md
@@ -4,6 +4,8 @@
 
 ## keeping track of federator remotes
 
+**Since [PR#3260](https://github.com/wireapp/wire-server/pull/3260).**
+
 Federation can start and end.  These events need handlers to be called
 (like remove remote users from local conv), plus it is not convenient
 to edit and re-deploy config files every time that happens.  Hence
@@ -23,6 +25,3 @@ we may decide that brig needs to cache the table itself, but for now
 (`GET` is only used for the internal end-point to share it with other
 services) we hope to get away with the simple solution and always read
 from cassandra directly.
-
-Introduced in
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260).
diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md
index e2c8f2a339..0c385e33c8 100644
--- a/docs/src/developer/reference/config-options.md
+++ b/docs/src/developer/reference/config-options.md
@@ -391,31 +391,9 @@ settings:
   federationDomain: example.com
 ```
 
-### Federation strategy: whom to federate with?
+### Federation allow list
 
-As of the release containing [PR#3260](https://github.com/wireapp/wire-server/pull/3260), federator gets its configuration from brig (which has a cassandra instance and is thus better equipped to handle persistent dynamic data).  See {ref}`configuring-remote-connections` for the whole story.
-
-Federation is turned off by default.  You can also turn it off
-explicitly by setting federation strategy "allowedDomains", and making
-sure that `brig.federation_remotes` (the response of the
-[`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
-request is empty).
-
-```yaml
-# brig.yaml
-optSettings:
-  federationStrategy: allowedDomains
-```
-
-If you want to federate with all domains that ask, change this to:
-
-```yaml
-# brig.yaml
-optSettings:
-  federationStrategy: allowAll
-```
-
-If you want to federate selectively with a list of known peers, consult {ref}`configuring-remote-connections`.
+See {ref}`configuring-remote-connections` (since [PR#3260](https://github.com/wireapp/wire-server/pull/3260)).
 
 ### Federation TLS Config
 
@@ -600,11 +578,7 @@ any key package whose expiry date is set further than 15 days after upload time
 
 #### Restrict user search
 
-You can configure search policies on a peer-by-peer basis, but using
-the config file for that is not supported any more since the release
-containing
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260).  See
-{ref}`configuring-remote-connections` for how to do this now.
+See {ref}`configuring-remote-connections` (since [PR#3260](https://github.com/wireapp/wire-server/pull/3260)).
 
 
 ### API Versioning
diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index e87feb9caa..9b28b9f784 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -395,6 +395,30 @@ cargohold:
       federationDomain: example.com # your chosen "backend domain"
 ```
 
+(configure-federation-strategy-in-brig)=
+
+### Configure federation strategy (whom to federate with) in brig
+
+(**This section is valid as of the release containing [PR#3260](https://github.com/wireapp/wire-server/pull/3260).**)
+
+You also need to define the federation strategy (whom to federate
+with), and the frequency with which the other backend services will
+refresh their cache of this configuration.
+
+``` yaml
+# override values for wire-server
+# (e.g. under ./helm_vars/wire-server/values.yaml)
+brig:
+  config:
+    optSettings:
+      setFederationStrategy: AllowNone # [AllowAll | AllowDynamic | AllowNone]
+      setFederationDomainConfigsUpdateFreq: 10 # seconds
+```
+
+The default of `AllowNone` probably doesn't make sense if you are
+reading this.  See {ref}`configuring-remote-connections` for details
+on the alternatives.
+
 ### Configure federator process to run and allow incoming traffic
 
 For federation to work, the `federator` subchart of wire-server has to
diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index 299b34be21..0f7fba5aa6 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -158,23 +158,63 @@ search request from *Alice*, one of its clients.
 
 ## Configuring Remote Connections
 
-Up to the release containing
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260), the
-config files of the individual services statically contained the
-domains of remote connections.  Starting with this release, this and
-all information about remote connections is stored in the database,
-and there is an internal REST API for adding and removing remotes:
+**Since [PR#3260](https://github.com/wireapp/wire-server/pull/3260).**
+
+
+Related: {ref}`configure-federation-strategy-in-brig`.  (TODO: or move this entire section there?)
+
+
+Brig keeps track of the following information for all services that
+need to know:
+
+* Federation strategy
+  - `allowNone`: federation is effectively disabled
+  - `allowAll`: no restriction on whom to federate with
+  - `allowDynamic`: only allow federating with a domain list maintained via an internal CRUD API (see below).
+* Settings for remote domains
+  - domain
+  - search policy: valid values are:
+    - `no_search`: No users are returned by federated searches.  default.
+    - `exact_handle_search`: Only users where the handle exactly matches are returned.
+    - `full_search`: Additionally to `exact_handle_search`, users are found by a freetext search on handle and display name.
+
+
+
+
+
+
+does anybody know off the top of their heads: is [this section](https://wearezeta.atlassian.net/wiki/spaces/BAC/pages/288620677/Processes+shared+with+CS#Different-search-visibility-per-team) still up to date?  and is stern?  [this page](https://docs.wire.com/developer/reference/config-options.html#federated-domain-specific-configuration-settings) tells a different story...
+
+
+
+
+* UpdateFrequency
+
+
+
+
+The federation strategy (ie., whom to federate with) and the
+information about remote backends are maintained by brig and made
+available to other services via these CRUD end-points:
 
 * [`POST`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/post_i_federation_remotes)
 * [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
-* [`PUT`](TODO)
+* [`PUT`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/put_i_federation_remotes__domain_)
 * [`DELETE`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/delete_i_federation_remotes__domain_)
+  **WARNING:** If you delete a connection, all users from that remote
+  will be removed from local conversations, and all conversations hosted
+  by that remote will be removed from the local backend.  Connections
+  between local and remote users that are removed will be archived, and
+  can be re-established should you decide to add the same backend later.
+
+The list of remotes is stored in the cassandra in
+`brig.federation_remotes`.  Brig's config file contains the federation
+strategy and update frequency that will be assumed by the pulling
+services.
+
+.  Relevant
+config options:
 
-**WARNING:** If you delete a connection, all users from that remote
-will be removed from local conversations, and all conversations hosted
-by that remote will be removed from the local backend.  Connections
-between local and remote users that are removed will be archived, and
-can be re-established should you decide to add the same backend later.
 
 {-
 TODO: this paragraph still annoys me.  move strategy to brig, too?  or
@@ -182,7 +222,7 @@ at least to a different syntax, and force admin to use both old and
 new syntax until transition period is over?  just to avoid the
 confusing bogus `:` at the end of the flag.
 
-The federation strategy (allow all or allow list) is still configured
+The federation strategy (allow all or allow dynamic) is still configured
 in federator, only the list of allowed hosts is ignored; if you select
 "allow all" (or if you disable federation), the list of known backends
 maintained by brig is mostly ignored, but e.g., search policy is still
@@ -331,17 +371,6 @@ optSettings:
       search_policy: no_search
 ```
 
-Valid values for `search_policy` are:
-- `no_search`: No users are returned by federated searches.
-- `exact_handle_search`: Only users where the handle exactly matches are returned.
-- `full_search`: Additionally to `exact_handle_search`, users are found by a freetext search on handle and display name.
-
-If there is no configuration for a domain, it's defaulted to `no_search`.
-
-
-
-
-does anybody know off the top of their heads: is [this section](https://wearezeta.atlassian.net/wiki/spaces/BAC/pages/288620677/Processes+shared+with+CS#Different-search-visibility-per-team) still up to date?  and is stern?  [this page](https://docs.wire.com/developer/reference/config-options.html#federated-domain-specific-configuration-settings) tells a different story...
 
 
 TODO: explain setFederationDomainConfigsUpdateFreq

From 17ab5da13199b0b866fb1f1d82d4ea800514fb82 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 25 May 2023 13:53:49 +1000
Subject: [PATCH 114/220] Updating with PR feedback

---
 .../wire-api/src/Wire/API/FederationUpdate.hs | 13 +++++---
 services/galley/src/Galley/App.hs             |  5 ++-
 services/galley/src/Galley/Run.hs             | 31 +++++--------------
 services/galley/test/integration/API/Util.hs  |  3 +-
 4 files changed, 20 insertions(+), 32 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index af7d24cb47..38cc134967 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -7,6 +7,7 @@ where
 import Control.Concurrent.Async
 import Control.Exception (ErrorCall (ErrorCall), throwIO)
 import qualified Control.Retry as R
+import qualified Data.Set as Set
 import Data.Text (unpack)
 import Imports
 import Network.HTTP.Client (defaultManagerSettings, newManager)
@@ -14,7 +15,7 @@ import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), ClientError, Sc
 import Servant.Client.Internal.HttpClient (ClientM, defaultMakeClientRequest)
 import qualified System.Logger as L
 import Util.Options (Endpoint (..))
-import Wire.API.Routes.FederationDomainConfig (FederationDomainConfigs (updateInterval))
+import Wire.API.Routes.FederationDomainConfig (FederationDomainConfig (domain), FederationDomainConfigs (remotes, updateInterval))
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
 
@@ -56,12 +57,16 @@ getAllowedDomainsLoop logger clientEnv callback env = forever $ do
       L.log logger L.Fatal $
         L.msg (L.val "Could not retrieve an updated list of federation domains from Brig; I'll keep trying!")
           L.~~ "error" L..= show e
-    Right cfg -> do
+    Right new -> do
       old <- readIORef env
-      callback old cfg
-      atomicWriteIORef env cfg
+      unless (domainListsEqual old new) $ callback old new
+      atomicWriteIORef env new
   delay <- updateInterval <$> readIORef env
   threadDelay (delay * 1_000_000)
+  where
+    domainListsEqual o n =
+      Set.fromList (domain <$> remotes o)
+        == Set.fromList (domain <$> remotes n)
 
 updateFedDomains :: Endpoint -> L.Logger -> FedUpdateCallback -> IO (IORef FederationDomainConfigs, Async ())
 updateFedDomains (Endpoint h p) log' cb = do
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index c77ba37eea..6548d5769b 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -151,9 +151,8 @@ validateOptions l o = do
   when (settings ^. setMaxTeamSize < optFanoutLimit) $
     error "setMaxTeamSize cannot be < setTruncationLimit"
 
-createEnv :: Metrics -> Opts -> IORef FederationDomainConfigs -> IO Env
-createEnv m o r = do
-  l <- Logger.mkLogger (o ^. optLogLevel) (o ^. optLogNetStrings) (o ^. optLogFormat)
+createEnv :: Metrics -> Opts -> Logger -> IORef FederationDomainConfigs -> IO Env
+createEnv m o l r = do
   cass <- initCassandra o l
   mgr <- initHttpManager o
   h2mgr <- initHttp2Manager
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index ea3be6ad48..30f91e0011 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -19,6 +19,7 @@
 module Galley.Run
   ( run,
     mkApp,
+    mkLogger,
   )
 where
 
@@ -39,7 +40,6 @@ import Data.Metrics.AWS (gaugeTokenRemaing)
 import qualified Data.Metrics.Middleware as M
 import Data.Metrics.Servant (servantPlusWAIPrometheusMiddleware)
 import Data.Misc (portNumber)
-import qualified Data.Set as Set
 import Data.String.Conversions (cs)
 import Data.Text (unpack)
 import qualified Galley.API as API
@@ -71,13 +71,9 @@ import Wire.API.Routes.Version.Wai
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
-  (ioref, _) <- lift $ do
-    -- Duplicating the logger from App.createEnv so that we don't have to deal
-    -- with recursive monadic actions to get the logger before we have the initial
-    -- IORef of values from brig. It's just easier this way.
-    l <- mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
-    updateFedDomains (opts ^. optBrig) l callback
-  (app, env) <- mkApp opts ioref
+  l <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
+  (ioref, _) <- lift $ updateFedDomains (opts ^. optBrig) l $ \_ _ -> pure ()
+  (app, env) <- mkApp opts ioref l
   settings <-
     lift $
       newSettings $
@@ -94,15 +90,12 @@ run opts = lowerCodensity $ do
   void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
-mkApp :: Opts -> IORef FederationDomainConfigs -> Codensity IO (Application, Env)
-mkApp opts fedDoms =
+mkApp :: Opts -> IORef FederationDomainConfigs -> Log.Logger -> Codensity IO (Application, Env)
+mkApp opts fedDoms logger =
   do
     metrics <- lift $ M.metrics
-    env <- lift $ App.createEnv metrics opts fedDoms
+    env <- lift $ App.createEnv metrics opts logger fedDoms
     lift $ runClient (env ^. cstate) $ versionCheck schemaVersion
-
-    let logger = env ^. App.applog
-
     let middlewares =
           versionMiddleware (opts ^. optSettings . setDisabledAPIVersions . traverse)
             . servantPlusWAIPrometheusMiddleware API.sitemap (Proxy @CombinedAPI)
@@ -179,13 +172,3 @@ collectAuthMetrics m env = do
       mbRemaining <- readAuthExpiration env
       gaugeTokenRemaing m mbRemaining
       threadDelay 1_000_000
-
-callback :: FedUpdateCallback
-callback old new = unless (domainListsEqual old new) $ do
-  -- TODO: perform the database updates here
-  -- This code will only run when there is a change in the domain lists
-  pure ()
-  where
-    domainListsEqual o n =
-      Set.fromList (domain <$> remotes o)
-        == Set.fromList (domain <$> remotes n)
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index 82893604a2..35460608f9 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -2458,7 +2458,8 @@ instance HasSettingsOverrides TestM where
     let opts = f (ts ^. tsGConf)
     liftIO . lowerCodensity $ do
       ioref <- newIORef defFederationDomainConfigs
-      (galleyApp, _env) <- Run.mkApp opts ioref
+      logger <- lift $ Run.mkLogger (opts ^. Opts.optLogLevel) (opts ^. Opts.optLogNetStrings) (opts ^. Opts.optLogFormat)
+      (galleyApp, _env) <- Run.mkApp opts ioref logger
       port' <- withMockServer galleyApp
       liftIO $
         runReaderT

From fbfac871a1368fcea58d96e40f6875f28d301094 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 25 May 2023 15:02:52 +1000
Subject: [PATCH 115/220] Have federation domain tests use MakesValue more

---
 integration/test/API/BrigInternal.hs | 15 +++++++++------
 integration/test/Test/Brig.hs        | 20 +++++++++++---------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 12ba3c5810..7efacf65f4 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -58,22 +58,25 @@ instance ToJSON FedConn where
         "search_policy" .= s
       ]
 
+instance MakesValue FedConn where
+  make = pure . toJSON
+
 instance FromJSON FedConn where
   parseJSON = withObject "FedConn" $ \obj -> do
     FedConn
       <$> obj .: fromString "domain"
       <*> obj .: fromString "search_policy"
 
-createFedConn :: (HasCallStack, MakesValue dom) => dom -> FedConn -> App Response
+createFedConn :: (HasCallStack, MakesValue dom, MakesValue fedConn) => dom -> fedConn -> App Response
 createFedConn dom fedConn = do
   res <- createFedConn' dom fedConn
   res.status `shouldMatchRange` (200, 299)
   pure res
 
-createFedConn' :: (HasCallStack, MakesValue dom) => dom -> FedConn -> App Response
+createFedConn' :: (HasCallStack, MakesValue dom, MakesValue fedConn) => dom -> fedConn -> App Response
 createFedConn' dom fedConn = do
   req <- rawBaseRequest dom Brig Unversioned "/i/federation/remotes"
-  submit "POST" $ req & addJSON fedConn
+  make fedConn >>= \v -> submit "POST" $ req & addJSON v
 
 readFedConns :: (HasCallStack, MakesValue dom) => dom -> App Response
 readFedConns dom = do
@@ -86,16 +89,16 @@ readFedConns' dom = do
   req <- rawBaseRequest dom Brig Unversioned "/i/federation/remotes"
   submit "GET" req
 
-updateFedConn :: (HasCallStack, MakesValue owndom) => owndom -> String -> FedConn -> App Response
+updateFedConn :: (HasCallStack, MakesValue owndom, MakesValue fedConn) => owndom -> String -> fedConn -> App Response
 updateFedConn owndom dom fedConn = do
   res <- updateFedConn' owndom dom fedConn
   res.status `shouldMatchRange` (200, 299)
   pure res
 
-updateFedConn' :: (HasCallStack, MakesValue owndom) => owndom -> String -> FedConn -> App Response
+updateFedConn' :: (HasCallStack, MakesValue owndom, MakesValue fedConn) => owndom -> String -> fedConn -> App Response
 updateFedConn' owndom dom fedConn = do
   req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
-  submit "PUT" (fedConn `addJSON` req)
+  make fedConn >>= \v -> submit "PUT" $ addJSON v req
 
 deleteFedConn :: (HasCallStack, MakesValue owndom) => owndom -> String -> App Response
 deleteFedConn owndom dom = do
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index ae21bff68f..6a8f6a17ad 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -4,7 +4,7 @@ import qualified API.Brig as Public
 import qualified API.BrigInternal as Internal
 import qualified API.Common as API
 import qualified API.GalleyInternal as Internal
-import Data.Aeson.Types (parseMaybe)
+import Data.Aeson.Types
 import Data.String.Conversions
 import GHC.Stack
 import SetupHelpers
@@ -22,14 +22,16 @@ testSearchContactForExternalUsers = do
   bindResponse (Public.searchContacts partner (owner %. "name")) $ \resp ->
     resp.status `shouldMatchInt` 403
 
+-- Parse a given JSON value to a structure.
+-- Calls `assertFailure` if the value cannot
+-- be parsed to the desired type.
+parseJSONApp :: FromJSON a => Value -> App a
+parseJSONApp = either assertFailure pure . parseEither parseJSON
+
 testCrudFederationRemotes :: HasCallStack => App ()
 testCrudFederationRemotes = do
   let parseFedConns :: HasCallStack => Response -> App [Internal.FedConn]
-      parseFedConns resp = do
-        -- TODO: not idiomatic!  try `getJSON 200 resp %. "remotes" & asList & mapM asObjOrSomething`
-        -- Some ideas: There is asList to assert that a Value is actually a an array of Values. Then you can sort that to have a defined order.
-        fromJust . parseMaybe parseJSON . fromJust <$> ((`lookupField` "remotes") =<< getJSON 200 resp)
-
+      parseFedConns resp = getJSON 200 resp %. "remotes" & asList >>= mapM parseJSONApp
       addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
       addOnce fedConn want = do
         res <- Internal.createFedConn OwnDomain fedConn
@@ -37,12 +39,12 @@ testCrudFederationRemotes = do
         res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
         sort res2 `shouldMatch` sort want
 
-      addFail :: HasCallStack => Internal.FedConn -> App ()
+      addFail :: HasCallStack => MakesValue fedConn => fedConn -> App ()
       addFail fedConn = do
         res <- Internal.createFedConn' OwnDomain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
-      deleteOnce :: HasCallStack => String -> [Internal.FedConn] -> App ()
+      deleteOnce :: String -> [Internal.FedConn] -> App ()
       deleteOnce domain want = do
         res <- Internal.deleteFedConn OwnDomain domain
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
@@ -61,7 +63,7 @@ testCrudFederationRemotes = do
         res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
         sort res2 `shouldMatch` sort want
 
-      updateFail :: HasCallStack => String -> Internal.FedConn -> App ()
+      updateFail :: (MakesValue fedConn, HasCallStack) => String -> fedConn -> App ()
       updateFail domain fedConn = do
         res <- Internal.updateFedConn' OwnDomain domain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533

From 8cdd041ad2649df73c6c4a5da3aef7ea782371e4 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 25 May 2023 15:33:44 +1000
Subject: [PATCH 116/220] FS-1115: Removing more FedConn type specific code
 from federation tests

---
 integration/test/Test/Brig.hs | 43 +++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 6a8f6a17ad..d9b83b7bec 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -22,46 +22,47 @@ testSearchContactForExternalUsers = do
   bindResponse (Public.searchContacts partner (owner %. "name")) $ \resp ->
     resp.status `shouldMatchInt` 403
 
--- Parse a given JSON value to a structure.
--- Calls `assertFailure` if the value cannot
--- be parsed to the desired type.
-parseJSONApp :: FromJSON a => Value -> App a
-parseJSONApp = either assertFailure pure . parseEither parseJSON
-
 testCrudFederationRemotes :: HasCallStack => App ()
 testCrudFederationRemotes = do
-  let parseFedConns :: HasCallStack => Response -> App [Internal.FedConn]
-      parseFedConns resp = getJSON 200 resp %. "remotes" & asList >>= mapM parseJSONApp
-      addOnce :: HasCallStack => Internal.FedConn -> [Internal.FedConn] -> App ()
+  let parseFedConns :: HasCallStack => Response -> App [Value]
+      parseFedConns resp =
+        -- Pick out the list of federation domain configs
+        getJSON 200 resp %. "remotes" & asList
+        -- Enforce that the values are objects and not something else
+        >>= traverse (fmap Object . asObject)
+      addOnce :: (MakesValue fedConn, HasCallStack) => fedConn -> [fedConn] -> App ()
       addOnce fedConn want = do
         res <- Internal.createFedConn OwnDomain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        sort res2 `shouldMatch` sort want
+        want' <- traverse make want
+        sort res2 `shouldMatch` sort want'
 
       addFail :: HasCallStack => MakesValue fedConn => fedConn -> App ()
       addFail fedConn = do
         res <- Internal.createFedConn' OwnDomain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
-      deleteOnce :: String -> [Internal.FedConn] -> App ()
+      deleteOnce :: MakesValue fedConn => String -> [fedConn] -> App ()
       deleteOnce domain want = do
         res <- Internal.deleteFedConn OwnDomain domain
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        sort res2 `shouldMatch` sort want
+        want' <- traverse make want
+        sort res2 `shouldMatch` sort want'
 
       deleteFail :: HasCallStack => String -> App ()
       deleteFail del = do
         res <- Internal.deleteFedConn' OwnDomain del
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
-      updateOnce :: HasCallStack => String -> Internal.FedConn -> [Internal.FedConn] -> App ()
+      updateOnce :: (MakesValue fedConn, HasCallStack) => String -> fedConn -> [fedConn] -> App ()
       updateOnce domain fedConn want = do
         res <- Internal.updateFedConn OwnDomain domain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        sort res2 `shouldMatch` sort want
+        want' <- traverse make want
+        sort res2 `shouldMatch` sort want'
 
       updateFail :: (MakesValue fedConn, HasCallStack) => String -> fedConn -> App ()
       updateFail domain fedConn = do
@@ -69,16 +70,20 @@ testCrudFederationRemotes = do
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
   let remote1, remote1', remote1'' :: Internal.FedConn
+      remote1J, remote1J', remote1J'' :: Value
       remote1 = Internal.FedConn (cs "good.example.com") "no_search"
       remote1' = remote1 {Internal.searchStrategy = "full_search"}
       remote1'' = remote1 {Internal.domain = "meh.example.com"}
+      remote1J = toJSON remote1
+      remote1J' = toJSON remote1'
+      remote1J'' = toJSON remote1''
 
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
 
   resetFedConns OwnDomain
   cfgRemotes <- parseFedConns =<< Internal.readFedConns OwnDomain
-  cfgRemotes `shouldMatch` [cfgRemotesExpect]
+  cfgRemotes `shouldMatch` [toJSON cfgRemotesExpect]
   -- entries present in the config file can be idempotently added if identical, but cannot be
   -- updated, deleted or updated.
   addOnce cfgRemotesExpect [cfgRemotesExpect]
@@ -86,11 +91,11 @@ testCrudFederationRemotes = do
   deleteFail (Internal.domain cfgRemotesExpect)
   updateFail (Internal.domain cfgRemotesExpect) (cfgRemotesExpect {Internal.searchStrategy = "no_search"})
   -- create
-  addOnce remote1 (remote1 : cfgRemotes)
-  addOnce remote1 (remote1 : cfgRemotes) -- idempotency
+  addOnce remote1J $ (remote1J : cfgRemotes)
+  addOnce remote1J $ (remote1J : cfgRemotes) -- idempotency
   -- update
-  updateOnce (Internal.domain remote1) remote1' (remote1' : cfgRemotes)
-  updateFail (Internal.domain remote1) remote1''
+  updateOnce (Internal.domain remote1) remote1J' (remote1J' : cfgRemotes)
+  updateFail (Internal.domain remote1) remote1J''
   -- delete
   deleteOnce (Internal.domain remote1) cfgRemotes
   deleteOnce (Internal.domain remote1) cfgRemotes -- idempotency

From 1ffaab20ba81a49c0c8389779f35c34f64f18637 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 25 May 2023 15:43:10 +1000
Subject: [PATCH 117/220] More code leaning on typeclasses

---
 integration/test/Test/Brig.hs | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index d9b83b7bec..5bb93e8be4 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -30,39 +30,36 @@ testCrudFederationRemotes = do
         getJSON 200 resp %. "remotes" & asList
         -- Enforce that the values are objects and not something else
         >>= traverse (fmap Object . asObject)
-      addOnce :: (MakesValue fedConn, HasCallStack) => fedConn -> [fedConn] -> App ()
+      addOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => fedConn -> [fedConn2] -> App ()
       addOnce fedConn want = do
         res <- Internal.createFedConn OwnDomain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        want' <- traverse make want
-        sort res2 `shouldMatch` sort want'
+        sort res2 `shouldMatch` sort want
 
       addFail :: HasCallStack => MakesValue fedConn => fedConn -> App ()
       addFail fedConn = do
         res <- Internal.createFedConn' OwnDomain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
-      deleteOnce :: MakesValue fedConn => String -> [fedConn] -> App ()
+      deleteOnce :: (Ord fedConn, ToJSON fedConn, MakesValue fedConn) => String -> [fedConn] -> App ()
       deleteOnce domain want = do
         res <- Internal.deleteFedConn OwnDomain domain
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        want' <- traverse make want
-        sort res2 `shouldMatch` sort want'
+        sort res2 `shouldMatch` sort want
 
       deleteFail :: HasCallStack => String -> App ()
       deleteFail del = do
         res <- Internal.deleteFedConn' OwnDomain del
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
-      updateOnce :: (MakesValue fedConn, HasCallStack) => String -> fedConn -> [fedConn] -> App ()
+      updateOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => String -> fedConn -> [fedConn2] -> App ()
       updateOnce domain fedConn want = do
         res <- Internal.updateFedConn OwnDomain domain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
         res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        want' <- traverse make want
-        sort res2 `shouldMatch` sort want'
+        sort res2 `shouldMatch` sort want
 
       updateFail :: (MakesValue fedConn, HasCallStack) => String -> fedConn -> App ()
       updateFail domain fedConn = do
@@ -70,20 +67,19 @@ testCrudFederationRemotes = do
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
   let remote1, remote1', remote1'' :: Internal.FedConn
-      remote1J, remote1J', remote1J'' :: Value
       remote1 = Internal.FedConn (cs "good.example.com") "no_search"
       remote1' = remote1 {Internal.searchStrategy = "full_search"}
       remote1'' = remote1 {Internal.domain = "meh.example.com"}
-      remote1J = toJSON remote1
-      remote1J' = toJSON remote1'
-      remote1J'' = toJSON remote1''
 
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
 
+  remote1J  <- make remote1
+  remote1J' <- make remote1'
+
   resetFedConns OwnDomain
   cfgRemotes <- parseFedConns =<< Internal.readFedConns OwnDomain
-  cfgRemotes `shouldMatch` [toJSON cfgRemotesExpect]
+  cfgRemotes `shouldMatch` [cfgRemotesExpect]
   -- entries present in the config file can be idempotently added if identical, but cannot be
   -- updated, deleted or updated.
   addOnce cfgRemotesExpect [cfgRemotesExpect]
@@ -91,11 +87,11 @@ testCrudFederationRemotes = do
   deleteFail (Internal.domain cfgRemotesExpect)
   updateFail (Internal.domain cfgRemotesExpect) (cfgRemotesExpect {Internal.searchStrategy = "no_search"})
   -- create
-  addOnce remote1J $ (remote1J : cfgRemotes)
-  addOnce remote1J $ (remote1J : cfgRemotes) -- idempotency
+  addOnce remote1 $ (remote1J : cfgRemotes)
+  addOnce remote1 $ (remote1J : cfgRemotes) -- idempotency
   -- update
-  updateOnce (Internal.domain remote1) remote1J' (remote1J' : cfgRemotes)
-  updateFail (Internal.domain remote1) remote1J''
+  updateOnce (Internal.domain remote1) remote1' (remote1J' : cfgRemotes)
+  updateFail (Internal.domain remote1) remote1''
   -- delete
   deleteOnce (Internal.domain remote1) cfgRemotes
   deleteOnce (Internal.domain remote1) cfgRemotes -- idempotency

From e8a7f8b92afe6eca30c71ce5880e2c0d32384fdc Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 25 May 2023 16:45:34 +1000
Subject: [PATCH 118/220] Pre-emptive rework before merging

---
 services/galley/src/Galley/Run.hs             | 60 +++++++------------
 .../galley/test/integration/Federation.hs     |  9 +--
 2 files changed, 27 insertions(+), 42 deletions(-)

diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index bb7bf33d0e..2d1b2f9261 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -50,7 +50,6 @@ import Galley.App
 import qualified Galley.App as App
 import Galley.Aws (awsEnv)
 import Galley.Cassandra
-import Galley.Env (fedDomains)
 import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
@@ -62,15 +61,8 @@ import qualified Network.Wai.Middleware.Gunzip as GZip
 import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
 import Servant hiding (route)
-import Servant.Client
-  ( BaseUrl (BaseUrl),
-    ClientEnv (ClientEnv),
-    Scheme (Http),
-    defaultMakeClientRequest
-  )
 import qualified System.Logger as Log
 import Util.Options
-import Wire.API.FederationUpdate
 import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
@@ -110,7 +102,7 @@ run opts = lowerCodensity $ do
 
   forM_ (env ^. aEnv) $ \aws ->
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
-  void $ Codensity $ Async.withAsync $ runApp env updateFedDomains
+  -- void $ Codensity $ Async.withAsync $ runApp env updateFedDomains
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
   void $ Codensity $ Async.withAsync $ runApp env undefined
@@ -202,24 +194,14 @@ collectAuthMetrics m env = do
       gaugeTokenRemaing m mbRemaining
       threadDelay 1_000_000
 
-updateFedDomains :: App ()
-updateFedDomains = do
-  env <- ask
-  let ioref              = env ^. fedDomains
-      logger             = env ^. applog
-      manager'           = env ^. manager
-      Endpoint host port = env ^. brig
-      baseUrl   = BaseUrl Http (unpack host) (fromIntegral port) ""
-      clientEnv = ClientEnv manager' baseUrl Nothing defaultMakeClientRequest
-  liftIO $ getAllowedDomainsLoop logger clientEnv ioref $ updateFedDomainsCallback env
-
 -- Build the map, keyed by conversations to the list of members
 insertIntoMap :: (ConvId, a) -> Map ConvId (N.NonEmpty a) -> Map ConvId (N.NonEmpty a)
 insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
 
-deleteFederationDomainRemote :: Env -> Domain -> IO ()
-deleteFederationDomainRemote env dom = do
-  remoteUsers <- evalGalleyToIO env $ E.getRemoteMembersByDomain dom
+deleteFederationDomainRemote :: Domain -> App ()
+deleteFederationDomainRemote dom = do
+  env <- ask
+  remoteUsers <- liftIO $ evalGalleyToIO env $ E.getRemoteMembersByDomain dom
   let lCnvMap = foldr insertIntoMap mempty remoteUsers
   for_ (Map.toList lCnvMap) $ \(cnvId, rUsers) -> do
     let lCnvId = toLocalUnsafe dom cnvId
@@ -227,7 +209,7 @@ deleteFederationDomainRemote env dom = do
     -- send out to all of the local clients that are a party
     -- to the conversation. However we also don't want to DOS
     -- clients. Maybe suppress and send out a bulk version?
-    evalGalleyToIO env
+    liftIO $ evalGalleyToIO env
       $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Federation domain removal: Remove from conversation error")
       . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
       . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Federation domain removal: Action denied, remove conversation member")
@@ -259,14 +241,16 @@ deleteFederationDomainRemote env dom = do
             undefined
             ()
 
-deleteFederationDomainLocal :: Env -> Domain -> IO ()
-deleteFederationDomainLocal env dom = do
-  localUsers <- evalGalleyToIO env $ E.getLocalMembersByDomain dom
+deleteFederationDomainLocal :: Domain -> App ()
+deleteFederationDomainLocal dom = do
+  env <- ask
+  localUsers <- liftIO $ evalGalleyToIO env $ E.getLocalMembersByDomain dom
   -- As above, build the map so we can get all local users per conversation
   let rCnvMap = foldr insertIntoMap mempty localUsers
+      localDomain = env ^. options . optSettings . setFederationDomain
   -- Process each user.
   for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
-    evalGalleyToIO env
+    liftIO $ evalGalleyToIO env
       $ mapError @NoChanges (const (InternalErrorWithDescription  "No Changes: Could not remove a local member from a remote conversation."))
       $ do
         now <- liftIO $ getCurrentTime
@@ -288,11 +272,10 @@ deleteFederationDomainLocal env dom = do
           onConversationUpdated dom convUpdate
           -- let rcnv = toRemoteUnsafe dom cnv
           -- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
-  where
-    localDomain = env ^. options . optSettings . setFederationDomain
+    
 
-deleteFederationDomain :: Env -> Set FederationDomainConfig -> IO ()
-deleteFederationDomain env deletedDomains = do
+deleteFederationDomain :: Set FederationDomainConfig -> App ()
+deleteFederationDomain deletedDomains = do
   for_ deletedDomains $ \fedDomCfg -> do
     -- https://wearezeta.atlassian.net/browse/FS-1179
     -- * Remove remote users for the given domain from all conversations owned by the current host
@@ -302,15 +285,16 @@ deleteFederationDomain env deletedDomains = do
     -- * Delete all connections from local users to users for the remote domain
     -- Get all remote users for the given domain, along with conversation IDs that they are in
     let dom = domain fedDomCfg
-    deleteFederationDomainRemote env dom
+    deleteFederationDomainRemote dom
     -- Get all local users for the given domain, along with remote conversation IDs that they are in
-    deleteFederationDomainLocal env dom
+    deleteFederationDomainLocal dom
     -- Remove the remote one-on-one conversations between local members and remote members for the given domain.
     -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
-    runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
+    env <- ask
+    liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
     
-updateFedDomainsCallback :: Env -> FederationDomainConfigs -> FederationDomainConfigs -> IO ()
-updateFedDomainsCallback env old new = do
+updateFedDomainsCallback :: FederationDomainConfigs -> FederationDomainConfigs -> App ()
+updateFedDomainsCallback old new = do
   -- This code will only run when there is a change in the domain lists
   let fromFedList = Set.fromList . remotes
       prevDoms = fromFedList old
@@ -323,5 +307,5 @@ updateFedDomainsCallback env old new = do
     -- the domain list is changing frequently.
     -- FS-1179 is handling this part.
     let deletedDomains = Set.difference prevDoms currDoms
-    deleteFederationDomain env deletedDomains
+    deleteFederationDomain deletedDomains
 
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index f3924872a4..67ce757029 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -19,6 +19,7 @@ import Galley.Options (optSettings, setFederationDomain)
 import Galley.Env
 import UnliftIO.Retry
 import Control.Monad.Catch
+import Galley.Monad
 
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
@@ -47,7 +48,7 @@ updateFedDomainsTest = do
   updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval  
 
   -- Removing multiple domains
-  -- liftIO $ updateFedDomainsCallback env old new
+  -- updateFedDomainsCallback old new
 
 constHandlers :: MonadIO m => [RetryStatus -> Handler m Bool]
 constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
@@ -72,7 +73,7 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
   _ <- postQualifiedMembers alice (remoteCharlie <| remoteBob :| []) convId
   liftIO $ threadDelay $ 3  * 1000000
   -- Remove the remote user from the local domain
-  liftIO $ updateFedDomainsCallback env old new
+  liftIO $ runApp env $ updateFedDomainsCallback old new
   -- Check that the conversation still exists.
   getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
     const 200 === statusCode
@@ -100,7 +101,7 @@ updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval = do
   _ <- postQualifiedMembers alice (remoteBob :| []) convId
 
   -- No-op
-  liftIO $ updateFedDomainsCallback env old new
+  liftIO $ runApp env $ updateFedDomainsCallback old new
   -- Check that the conversation still exists.
   getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
     const 200 === statusCode
@@ -125,7 +126,7 @@ updateFedDomainsTestNoop env remoteDomain interval = do
   connectWithRemoteUser alice remoteBob
   _ <- postQualifiedMembers alice (remoteBob :| []) convId
   -- No-op
-  liftIO $ updateFedDomainsCallback env old new
+  liftIO $ runApp env $ updateFedDomainsCallback old new
   -- Check that the conversation still exists.
   getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
     const 200 === statusCode

From 56f7954a5ebffb13177f8b312f811745c26ffc83 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 25 May 2023 09:14:30 +0200
Subject: [PATCH 119/220] Mark flaky test case.

---
 services/brig/test/integration/API/Federation.hs | 2 +-
 services/brig/test/integration/Util.hs           | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs
index 27f01e2bf0..20cf31c64d 100644
--- a/services/brig/test/integration/API/Federation.hs
+++ b/services/brig/test/integration/API/Federation.hs
@@ -74,7 +74,7 @@ tests m opts brig cannon fedBrigClient =
         test m "POST /federation/search-users : Found (multiple users)" (testFulltextSearchMultipleUsers opts brig),
         test m "POST /federation/search-users : NotFound" (testSearchNotFound opts),
         test m "POST /federation/search-users : Empty Input - NotFound" (testSearchNotFoundEmpty opts),
-        test m "POST /federation/search-users : configured restrictions" (testSearchRestrictions opts brig),
+        flakyTest m "POST /federation/search-users : configured restrictions" (testSearchRestrictions opts brig),
         test m "POST /federation/get-user-by-handle : configured restrictions" (testGetUserByHandleRestrictions opts brig),
         test m "POST /federation/get-user-by-handle : Found" (testGetUserByHandleSuccess opts brig),
         test m "POST /federation/get-user-by-handle : NotFound" (testGetUserByHandleNotFound opts),
diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs
index 29534bcdbf..7b52777ea5 100644
--- a/services/brig/test/integration/Util.hs
+++ b/services/brig/test/integration/Util.hs
@@ -100,6 +100,7 @@ import Test.Tasty (TestName, TestTree)
 import Test.Tasty.Cannon
 import qualified Test.Tasty.Cannon as WS
 import Test.Tasty.HUnit
+import Test.Tasty.Pending (flakyTestCase)
 import Text.Printf (printf)
 import qualified UnliftIO.Async as Async
 import Util.Options
@@ -227,6 +228,9 @@ instance ToJSON SESNotification where
 test :: Manager -> TestName -> Http a -> TestTree
 test m n h = testCase n (void $ runHttpT m h)
 
+flakyTest :: Manager -> TestName -> Http a -> TestTree
+flakyTest m n h = flakyTestCase n (void $ runHttpT m h)
+
 twoRandomUsers :: (MonadCatch m, MonadIO m, MonadHttp m, HasCallStack) => Brig -> m (Qualified UserId, UserId, Qualified UserId, UserId)
 twoRandomUsers brig = do
   quid1 <- userQualifiedId <$> randomUser brig

From 01ef6ff32b8378052682768ddee207dff09dbb0c Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 25 May 2023 12:34:17 +0200
Subject: [PATCH 120/220] docs.

---
 .../developer/federation-design-aspects.md    |  18 +-
 .../src/developer/reference/config-options.md |   4 +-
 docs/src/understand/configure-federation.md   | 126 +++++++++-
 .../federation/backend-communication.md       | 221 ------------------
 docs/src/understand/searchability.md          |  27 +--
 5 files changed, 138 insertions(+), 258 deletions(-)

diff --git a/docs/src/developer/developer/federation-design-aspects.md b/docs/src/developer/developer/federation-design-aspects.md
index c2bccf47c0..0d2805e472 100644
--- a/docs/src/developer/developer/federation-design-aspects.md
+++ b/docs/src/developer/developer/federation-design-aspects.md
@@ -14,14 +14,16 @@ service keeps a cache in an `IORef` in its `Env` (this information is
 needed in many end-points, so it has to remain as fast as read access
 to `Env`).
 
-This section elaborates on the implementation.  See
-{ref}`configuring-remote-connections` for the administrator's point of
-view.  If you haven't done so, go read that section now!
+See {ref}`configure-federation-strategy-in-brig` for the
+administrator's point of view.  If you haven't done so, go read that
+section now!
 
 The state is persisted in cassandra table `brig.federation_remotes`.
 brig provides the contents via an internal CRUD API (see
-{ref}`configuring-remote-connections` for the links).  In the future,
-we may decide that brig needs to cache the table itself, but for now
-(`GET` is only used for the internal end-point to share it with other
-services) we hope to get away with the simple solution and always read
-from cassandra directly.
+{ref}`configure-federation-strategy-in-brig` for the links).  In the
+future, we may decide that brig needs to cache the table itself, but
+for now (`GET` is only used for the internal end-point to share it
+with other services) we hope to get away with the simple solution and
+always read from cassandra directly.
+
+(More details to be added?)
diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md
index 0c385e33c8..eb08e5e38c 100644
--- a/docs/src/developer/reference/config-options.md
+++ b/docs/src/developer/reference/config-options.md
@@ -393,7 +393,7 @@ settings:
 
 ### Federation allow list
 
-See {ref}`configuring-remote-connections` (since [PR#3260](https://github.com/wireapp/wire-server/pull/3260)).
+See {ref}`configure-federation-strategy-in-brig` (since [PR#3260](https://github.com/wireapp/wire-server/pull/3260)).
 
 ### Federation TLS Config
 
@@ -578,7 +578,7 @@ any key package whose expiry date is set further than 15 days after upload time
 
 #### Restrict user search
 
-See {ref}`configuring-remote-connections` (since [PR#3260](https://github.com/wireapp/wire-server/pull/3260)).
+See {ref}`configure-federation-strategy-in-brig` (since [PR#3260](https://github.com/wireapp/wire-server/pull/3260)).
 
 
 ### API Versioning
diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index 9b28b9f784..bd34169275 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -399,7 +399,7 @@ cargohold:
 
 ### Configure federation strategy (whom to federate with) in brig
 
-(**This section is valid as of the release containing [PR#3260](https://github.com/wireapp/wire-server/pull/3260).**)
+**Since [PR#3260](https://github.com/wireapp/wire-server/pull/3260).**
 
 You also need to define the federation strategy (whom to federate
 with), and the frequency with which the other backend services will
@@ -411,13 +411,129 @@ refresh their cache of this configuration.
 brig:
   config:
     optSettings:
-      setFederationStrategy: AllowNone # [AllowAll | AllowDynamic | AllowNone]
+      setFederationStrategy: allowNone # [allowAll | allowDynamic | allowNone]
       setFederationDomainConfigsUpdateFreq: 10 # seconds
 ```
 
-The default of `AllowNone` probably doesn't make sense if you are
-reading this.  See {ref}`configuring-remote-connections` for details
-on the alternatives.
+The default strategy of `allowNone` effectively disables federation
+(and probably isn't what you want if you are reading this).
+`allowAll` federates with any backend that requests contact or that a
+user uses in a search.  `allowDynamic` only federates with known
+remote backends listed in cassandra.
+
+The update frequence determines how often other services will refresh
+the information about remote connections from brig.
+
+More information about individual remote connections is stored in
+brig's cassandra, and maintained via internal brig api end-points by
+the sysadmin:
+
+* [`POST`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/post_i_federation_remotes)
+
+  - after adding a new remote backend, wait for the other end to do
+    the same with you, and then wait a few moments for things to
+    stabilize (at least `update_interval * 2`; see below).
+
+* [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
+
+  - this serves an object with 3 fields:
+     - `remotes` (from cassandra): the list of remote domains with search strategy (and
+       possibly other information in the future);
+     - `strategy` (from config): one of `allowNone`, `allowDynamic`, `allowAll` (see above)
+     - `update_interval` (from config): the suggested update frequency with which calling
+       services should refresh their information.
+
+  - It doesn't serve the local domain, which needs to be configured
+   for every service that needs to know it individually.  This may
+   change in the future.
+
+  - This end-point enjoys a comparably high amount of traffic.  If you
+   have many a large instance (say, >100 pods), *and* you set a very
+   short update interval (<10s), you should monitor brig's service and
+   database load closely in the beginning.
+
+* [`PUT`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/put_i_federation_remotes__domain_)
+
+* [`DELETE`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/delete_i_federation_remotes__domain_)
+  - **WARNING:** If you delete a connection, all users from that
+   remote will be removed from local conversations, and all
+   conversations hosted by that remote will be removed from the local
+   backend.  Connections between local and remote users that are
+   removed will be archived, and can be re-established should you
+   decide to add the same backend later.
+
+The `remotes` list looks like this:
+
+```
+[
+  {
+    "domain": "wire.example.com",
+    "search_policy": "full_search"
+  },
+  {
+    "domain": "evil.example.com"
+  },
+  ...
+]
+```
+
+It serves two purposes:
+
+1. If federation strategy is `allowDynamic`, only backends that are
+  listed can be reached by us and can reach us;
+
+2. Independently of the federation strategy, the list provides
+  information about remote backends that may change dynamically (at
+  the time of writing this: search policy, see
+  {ref}`searching-users-on-another-federated-backend`)
+
+The search policy for a remote backend can be:
+
+- `no_search`: No users are returned by federated searches.  default.
+- `exact_handle_search`: Only users where the handle exactly matches are returned.
+- `full_search`: Additionally to `exact_handle_search`, users are found by a freetext search on handle and display name.
+
+Default is `no_search`.
+
+Also see {ref}`configuring-remote-connections-dev-perspective` for the
+developer's point of view on this topic.
+
+#### If your instance has been federating before
+
+Only needed if your instance has been federating with other instances
+prior to [PR#3260](https://github.com/wireapp/wire-server/pull/3260).
+
+The new configuration process ignores the federation policy set in the
+federator config under TODO NOISE FROM HERE ON OUT ***
+
+TODO: you need to update config files!
+  - complete list of search policies, no more defaults
+  - new fed strategy syntax (keep the old, just copy)
+  - later, remove the old syntax in brig, federator.
+
+As of the release containing
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260),
+[`federationStrategy`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/federator/values.yaml#L44-L45)
+in the federation config file is ignored, and brig's cassandra is used
+instead.  Furthermore, for a transition period,
+[`setFederationDomainConfigs`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/brig/templates/configmap.yaml#L250-L252)
+from the brig config file also remains being honored.  Attempting to
+delete entries that occur in the config file will trigger an error;
+delete from the config file first, then from cassandra.
+
+In the future, wire-server will stop honoring the config file data,
+and solely rely on brig's cassandra.  From that point onward, you can
+delete any connection, whether listed in the config file or not.
+Watch out for the release notes to learn when this will happen.
+(Something like *"[Federation only] support for remote configuration
+in config file is discontinued.  Before upgrading to this release,
+upgrade to the release containing
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260) first.
+After upgrading to this release, `setFederationDomainConfigs` in brig's
+config file will be ignored, and you should remove it at your
+convenience.*)
+
+
 
 ### Configure federator process to run and allow incoming traffic
 
diff --git a/docs/src/understand/federation/backend-communication.md b/docs/src/understand/federation/backend-communication.md
index 0f7fba5aa6..a71c6e158b 100644
--- a/docs/src/understand/federation/backend-communication.md
+++ b/docs/src/understand/federation/backend-communication.md
@@ -153,224 +153,3 @@ search request from *Alice*, one of its clients.
 :width: 100%
 :align: center
 ```
-
-(configuring-remote-connections)=
-
-## Configuring Remote Connections
-
-**Since [PR#3260](https://github.com/wireapp/wire-server/pull/3260).**
-
-
-Related: {ref}`configure-federation-strategy-in-brig`.  (TODO: or move this entire section there?)
-
-
-Brig keeps track of the following information for all services that
-need to know:
-
-* Federation strategy
-  - `allowNone`: federation is effectively disabled
-  - `allowAll`: no restriction on whom to federate with
-  - `allowDynamic`: only allow federating with a domain list maintained via an internal CRUD API (see below).
-* Settings for remote domains
-  - domain
-  - search policy: valid values are:
-    - `no_search`: No users are returned by federated searches.  default.
-    - `exact_handle_search`: Only users where the handle exactly matches are returned.
-    - `full_search`: Additionally to `exact_handle_search`, users are found by a freetext search on handle and display name.
-
-
-
-
-
-
-does anybody know off the top of their heads: is [this section](https://wearezeta.atlassian.net/wiki/spaces/BAC/pages/288620677/Processes+shared+with+CS#Different-search-visibility-per-team) still up to date?  and is stern?  [this page](https://docs.wire.com/developer/reference/config-options.html#federated-domain-specific-configuration-settings) tells a different story...
-
-
-
-
-* UpdateFrequency
-
-
-
-
-The federation strategy (ie., whom to federate with) and the
-information about remote backends are maintained by brig and made
-available to other services via these CRUD end-points:
-
-* [`POST`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/post_i_federation_remotes)
-* [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
-* [`PUT`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/put_i_federation_remotes__domain_)
-* [`DELETE`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/delete_i_federation_remotes__domain_)
-  **WARNING:** If you delete a connection, all users from that remote
-  will be removed from local conversations, and all conversations hosted
-  by that remote will be removed from the local backend.  Connections
-  between local and remote users that are removed will be archived, and
-  can be re-established should you decide to add the same backend later.
-
-The list of remotes is stored in the cassandra in
-`brig.federation_remotes`.  Brig's config file contains the federation
-strategy and update frequency that will be assumed by the pulling
-services.
-
-.  Relevant
-config options:
-
-
-{-
-TODO: this paragraph still annoys me.  move strategy to brig, too?  or
-at least to a different syntax, and force admin to use both old and
-new syntax until transition period is over?  just to avoid the
-confusing bogus `:` at the end of the flag.
-
-The federation strategy (allow all or allow dynamic) is still configured
-in federator, only the list of allowed hosts is ignored; if you select
-"allow all" (or if you disable federation), the list of known backends
-maintained by brig is mostly ignored, but e.g., search policy is still
-considered by brig itself.
--}
-
-{-
-
-TODO: explain how brig doesn't cache, but always read from the
-database, and that if you have update cycles of <10? secs, and/or
-clusters with >100? pods, you should monitor the load a little after
-upgrade.
-
--}
-
-TODO: explain how things need a while to stabilize (configurable), but
-that the other backend also needs to know us in order to be reachable.
-(how do we handle one backend being known to the other first for a few
-minutes / hours?)
-
-
-Update intervals are currently supplied by Brig in same response that
-carries the federation domain lists. This allows for simplified control
-of the update times and minimises changes to both services and their
-configuration files.
-
-
-See {ref}`configuring-remote-connections-dev-perspective` for the
-developer's point of view on this topic.
-
-### Transitioning from config file to database state
-
-transitioning is only necessary if you (1) upgrade and not install fresh; and (2) already have federation enabled before the upgrade.
-
-TODO: you need to update config files!
-  - complete list of search policies, no more defaults
-  - new fed strategy syntax (keep the old, just copy)
-  - later, remove the old syntax in brig, federator.
-
-As of the release containing
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260),
-[`federationStrategy`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/federator/values.yaml#L44-L45)
-in the federation config file is ignored, and brig's cassandra is used
-instead.  Furthermore, for a transition period,
-[`setFederationDomainConfigs`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/brig/templates/configmap.yaml#L250-L252)
-from the brig config file also remains being honored.  Attempting to
-delete entries that occur in the config file will trigger an error;
-delete from the config file first, then from cassandra.
-
-In the future, wire-server will stop honoring the config file data,
-and solely rely on brig's cassandra.  From that point onward, you can
-delete any connection, whether listed in the config file or not.
-Watch out for the release notes to learn when this will happen.
-(Something like *"[Federation only] support for remote configuration
-in config file is discontinued.  Before upgrading to this release,
-upgrade to the release containing
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260) first.
-After upgrading to this release, `setFederationDomainConfigs` in brig's
-config file will be ignored, and you should remove it at your
-convenience.  See
-[docs](https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
-for details."*)
-
-
-
-### noise!!!
-
-
-
-### Federation allow list
-
-As of 2021-07, federation (whatever is implemented by the time you read this) is turned off by default by means of having an empty allow list:
-
-```yaml
-# federator.yaml
-optSettings:
-  federationStrategy:
-    allowedDomains: []
-```
-
-You can choose to federate with a specific list of allowed servers:
-
-
-```yaml
-# federator.yaml
-optSettings:
-  federationStrategy:
-    allowedDomains:
-      - server1.example.com
-      - server2.example.com
-```
-
-or, you can federate with everyone:
-
-```yaml
-# federator.yaml
-optSettings:
-  federationStrategy:
-    # note the 'empty' value after 'allowAll'
-    allowAll:
-
-# when configuring helm charts, this becomes (note 'true' after 'allowAll')
-# inside helm_vars/wire-server:
-federator:
-  optSettings:
-    federationStrategy:
-      allowAll: true
-```
-
-
-
-
-this is deprecated:
-
-```
-  setFederationDomainConfigs:
-    - domain: example.com
-      search_policy: no_search
-
-```
-
-
-
-
-**This section is deprecated .  See
-https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
-for details.**
-
-#### Restrict user search
-
-TODO: deprecate this, also rename this section.  it's about federation now.
-
-TODO: should we consider the federation strategy from federator in the
-union returned by brig for a transition period as well?  (if not, we
-need to insist on updating brig's config before this upgrade.  no
-remote backend may be unlisted and use the search policy default.  we
-should also crash on startup when somebody tries that.)
-
-The lookup and search of users on a wire instance can be configured. This can be done per federated domain.
-
-```yaml
-# [brig.yaml]
-optSettings:
-  setFederationDomainConfigs:
-    - domain: example.com
-      search_policy: no_search
-```
-
-
-
-TODO: explain setFederationDomainConfigsUpdateFreq
diff --git a/docs/src/understand/searchability.md b/docs/src/understand/searchability.md
index 2f37fdcc09..8a444d5db6 100644
--- a/docs/src/understand/searchability.md
+++ b/docs/src/understand/searchability.md
@@ -99,29 +99,13 @@ galley:
 
 This default value applies to all teams for which no explicit configuration of the `TeamSearchVisibility` has been set.
 
-## Searching users on another federated backend
-
-
-Allowing search is done at the backend configuration level by the sysadmin:
+(searching-users-on-another-federated-backend)=
 
-- A configuration setting `FederatedUserSearchPolicy` per federating domain with these possible values:
-
-  - `no_search` The federating backend is not allowed to search any users (either by exact handle or full-text).
-  - `exact_handle_search` The federating backend may only search by exact handle
-  - `full_search` The federating backend may search users by full text search on display name and handle. The search search results are additionally affected by `SearchVisibilityInbound` setting of each team on the backend.
+## Searching users on another federated backend
 
-  The configuration value `FederatedUserSearchPolicy` is per federated domain, e.g. in the values of the wire-server chart:
-  
-  ```yaml
-  brig:
-    config:
-      optSettings:
-        setFederationDomainConfigs:
-          - domain: a.example.com
-            search_policy: no_search
-          - domain: a.example.com
-            search_policy: full_search
-  ```
+- Setting the search policy for individual remote federated backends
+  is done via a internal brig api end-points by a sysadmin (see
+  {ref}`configure-federation-strategy-in-brig`}.
 
 - The `SearchVisibilityInbound` setting applies. Since the default value for teams is `SearchableByOwnTeam` this means that for a team to be full-text searchable by users on a federating backend both
 
@@ -271,4 +255,3 @@ settings:
   featureFlags:
     teamSearchVisibility: disabled-by-default # or enabled-by-default
 ```
-

From a12cf1bb090ebbf12718a6bcaa66120a9dc45b94 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Fri, 26 May 2023 17:23:33 +1000
Subject: [PATCH 121/220] Formatting

---
 integration/test/Test/Brig.hs                 |   9 +-
 services/brig/src/Brig/API/Federation.hs      |   2 +-
 services/galley/src/Galley/API/Action.hs      |  25 ++--
 services/galley/src/Galley/API/Util.hs        |  15 ++-
 .../galley/src/Galley/Cassandra/Connection.hs |  14 +-
 .../src/Galley/Cassandra/Conversation.hs      |   2 +-
 .../galley/src/Galley/Effects/MemberStore.hs  |   2 +-
 services/galley/src/Galley/Run.hs             | 125 ++++++++++--------
 .../galley/test/integration/Federation.hs     |  62 +++++----
 services/galley/test/integration/Main.hs      |   4 +-
 10 files changed, 142 insertions(+), 118 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 5bb93e8be4..66502f1858 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -27,9 +27,10 @@ testCrudFederationRemotes = do
   let parseFedConns :: HasCallStack => Response -> App [Value]
       parseFedConns resp =
         -- Pick out the list of federation domain configs
-        getJSON 200 resp %. "remotes" & asList
-        -- Enforce that the values are objects and not something else
-        >>= traverse (fmap Object . asObject)
+        getJSON 200 resp %. "remotes"
+          & asList
+          -- Enforce that the values are objects and not something else
+          >>= traverse (fmap Object . asObject)
       addOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => fedConn -> [fedConn2] -> App ()
       addOnce fedConn want = do
         res <- Internal.createFedConn OwnDomain fedConn
@@ -74,7 +75,7 @@ testCrudFederationRemotes = do
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
 
-  remote1J  <- make remote1
+  remote1J <- make remote1
   remote1J' <- make remote1'
 
   resetFedConns OwnDomain
diff --git a/services/brig/src/Brig/API/Federation.hs b/services/brig/src/Brig/API/Federation.hs
index fd00e6136d..61edef6ae9 100644
--- a/services/brig/src/Brig/API/Federation.hs
+++ b/services/brig/src/Brig/API/Federation.hs
@@ -219,4 +219,4 @@ onUserDeleted origDomain udcn = lift $ do
     pooledForConcurrentlyN_ 16 (nonEmpty acceptedLocals) $ \(List1 -> recipients) ->
       notify event (tUnqualified deletedUser) Push.RouteDirect Nothing (pure recipients)
   wrapClient $ Data.deleteRemoteConnections deletedUser connections
-  pure EmptyResponse
\ No newline at end of file
+  pure EmptyResponse
diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs
index 95115b0355..6e4c41522b 100644
--- a/services/galley/src/Galley/API/Action.hs
+++ b/services/galley/src/Galley/API/Action.hs
@@ -86,6 +86,7 @@ import Polysemy.Error
 import Polysemy.Input
 import Polysemy.TinyLog
 import qualified Polysemy.TinyLog as P
+import qualified Polysemy.TinyLog as TinyLog
 import qualified System.Logger as Log
 import Wire.API.Conversation hiding (Conversation, Member)
 import Wire.API.Conversation.Action
@@ -102,7 +103,6 @@ import Wire.API.Team.LegalHold
 import Wire.API.Team.Member
 import Wire.API.Unreachable
 import qualified Wire.API.User as User
-import qualified Polysemy.TinyLog as TinyLog
 
 data NoChanges = NoChanges
 
@@ -335,12 +335,19 @@ performAction tag origUser lconv action = do
       pure (mempty, action)
     SConversationRemoveMembersTag -> do
       let presentVictims = filter (isConvMemberL lconv) (toList action)
-      _ <- error $
-        "-----------------------------\n\n\n" <>
-        "lconv = " <> show lconv <> "\n\n\n" <>
-        "action = " <> show action <> "\n\n\n" <>
-        "presentVictims = " <> show presentVictims <> "\n\n\n" <>
-        "-----------------------------"
+      _ <-
+        error $
+          "-----------------------------\n\n\n"
+            <> "lconv = "
+            <> show lconv
+            <> "\n\n\n"
+            <> "action = "
+            <> show action
+            <> "\n\n\n"
+            <> "presentVictims = "
+            <> show presentVictims
+            <> "\n\n\n"
+            <> "-----------------------------"
       TinyLog.err $ Log.msg ("action" :: String) . Log.field "values" (show action)
       TinyLog.err $ Log.msg ("presentVictims" :: String) . Log.field "values" (show presentVictims)
       when (null presentVictims) noChanges
@@ -659,7 +666,6 @@ updateLocalConversationUnchecked lconv qusr con action = do
     (convBotsAndMembers conv <> extraTargets)
     action'
 
-
 -- | Similar to 'updateLocalConversationUnchecked', but skips performing
 -- user authorisation checks. This is written for use in de-federation code
 -- where conversations for many users will be torn down at once and must work.
@@ -668,7 +674,8 @@ updateLocalConversationUnchecked lconv qusr con action = do
 updateLocalConversationUserUnchecked ::
   forall tag r.
   ( SingI tag,
-    HasConversationActionEffects tag r
+    HasConversationActionEffects tag r,
+    Member (Error FederationError) r
   ) =>
   Local Conversation ->
   Qualified UserId ->
diff --git a/services/galley/src/Galley/API/Util.hs b/services/galley/src/Galley/API/Util.hs
index 754cd91b3a..7263238a73 100644
--- a/services/galley/src/Galley/API/Util.hs
+++ b/services/galley/src/Galley/API/Util.hs
@@ -376,13 +376,14 @@ instance IsConvMemberId (Local UserId) LocalMember where
 
 instance IsConvMemberId (Remote UserId) RemoteMember where
   getConvMember _ conv u = find ((u ==) . rmId) (Data.convRemoteMembers conv)
-    -- error $
-    --   "----------------" <>
-    --   "\n\n\nconv = " <> show conv <>
-    --   "\n\n\nu = " <> show u <>
-    --   "\n\n\nresult = " <> show (find ((u ==) . rmId) (Data.convRemoteMembers conv)) <>
-    --   "\n\n\n" <>
-    --   "----------------"
+
+-- error $
+--   "----------------" <>
+--   "\n\n\nconv = " <> show conv <>
+--   "\n\n\nu = " <> show u <>
+--   "\n\n\nresult = " <> show (find ((u ==) . rmId) (Data.convRemoteMembers conv)) <>
+--   "\n\n\n" <>
+--   "----------------"
 
 instance IsConvMemberId (Qualified UserId) (Either LocalMember RemoteMember) where
   getConvMember loc conv =
diff --git a/services/galley/src/Galley/Cassandra/Connection.hs b/services/galley/src/Galley/Cassandra/Connection.hs
index 10a4f786aa..56301aee97 100644
--- a/services/galley/src/Galley/Cassandra/Connection.hs
+++ b/services/galley/src/Galley/Cassandra/Connection.hs
@@ -1,20 +1,20 @@
 module Galley.Cassandra.Connection where
 
-import Cassandra (MonadClient, PrepQuery, W, params, Consistency (LocalQuorum), retry, x1, write)
-import Imports
+import Cassandra (Consistency (LocalQuorum), MonadClient, PrepQuery, W, params, retry, write, x1)
 import Data.Domain
 import Galley.Cassandra.Instances ()
+import Imports
 
 -- Queries targeting this table are usually in Brig, but I've put this one
 -- here so that we don't have yet another network call to Brig when most
 -- everything is already happening in galley
 
-deleteRemoteConnectionsByDomain
-  :: MonadClient m
-  => Domain
-  -> m ()
+deleteRemoteConnectionsByDomain ::
+  MonadClient m =>
+  Domain ->
+  m ()
 deleteRemoteConnectionsByDomain domain =
   retry x1 . write remoteConnectionsDeleteByDomain $ params LocalQuorum $ pure domain
 
 remoteConnectionsDeleteByDomain :: PrepQuery W (Identity Domain) ()
-remoteConnectionsDeleteByDomain = "DELETE FROM connection_remote where right_domain = ?"
\ No newline at end of file
+remoteConnectionsDeleteByDomain = "DELETE FROM connection_remote where right_domain = ?"
diff --git a/services/galley/src/Galley/Cassandra/Conversation.hs b/services/galley/src/Galley/Cassandra/Conversation.hs
index 05bcd01b8f..d4d82fcd45 100644
--- a/services/galley/src/Galley/Cassandra/Conversation.hs
+++ b/services/galley/src/Galley/Cassandra/Conversation.hs
@@ -405,4 +405,4 @@ interpretConversationStoreToCassandra = interpret $ \case
   SetGroupId gId cid -> embedClient $ mapGroupId gId cid
   SetPublicGroupState cid gib -> embedClient $ setPublicGroupState cid gib
   AcquireCommitLock gId epoch ttl -> embedClient $ acquireCommitLock gId epoch ttl
-  ReleaseCommitLock gId epoch -> embedClient $ releaseCommitLock gId epoch
\ No newline at end of file
+  ReleaseCommitLock gId epoch -> embedClient $ releaseCommitLock gId epoch
diff --git a/services/galley/src/Galley/Effects/MemberStore.hs b/services/galley/src/Galley/Effects/MemberStore.hs
index a62d3f653b..e8e6e2e34c 100644
--- a/services/galley/src/Galley/Effects/MemberStore.hs
+++ b/services/galley/src/Galley/Effects/MemberStore.hs
@@ -50,6 +50,7 @@ module Galley.Effects.MemberStore
   )
 where
 
+import Data.Domain
 import Data.Id
 import Data.Qualified
 import Galley.Data.Services
@@ -62,7 +63,6 @@ import Wire.API.Conversation.Member hiding (Member)
 import Wire.API.MLS.Group
 import Wire.API.MLS.KeyPackage
 import Wire.API.Provider.Service
-import Data.Domain
 
 data MemberStore m a where
   CreateMembers :: ToUserRole u => ConvId -> UserList u -> MemberStore m ([LocalMember], [RemoteMember])
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 8b38335ce2..0639fb6289 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -20,7 +20,7 @@ module Galley.Run
   ( run,
     mkApp,
     updateFedDomainsCallback,
-    mkLogger
+    mkLogger,
   )
 where
 
@@ -35,24 +35,38 @@ import Control.Lens (view, (.~), (^.))
 import Control.Monad.Codensity
 import qualified Data.Aeson as Aeson
 import Data.Default
+import Data.Domain (Domain)
 import Data.Id
+import qualified Data.List.NonEmpty as N
 import qualified Data.Map as Map
 import Data.Metrics (Metrics)
 import Data.Metrics.AWS (gaugeTokenRemaing)
 import qualified Data.Metrics.Middleware as M
 import Data.Metrics.Servant (servantPlusWAIPrometheusMiddleware)
 import Data.Misc (portNumber)
+import Data.Qualified
+import qualified Data.Set as Set
+import Data.Singletons
 import Data.String.Conversions (cs)
 import Data.Text (unpack)
+import Data.Time (getCurrentTime)
 import qualified Galley.API as API
+import Galley.API.Action
+import Galley.API.Error
+import Galley.API.Federation
 import Galley.API.Internal
+import Galley.API.Util (getConversationWithError)
 import Galley.App
 import qualified Galley.App as App
 import Galley.Aws (awsEnv)
 import Galley.Cassandra
+import Galley.Cassandra.Connection
+import Galley.Data.Conversation.Types (convMetadata)
+import qualified Galley.Effects.MemberStore as E
 import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
+import Galley.Types.Conversations.Members
 import Imports
 import qualified Network.HTTP.Media.RenderHeader as HTTPMedia
 import qualified Network.HTTP.Types as HTTP
@@ -60,39 +74,29 @@ import Network.Wai
 import qualified Network.Wai.Middleware.Gunzip as GZip
 import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
+import Polysemy.Error
 import Servant hiding (route)
 import qualified System.Logger as Log
 import System.Logger.Extended (mkLogger)
 import Util.Options
+import Wire.API.Conversation (ConvType (ConnectConv, One2OneConv), cnvmType)
+import Wire.API.Conversation.Action
+import Wire.API.Conversation.Role
+import Wire.API.Error
+import Wire.API.Error.Galley
+import qualified Wire.API.Federation.API.Galley as F
+import Wire.API.FederationUpdate
 import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
-import qualified Data.List.NonEmpty as N
-import Galley.API.Action
-import Galley.Types.Conversations.Members
-import Data.Qualified
-import Wire.API.Error.Galley
-import Wire.API.Conversation.Role
-import qualified Wire.API.Federation.API.Galley as F
-import Wire.API.Error
-import Polysemy.Error
-import Galley.API.Error
-import qualified Galley.Effects.MemberStore as E
-import Data.Time (getCurrentTime)
-import Wire.API.Conversation.Action
-import Galley.API.Federation
-import Data.Singletons
-import Wire.API.Conversation (ConvType(One2OneConv), cnvmType, ConvType (ConnectConv, One2OneConv))
-import Galley.Data.Conversation.Types (convMetadata)
-import Galley.API.Util (getConversationWithError)
-import Data.Domain (Domain)
-import Galley.Cassandra.Connection
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
   l <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
-  (ioref, _) <- lift $ updateFedDomains (opts ^. optBrig) l $ \_ _ -> pure ()
+  (ioref, _) <- lift $
+    updateFedDomains (opts ^. optBrig) l $
+      \_ _ -> pure () -- TODO: Push the domain to be deleted to a RabbitMQ queue
   (app, env) <- mkApp opts ioref l
   settings <-
     lift $
@@ -105,7 +109,8 @@ run opts = lowerCodensity $ do
 
   forM_ (env ^. aEnv) $ \aws ->
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
-  -- void $ Codensity $ Async.withAsync $ runApp env updateFedDomains
+  -- TODO: Run a loop that pulls from RabbitMQ and performs the deletion
+  -- void $ Codensity $ Async.withAsync $ runApp env $ _ deleteFederationDomain
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
   void $ Codensity $ Async.withAsync $ runApp env undefined
@@ -209,13 +214,14 @@ deleteFederationDomainRemote dom = do
     -- send out to all of the local clients that are a party
     -- to the conversation. However we also don't want to DOS
     -- clients. Maybe suppress and send out a bulk version?
-    liftIO $ evalGalleyToIO env
+    liftIO
+      $ evalGalleyToIO env
       $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Federation domain removal: Remove from conversation error")
-      . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
-      . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription  "Federation domain removal: Action denied, remove conversation member")
-      . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription  "Federation domain removal: Invalid operation")
-      . mapToRuntimeError @'NotATeamMember (InternalErrorWithDescription  "Federation domain removal: Not a team member")
-      . mapError @NoChanges (const (InternalErrorWithDescription  "Federation domain removal: No changes"))
+        . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
+        . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription "Federation domain removal: Action denied, remove conversation member")
+        . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription "Federation domain removal: Invalid operation")
+        . mapToRuntimeError @'NotATeamMember (InternalErrorWithDescription "Federation domain removal: Not a team member")
+        . mapError @NoChanges (const (InternalErrorWithDescription "Federation domain removal: No changes"))
       -- This is allowed to send notifications to _local_ clients.
       -- But we are suppressing those events as we don't want to
       -- DOS our users if a large and deeply interconnected federation
@@ -227,8 +233,9 @@ deleteFederationDomainRemote dom = do
         updateLocalConversationUserUnchecked
           @'ConversationRemoveMembersTag
           lConv
-          undefined $ -- This field can be undefined as the path for ConversationRemoveMembersTag doens't use it
-          tUntagged . rmId <$> rUsers
+          undefined
+          $ tUntagged . rmId <$> rUsers -- This field can be undefined as the path for ConversationRemoveMembersTag doens't use it
+
         -- Check if the conversation if type 2 or 3, one-on-one conversations.
         -- If it is, then we need to remove the entire conversation as users
         -- aren't able to delete those types of conversations themselves.
@@ -250,39 +257,41 @@ deleteFederationDomainLocal dom = do
       localDomain = env ^. options . optSettings . setFederationDomain
   -- Process each user.
   for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
-    liftIO $ evalGalleyToIO env
-      $ mapError @NoChanges (const (InternalErrorWithDescription  "No Changes: Could not remove a local member from a remote conversation."))
-      $ do
-        now <- liftIO $ getCurrentTime
-        for_ lUsers $ \user -> do
-          let lUser = toLocalUnsafe localDomain user
-              convUpdate = F.ConversationUpdate
-                { cuTime = now
-                , cuOrigUserId = tUntagged lUser
-                , cuConvId = cnv
-                , cuAlreadyPresentUsers = mempty
-                , cuAction = SomeConversationAction (sing @'ConversationDeleteTag) ()
-                }
-          -- These functions are used directly rather than as part of a larger conversation
-          -- delete function, as we don't have an originating user, and we can't send data
-          -- to the remote backend.
-          -- We don't need to check the conversation type here, as we can't tell the
-          -- remote federation server to delete the conversation. They will have to do a
-          -- similar processing run for removing the local domain from their federation list.
-          onConversationUpdated dom convUpdate
-          -- let rcnv = toRemoteUnsafe dom cnv
-          -- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
-    
+    liftIO $
+      evalGalleyToIO env $
+        mapError @NoChanges (const (InternalErrorWithDescription "No Changes: Could not remove a local member from a remote conversation.")) $
+          do
+            now <- liftIO $ getCurrentTime
+            for_ lUsers $ \user -> do
+              let lUser = toLocalUnsafe localDomain user
+                  convUpdate =
+                    F.ConversationUpdate
+                      { cuTime = now,
+                        cuOrigUserId = tUntagged lUser,
+                        cuConvId = cnv,
+                        cuAlreadyPresentUsers = mempty,
+                        cuAction = SomeConversationAction (sing @'ConversationDeleteTag) ()
+                      }
+              -- These functions are used directly rather than as part of a larger conversation
+              -- delete function, as we don't have an originating user, and we can't send data
+              -- to the remote backend.
+              -- We don't need to check the conversation type here, as we can't tell the
+              -- remote federation server to delete the conversation. They will have to do a
+              -- similar processing run for removing the local domain from their federation list.
+              onConversationUpdated dom convUpdate
+
+-- let rcnv = toRemoteUnsafe dom cnv
+-- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
 
 deleteFederationDomain :: Set FederationDomainConfig -> App ()
 deleteFederationDomain deletedDomains = do
   for_ deletedDomains $ \fedDomCfg -> do
     -- https://wearezeta.atlassian.net/browse/FS-1179
-    -- * Remove remote users for the given domain from all conversations owned by the current host
-    -- * Remove all local users from remote conversations owned by the given domain.
+    -- \* Remove remote users for the given domain from all conversations owned by the current host
+    -- \* Remove all local users from remote conversations owned by the given domain.
     --   NOTE: This is NOT sent to other backends, as this information is not authoratative, but is
     --   good enough to tell local users about the federation connection being removed.
-    -- * Delete all connections from local users to users for the remote domain
+    -- \* Delete all connections from local users to users for the remote domain
     -- Get all remote users for the given domain, along with conversation IDs that they are in
     let dom = domain fedDomCfg
     deleteFederationDomainRemote dom
@@ -292,7 +301,7 @@ deleteFederationDomain deletedDomains = do
     -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
     env <- ask
     liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
-    
+
 updateFedDomainsCallback :: FederationDomainConfigs -> FederationDomainConfigs -> App ()
 updateFedDomainsCallback old new = do
   -- This code will only run when there is a change in the domain lists
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 67ce757029..8d824ff6cb 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -1,25 +1,25 @@
 module Federation where
 
-import TestSetup
+import API.Util
+import Bilge.Assert
+import Bilge.Response
 import Control.Lens ((^.))
-import Imports
-import Galley.Run
+import Control.Monad.Catch
 import Control.Monad.Codensity (lowerCodensity)
-import Wire.API.Routes.FederationDomainConfig
 import Data.Domain
-import Wire.API.User.Search
-import API.Util
 import Data.Id
 import Data.List.NonEmpty
 import Data.Qualified
-import Wire.API.Conversation
-import Bilge.Assert
-import Bilge.Response
-import Galley.Options (optSettings, setFederationDomain)
 import Galley.Env
-import UnliftIO.Retry
-import Control.Monad.Catch
 import Galley.Monad
+import Galley.Options
+import Galley.Run
+import Imports
+import TestSetup
+import UnliftIO.Retry
+import Wire.API.Conversation
+import Wire.API.Routes.FederationDomainConfig
+import Wire.API.User.Search
 
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
@@ -29,10 +29,13 @@ updateFedDomainsTest = do
   s <- ask
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
-  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  -- But this is how the env is made, so it is what we do
+  l <- liftIO $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
+  r <- newIORef defFederationDomainConfigs
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts r l
   -- Common variables.
   let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
-      remoteDomain  = Domain "far-away.example.com"
+      remoteDomain = Domain "far-away.example.com"
       remoteDomain2 = Domain "far-away-two.example.com"
 
   -- Setup a conversation for a known remote domain.
@@ -42,13 +45,13 @@ updateFedDomainsTest = do
   -- updateFedDomainsTestNoop env remoteDomain interval
 
   -- Adding a new federation domain, this too should be a no-op
-  -- updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval  
+  -- updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval
 
   -- Removing a single domain
-  updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval  
+  updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval
 
-  -- Removing multiple domains
-  -- updateFedDomainsCallback old new
+-- Removing multiple domains
+-- updateFedDomainsCallback old new
 
 constHandlers :: MonadIO m => [RetryStatus -> Handler m Bool]
 constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
@@ -58,8 +61,8 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
   s <- ask
   let opts = s ^. tsGConf
       localDomain = opts ^. optSettings . setFederationDomain
-      old = FederationDomainConfigs AllowList [FederationDomainConfig remoteDomain FullSearch, FederationDomainConfig remoteDomain2 FullSearch] interval
-      new = old { remotes = [FederationDomainConfig remoteDomain2 FullSearch] }
+      old = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain FullSearch, FederationDomainConfig remoteDomain2 FullSearch] interval
+      new = old {remotes = [FederationDomainConfig remoteDomain2 FullSearch]}
   qalice <- randomQualifiedUser
   bobId <- randomId
   charlieId <- randomId
@@ -68,10 +71,11 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
       remoteCharlie = Qualified charlieId remoteDomain2
   -- Create a conversation
   convId <- decodeConvId <$> postConv alice [] (Just "remote gossip") [] Nothing Nothing
+  let qConvId = Qualified convId localDomain
   connectWithRemoteUser alice remoteBob
   connectWithRemoteUser alice remoteCharlie
-  _ <- postQualifiedMembers alice (remoteCharlie <| remoteBob :| []) convId
-  liftIO $ threadDelay $ 3  * 1000000
+  _ <- postQualifiedMembers alice (remoteCharlie <| remoteBob :| []) qConvId
+  liftIO $ threadDelay $ 3 * 1000000
   -- Remove the remote user from the local domain
   liftIO $ runApp env $ updateFedDomainsCallback old new
   -- Check that the conversation still exists.
@@ -89,16 +93,17 @@ updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval = do
   s <- ask
   let opts = s ^. tsGConf
       localDomain = opts ^. optSettings . setFederationDomain
-      old = FederationDomainConfigs AllowList [FederationDomainConfig remoteDomain FullSearch] interval
-      new = old { remotes = FederationDomainConfig remoteDomain2 FullSearch : remotes old }
+      old = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain FullSearch] interval
+      new = old {remotes = FederationDomainConfig remoteDomain2 FullSearch : remotes old}
   qalice <- randomQualifiedUser
   bobId <- randomId
   let alice = qUnqualified qalice
       remoteBob = Qualified bobId remoteDomain
   -- Create a conversation
   convId <- decodeConvId <$> postConv alice [] (Just "remote gossip") [] Nothing Nothing
+  let qConvId = Qualified convId localDomain
   connectWithRemoteUser alice remoteBob
-  _ <- postQualifiedMembers alice (remoteBob :| []) convId
+  _ <- postQualifiedMembers alice (remoteBob :| []) qConvId
 
   -- No-op
   liftIO $ runApp env $ updateFedDomainsCallback old new
@@ -115,7 +120,7 @@ updateFedDomainsTestNoop env remoteDomain interval = do
   s <- ask
   let opts = s ^. tsGConf
       localDomain = opts ^. optSettings . setFederationDomain
-      old = FederationDomainConfigs AllowList [FederationDomainConfig remoteDomain FullSearch] interval
+      old = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain FullSearch] interval
       new = old
   qalice <- randomQualifiedUser
   bobId <- randomId
@@ -123,8 +128,9 @@ updateFedDomainsTestNoop env remoteDomain interval = do
       remoteBob = Qualified bobId remoteDomain
   -- Create a conversation
   convId <- decodeConvId <$> postConv alice [] (Just "remote gossip") [] Nothing Nothing
+  let qConvId = Qualified convId localDomain
   connectWithRemoteUser alice remoteBob
-  _ <- postQualifiedMembers alice (remoteBob :| []) convId
+  _ <- postQualifiedMembers alice (remoteBob :| []) qConvId
   -- No-op
   liftIO $ runApp env $ updateFedDomainsCallback old new
   -- Check that the conversation still exists.
@@ -133,4 +139,4 @@ updateFedDomainsTestNoop env remoteDomain interval = do
     let findRemote :: Conversation -> Maybe (Qualified UserId)
         findRemote = find (== remoteBob) . fmap omQualifiedId . cmOthers . cnvMembers
     const (Right $ pure remoteBob) === (fmap findRemote <$> responseJsonEither)
-    const (Right qalice) === (fmap (memId . cmSelf . cnvMembers) <$> responseJsonEither)
\ No newline at end of file
+    const (Right qalice) === (fmap (memId . cmSelf . cnvMembers) <$> responseJsonEither)
diff --git a/services/galley/test/integration/Main.hs b/services/galley/test/integration/Main.hs
index 8351db5040..0a5ced3f36 100644
--- a/services/galley/test/integration/Main.hs
+++ b/services/galley/test/integration/Main.hs
@@ -34,6 +34,7 @@ import Data.Tagged
 import Data.Text (pack)
 import Data.Text.Encoding (encodeUtf8)
 import Data.Yaml (decodeFileEither)
+import Federation
 import Galley.API (sitemap)
 import qualified Galley.Aws as Aws
 import Galley.Options
@@ -47,13 +48,12 @@ import qualified System.Logger.Class as Logger
 import Test.Tasty
 import Test.Tasty.HUnit
 import Test.Tasty.Options
+import TestHelpers (test)
 import TestSetup
 import Util.Options
 import Util.Options.Common
 import Util.Test
 import qualified Util.Test.SQS as SQS
-import Federation
-import TestHelpers (test)
 
 newtype ServiceConfigFile = ServiceConfigFile String
   deriving (Eq, Ord, Typeable)

From e779c861c3b710286d8ed1354f62ac5c207c5f8d Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 29 May 2023 16:56:19 +1000
Subject: [PATCH 122/220] FS-1179: Setting up the receiving RabbitMQ loop.

Setup a reading loop for RabbitMQ to get the domains that need to be
deleted. We're using a queue here so that we have a non-volatile store
for these domains that can automatically handle galley instances dying
or being scaled down.
---
 .../wire-api/src/Wire/API/FederationUpdate.hs | 10 ++-
 services/galley/galley.cabal                  |  1 +
 services/galley/src/Galley/Options.hs         | 22 ++++-
 services/galley/src/Galley/Run.hs             | 86 +++++++++++++++----
 4 files changed, 101 insertions(+), 18 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 38cc134967..63be77b820 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -1,6 +1,8 @@
 module Wire.API.FederationUpdate
   ( FedUpdateCallback,
     updateFedDomains,
+    getAllowedDomainsInitial,
+    updateFedDomains'
   )
 where
 
@@ -72,7 +74,11 @@ updateFedDomains :: Endpoint -> L.Logger -> FedUpdateCallback -> IO (IORef Feder
 updateFedDomains (Endpoint h p) log' cb = do
   clientEnv <- newManager defaultManagerSettings <&> \mgr -> ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
   ioref <- newIORef =<< getAllowedDomainsInitial log' clientEnv
-  updateDomainsThread <- async $ getAllowedDomainsLoop log' clientEnv cb ioref
-  pure (ioref, updateDomainsThread)
+  updateFedDomains' ioref clientEnv log' cb
   where
     baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
+
+updateFedDomains' :: IORef FederationDomainConfigs -> ClientEnv -> L.Logger -> FedUpdateCallback -> IO (IORef FederationDomainConfigs, Async ())
+updateFedDomains' ioref clientEnv log' cb = do
+  updateDomainsThread <- async $ getAllowedDomainsLoop log' clientEnv cb ioref
+  pure (ioref, updateDomainsThread)
\ No newline at end of file
diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index 27b2ab430a..ce0b2d4004 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -196,6 +196,7 @@ library
       aeson                  >=2.0.1.0
     , amazonka               >=1.4.5
     , amazonka-sqs           >=1.4.5
+    , amqp
     , asn1-encoding
     , asn1-types
     , async                  >=2.0
diff --git a/services/galley/src/Galley/Options.hs b/services/galley/src/Galley/Options.hs
index d9776451eb..ee19066ec2 100644
--- a/services/galley/src/Galley/Options.hs
+++ b/services/galley/src/Galley/Options.hs
@@ -52,6 +52,12 @@ module Galley.Options
     optLogLevel,
     optLogNetStrings,
     optLogFormat,
+    optRabbitMq,
+    RabbitMqOpts,
+    rmqHost,
+    rmqPort,
+    rmqVhost,
+    rmqQueue
   )
 where
 
@@ -145,6 +151,18 @@ deriveFromJSON toOptionFieldName ''JournalOpts
 
 makeLenses ''JournalOpts
 
+-- Based on Wire.BackgroundWorker.Options
+data RabbitMqOpts = RabbitMqOpts
+  { _rmqHost :: !String,
+    _rmqPort :: !Int,
+    _rmqVhost :: !Text,
+    _rmqQueue :: !Text
+  }
+  deriving (Show, Generic)
+makeLenses ''RabbitMqOpts
+deriveFromJSON toOptionFieldName ''RabbitMqOpts
+
+
 data Opts = Opts
   { -- | Host and port to bind to
     _optGalley :: !Endpoint,
@@ -172,7 +190,9 @@ data Opts = Opts
     --  <http://cr.yp.to/proto/netstrings.txt>
     _optLogNetStrings :: !(Maybe (Last Bool)),
     -- | What log format to use
-    _optLogFormat :: !(Maybe (Last LogFormat))
+    _optLogFormat :: !(Maybe (Last LogFormat)),
+    -- | RabbitMQ
+    _optRabbitMq :: !(Maybe RabbitMqOpts)
   }
 
 deriveFromJSON toOptionFieldName ''Opts
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 0639fb6289..32400f6b84 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -90,13 +90,22 @@ import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
+import Network.AMQP.Extended (openConnectionWithRetries, RabbitMqHooks (RabbitMqHooks))
+import qualified Network.AMQP.Extended as AMQP
+import qualified Network.AMQP as AMQP
+import qualified Network.AMQP.Types as AMQP
+import Network.HTTP.Client (defaultManagerSettings, newManager)
+import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http))
+import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
+import qualified System.Logger as L
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
   l <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
-  (ioref, _) <- lift $
-    updateFedDomains (opts ^. optBrig) l $
-      \_ _ -> pure () -- TODO: Push the domain to be deleted to a RabbitMQ queue
+  let Endpoint h p = opts ^. optBrig
+  clientEnv <- liftIO $ newManager defaultManagerSettings <&> \mgr ->
+    ClientEnv mgr (BaseUrl  Http (unpack h) (fromIntegral p) "") Nothing defaultMakeClientRequest
+  ioref <- liftIO $ newIORef =<< getAllowedDomainsInitial l clientEnv
   (app, env) <- mkApp opts ioref l
   settings <-
     lift $
@@ -109,13 +118,57 @@ run opts = lowerCodensity $ do
 
   forM_ (env ^. aEnv) $ \aws ->
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
+
   -- TODO: Run a loop that pulls from RabbitMQ and performs the deletion
-  -- void $ Codensity $ Async.withAsync $ runApp env $ _ deleteFederationDomain
+  maybe
+    (pure ())
+    (\rmq -> void $ Codensity $ Async.withAsync $ readFromRabbitMQ l env rmq)
+    $ opts ^. optRabbitMq
+
+  -- TODO: Run a loop that polls brig for the new domain list, updating the IORef
+  -- The remote domain deletion is tracked with a rabbitmq queue. This is so that
+  -- we can ensure only one galley instance is removing domains and that they aren't
+  -- stepping on each others toes. It also allows for the galley instance to go down
+  -- without losing information about which domains still need to be removed.
+  void $ Codensity $ Async.withAsync $ updateFedDomains' ioref clientEnv l $ \_old _new -> do
+    -- TODO: Push the deleted domains to rabbit so they can be deleted.
+    pure ()
+
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
   void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
+
+readFromRabbitMQ :: L.Logger -> Env -> RabbitMqOpts -> IO ()
+readFromRabbitMQ l env rmq = openConnectionWithRetries l host port vhost $ RabbitMqHooks
+  { AMQP.onConnectionClose = pure () -- Log that the channel closed?
+  , AMQP.onChannelException = const $ pure () -- Log the exception?
+  , AMQP.onNewChannel = \channel -> do
+    -- Ensure that the queue exists and is single active consumer.
+    -- Queue declaration is idempotent
+    let headers = AMQP.FieldTable $ Map.fromList [("x-single-active-consumer", AMQP.FVBool True)]
+    void $ AMQP.declareQueue channel $ AMQP.newQueue { AMQP.queueName = rmq ^. rmqQueue, AMQP.queueHeaders = headers }
+    -- Read messages from RabbitMQ, process the message, and ACK or NACK it as appropriate.
+    void $ AMQP.consumeMsgs channel queue AMQP.Ack $ \(message, envelope) -> do
+      case Aeson.eitherDecode (AMQP.msgBody message) of
+        Left e -> do
+          Log.err l $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
+          AMQP.nackEnv envelope
+        Right dom -> do
+          runApp env $ do
+            deleteFederationDomainRemote dom
+            deleteFederationDomainLocal dom
+            deleteFederationDomainOneOnOne dom
+          AMQP.ackEnv envelope
+  }
+  where
+    host = rmq ^. rmqHost
+    port = rmq ^. rmqPort
+    vhost = rmq ^. rmqVhost
+    queue = rmq ^. rmqQueue
+
+
 mkApp :: Opts -> IORef FederationDomainConfigs -> Log.Logger -> Codensity IO (Application, Env)
 mkApp opts fedDoms logger =
   do
@@ -283,6 +336,11 @@ deleteFederationDomainLocal dom = do
 -- let rcnv = toRemoteUnsafe dom cnv
 -- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
 
+deleteFederationDomainOneOnOne :: Domain -> App ()
+deleteFederationDomainOneOnOne dom = do
+  env <- ask
+  liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
+
 deleteFederationDomain :: Set FederationDomainConfig -> App ()
 deleteFederationDomain deletedDomains = do
   for_ deletedDomains $ \fedDomCfg -> do
@@ -299,8 +357,7 @@ deleteFederationDomain deletedDomains = do
     deleteFederationDomainLocal dom
     -- Remove the remote one-on-one conversations between local members and remote members for the given domain.
     -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
-    env <- ask
-    liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
+    deleteFederationDomainOneOnOne dom    
 
 updateFedDomainsCallback :: FederationDomainConfigs -> FederationDomainConfigs -> App ()
 updateFedDomainsCallback old new = do
@@ -308,12 +365,11 @@ updateFedDomainsCallback old new = do
   let fromFedList = Set.fromList . remotes
       prevDoms = fromFedList old
       currDoms = fromFedList new
-  unless (prevDoms == currDoms) $ do
-    -- Perform updates before rewriting the tvar
-    -- This means that if the update fails on a
-    -- particular invocation, it can be run again
-    -- on the next firing as it isn't likely that
-    -- the domain list is changing frequently.
-    -- FS-1179 is handling this part.
-    let deletedDomains = Set.difference prevDoms currDoms
-    deleteFederationDomain deletedDomains
+      deletedDomains = Set.difference prevDoms currDoms
+  -- Perform updates before rewriting the ioref
+  -- This means that if the update fails on a
+  -- particular invocation, it can be run again
+  -- on the next firing as it isn't likely that
+  -- the domain list is changing frequently.
+  -- FS-1179 is handling this part.
+  deleteFederationDomain deletedDomains

From ced3efb08f324adda5b4f0e707e868ad58eb1501 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 29 May 2023 19:14:00 +1000
Subject: [PATCH 123/220] wip

---
 .../wire-api/src/Wire/API/FederationUpdate.hs |   8 +-
 services/galley/src/Galley/Run.hs             | 128 ++++++++++++------
 2 files changed, 90 insertions(+), 46 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 63be77b820..23d040a2b4 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -74,11 +74,9 @@ updateFedDomains :: Endpoint -> L.Logger -> FedUpdateCallback -> IO (IORef Feder
 updateFedDomains (Endpoint h p) log' cb = do
   clientEnv <- newManager defaultManagerSettings <&> \mgr -> ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
   ioref <- newIORef =<< getAllowedDomainsInitial log' clientEnv
-  updateFedDomains' ioref clientEnv log' cb
+  (ioref,) <$> updateFedDomains' ioref clientEnv log' cb
   where
     baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
 
-updateFedDomains' :: IORef FederationDomainConfigs -> ClientEnv -> L.Logger -> FedUpdateCallback -> IO (IORef FederationDomainConfigs, Async ())
-updateFedDomains' ioref clientEnv log' cb = do
-  updateDomainsThread <- async $ getAllowedDomainsLoop log' clientEnv cb ioref
-  pure (ioref, updateDomainsThread)
\ No newline at end of file
+updateFedDomains' :: IORef FederationDomainConfigs -> ClientEnv -> L.Logger -> FedUpdateCallback -> IO (Async ())
+updateFedDomains' ioref clientEnv log' cb = async $ getAllowedDomainsLoop log' clientEnv cb ioref
\ No newline at end of file
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 32400f6b84..e63872492a 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -97,7 +97,9 @@ import qualified Network.AMQP.Types as AMQP
 import Network.HTTP.Client (defaultManagerSettings, newManager)
 import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http))
 import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
-import qualified System.Logger as L
+
+-- This type is used to tie the amqp sending and receiving message types together.
+type MsgData = Domain
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
@@ -119,20 +121,86 @@ run opts = lowerCodensity $ do
   forM_ (env ^. aEnv) $ \aws ->
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
 
-  -- TODO: Run a loop that pulls from RabbitMQ and performs the deletion
   maybe
-    (pure ())
-    (\rmq -> void $ Codensity $ Async.withAsync $ readFromRabbitMQ l env rmq)
-    $ opts ^. optRabbitMq
+    -- Update federation domains without talking to rabbitmq
+    -- This doesn't need a call back as there isn't anything
+    -- we need to do beyond updating the IORef, which is done
+    -- for us.
+    (void $ Codensity $ Async.withAsync $ updateFedDomains' ioref clientEnv l $ \_ _ -> pure ())
+    (\rmq -> void $ Codensity $ Async.withAsync $ do
+      -- This ioref is needed so that we can kill the async thread that
+      -- is forked by updateFedDomains'
+      threadRef <- newIORef Nothing
+      let killForkedThread = readIORef threadRef >>= maybe
+            (pure ())
+            (\t -> do
+              Async.cancel t
+              atomicWriteIORef threadRef Nothing
+            )
+          mqh = rmq ^. rmqHost
+          mqp = rmq ^. rmqPort
+          mqv = rmq ^. rmqVhost
+          mqq = rmq ^. rmqQueue
+      openConnectionWithRetries l mqh mqp mqv $ RabbitMqHooks
+        { AMQP.onConnectionClose = do
+          -- Log that the channel closed?
+          killForkedThread
+        , AMQP.onChannelException = const $ do
+          -- Log the exception?
+          killForkedThread
+        , AMQP.onNewChannel = \channel -> do
+          -- NOTE: `amqp` uses ChanThreadKilledException to signal that this channel is closed
+          -- This exception should _NOT_ be caught, or if it is it needs to be rethrown. This
+          -- will kill the thread. This handler is then used to start a new one.
 
-  -- TODO: Run a loop that polls brig for the new domain list, updating the IORef
-  -- The remote domain deletion is tracked with a rabbitmq queue. This is so that
-  -- we can ensure only one galley instance is removing domains and that they aren't
-  -- stepping on each others toes. It also allows for the galley instance to go down
-  -- without losing information about which domains still need to be removed.
-  void $ Codensity $ Async.withAsync $ updateFedDomains' ioref clientEnv l $ \_old _new -> do
-    -- TODO: Push the deleted domains to rabbit so they can be deleted.
-    pure ()
+          -- Ensure that the queue exists and is single active consumer.
+          -- Queue declaration is idempotent
+          let headers = AMQP.FieldTable $ Map.fromList [("x-single-active-consumer", AMQP.FVBool True)]
+          void $ AMQP.declareQueue channel $ AMQP.newQueue { AMQP.queueName = rmq ^. rmqQueue, AMQP.queueHeaders = headers }
+
+          -- Update federation domains, write deleted domains to rabbitmq
+          -- Push this thread id somewhere so we can make sure it is killed with
+          -- this channel thread. We don't want to leak those resources.
+          threadId <- updateFedDomains' ioref clientEnv l $ \old new -> do
+            let fromFedList = Set.fromList . remotes
+                prevDoms = fromFedList old
+                currDoms = fromFedList new
+                deletedDomains = Set.difference prevDoms currDoms
+            -- Write to the queue
+            -- NOTE: This type must be compatible with what is being read from the queue.
+            for_ deletedDomains $ \fedCfg -> do
+              -- We're using the default exchange. This will deliver the
+              -- message to the queue name used for the routing key
+              void $ AMQP.publishMsg channel "" mqq $ AMQP.newMsg
+                { AMQP.msgBody = Aeson.encode @MsgData $ domain fedCfg
+                , AMQP.msgDeliveryMode = pure AMQP.Persistent
+                }
+          atomicWriteIORef threadRef $ pure threadId          
+
+          -- Read messages from RabbitMQ, process the message, and ACK or NACK it as appropriate.
+          -- This is automatically killed by `amqp`, we don't need to handle it.
+          --
+          -- We can run this on every galley instance, and rabbitmq will handle the single
+          -- consumer constraint for us. This is done via the x-single-active-consumer header
+          -- that is set when the queue is created. When the active consumer disconnects for
+          -- whatever reason, rabbit will pick another of the subscribed clients to be the new
+          -- active consumer.
+          void $ AMQP.consumeMsgs channel mqq AMQP.Ack $ \(message, envelope) -> 
+            case Aeson.eitherDecode @MsgData (AMQP.msgBody message) of
+              Left e -> do
+                Log.err l $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
+                AMQP.nackEnv envelope
+              Right dom -> do
+                runApp env $ do
+                  deleteFederationDomainRemote dom
+                  deleteFederationDomainLocal dom
+                  deleteFederationDomainOneOnOne dom
+                AMQP.ackEnv envelope
+          -- Keep this thread around until it is killed.
+          forever $ threadDelay maxBound 
+        }
+    )
+    $ opts ^. optRabbitMq
 
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
@@ -140,34 +208,6 @@ run opts = lowerCodensity $ do
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
 
-readFromRabbitMQ :: L.Logger -> Env -> RabbitMqOpts -> IO ()
-readFromRabbitMQ l env rmq = openConnectionWithRetries l host port vhost $ RabbitMqHooks
-  { AMQP.onConnectionClose = pure () -- Log that the channel closed?
-  , AMQP.onChannelException = const $ pure () -- Log the exception?
-  , AMQP.onNewChannel = \channel -> do
-    -- Ensure that the queue exists and is single active consumer.
-    -- Queue declaration is idempotent
-    let headers = AMQP.FieldTable $ Map.fromList [("x-single-active-consumer", AMQP.FVBool True)]
-    void $ AMQP.declareQueue channel $ AMQP.newQueue { AMQP.queueName = rmq ^. rmqQueue, AMQP.queueHeaders = headers }
-    -- Read messages from RabbitMQ, process the message, and ACK or NACK it as appropriate.
-    void $ AMQP.consumeMsgs channel queue AMQP.Ack $ \(message, envelope) -> do
-      case Aeson.eitherDecode (AMQP.msgBody message) of
-        Left e -> do
-          Log.err l $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
-          AMQP.nackEnv envelope
-        Right dom -> do
-          runApp env $ do
-            deleteFederationDomainRemote dom
-            deleteFederationDomainLocal dom
-            deleteFederationDomainOneOnOne dom
-          AMQP.ackEnv envelope
-  }
-  where
-    host = rmq ^. rmqHost
-    port = rmq ^. rmqPort
-    vhost = rmq ^. rmqVhost
-    queue = rmq ^. rmqQueue
-
 
 mkApp :: Opts -> IORef FederationDomainConfigs -> Log.Logger -> Codensity IO (Application, Env)
 mkApp opts fedDoms logger =
@@ -341,6 +381,12 @@ deleteFederationDomainOneOnOne dom = do
   env <- ask
   liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
 
+
+
+
+-------
+-- TODO: Delete these functions
+
 deleteFederationDomain :: Set FederationDomainConfig -> App ()
 deleteFederationDomain deletedDomains = do
   for_ deletedDomains $ \fedDomCfg -> do

From 82ebc0e74e42f98188a0695dc1ea2ef092696d90 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 29 May 2023 20:30:04 +1000
Subject: [PATCH 124/220] FS-1179: Refactoring the RabbitMQ code

Refactoring RabbitMQ code so that there is a single channel for both
production and consumption of messages. There is also a function for
"simple" processing where a message queuing system isn't available, but
it still needs to have persistence and task co-ordination defined for
it. This may not be a problem if we decide to mandate the use of
RabbitMQ, or a similar queue.
---
 services/galley/src/Galley/Run.hs | 185 ++++++++++++++++++------------
 1 file changed, 111 insertions(+), 74 deletions(-)

diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index e63872492a..56de3cb1fb 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -97,6 +97,7 @@ import qualified Network.AMQP.Types as AMQP
 import Network.HTTP.Client (defaultManagerSettings, newManager)
 import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http))
 import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
+import Galley.Env
 
 -- This type is used to tie the amqp sending and receiving message types together.
 type MsgData = Domain
@@ -126,80 +127,8 @@ run opts = lowerCodensity $ do
     -- This doesn't need a call back as there isn't anything
     -- we need to do beyond updating the IORef, which is done
     -- for us.
-    (void $ Codensity $ Async.withAsync $ updateFedDomains' ioref clientEnv l $ \_ _ -> pure ())
-    (\rmq -> void $ Codensity $ Async.withAsync $ do
-      -- This ioref is needed so that we can kill the async thread that
-      -- is forked by updateFedDomains'
-      threadRef <- newIORef Nothing
-      let killForkedThread = readIORef threadRef >>= maybe
-            (pure ())
-            (\t -> do
-              Async.cancel t
-              atomicWriteIORef threadRef Nothing
-            )
-          mqh = rmq ^. rmqHost
-          mqp = rmq ^. rmqPort
-          mqv = rmq ^. rmqVhost
-          mqq = rmq ^. rmqQueue
-      openConnectionWithRetries l mqh mqp mqv $ RabbitMqHooks
-        { AMQP.onConnectionClose = do
-          -- Log that the channel closed?
-          killForkedThread
-        , AMQP.onChannelException = const $ do
-          -- Log the exception?
-          killForkedThread
-        , AMQP.onNewChannel = \channel -> do
-          -- NOTE: `amqp` uses ChanThreadKilledException to signal that this channel is closed
-          -- This exception should _NOT_ be caught, or if it is it needs to be rethrown. This
-          -- will kill the thread. This handler is then used to start a new one.
-
-          -- Ensure that the queue exists and is single active consumer.
-          -- Queue declaration is idempotent
-          let headers = AMQP.FieldTable $ Map.fromList [("x-single-active-consumer", AMQP.FVBool True)]
-          void $ AMQP.declareQueue channel $ AMQP.newQueue { AMQP.queueName = rmq ^. rmqQueue, AMQP.queueHeaders = headers }
-
-          -- Update federation domains, write deleted domains to rabbitmq
-          -- Push this thread id somewhere so we can make sure it is killed with
-          -- this channel thread. We don't want to leak those resources.
-          threadId <- updateFedDomains' ioref clientEnv l $ \old new -> do
-            let fromFedList = Set.fromList . remotes
-                prevDoms = fromFedList old
-                currDoms = fromFedList new
-                deletedDomains = Set.difference prevDoms currDoms
-            -- Write to the queue
-            -- NOTE: This type must be compatible with what is being read from the queue.
-            for_ deletedDomains $ \fedCfg -> do
-              -- We're using the default exchange. This will deliver the
-              -- message to the queue name used for the routing key
-              void $ AMQP.publishMsg channel "" mqq $ AMQP.newMsg
-                { AMQP.msgBody = Aeson.encode @MsgData $ domain fedCfg
-                , AMQP.msgDeliveryMode = pure AMQP.Persistent
-                }
-          atomicWriteIORef threadRef $ pure threadId          
-
-          -- Read messages from RabbitMQ, process the message, and ACK or NACK it as appropriate.
-          -- This is automatically killed by `amqp`, we don't need to handle it.
-          --
-          -- We can run this on every galley instance, and rabbitmq will handle the single
-          -- consumer constraint for us. This is done via the x-single-active-consumer header
-          -- that is set when the queue is created. When the active consumer disconnects for
-          -- whatever reason, rabbit will pick another of the subscribed clients to be the new
-          -- active consumer.
-          void $ AMQP.consumeMsgs channel mqq AMQP.Ack $ \(message, envelope) -> 
-            case Aeson.eitherDecode @MsgData (AMQP.msgBody message) of
-              Left e -> do
-                Log.err l $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
-                AMQP.nackEnv envelope
-              Right dom -> do
-                runApp env $ do
-                  deleteFederationDomainRemote dom
-                  deleteFederationDomainLocal dom
-                  deleteFederationDomainOneOnOne dom
-                AMQP.ackEnv envelope
-          -- Keep this thread around until it is killed.
-          forever $ threadDelay maxBound 
-        }
-    )
+    (simpleFederationUpdate ioref clientEnv l)
+    (complexFederationUpdate env clientEnv)
     $ opts ^. optRabbitMq
 
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
@@ -207,7 +136,115 @@ run opts = lowerCodensity $ do
   void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
+-- The simple update where rabbitmq isn't defined, so our processing is cut down a lot.
+--
+-- TODO: We still need to delete things, so we'll have to come up with some other processing
+-- and non-volatile data storage to replace rabbit
+simpleFederationUpdate :: IORef FederationDomainConfigs -> ClientEnv -> Log.Logger -> Codensity IO ()
+simpleFederationUpdate ioref clientEnv l = void $ Codensity $ Async.withAsync $
+  updateFedDomains' ioref clientEnv l $ \_ _ -> pure ()
+
+-- Complex handling. Most of the complexity comes from interweaving both rabbit queue handling
+-- and the HTTP calls out to brig for the new lists of federation domains. Rabbit handles the
+-- heavey lifting of ensuring single threaded processing of the domains to be deleted.
+complexFederationUpdate
+  :: Env
+  -> ClientEnv
+  -> RabbitMqOpts
+  -> Codensity IO ()
+complexFederationUpdate env clientEnv rmq = void $ Codensity $ Async.withAsync $ do
+  -- This ioref is needed so that we can kill the async thread that
+  -- is forked by updateFedDomains'
+  threadRef <- newIORef Nothing
+  let
+      mqh = rmq ^. rmqHost
+      mqp = rmq ^. rmqPort
+      mqv = rmq ^. rmqVhost
+      mqq = rmq ^. rmqQueue
+  openConnectionWithRetries (env ^. applog) mqh mqp mqv $ RabbitMqHooks
+    { AMQP.onConnectionClose = do
+      Log.info (env ^. applog) $ Log.msg @Text "AMQP connection closed"
+      killForkedThread threadRef
+    , AMQP.onChannelException = \e -> do
+      Log.err (env ^. applog) $ Log.msg @Text "AMQP channel exception" . Log.field "exception" (show e)
+      killForkedThread threadRef
+    , AMQP.onNewChannel = \channel -> do
+      -- NOTE: `amqp` uses ChanThreadKilledException to signal that this channel is closed
+      -- This exception should _NOT_ be caught, or if it is it needs to be rethrown. This
+      -- will kill the thread. This handler is then used to start a new one.
+      ensureQueue channel mqq
+      writeRabbitMq clientEnv env channel mqq threadRef
+      readRabbitMq channel mqq env
+
+      -- Keep this thread around until it is killed.
+      forever $ threadDelay maxBound 
+    }
+  where
+    killForkedThread ref = 
+      readIORef ref >>= maybe
+        (pure ())
+        (\t -> do
+          Async.cancel t
+          atomicWriteIORef ref Nothing
+        )
 
+-- Ensure that the queue exists and is single active consumer.
+-- Queue declaration is idempotent
+ensureQueue :: AMQP.Channel -> Text -> IO ()
+ensureQueue channel mqq = do
+  void $ AMQP.declareQueue channel $ AMQP.newQueue { AMQP.queueName = mqq, AMQP.queueHeaders = headers }
+  where
+    headers = AMQP.FieldTable $ Map.fromList [("x-single-active-consumer", AMQP.FVBool True)]
+
+-- Update federation domains, write deleted domains to rabbitmq
+-- Push this thread id somewhere so we can make sure it is killed with
+-- this channel thread. We don't want to leak those resources.
+writeRabbitMq
+  :: ClientEnv
+  -> Env
+  -> AMQP.Channel
+  -> Text
+  -> IORef (Maybe (Async.Async ()))
+  -> IO ()
+writeRabbitMq clientEnv env channel mqq threadRef = do
+  threadId <- updateFedDomains' ioref clientEnv (env ^. applog) $ \old new -> do
+    let fromFedList = Set.fromList . remotes
+        prevDoms = fromFedList old
+        currDoms = fromFedList new
+        deletedDomains = Set.difference prevDoms currDoms
+    -- Write to the queue
+    -- NOTE: This type must be compatible with what is being read from the queue.
+    for_ deletedDomains $ \fedCfg -> do
+      -- We're using the default exchange. This will deliver the
+      -- message to the queue name used for the routing key
+      void $ AMQP.publishMsg channel "" mqq $ AMQP.newMsg
+        { AMQP.msgBody = Aeson.encode @MsgData $ domain fedCfg
+        , AMQP.msgDeliveryMode = pure AMQP.Persistent
+        }
+  atomicWriteIORef threadRef $ pure threadId
+  where
+    ioref = env ^. fedDomains
+
+-- Read messages from RabbitMQ, process the message, and ACK or NACK it as appropriate.
+-- This is automatically killed by `amqp`, we don't need to handle it.
+--
+-- We can run this on every galley instance, and rabbitmq will handle the single
+-- consumer constraint for us. This is done via the x-single-active-consumer header
+-- that is set when the queue is created. When the active consumer disconnects for
+-- whatever reason, rabbit will pick another of the subscribed clients to be the new
+-- active consumer.
+readRabbitMq :: AMQP.Channel -> Text -> Env -> IO ()
+readRabbitMq channel mqq env = void $ AMQP.consumeMsgs channel mqq AMQP.Ack $ \(message, envelope) ->
+  case Aeson.eitherDecode @MsgData (AMQP.msgBody message) of
+    Left e -> do
+      Log.err (env ^. applog) $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
+      AMQP.nackEnv envelope
+    Right dom -> do
+      runApp env $ do
+        deleteFederationDomainRemote dom
+        deleteFederationDomainLocal dom
+        deleteFederationDomainOneOnOne dom
+      AMQP.ackEnv envelope
 
 mkApp :: Opts -> IORef FederationDomainConfigs -> Log.Logger -> Codensity IO (Application, Env)
 mkApp opts fedDoms logger =

From 803b0dfd97e2198ab6ad824c9bb8e74de6d62d5f Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 29 May 2023 20:42:56 +1000
Subject: [PATCH 125/220] Code formatting

---
 .../wire-api/src/Wire/API/FederationUpdate.hs |   4 +-
 services/galley/default.nix                   |   2 +
 services/galley/src/Galley/Options.hs         |   4 +-
 services/galley/src/Galley/Run.hs             | 125 +++++++++---------
 4 files changed, 71 insertions(+), 64 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 23d040a2b4..1f609baa0b 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -2,7 +2,7 @@ module Wire.API.FederationUpdate
   ( FedUpdateCallback,
     updateFedDomains,
     getAllowedDomainsInitial,
-    updateFedDomains'
+    updateFedDomains',
   )
 where
 
@@ -79,4 +79,4 @@ updateFedDomains (Endpoint h p) log' cb = do
     baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
 
 updateFedDomains' :: IORef FederationDomainConfigs -> ClientEnv -> L.Logger -> FedUpdateCallback -> IO (Async ())
-updateFedDomains' ioref clientEnv log' cb = async $ getAllowedDomainsLoop log' clientEnv cb ioref
\ No newline at end of file
+updateFedDomains' ioref clientEnv log' cb = async $ getAllowedDomainsLoop log' clientEnv cb ioref
diff --git a/services/galley/default.nix b/services/galley/default.nix
index 26d43456e4..82b846c308 100644
--- a/services/galley/default.nix
+++ b/services/galley/default.nix
@@ -7,6 +7,7 @@
 , aeson-qq
 , amazonka
 , amazonka-sqs
+, amqp
 , asn1-encoding
 , asn1-types
 , async
@@ -136,6 +137,7 @@ mkDerivation {
     aeson
     amazonka
     amazonka-sqs
+    amqp
     asn1-encoding
     asn1-types
     async
diff --git a/services/galley/src/Galley/Options.hs b/services/galley/src/Galley/Options.hs
index ee19066ec2..c5e228b59d 100644
--- a/services/galley/src/Galley/Options.hs
+++ b/services/galley/src/Galley/Options.hs
@@ -57,7 +57,7 @@ module Galley.Options
     rmqHost,
     rmqPort,
     rmqVhost,
-    rmqQueue
+    rmqQueue,
   )
 where
 
@@ -159,10 +159,10 @@ data RabbitMqOpts = RabbitMqOpts
     _rmqQueue :: !Text
   }
   deriving (Show, Generic)
+
 makeLenses ''RabbitMqOpts
 deriveFromJSON toOptionFieldName ''RabbitMqOpts
 
-
 data Opts = Opts
   { -- | Host and port to bind to
     _optGalley :: !Endpoint,
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 56de3cb1fb..87e624130a 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -63,11 +63,17 @@ import Galley.Cassandra
 import Galley.Cassandra.Connection
 import Galley.Data.Conversation.Types (convMetadata)
 import qualified Galley.Effects.MemberStore as E
+import Galley.Env
 import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
 import Galley.Types.Conversations.Members
 import Imports
+import qualified Network.AMQP as AMQP
+import Network.AMQP.Extended (RabbitMqHooks (RabbitMqHooks), openConnectionWithRetries)
+import qualified Network.AMQP.Extended as AMQP
+import qualified Network.AMQP.Types as AMQP
+import Network.HTTP.Client (defaultManagerSettings, newManager)
 import qualified Network.HTTP.Media.RenderHeader as HTTPMedia
 import qualified Network.HTTP.Types as HTTP
 import Network.Wai
@@ -76,6 +82,8 @@ import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
 import Polysemy.Error
 import Servant hiding (route)
+import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http))
+import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
 import qualified System.Logger as Log
 import System.Logger.Extended (mkLogger)
 import Util.Options
@@ -90,14 +98,6 @@ import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
-import Network.AMQP.Extended (openConnectionWithRetries, RabbitMqHooks (RabbitMqHooks))
-import qualified Network.AMQP.Extended as AMQP
-import qualified Network.AMQP as AMQP
-import qualified Network.AMQP.Types as AMQP
-import Network.HTTP.Client (defaultManagerSettings, newManager)
-import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http))
-import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
-import Galley.Env
 
 -- This type is used to tie the amqp sending and receiving message types together.
 type MsgData = Domain
@@ -106,8 +106,10 @@ run :: Opts -> IO ()
 run opts = lowerCodensity $ do
   l <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
   let Endpoint h p = opts ^. optBrig
-  clientEnv <- liftIO $ newManager defaultManagerSettings <&> \mgr ->
-    ClientEnv mgr (BaseUrl  Http (unpack h) (fromIntegral p) "") Nothing defaultMakeClientRequest
+  clientEnv <-
+    liftIO $
+      newManager defaultManagerSettings <&> \mgr ->
+        ClientEnv mgr (BaseUrl Http (unpack h) (fromIntegral p) "") Nothing defaultMakeClientRequest
   ioref <- liftIO $ newIORef =<< getAllowedDomainsInitial l clientEnv
   (app, env) <- mkApp opts ioref l
   settings <-
@@ -141,71 +143,75 @@ run opts = lowerCodensity $ do
 -- TODO: We still need to delete things, so we'll have to come up with some other processing
 -- and non-volatile data storage to replace rabbit
 simpleFederationUpdate :: IORef FederationDomainConfigs -> ClientEnv -> Log.Logger -> Codensity IO ()
-simpleFederationUpdate ioref clientEnv l = void $ Codensity $ Async.withAsync $
-  updateFedDomains' ioref clientEnv l $ \_ _ -> pure ()
+simpleFederationUpdate ioref clientEnv l = void $
+  Codensity $
+    Async.withAsync $
+      updateFedDomains' ioref clientEnv l $
+        \_ _ -> pure ()
 
 -- Complex handling. Most of the complexity comes from interweaving both rabbit queue handling
 -- and the HTTP calls out to brig for the new lists of federation domains. Rabbit handles the
 -- heavey lifting of ensuring single threaded processing of the domains to be deleted.
-complexFederationUpdate
-  :: Env
-  -> ClientEnv
-  -> RabbitMqOpts
-  -> Codensity IO ()
+complexFederationUpdate ::
+  Env ->
+  ClientEnv ->
+  RabbitMqOpts ->
+  Codensity IO ()
 complexFederationUpdate env clientEnv rmq = void $ Codensity $ Async.withAsync $ do
   -- This ioref is needed so that we can kill the async thread that
   -- is forked by updateFedDomains'
   threadRef <- newIORef Nothing
-  let
-      mqh = rmq ^. rmqHost
+  let mqh = rmq ^. rmqHost
       mqp = rmq ^. rmqPort
       mqv = rmq ^. rmqVhost
       mqq = rmq ^. rmqQueue
-  openConnectionWithRetries (env ^. applog) mqh mqp mqv $ RabbitMqHooks
-    { AMQP.onConnectionClose = do
-      Log.info (env ^. applog) $ Log.msg @Text "AMQP connection closed"
-      killForkedThread threadRef
-    , AMQP.onChannelException = \e -> do
-      Log.err (env ^. applog) $ Log.msg @Text "AMQP channel exception" . Log.field "exception" (show e)
-      killForkedThread threadRef
-    , AMQP.onNewChannel = \channel -> do
-      -- NOTE: `amqp` uses ChanThreadKilledException to signal that this channel is closed
-      -- This exception should _NOT_ be caught, or if it is it needs to be rethrown. This
-      -- will kill the thread. This handler is then used to start a new one.
-      ensureQueue channel mqq
-      writeRabbitMq clientEnv env channel mqq threadRef
-      readRabbitMq channel mqq env
-
-      -- Keep this thread around until it is killed.
-      forever $ threadDelay maxBound 
-    }
+  openConnectionWithRetries (env ^. applog) mqh mqp mqv $
+    RabbitMqHooks
+      { AMQP.onConnectionClose = do
+          Log.info (env ^. applog) $ Log.msg @Text "AMQP connection closed"
+          killForkedThread threadRef,
+        AMQP.onChannelException = \e -> do
+          Log.err (env ^. applog) $ Log.msg @Text "AMQP channel exception" . Log.field "exception" (show e)
+          killForkedThread threadRef,
+        AMQP.onNewChannel = \channel -> do
+          -- NOTE: `amqp` uses ChanThreadKilledException to signal that this channel is closed
+          -- This exception should _NOT_ be caught, or if it is it needs to be rethrown. This
+          -- will kill the thread. This handler is then used to start a new one.
+          ensureQueue channel mqq
+          writeRabbitMq clientEnv env channel mqq threadRef
+          readRabbitMq channel mqq env
+
+          -- Keep this thread around until it is killed.
+          forever $ threadDelay maxBound
+      }
   where
-    killForkedThread ref = 
-      readIORef ref >>= maybe
-        (pure ())
-        (\t -> do
-          Async.cancel t
-          atomicWriteIORef ref Nothing
-        )
+    killForkedThread ref =
+      readIORef ref
+        >>= maybe
+          (pure ())
+          ( \t -> do
+              Async.cancel t
+              atomicWriteIORef ref Nothing
+          )
 
 -- Ensure that the queue exists and is single active consumer.
 -- Queue declaration is idempotent
 ensureQueue :: AMQP.Channel -> Text -> IO ()
 ensureQueue channel mqq = do
-  void $ AMQP.declareQueue channel $ AMQP.newQueue { AMQP.queueName = mqq, AMQP.queueHeaders = headers }
+  void $ AMQP.declareQueue channel $ AMQP.newQueue {AMQP.queueName = mqq, AMQP.queueHeaders = headers}
   where
     headers = AMQP.FieldTable $ Map.fromList [("x-single-active-consumer", AMQP.FVBool True)]
 
 -- Update federation domains, write deleted domains to rabbitmq
 -- Push this thread id somewhere so we can make sure it is killed with
 -- this channel thread. We don't want to leak those resources.
-writeRabbitMq
-  :: ClientEnv
-  -> Env
-  -> AMQP.Channel
-  -> Text
-  -> IORef (Maybe (Async.Async ()))
-  -> IO ()
+writeRabbitMq ::
+  ClientEnv ->
+  Env ->
+  AMQP.Channel ->
+  Text ->
+  IORef (Maybe (Async.Async ())) ->
+  IO ()
 writeRabbitMq clientEnv env channel mqq threadRef = do
   threadId <- updateFedDomains' ioref clientEnv (env ^. applog) $ \old new -> do
     let fromFedList = Set.fromList . remotes
@@ -217,10 +223,12 @@ writeRabbitMq clientEnv env channel mqq threadRef = do
     for_ deletedDomains $ \fedCfg -> do
       -- We're using the default exchange. This will deliver the
       -- message to the queue name used for the routing key
-      void $ AMQP.publishMsg channel "" mqq $ AMQP.newMsg
-        { AMQP.msgBody = Aeson.encode @MsgData $ domain fedCfg
-        , AMQP.msgDeliveryMode = pure AMQP.Persistent
-        }
+      void $
+        AMQP.publishMsg channel "" mqq $
+          AMQP.newMsg
+            { AMQP.msgBody = Aeson.encode @MsgData $ domain fedCfg,
+              AMQP.msgDeliveryMode = pure AMQP.Persistent
+            }
   atomicWriteIORef threadRef $ pure threadId
   where
     ioref = env ^. fedDomains
@@ -418,9 +426,6 @@ deleteFederationDomainOneOnOne dom = do
   env <- ask
   liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
 
-
-
-
 -------
 -- TODO: Delete these functions
 
@@ -440,7 +445,7 @@ deleteFederationDomain deletedDomains = do
     deleteFederationDomainLocal dom
     -- Remove the remote one-on-one conversations between local members and remote members for the given domain.
     -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
-    deleteFederationDomainOneOnOne dom    
+    deleteFederationDomainOneOnOne dom
 
 updateFedDomainsCallback :: FederationDomainConfigs -> FederationDomainConfigs -> App ()
 updateFedDomainsCallback old new = do

From 5c1b3d1c0e0f842f51201a5e3456414ff3d3bacd Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 31 May 2023 19:50:29 +1000
Subject: [PATCH 126/220] FS-1179: Fixing tests using the wrong domain.

Using the wrong domain in the galley code, not the test directly, was
causing errors that were annoying to debug.
---
 services/galley/galley.cabal                  |  1 +
 services/galley/src/Galley/API/Action.hs      | 26 +++---
 services/galley/src/Galley/API/Util.hs        |  8 --
 services/galley/src/Galley/Run.hs             | 50 +++--------
 .../galley/test/integration/Federation.hs     | 88 ++++++++++++++++---
 services/galley/test/integration/Main.hs      |  3 +-
 services/galley/test/integration/RabbitMQ.hs  |  1 +
 7 files changed, 103 insertions(+), 74 deletions(-)
 create mode 100644 services/galley/test/integration/RabbitMQ.hs

diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index ce0b2d4004..00a3da313e 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -370,6 +370,7 @@ executable galley-integration
     API.Util.TeamFeature
     Federation
     Main
+    RabbitMQ
     TestHelpers
     TestSetup
 
diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs
index 6e4c41522b..b309ab5cfd 100644
--- a/services/galley/src/Galley/API/Action.hs
+++ b/services/galley/src/Galley/API/Action.hs
@@ -335,19 +335,19 @@ performAction tag origUser lconv action = do
       pure (mempty, action)
     SConversationRemoveMembersTag -> do
       let presentVictims = filter (isConvMemberL lconv) (toList action)
-      _ <-
-        error $
-          "-----------------------------\n\n\n"
-            <> "lconv = "
-            <> show lconv
-            <> "\n\n\n"
-            <> "action = "
-            <> show action
-            <> "\n\n\n"
-            <> "presentVictims = "
-            <> show presentVictims
-            <> "\n\n\n"
-            <> "-----------------------------"
+      -- _ <-
+      --   error $
+      --     "-----------------------------\n\n\n"
+      --       <> "lconv = "
+      --       <> show lconv
+      --       <> "\n\n\n"
+      --       <> "action = "
+      --       <> show action
+      --       <> "\n\n\n"
+      --       <> "presentVictims = "
+      --       <> show presentVictims
+      --       <> "\n\n\n"
+      --       <> "-----------------------------"
       TinyLog.err $ Log.msg ("action" :: String) . Log.field "values" (show action)
       TinyLog.err $ Log.msg ("presentVictims" :: String) . Log.field "values" (show presentVictims)
       when (null presentVictims) noChanges
diff --git a/services/galley/src/Galley/API/Util.hs b/services/galley/src/Galley/API/Util.hs
index 7263238a73..9ab0748eef 100644
--- a/services/galley/src/Galley/API/Util.hs
+++ b/services/galley/src/Galley/API/Util.hs
@@ -377,14 +377,6 @@ instance IsConvMemberId (Local UserId) LocalMember where
 instance IsConvMemberId (Remote UserId) RemoteMember where
   getConvMember _ conv u = find ((u ==) . rmId) (Data.convRemoteMembers conv)
 
--- error $
---   "----------------" <>
---   "\n\n\nconv = " <> show conv <>
---   "\n\n\nu = " <> show u <>
---   "\n\n\nresult = " <> show (find ((u ==) . rmId) (Data.convRemoteMembers conv)) <>
---   "\n\n\n" <>
---   "----------------"
-
 instance IsConvMemberId (Qualified UserId) (Either LocalMember RemoteMember) where
   getConvMember loc conv =
     foldQualified
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 87e624130a..1c2a607475 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -19,8 +19,11 @@
 module Galley.Run
   ( run,
     mkApp,
-    updateFedDomainsCallback,
     mkLogger,
+    -- Exported for tests
+    deleteFederationDomainRemote,
+    deleteFederationDomainLocal,
+    deleteFederationDomainOneOnOne,
   )
 where
 
@@ -341,13 +344,15 @@ collectAuthMetrics m env = do
 insertIntoMap :: (ConvId, a) -> Map ConvId (N.NonEmpty a) -> Map ConvId (N.NonEmpty a)
 insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
 
+-- Remove remote members from local conversations
 deleteFederationDomainRemote :: Domain -> App ()
 deleteFederationDomainRemote dom = do
   env <- ask
   remoteUsers <- liftIO $ evalGalleyToIO env $ E.getRemoteMembersByDomain dom
   let lCnvMap = foldr insertIntoMap mempty remoteUsers
+      localDomain = env ^. options . optSettings . setFederationDomain
   for_ (Map.toList lCnvMap) $ \(cnvId, rUsers) -> do
-    let lCnvId = toLocalUnsafe dom cnvId
+    let lCnvId = toLocalUnsafe localDomain cnvId
     -- This value contains an event that we might need to
     -- send out to all of the local clients that are a party
     -- to the conversation. However we also don't want to DOS
@@ -367,7 +372,7 @@ deleteFederationDomainRemote dom = do
       -- to each client isn't something we want to be doing.
       $ do
         conv <- getConversationWithError lCnvId
-        let lConv = toLocalUnsafe dom conv
+        let lConv = toLocalUnsafe localDomain conv
         updateLocalConversationUserUnchecked
           @'ConversationRemoveMembersTag
           lConv
@@ -386,6 +391,7 @@ deleteFederationDomainRemote dom = do
             undefined
             ()
 
+-- Remove local members from remote conversations
 deleteFederationDomainLocal :: Domain -> App ()
 deleteFederationDomainLocal dom = do
   env <- ask
@@ -424,40 +430,4 @@ deleteFederationDomainLocal dom = do
 deleteFederationDomainOneOnOne :: Domain -> App ()
 deleteFederationDomainOneOnOne dom = do
   env <- ask
-  liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
-
--------
--- TODO: Delete these functions
-
-deleteFederationDomain :: Set FederationDomainConfig -> App ()
-deleteFederationDomain deletedDomains = do
-  for_ deletedDomains $ \fedDomCfg -> do
-    -- https://wearezeta.atlassian.net/browse/FS-1179
-    -- \* Remove remote users for the given domain from all conversations owned by the current host
-    -- \* Remove all local users from remote conversations owned by the given domain.
-    --   NOTE: This is NOT sent to other backends, as this information is not authoratative, but is
-    --   good enough to tell local users about the federation connection being removed.
-    -- \* Delete all connections from local users to users for the remote domain
-    -- Get all remote users for the given domain, along with conversation IDs that they are in
-    let dom = domain fedDomCfg
-    deleteFederationDomainRemote dom
-    -- Get all local users for the given domain, along with remote conversation IDs that they are in
-    deleteFederationDomainLocal dom
-    -- Remove the remote one-on-one conversations between local members and remote members for the given domain.
-    -- NOTE: We cannot tell the remote backend about these changes as we are no longer federated.
-    deleteFederationDomainOneOnOne dom
-
-updateFedDomainsCallback :: FederationDomainConfigs -> FederationDomainConfigs -> App ()
-updateFedDomainsCallback old new = do
-  -- This code will only run when there is a change in the domain lists
-  let fromFedList = Set.fromList . remotes
-      prevDoms = fromFedList old
-      currDoms = fromFedList new
-      deletedDomains = Set.difference prevDoms currDoms
-  -- Perform updates before rewriting the ioref
-  -- This means that if the update fails on a
-  -- particular invocation, it can be run again
-  -- on the next firing as it isn't likely that
-  -- the domain list is changing frequently.
-  -- FS-1179 is handling this part.
-  deleteFederationDomain deletedDomains
+  liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
\ No newline at end of file
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 8d824ff6cb..11aeb69c79 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -20,10 +20,45 @@ import UnliftIO.Retry
 import Wire.API.Conversation
 import Wire.API.Routes.FederationDomainConfig
 import Wire.API.User.Search
+import qualified Data.Set as Set
+import Test.Tasty.HUnit
+import Galley.API.Util
+import qualified Data.UUID as UUID
+import qualified Galley.Data.Conversation.Types as Types
+import Galley.Types.Conversations.Members (defMemberStatus, LocalMember (..), RemoteMember (..))
+import Wire.API.Conversation.Role (roleNameWireMember)
+import Wire.API.Conversation.Protocol (Protocol(..))
 
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
 
+isConvMemberLTests :: TestM ()
+isConvMemberLTests = do
+    s <- ask
+    let opts = s ^. tsGConf
+        localDomain = opts ^. optSettings . setFederationDomain
+        remoteDomain = Domain "far-away.example.com"
+        convId = Id $ fromJust $ UUID.fromString "8cc34301-6949-46c5-bb93-00a72268e2f5"
+        convLocalMembers = [LocalMember userId defMemberStatus Nothing roleNameWireMember]
+        convRemoteMembers = [RemoteMember rUserId roleNameWireMember]
+        lconv = toLocalUnsafe localDomain $ Types.Conversation
+            convId
+            convLocalMembers
+            convRemoteMembers
+            False
+            (defConversationMetadata userId)
+            ProtocolProteus
+        lUserId :: Local UserId
+        lUserId = toLocalUnsafe localDomain $ Id $ fromJust $ UUID.fromString "217352c0-8b2b-4653-ac76-a88d19490dad" -- A random V4 UUID
+        userId = qUnqualified $ tUntagged lUserId
+        rUserId :: Remote UserId
+        rUserId = toRemoteUnsafe remoteDomain $ Id $ fromJust $ UUID.fromString "d87745f5-dfe7-4ff0-8772-b9c22118b372"
+    liftIO $ assertBool "UserId" $ isConvMemberL lconv userId
+    liftIO $ assertBool "Local UserId" $ isConvMemberL lconv lUserId
+    liftIO $ assertBool "Remote UserId" $ isConvMemberL lconv rUserId
+    liftIO $ assertBool "Qualified UserId (local)" $ isConvMemberL lconv $ tUntagged lUserId
+    liftIO $ assertBool "Qualified UserId (remote)" $ isConvMemberL lconv $ tUntagged rUserId
+
 updateFedDomainsTest :: TestM ()
 updateFedDomainsTest = do
   s <- ask
@@ -35,9 +70,10 @@ updateFedDomainsTest = do
   (_, env) <- liftIO $ lowerCodensity $ mkApp opts r l
   -- Common variables.
   let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
-      remoteDomain = Domain "far-away.example.com"
-      remoteDomain2 = Domain "far-away-two.example.com"
-
+      remoteDomain = Domain "far-away.example.org"
+      remoteDomain2 = Domain "far-away-two.example.net"
+  liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
+  liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
   -- Setup a conversation for a known remote domain.
   -- Include that domain in the old and new lists so
   -- if the function is acting up we know it will be
@@ -53,6 +89,23 @@ updateFedDomainsTest = do
 -- Removing multiple domains
 -- updateFedDomainsCallback old new
 
+fromFedList :: FederationDomainConfigs -> Set Domain
+fromFedList = Set.fromList . fmap domain . remotes
+
+-- Bundle all of the deletes together for easy calling
+deleteFederationDomain :: Domain -> App ()
+deleteFederationDomain d = do
+    deleteFederationDomainRemote d
+    deleteFederationDomainLocal d
+    deleteFederationDomainOneOnOne d
+
+deleteFederationDomains :: FederationDomainConfigs -> FederationDomainConfigs -> App ()
+deleteFederationDomains old new = do
+    let prev = fromFedList old
+        curr = fromFedList new
+        deletedDomains = Set.difference prev curr
+    for_ deletedDomains deleteFederationDomain
+
 constHandlers :: MonadIO m => [RetryStatus -> Handler m Bool]
 constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
 
@@ -61,8 +114,8 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
   s <- ask
   let opts = s ^. tsGConf
       localDomain = opts ^. optSettings . setFederationDomain
-      old = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain FullSearch, FederationDomainConfig remoteDomain2 FullSearch] interval
-      new = old {remotes = [FederationDomainConfig remoteDomain2 FullSearch]}
+      new = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain2 FullSearch] interval
+      old = new {remotes = FederationDomainConfig remoteDomain FullSearch : remotes new}
   qalice <- randomQualifiedUser
   bobId <- randomId
   charlieId <- randomId
@@ -70,14 +123,14 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
       remoteBob = Qualified bobId remoteDomain
       remoteCharlie = Qualified charlieId remoteDomain2
   -- Create a conversation
-  convId <- decodeConvId <$> postConv alice [] (Just "remote gossip") [] Nothing Nothing
-  let qConvId = Qualified convId localDomain
+  conv <- postConv alice [] (Just "remote gossip") [] Nothing Nothing
+  let qConvId = decodeQualifiedConvId conv
+      convId = qUnqualified qConvId
   connectWithRemoteUser alice remoteBob
   connectWithRemoteUser alice remoteCharlie
   _ <- postQualifiedMembers alice (remoteCharlie <| remoteBob :| []) qConvId
-  liftIO $ threadDelay $ 3 * 1000000
   -- Remove the remote user from the local domain
-  liftIO $ runApp env $ updateFedDomainsCallback old new
+  liftIO $ runApp env $ deleteFederationDomains old new
   -- Check that the conversation still exists.
   getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
     const 200 === statusCode
@@ -95,18 +148,29 @@ updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval = do
       localDomain = opts ^. optSettings . setFederationDomain
       old = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain FullSearch] interval
       new = old {remotes = FederationDomainConfig remoteDomain2 FullSearch : remotes old}
+      -- Just check against the domains, as the search
+      -- strategies are outside of this testing scope
+      newDoms = domain <$> new.remotes
+      oldDoms = domain <$> old.remotes
+  liftIO $ assertBool "old and new are different" $ oldDoms /= newDoms
+  liftIO $ assertBool "old is shorter than new" $ Imports.length oldDoms < Imports.length newDoms
+  liftIO $ assertBool "new contains old" $ all (`elem` newDoms) oldDoms
+  liftIO $ assertBool "new elements not in old" $ any (`notElem` oldDoms) newDoms
   qalice <- randomQualifiedUser
   bobId <- randomId
   let alice = qUnqualified qalice
       remoteBob = Qualified bobId remoteDomain
   -- Create a conversation
-  convId <- decodeConvId <$> postConv alice [] (Just "remote gossip") [] Nothing Nothing
+
+  conv <- postConv alice [] (Just "remote gossip") [] Nothing Nothing
+  liftIO $ assertBool ("conv = " <> show conv) False
+  let convId = decodeConvId conv
   let qConvId = Qualified convId localDomain
   connectWithRemoteUser alice remoteBob
   _ <- postQualifiedMembers alice (remoteBob :| []) qConvId
 
   -- No-op
-  liftIO $ runApp env $ updateFedDomainsCallback old new
+  liftIO $ runApp env $ deleteFederationDomains old new
   -- Check that the conversation still exists.
   getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
     const 200 === statusCode
@@ -132,7 +196,7 @@ updateFedDomainsTestNoop env remoteDomain interval = do
   connectWithRemoteUser alice remoteBob
   _ <- postQualifiedMembers alice (remoteBob :| []) qConvId
   -- No-op
-  liftIO $ runApp env $ updateFedDomainsCallback old new
+  liftIO $ runApp env $ deleteFederationDomains old new
   -- Check that the conversation still exists.
   getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
     const 200 === statusCode
diff --git a/services/galley/test/integration/Main.hs b/services/galley/test/integration/Main.hs
index 0a5ced3f36..519cf856b2 100644
--- a/services/galley/test/integration/Main.hs
+++ b/services/galley/test/integration/Main.hs
@@ -96,7 +96,8 @@ main = withOpenSSL $ runTests go
               mempty
               (pathsConsistencyCheck . treeToPaths . compile $ Galley.API.sitemap),
           API.tests setup,
-          test setup "Federation Domains" updateFedDomainsTest
+          test setup "Federation Domains" updateFedDomainsTest,
+          test setup "isConvMemberL" isConvMemberLTests
         ]
     getOpts gFile iFile = do
       m <- newManager tlsManagerSettings {managerResponseTimeout = responseTimeoutMicro 300000000}
diff --git a/services/galley/test/integration/RabbitMQ.hs b/services/galley/test/integration/RabbitMQ.hs
new file mode 100644
index 0000000000..44dbf76a71
--- /dev/null
+++ b/services/galley/test/integration/RabbitMQ.hs
@@ -0,0 +1 @@
+module RabbitMQ where
\ No newline at end of file

From 28d27d6abadd760b8ff3633c44f6ad780fad1d3d Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Sun, 4 Jun 2023 14:31:24 +0200
Subject: [PATCH 127/220] docs.

---
 docs/src/understand/configure-federation.md | 82 +++++++++++++--------
 1 file changed, 50 insertions(+), 32 deletions(-)

diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index bd34169275..fe61db0802 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -500,40 +500,58 @@ developer's point of view on this topic.
 
 #### If your instance has been federating before
 
-Only needed if your instance has been federating with other instances
-prior to [PR#3260](https://github.com/wireapp/wire-server/pull/3260).
-
-The new configuration process ignores the federation policy set in the
-federator config under TODO NOISE FROM HERE ON OUT ***
-
-TODO: you need to update config files!
-  - complete list of search policies, no more defaults
-  - new fed strategy syntax (keep the old, just copy)
-  - later, remove the old syntax in brig, federator.
-
-As of the release containing
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260),
-[`federationStrategy`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/federator/values.yaml#L44-L45)
-in the federation config file is ignored, and brig's cassandra is used
-instead.  Furthermore, for a transition period,
-[`setFederationDomainConfigs`](https://github.com/wireapp/wire-server/blob/4a4ba8dd54586e1d85fe4af609990d79ae3d8cc2/charts/brig/templates/configmap.yaml#L250-L252)
-from the brig config file also remains being honored.  Attempting to
-delete entries that occur in the config file will trigger an error;
-delete from the config file first, then from cassandra.
-
-In the future, wire-server will stop honoring the config file data,
-and solely rely on brig's cassandra.  From that point onward, you can
-delete any connection, whether listed in the config file or not.
-Watch out for the release notes to learn when this will happen.
-(Something like *"[Federation only] support for remote configuration
-in config file is discontinued.  Before upgrading to this release,
-upgrade to the release containing
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260) first.
-After upgrading to this release, `setFederationDomainConfigs` in brig's
-config file will be ignored, and you should remove it at your
-convenience.*)
+You only need to read this section if your instance has been
+federating with other instances prior to
+[PR#3260](https://github.com/wireapp/wire-server/pull/3260), and you
+are upgrading to the release containing that PR.
+
+From now on the federation policy set in the federator config under
+`federationStrategy` is ignored.  Instead, the federation strategy is
+pulled by all services from brig, who in turn gets it from a
+combination of config file and database (see
+{ref}`configure-federation-strategy-in-brig` above).
+
+In order to achieve a zero-downtime upgrade, follow these steps:
+
+1. Update the brig config values file as described above.
+
+2. If you have chosen `brig.config.optSettings.setFederationStrategy:
+  allowDynamic` and/or if you had previously configured search
+  policies under `brig.config.optSettings.setFederationDomainConfigs`,
+  update brig config values again.
+
+    This is to cover the time window
+    between upgrading the brig pods and populating cassandra with the
+    information needed.  Example:
+
+    ```yaml
+    brig:
+      config:
+        optSettings:
+          setFederationDomainConfigs:
+          - domain: red.example.com
+            search_policy: full_search
+          - domain: blue.example.com
+            search_policy: no_search
+    ```
+
+    Any later lookup of this information will return the union of what
+    is in cassandra and what is in the config file.  Any attempt to
+    write data to cassandra that contradicts data in the config file
+    will result in an error.  Before you change any remote domain
+    config, remove it from the config file.
+
+3. Populate cassandra with remote domain configs as described above.
 
+4. At any time after you are done with the upgrade and have convinced
+  yourself everything went smoothly, remove outdated brig and
+  federator config values, in particular:
+    - `brig.config.optSettings.setFederationDomainConfigs`
+    - `federator.config.optSettings.federationStrategy`
 
+    At a later point, wire-server will start ignoring
+    `setFederationDomainConfigs` altogether (follow future entries in
+    the changelog to learn when that happens).
 
 ### Configure federator process to run and allow incoming traffic
 

From d334aa6b4b3578fbdbc905ba503bd647324ae128 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Sun, 4 Jun 2023 14:43:43 +0200
Subject: [PATCH 128/220] Changelog.

---
 changelog.d/0-release-notes/pr-3260 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/0-release-notes/pr-3260 b/changelog.d/0-release-notes/pr-3260
index 5b566bff43..a2299fffcb 100644
--- a/changelog.d/0-release-notes/pr-3260
+++ b/changelog.d/0-release-notes/pr-3260
@@ -1 +1 @@
-Federation only: from this release on, remote connections should be configured via an internal REST API; the config files will be honored for a transition period, but will be ignored starting in a future release.  YOU NEED TO UPDATE YOUR BRIG HELM CHART BEFORE DEPLOYING THIS RELEASE.  [Details in the docs.](https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections)
\ No newline at end of file
+Federation only: from this release on, remote connections should be configured via an internal REST API; the config files will be honored for a transition period, but will be ignored starting in a future release.  YOU NEED TO UPDATE YOUR BRIG HELM CHART BEFORE DEPLOYING THIS RELEASE.  [Details in the docs.](http://docs.wire.com/understand/configure-federation.html#if-your-instance-has-been-federating-before)
\ No newline at end of file

From 171a31e273325cf3a7319339cc95277793434f28 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Sun, 4 Jun 2023 14:52:23 +0200
Subject: [PATCH 129/220] docs.

---
 docs/src/understand/configure-federation.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index fe61db0802..679cd24008 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -516,13 +516,10 @@ In order to achieve a zero-downtime upgrade, follow these steps:
 1. Update the brig config values file as described above.
 
 2. If you have chosen `brig.config.optSettings.setFederationStrategy:
-  allowDynamic` and/or if you had previously configured search
-  policies under `brig.config.optSettings.setFederationDomainConfigs`,
-  update brig config values again.
-
-    This is to cover the time window
-    between upgrading the brig pods and populating cassandra with the
-    information needed.  Example:
+  allowDynamic` you need to make sure the list of all domains you want
+  to allow federation with is complete (before, there was a search
+  policy default; now wire will stop federating with removes that are
+  not listed here).  Example:
 
     ```yaml
     brig:
@@ -535,6 +532,10 @@ In order to achieve a zero-downtime upgrade, follow these steps:
             search_policy: no_search
     ```
 
+    This change is to cover the time window between upgrading the brig
+    pods and populating cassandra with the information needed (see
+    Step 3 below).
+
     Any later lookup of this information will return the union of what
     is in cassandra and what is in the config file.  Any attempt to
     write data to cassandra that contradicts data in the config file

From 125ba0ab63d1bedd8f1c62cbbf3c1a145688e83b Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Sun, 4 Jun 2023 14:52:41 +0200
Subject: [PATCH 130/220] sanitize-pr

---
 cassandra-schema.cql          | 1406 ++++++++++++++++-----------------
 integration/test/Test/Brig.hs |    9 +-
 2 files changed, 708 insertions(+), 707 deletions(-)

diff --git a/cassandra-schema.cql b/cassandra-schema.cql
index da6f215fb4..8ca8c9b35e 100644
--- a/cassandra-schema.cql
+++ b/cassandra-schema.cql
@@ -1,26 +1,24 @@
 -- automatically generated with `make git-add-cassandra-schema`
 
-CREATE KEYSPACE galley_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
+CREATE KEYSPACE brig_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
 
-CREATE TYPE galley_test.permissions (
-    self bigint,
-    copy bigint
+CREATE TYPE brig_test.asset (
+    typ int,
+    key text,
+    size int
 );
 
-CREATE TYPE galley_test.pubkey (
+CREATE TYPE brig_test.pubkey (
     typ int,
     size int,
     pem blob
 );
 
-CREATE TABLE galley_test.meta (
-    id int,
-    version int,
-    date timestamp,
-    descr text,
-    PRIMARY KEY (id, version)
-) WITH CLUSTERING ORDER BY (version ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.team_invitation_info (
+    code ascii PRIMARY KEY,
+    id uuid,
+    team uuid
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -35,12 +33,10 @@ CREATE TABLE galley_test.meta (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.team_conv (
-    team uuid,
-    conv uuid,
-    PRIMARY KEY (team, conv)
-) WITH CLUSTERING ORDER BY (conv ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.provider_keys (
+    key text PRIMARY KEY,
+    provider uuid
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -55,19 +51,20 @@ CREATE TABLE galley_test.team_conv (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.user_team (
-    user uuid,
-    team uuid,
-    PRIMARY KEY (user, team)
-) WITH CLUSTERING ORDER BY (team ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.oauth_refresh_token (
+    id uuid PRIMARY KEY,
+    client uuid,
+    created_at timestamp,
+    scope set<text>,
+    user uuid
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 0
+    AND default_time_to_live = 14515200
     AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
@@ -75,15 +72,13 @@ CREATE TABLE galley_test.user_team (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.service (
-    provider uuid,
-    id uuid,
-    auth_token ascii,
-    base_url blob,
-    enabled boolean,
-    fingerprints set<blob>,
-    PRIMARY KEY (provider, id)
-) WITH CLUSTERING ORDER BY (id ASC)
+CREATE TABLE brig_test.team_invitation_email (
+    email text,
+    team uuid,
+    code ascii,
+    invitation uuid,
+    PRIMARY KEY (email, team)
+) WITH CLUSTERING ORDER BY (team ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -99,14 +94,10 @@ CREATE TABLE galley_test.service (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.data_migration (
-    id int,
-    version int,
-    date timestamp,
-    descr text,
-    PRIMARY KEY (id, version)
-) WITH CLUSTERING ORDER BY (version ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.rich_info (
+    user uuid PRIMARY KEY,
+    json blob
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -121,40 +112,10 @@ CREATE TABLE galley_test.data_migration (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.team_features (
-    team_id uuid PRIMARY KEY,
-    app_lock_enforce int,
-    app_lock_inactivity_timeout_secs int,
-    app_lock_status int,
-    conference_calling int,
-    digital_signatures int,
-    expose_invitation_urls_to_team_admin int,
-    file_sharing int,
-    file_sharing_lock_status int,
-    guest_links_lock_status int,
-    guest_links_status int,
-    legalhold_status int,
-    mls_allowed_ciphersuites set<int>,
-    mls_default_ciphersuite int,
-    mls_default_protocol int,
-    mls_e2eid_acme_discovery_url blob,
-    mls_e2eid_grace_period int,
-    mls_e2eid_lock_status int,
-    mls_e2eid_status int,
-    mls_e2eid_ver_exp timestamp,
-    mls_protocol_toggle_users set<uuid>,
-    mls_status int,
-    outlook_cal_integration_lock_status int,
-    outlook_cal_integration_status int,
-    search_visibility_inbound_status int,
-    search_visibility_status int,
-    self_deleting_messages_lock_status int,
-    self_deleting_messages_status int,
-    self_deleting_messages_ttl int,
-    snd_factor_password_challenge_lock_status int,
-    snd_factor_password_challenge_status int,
-    sso_status int,
-    validate_saml_emails int
+CREATE TABLE brig_test.user_keys_hash (
+    key blob PRIMARY KEY,
+    key_type int,
+    user uuid
 ) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -170,22 +131,14 @@ CREATE TABLE galley_test.team_features (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.member (
-    conv uuid,
-    user uuid,
-    conversation_role text,
-    hidden boolean,
-    hidden_ref text,
-    otr_archived boolean,
-    otr_archived_ref text,
-    otr_muted boolean,
-    otr_muted_ref text,
-    otr_muted_status int,
-    provider uuid,
+CREATE TABLE brig_test.service_tag (
+    bucket int,
+    tag bigint,
+    name text,
     service uuid,
-    status int,
-    PRIMARY KEY (conv, user)
-) WITH CLUSTERING ORDER BY (user ASC)
+    provider uuid,
+    PRIMARY KEY ((bucket, tag), name, service)
+) WITH CLUSTERING ORDER BY (name ASC, service ASC)
     AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -201,11 +154,14 @@ CREATE TABLE galley_test.member (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.custom_backend (
-    domain text PRIMARY KEY,
-    config_json_url blob,
-    webapp_welcome_url blob
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.meta (
+    id int,
+    version int,
+    date timestamp,
+    descr text,
+    PRIMARY KEY (id, version)
+) WITH CLUSTERING ORDER BY (version ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -220,19 +176,10 @@ CREATE TABLE galley_test.custom_backend (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.user_remote_conv (
-    user uuid,
-    conv_remote_domain text,
-    conv_remote_id uuid,
-    hidden boolean,
-    hidden_ref text,
-    otr_archived boolean,
-    otr_archived_ref text,
-    otr_muted_ref text,
-    otr_muted_status int,
-    PRIMARY KEY (user, conv_remote_domain, conv_remote_id)
-) WITH CLUSTERING ORDER BY (conv_remote_domain ASC, conv_remote_id ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.unique_claims (
+    value text PRIMARY KEY,
+    claims set<uuid>
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -240,19 +187,27 @@ CREATE TABLE galley_test.user_remote_conv (
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
     AND default_time_to_live = 0
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 0
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.legalhold_whitelisted (
-    team uuid PRIMARY KEY
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.user_cookies (
+    user uuid,
+    expires timestamp,
+    id bigint,
+    created timestamp,
+    label text,
+    succ_id bigint,
+    type int,
+    PRIMARY KEY (user, expires, id)
+) WITH CLUSTERING ORDER BY (expires ASC, id ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -264,13 +219,13 @@ CREATE TABLE galley_test.legalhold_whitelisted (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.member_remote_user (
-    conv uuid,
-    user_remote_domain text,
-    user_remote_id uuid,
-    conversation_role text,
-    PRIMARY KEY (conv, user_remote_domain, user_remote_id)
-) WITH CLUSTERING ORDER BY (user_remote_domain ASC, user_remote_id ASC)
+CREATE TABLE brig_test.mls_key_packages (
+    user uuid,
+    client text,
+    ref blob,
+    data blob,
+    PRIMARY KEY ((user, client), ref)
+) WITH CLUSTERING ORDER BY (ref ASC)
     AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -286,16 +241,14 @@ CREATE TABLE galley_test.member_remote_user (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.team_member (
-    team uuid,
-    user uuid,
-    invited_at timestamp,
-    invited_by uuid,
-    legalhold_status int,
-    perms frozen<permissions>,
-    PRIMARY KEY (team, user)
-) WITH CLUSTERING ORDER BY (user ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.mls_key_package_refs (
+    ref blob PRIMARY KEY,
+    client text,
+    conv uuid,
+    conv_domain text,
+    domain text,
+    user uuid
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -310,16 +263,13 @@ CREATE TABLE galley_test.team_member (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.team_notifications (
-    team uuid,
-    id timeuuid,
-    payload blob,
-    PRIMARY KEY (team, id)
-) WITH CLUSTERING ORDER BY (id ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.excluded_phones (
+    prefix text PRIMARY KEY,
+    comment text
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -331,16 +281,17 @@ CREATE TABLE galley_test.team_notifications (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.legalhold_pending_prekeys (
+CREATE TABLE brig_test.codes (
     user uuid,
-    key int,
-    data text,
-    PRIMARY KEY (user, key)
-) WITH CLUSTERING ORDER BY (key ASC)
-    AND bloom_filter_fp_chance = 0.1
+    scope int,
+    code text,
+    retries int,
+    PRIMARY KEY (user, scope)
+) WITH CLUSTERING ORDER BY (scope ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -352,14 +303,13 @@ CREATE TABLE galley_test.legalhold_pending_prekeys (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.group_id_conv_id (
-    group_id blob PRIMARY KEY,
-    conv_id uuid,
-    domain text
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.user_handle (
+    handle text PRIMARY KEY,
+    user uuid
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -371,19 +321,25 @@ CREATE TABLE galley_test.group_id_conv_id (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
--- NOTE: this table is unused. It was replaced by mls_group_member_client
-CREATE TABLE galley_test.member_client (
-    conv uuid,
-    user_domain text,
-    user uuid,
-    client text,
-    key_package_ref blob,
-    PRIMARY KEY (conv, user_domain, user, client)
-) WITH CLUSTERING ORDER BY (user_domain ASC, user ASC, client ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.service (
+    provider uuid,
+    id uuid,
+    assets list<frozen<asset>>,
+    auth_tokens list<ascii>,
+    base_url blob,
+    descr text,
+    enabled boolean,
+    fingerprints list<blob>,
+    name text,
+    pubkeys list<frozen<pubkey>>,
+    summary text,
+    tags set<bigint>,
+    PRIMARY KEY (provider, id)
+) WITH CLUSTERING ORDER BY (id ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -395,20 +351,19 @@ CREATE TABLE galley_test.member_client (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.legalhold_service (
-    team_id uuid PRIMARY KEY,
-    auth_token ascii,
-    base_url blob,
-    fingerprint blob,
-    pubkey pubkey
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.oauth_user_refresh_token (
+    user uuid,
+    token_id uuid,
+    PRIMARY KEY (user, token_id)
+) WITH CLUSTERING ORDER BY (token_id ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 0
+    AND default_time_to_live = 14515200
     AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
@@ -416,18 +371,14 @@ CREATE TABLE galley_test.legalhold_service (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.conversation_codes (
-    key ascii,
-    scope int,
-    conversation uuid,
-    password blob,
-    value ascii,
-    PRIMARY KEY (key, scope)
-) WITH CLUSTERING ORDER BY (scope ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.invitation_info (
+    code ascii PRIMARY KEY,
+    id uuid,
+    inviter uuid
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -439,14 +390,12 @@ CREATE TABLE galley_test.conversation_codes (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.mls_group_member_client (
-    group_id blob,
-    user_domain text,
-    user uuid,
-    client text,
-    key_package_ref blob,
-    PRIMARY KEY (group_id, user_domain, user, client)
-) WITH CLUSTERING ORDER BY (user_domain ASC, user ASC, client ASC)
+CREATE TABLE brig_test.service_whitelist (
+    team uuid,
+    provider uuid,
+    service uuid,
+    PRIMARY KEY (team, provider, service)
+) WITH CLUSTERING ORDER BY (provider ASC, service ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -462,10 +411,14 @@ CREATE TABLE galley_test.mls_group_member_client (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.clients (
-    user uuid PRIMARY KEY,
-    clients set<text>
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.provider (
+    id uuid PRIMARY KEY,
+    descr text,
+    email text,
+    name text,
+    password blob,
+    url blob
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -480,23 +433,9 @@ CREATE TABLE galley_test.clients (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.conversation (
-    conv uuid PRIMARY KEY,
-    access set<int>,
-    access_role int,
-    access_roles_v2 set<int>,
-    cipher_suite int,
-    creator uuid,
-    deleted boolean,
-    epoch bigint,
-    group_id blob,
-    message_timer bigint,
-    name text,
-    protocol int,
-    public_group_state blob,
-    receipt_mode int,
-    team uuid,
-    type int
+CREATE TABLE brig_test.user_keys (
+    key text PRIMARY KEY,
+    user uuid
 ) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -512,15 +451,17 @@ CREATE TABLE galley_test.conversation (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.mls_commit_locks (
-    group_id blob,
-    epoch bigint,
-    PRIMARY KEY (group_id, epoch)
-) WITH CLUSTERING ORDER BY (epoch ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.mls_public_keys (
+    user uuid,
+    client text,
+    sig_scheme text,
+    key blob,
+    PRIMARY KEY (user, client, sig_scheme)
+) WITH CLUSTERING ORDER BY (client ASC, sig_scheme ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -532,17 +473,10 @@ CREATE TABLE galley_test.mls_commit_locks (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.team (
-    team uuid PRIMARY KEY,
-    binding boolean,
-    creator uuid,
-    deleted boolean,
-    icon text,
-    icon_key text,
-    name text,
-    search_visibility int,
-    splash_screen text,
-    status int
+CREATE TABLE brig_test.invitee_info (
+    invitee uuid PRIMARY KEY,
+    conv uuid,
+    inviter uuid
 ) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -558,12 +492,33 @@ CREATE TABLE galley_test.team (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.billing_team_member (
-    team uuid,
+CREATE TABLE brig_test.nonce (
     user uuid,
-    PRIMARY KEY (team, user)
-) WITH CLUSTERING ORDER BY (user ASC)
+    key text,
+    nonce uuid,
+    PRIMARY KEY (user, key)
+) WITH CLUSTERING ORDER BY (key ASC)
     AND bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 300
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
+CREATE TABLE brig_test.login_codes (
+    user uuid PRIMARY KEY,
+    code text,
+    retries int,
+    timeout timestamp
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -578,15 +533,12 @@ CREATE TABLE galley_test.billing_team_member (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.mls_proposal_refs (
-    group_id blob,
-    epoch bigint,
-    ref blob,
-    origin int,
-    proposal blob,
-    PRIMARY KEY (group_id, epoch, ref)
-) WITH CLUSTERING ORDER BY (epoch ASC, ref ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.oauth_client (
+    id uuid PRIMARY KEY,
+    name text,
+    redirect_uri blob,
+    secret blob
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -601,15 +553,18 @@ CREATE TABLE galley_test.mls_proposal_refs (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE galley_test.user (
+CREATE TABLE brig_test.service_team (
+    provider uuid,
+    service uuid,
+    team uuid,
     user uuid,
     conv uuid,
-    PRIMARY KEY (user, conv)
-) WITH CLUSTERING ORDER BY (conv ASC)
-    AND bloom_filter_fp_chance = 0.1
+    PRIMARY KEY ((provider, service), team, user)
+) WITH CLUSTERING ORDER BY (team ASC, user ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -621,18 +576,9 @@ CREATE TABLE galley_test.user (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE KEYSPACE gundeck_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
-
-CREATE TABLE gundeck_test.push (
-    ptoken text,
-    app text,
-    transport int,
-    client text,
-    connection blob,
-    usr uuid,
-    PRIMARY KEY (ptoken, app, transport)
-) WITH CLUSTERING ORDER BY (app ASC, transport ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.blacklist (
+    key text PRIMARY KEY
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -647,35 +593,39 @@ CREATE TABLE gundeck_test.push (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE gundeck_test.notifications (
-    user uuid,
-    id timeuuid,
-    clients set<text>,
-    payload blob,
-    PRIMARY KEY (user, id)
-) WITH CLUSTERING ORDER BY (id ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.service_whitelist_rev (
+    provider uuid,
+    service uuid,
+    team uuid,
+    PRIMARY KEY ((provider, service), team)
+) WITH CLUSTERING ORDER BY (team ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy', 'compaction_window_size': '1', 'compaction_window_unit': 'DAYS', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
     AND default_time_to_live = 0
-    AND gc_grace_seconds = 0
+    AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE gundeck_test.meta (
-    id int,
-    version int,
-    date timestamp,
-    descr text,
-    PRIMARY KEY (id, version)
-) WITH CLUSTERING ORDER BY (version ASC)
+CREATE TABLE brig_test.team_invitation (
+    team uuid,
+    id uuid,
+    code ascii,
+    created_at timestamp,
+    created_by uuid,
+    email text,
+    name text,
+    phone text,
+    role int,
+    PRIMARY KEY (team, id)
+) WITH CLUSTERING ORDER BY (id ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -691,17 +641,32 @@ CREATE TABLE gundeck_test.meta (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE gundeck_test.user_push (
-    usr uuid,
-    ptoken text,
-    app text,
-    transport int,
-    arn text,
-    client text,
-    connection blob,
-    PRIMARY KEY (usr, ptoken, app, transport)
-) WITH CLUSTERING ORDER BY (ptoken ASC, app ASC, transport ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.user (
+    id uuid PRIMARY KEY,
+    accent list<float>,
+    accent_id int,
+    activated boolean,
+    assets list<frozen<asset>>,
+    country ascii,
+    email text,
+    email_unvalidated text,
+    expires timestamp,
+    feature_conference_calling int,
+    handle text,
+    language ascii,
+    managed_by int,
+    name text,
+    password blob,
+    phone text,
+    picture list<blob>,
+    provider uuid,
+    searchable boolean,
+    service uuid,
+    sso_id text,
+    status int,
+    supported_protocols int,
+    team uuid
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -716,25 +681,13 @@ CREATE TABLE gundeck_test.user_push (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE KEYSPACE brig_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
-
-CREATE TYPE brig_test.asset (
-    typ int,
-    key text,
-    size int
-);
-
-CREATE TYPE brig_test.pubkey (
-    typ int,
-    size int,
-    pem blob
-);
-
-CREATE TABLE brig_test.team_invitation_info (
-    code ascii PRIMARY KEY,
-    id uuid,
-    team uuid
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.vcodes_throttle (
+    key ascii,
+    scope int,
+    initial_delay int,
+    PRIMARY KEY (key, scope)
+) WITH CLUSTERING ORDER BY (scope ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -749,13 +702,16 @@ CREATE TABLE brig_test.team_invitation_info (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.provider_keys (
-    key text PRIMARY KEY,
-    provider uuid
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.properties (
+    user uuid,
+    key ascii,
+    value blob,
+    PRIMARY KEY (user, key)
+) WITH CLUSTERING ORDER BY (key ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -767,20 +723,22 @@ CREATE TABLE brig_test.provider_keys (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.oauth_refresh_token (
-    id uuid PRIMARY KEY,
-    client uuid,
-    created_at timestamp,
-    scope set<text>,
-    user uuid
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.service_user (
+    provider uuid,
+    service uuid,
+    user uuid,
+    conv uuid,
+    team uuid,
+    PRIMARY KEY ((provider, service), user)
+) WITH CLUSTERING ORDER BY (user ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 14515200
+    AND default_time_to_live = 0
     AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
@@ -788,17 +746,17 @@ CREATE TABLE brig_test.oauth_refresh_token (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.team_invitation_email (
-    email text,
-    team uuid,
-    code ascii,
-    invitation uuid,
-    PRIMARY KEY (email, team)
-) WITH CLUSTERING ORDER BY (team ASC)
+CREATE TABLE brig_test.prekeys (
+    user uuid,
+    client text,
+    key int,
+    data text,
+    PRIMARY KEY (user, client, key)
+) WITH CLUSTERING ORDER BY (client ASC, key ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -810,9 +768,13 @@ CREATE TABLE brig_test.team_invitation_email (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.rich_info (
-    user uuid PRIMARY KEY,
-    json blob
+CREATE TABLE brig_test.oauth_auth_code (
+    code ascii PRIMARY KEY,
+    client uuid,
+    code_challenge blob,
+    redirect_uri blob,
+    scope set<text>,
+    user uuid
 ) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -820,7 +782,7 @@ CREATE TABLE brig_test.rich_info (
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 0
+    AND default_time_to_live = 300
     AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
@@ -828,11 +790,22 @@ CREATE TABLE brig_test.rich_info (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.user_keys_hash (
-    key blob PRIMARY KEY,
-    key_type int,
-    user uuid
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.clients (
+    user uuid,
+    client text,
+    capabilities set<int>,
+    class int,
+    cookie text,
+    ip inet,
+    label text,
+    lat double,
+    lon double,
+    model text,
+    tstamp timestamp,
+    type int,
+    PRIMARY KEY (user, client)
+) WITH CLUSTERING ORDER BY (client ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -847,15 +820,10 @@ CREATE TABLE brig_test.user_keys_hash (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.service_tag (
-    bucket int,
-    tag bigint,
-    name text,
-    service uuid,
-    provider uuid,
-    PRIMARY KEY ((bucket, tag), name, service)
-) WITH CLUSTERING ORDER BY (name ASC, service ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.budget (
+    key text PRIMARY KEY,
+    budget int
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -863,24 +831,27 @@ CREATE TABLE brig_test.service_tag (
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
     AND default_time_to_live = 0
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 0
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.meta (
-    id int,
-    version int,
-    date timestamp,
-    descr text,
-    PRIMARY KEY (id, version)
-) WITH CLUSTERING ORDER BY (version ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.connection_remote (
+    left uuid,
+    right_domain text,
+    right_user uuid,
+    conv_domain text,
+    conv_id uuid,
+    last_update timestamp,
+    status int,
+    PRIMARY KEY (left, right_domain, right_user)
+) WITH CLUSTERING ORDER BY (right_domain ASC, right_user ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -892,34 +863,33 @@ CREATE TABLE brig_test.meta (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.unique_claims (
-    value text PRIMARY KEY,
-    claims set<uuid>
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.users_pending_activation (
+    user uuid PRIMARY KEY,
+    expires_at timestamp
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
     AND default_time_to_live = 0
-    AND gc_grace_seconds = 0
+    AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.user_cookies (
-    user uuid,
-    expires timestamp,
-    id bigint,
-    created timestamp,
-    label text,
-    succ_id bigint,
-    type int,
-    PRIMARY KEY (user, expires, id)
-) WITH CLUSTERING ORDER BY (expires ASC, id ASC)
+CREATE TABLE brig_test.connection (
+    left uuid,
+    right uuid,
+    conv uuid,
+    last_update timestamp,
+    message text,
+    status int,
+    PRIMARY KEY (left, right)
+) WITH CLUSTERING ORDER BY (right ASC)
     AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -934,15 +904,15 @@ CREATE TABLE brig_test.user_cookies (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
+CREATE INDEX conn_status ON brig_test.connection (status);
 
-CREATE TABLE brig_test.mls_key_packages (
-    user uuid,
-    client text,
-    ref blob,
-    data blob,
-    PRIMARY KEY ((user, client), ref)
-) WITH CLUSTERING ORDER BY (ref ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.password_reset (
+    key ascii PRIMARY KEY,
+    code ascii,
+    retries int,
+    timeout timestamp,
+    user uuid
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -957,17 +927,13 @@ CREATE TABLE brig_test.mls_key_packages (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.mls_key_package_refs (
-    ref blob PRIMARY KEY,
-    client text,
-    conv uuid,
-    conv_domain text,
-    domain text,
-    user uuid
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.federation_remotes (
+    domain text PRIMARY KEY,
+    search_policy int
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -979,10 +945,17 @@ CREATE TABLE brig_test.mls_key_package_refs (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.excluded_phones (
-    prefix text PRIMARY KEY,
-    comment text
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.invitation (
+    inviter uuid,
+    id uuid,
+    code ascii,
+    created_at timestamp,
+    email text,
+    name text,
+    phone text,
+    PRIMARY KEY (inviter, id)
+) WITH CLUSTERING ORDER BY (id ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -997,17 +970,18 @@ CREATE TABLE brig_test.excluded_phones (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.codes (
-    user uuid,
-    scope int,
-    code text,
-    retries int,
-    PRIMARY KEY (user, scope)
-) WITH CLUSTERING ORDER BY (scope ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE brig_test.activation_keys (
+    key ascii PRIMARY KEY,
+    challenge ascii,
+    code ascii,
+    key_text text,
+    key_type ascii,
+    retries int,
+    user uuid
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1019,10 +993,17 @@ CREATE TABLE brig_test.codes (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.user_handle (
-    handle text PRIMARY KEY,
-    user uuid
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE brig_test.vcodes (
+    key ascii,
+    scope int,
+    account uuid,
+    email text,
+    phone text,
+    retries int,
+    value ascii,
+    PRIMARY KEY (key, scope)
+) WITH CLUSTERING ORDER BY (scope ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -1030,28 +1011,20 @@ CREATE TABLE brig_test.user_handle (
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
     AND default_time_to_live = 0
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 0
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.service (
-    provider uuid,
-    id uuid,
-    assets list<frozen<asset>>,
-    auth_tokens list<ascii>,
-    base_url blob,
-    descr text,
-    enabled boolean,
-    fingerprints list<blob>,
+CREATE TABLE brig_test.service_prefix (
+    prefix text,
     name text,
-    pubkeys list<frozen<pubkey>>,
-    summary text,
-    tags set<bigint>,
-    PRIMARY KEY (provider, id)
-) WITH CLUSTERING ORDER BY (id ASC)
+    service uuid,
+    provider uuid,
+    PRIMARY KEY (prefix, name, service)
+) WITH CLUSTERING ORDER BY (name ASC, service ASC)
     AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1067,11 +1040,26 @@ CREATE TABLE brig_test.service (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.oauth_user_refresh_token (
-    user uuid,
-    token_id uuid,
-    PRIMARY KEY (user, token_id)
-) WITH CLUSTERING ORDER BY (token_id ASC)
+CREATE KEYSPACE galley_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
+
+CREATE TYPE galley_test.permissions (
+    self bigint,
+    copy bigint
+);
+
+CREATE TYPE galley_test.pubkey (
+    typ int,
+    size int,
+    pem blob
+);
+
+CREATE TABLE galley_test.meta (
+    id int,
+    version int,
+    date timestamp,
+    descr text,
+    PRIMARY KEY (id, version)
+) WITH CLUSTERING ORDER BY (version ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1079,7 +1067,7 @@ CREATE TABLE brig_test.oauth_user_refresh_token (
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 14515200
+    AND default_time_to_live = 0
     AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
@@ -1087,14 +1075,15 @@ CREATE TABLE brig_test.oauth_user_refresh_token (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.invitation_info (
-    code ascii PRIMARY KEY,
-    id uuid,
-    inviter uuid
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE galley_test.team_conv (
+    team uuid,
+    conv uuid,
+    PRIMARY KEY (team, conv)
+) WITH CLUSTERING ORDER BY (conv ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1106,12 +1095,35 @@ CREATE TABLE brig_test.invitation_info (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.service_whitelist (
+CREATE TABLE galley_test.user_team (
+    user uuid,
     team uuid,
+    PRIMARY KEY (user, team)
+) WITH CLUSTERING ORDER BY (team ASC)
+    AND bloom_filter_fp_chance = 0.1
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
+CREATE TABLE galley_test.service (
     provider uuid,
-    service uuid,
-    PRIMARY KEY (team, provider, service)
-) WITH CLUSTERING ORDER BY (provider ASC, service ASC)
+    id uuid,
+    auth_token ascii,
+    base_url blob,
+    enabled boolean,
+    fingerprints set<blob>,
+    PRIMARY KEY (provider, id)
+) WITH CLUSTERING ORDER BY (id ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1127,17 +1139,17 @@ CREATE TABLE brig_test.service_whitelist (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.provider (
-    id uuid PRIMARY KEY,
+CREATE TABLE galley_test.data_migration (
+    id int,
+    version int,
+    date timestamp,
     descr text,
-    email text,
-    name text,
-    password blob,
-    url blob
-) WITH bloom_filter_fp_chance = 0.1
+    PRIMARY KEY (id, version)
+) WITH CLUSTERING ORDER BY (version ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1149,9 +1161,40 @@ CREATE TABLE brig_test.provider (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.user_keys (
-    key text PRIMARY KEY,
-    user uuid
+CREATE TABLE galley_test.team_features (
+    team_id uuid PRIMARY KEY,
+    app_lock_enforce int,
+    app_lock_inactivity_timeout_secs int,
+    app_lock_status int,
+    conference_calling int,
+    digital_signatures int,
+    expose_invitation_urls_to_team_admin int,
+    file_sharing int,
+    file_sharing_lock_status int,
+    guest_links_lock_status int,
+    guest_links_status int,
+    legalhold_status int,
+    mls_allowed_ciphersuites set<int>,
+    mls_default_ciphersuite int,
+    mls_default_protocol int,
+    mls_e2eid_acme_discovery_url blob,
+    mls_e2eid_grace_period int,
+    mls_e2eid_lock_status int,
+    mls_e2eid_status int,
+    mls_e2eid_ver_exp timestamp,
+    mls_protocol_toggle_users set<uuid>,
+    mls_status int,
+    outlook_cal_integration_lock_status int,
+    outlook_cal_integration_status int,
+    search_visibility_inbound_status int,
+    search_visibility_status int,
+    self_deleting_messages_lock_status int,
+    self_deleting_messages_status int,
+    self_deleting_messages_ttl int,
+    snd_factor_password_challenge_lock_status int,
+    snd_factor_password_challenge_status int,
+    sso_status int,
+    validate_saml_emails int
 ) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1167,13 +1210,22 @@ CREATE TABLE brig_test.user_keys (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.mls_public_keys (
+CREATE TABLE galley_test.member (
+    conv uuid,
     user uuid,
-    client text,
-    sig_scheme text,
-    key blob,
-    PRIMARY KEY (user, client, sig_scheme)
-) WITH CLUSTERING ORDER BY (client ASC, sig_scheme ASC)
+    conversation_role text,
+    hidden boolean,
+    hidden_ref text,
+    otr_archived boolean,
+    otr_archived_ref text,
+    otr_muted boolean,
+    otr_muted_ref text,
+    otr_muted_status int,
+    provider uuid,
+    service uuid,
+    status int,
+    PRIMARY KEY (conv, user)
+) WITH CLUSTERING ORDER BY (user ASC)
     AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1189,14 +1241,14 @@ CREATE TABLE brig_test.mls_public_keys (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.invitee_info (
-    invitee uuid PRIMARY KEY,
-    conv uuid,
-    inviter uuid
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE galley_test.custom_backend (
+    domain text PRIMARY KEY,
+    config_json_url blob,
+    webapp_welcome_url blob
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1208,20 +1260,26 @@ CREATE TABLE brig_test.invitee_info (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.nonce (
+CREATE TABLE galley_test.user_remote_conv (
     user uuid,
-    key text,
-    nonce uuid,
-    PRIMARY KEY (user, key)
-) WITH CLUSTERING ORDER BY (key ASC)
-    AND bloom_filter_fp_chance = 0.01
+    conv_remote_domain text,
+    conv_remote_id uuid,
+    hidden boolean,
+    hidden_ref text,
+    otr_archived boolean,
+    otr_archived_ref text,
+    otr_muted_ref text,
+    otr_muted_status int,
+    PRIMARY KEY (user, conv_remote_domain, conv_remote_id)
+) WITH CLUSTERING ORDER BY (conv_remote_domain ASC, conv_remote_id ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 300
+    AND default_time_to_live = 0
     AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
@@ -1229,11 +1287,8 @@ CREATE TABLE brig_test.nonce (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.login_codes (
-    user uuid PRIMARY KEY,
-    code text,
-    retries int,
-    timeout timestamp
+CREATE TABLE galley_test.legalhold_whitelisted (
+    team uuid PRIMARY KEY
 ) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1249,15 +1304,17 @@ CREATE TABLE brig_test.login_codes (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.oauth_client (
-    id uuid PRIMARY KEY,
-    name text,
-    redirect_uri blob,
-    secret blob
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE galley_test.member_remote_user (
+    conv uuid,
+    user_remote_domain text,
+    user_remote_id uuid,
+    conversation_role text,
+    PRIMARY KEY (conv, user_remote_domain, user_remote_id)
+) WITH CLUSTERING ORDER BY (user_remote_domain ASC, user_remote_id ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1269,18 +1326,19 @@ CREATE TABLE brig_test.oauth_client (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.service_team (
-    provider uuid,
-    service uuid,
+CREATE TABLE galley_test.team_member (
     team uuid,
     user uuid,
-    conv uuid,
-    PRIMARY KEY ((provider, service), team, user)
-) WITH CLUSTERING ORDER BY (team ASC, user ASC)
-    AND bloom_filter_fp_chance = 0.01
+    invited_at timestamp,
+    invited_by uuid,
+    legalhold_status int,
+    perms frozen<permissions>,
+    PRIMARY KEY (team, user)
+) WITH CLUSTERING ORDER BY (user ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1292,9 +1350,13 @@ CREATE TABLE brig_test.service_team (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.blacklist (
-    key text PRIMARY KEY
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE galley_test.team_notifications (
+    team uuid,
+    id timeuuid,
+    payload blob,
+    PRIMARY KEY (team, id)
+) WITH CLUSTERING ORDER BY (id ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -1309,16 +1371,16 @@ CREATE TABLE brig_test.blacklist (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.service_whitelist_rev (
-    provider uuid,
-    service uuid,
-    team uuid,
-    PRIMARY KEY ((provider, service), team)
-) WITH CLUSTERING ORDER BY (team ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE galley_test.legalhold_pending_prekeys (
+    user uuid,
+    key int,
+    data text,
+    PRIMARY KEY (user, key)
+) WITH CLUSTERING ORDER BY (key ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1330,19 +1392,11 @@ CREATE TABLE brig_test.service_whitelist_rev (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.team_invitation (
-    team uuid,
-    id uuid,
-    code ascii,
-    created_at timestamp,
-    created_by uuid,
-    email text,
-    name text,
-    phone text,
-    role int,
-    PRIMARY KEY (team, id)
-) WITH CLUSTERING ORDER BY (id ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE galley_test.group_id_conv_id (
+    group_id blob PRIMARY KEY,
+    conv_id uuid,
+    domain text
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -1357,35 +1411,19 @@ CREATE TABLE brig_test.team_invitation (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.user (
-    id uuid PRIMARY KEY,
-    accent list<float>,
-    accent_id int,
-    activated boolean,
-    assets list<frozen<asset>>,
-    country ascii,
-    email text,
-    email_unvalidated text,
-    expires timestamp,
-    feature_conference_calling int,
-    handle text,
-    language ascii,
-    managed_by int,
-    name text,
-    password blob,
-    phone text,
-    picture list<blob>,
-    provider uuid,
-    searchable boolean,
-    service uuid,
-    sso_id text,
-    status int,
-    supported_protocols int,
-    team uuid
-) WITH bloom_filter_fp_chance = 0.1
+-- NOTE: this table is unused. It was replaced by mls_group_member_client
+CREATE TABLE galley_test.member_client (
+    conv uuid,
+    user_domain text,
+    user uuid,
+    client text,
+    key_package_ref blob,
+    PRIMARY KEY (conv, user_domain, user, client)
+) WITH CLUSTERING ORDER BY (user_domain ASC, user ASC, client ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1397,16 +1435,16 @@ CREATE TABLE brig_test.user (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.vcodes_throttle (
-    key ascii,
-    scope int,
-    initial_delay int,
-    PRIMARY KEY (key, scope)
-) WITH CLUSTERING ORDER BY (scope ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE galley_test.legalhold_service (
+    team_id uuid PRIMARY KEY,
+    auth_token ascii,
+    base_url blob,
+    fingerprint blob,
+    pubkey pubkey
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1418,16 +1456,18 @@ CREATE TABLE brig_test.vcodes_throttle (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.properties (
-    user uuid,
+CREATE TABLE galley_test.conversation_codes (
     key ascii,
-    value blob,
-    PRIMARY KEY (user, key)
-) WITH CLUSTERING ORDER BY (key ASC)
-    AND bloom_filter_fp_chance = 0.01
+    scope int,
+    conversation uuid,
+    password blob,
+    value ascii,
+    PRIMARY KEY (key, scope)
+) WITH CLUSTERING ORDER BY (scope ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1439,14 +1479,14 @@ CREATE TABLE brig_test.properties (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.service_user (
-    provider uuid,
-    service uuid,
+CREATE TABLE galley_test.mls_group_member_client (
+    group_id blob,
+    user_domain text,
     user uuid,
-    conv uuid,
-    team uuid,
-    PRIMARY KEY ((provider, service), user)
-) WITH CLUSTERING ORDER BY (user ASC)
+    client text,
+    key_package_ref blob,
+    PRIMARY KEY (group_id, user_domain, user, client)
+) WITH CLUSTERING ORDER BY (user_domain ASC, user ASC, client ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1462,14 +1502,10 @@ CREATE TABLE brig_test.service_user (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.prekeys (
-    user uuid,
-    client text,
-    key int,
-    data text,
-    PRIMARY KEY (user, client, key)
-) WITH CLUSTERING ORDER BY (client ASC, key ASC)
-    AND bloom_filter_fp_chance = 0.01
+CREATE TABLE galley_test.clients (
+    user uuid PRIMARY KEY,
+    clients set<text>
+) WITH bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -1484,21 +1520,31 @@ CREATE TABLE brig_test.prekeys (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.oauth_auth_code (
-    code ascii PRIMARY KEY,
-    client uuid,
-    code_challenge blob,
-    redirect_uri blob,
-    scope set<text>,
-    user uuid
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE galley_test.conversation (
+    conv uuid PRIMARY KEY,
+    access set<int>,
+    access_role int,
+    access_roles_v2 set<int>,
+    cipher_suite int,
+    creator uuid,
+    deleted boolean,
+    epoch bigint,
+    group_id blob,
+    message_timer bigint,
+    name text,
+    protocol int,
+    public_group_state blob,
+    receipt_mode int,
+    team uuid,
+    type int
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 300
+    AND default_time_to_live = 0
     AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
@@ -1506,25 +1552,15 @@ CREATE TABLE brig_test.oauth_auth_code (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.clients (
-    user uuid,
-    client text,
-    capabilities set<int>,
-    class int,
-    cookie text,
-    ip inet,
-    label text,
-    lat double,
-    lon double,
-    model text,
-    tstamp timestamp,
-    type int,
-    PRIMARY KEY (user, client)
-) WITH CLUSTERING ORDER BY (client ASC)
+CREATE TABLE galley_test.mls_commit_locks (
+    group_id blob,
+    epoch bigint,
+    PRIMARY KEY (group_id, epoch)
+) WITH CLUSTERING ORDER BY (epoch ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1536,9 +1572,17 @@ CREATE TABLE brig_test.clients (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.budget (
-    key text PRIMARY KEY,
-    budget int
+CREATE TABLE galley_test.team (
+    team uuid PRIMARY KEY,
+    binding boolean,
+    creator uuid,
+    deleted boolean,
+    icon text,
+    icon_key text,
+    name text,
+    search_visibility int,
+    splash_screen text,
+    status int
 ) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1547,27 +1591,22 @@ CREATE TABLE brig_test.budget (
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
     AND default_time_to_live = 0
-    AND gc_grace_seconds = 0
+    AND gc_grace_seconds = 864000
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.connection_remote (
-    left uuid,
-    right_domain text,
-    right_user uuid,
-    conv_domain text,
-    conv_id uuid,
-    last_update timestamp,
-    status int,
-    PRIMARY KEY (left, right_domain, right_user)
-) WITH CLUSTERING ORDER BY (right_domain ASC, right_user ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE galley_test.billing_team_member (
+    team uuid,
+    user uuid,
+    PRIMARY KEY (team, user)
+) WITH CLUSTERING ORDER BY (user ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1579,10 +1618,15 @@ CREATE TABLE brig_test.connection_remote (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.users_pending_activation (
-    user uuid PRIMARY KEY,
-    expires_at timestamp
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE galley_test.mls_proposal_refs (
+    group_id blob,
+    epoch bigint,
+    ref blob,
+    origin int,
+    proposal blob,
+    PRIMARY KEY (group_id, epoch, ref)
+) WITH CLUSTERING ORDER BY (epoch ASC, ref ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
@@ -1597,15 +1641,11 @@ CREATE TABLE brig_test.users_pending_activation (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.connection (
-    left uuid,
-    right uuid,
+CREATE TABLE galley_test.user (
+    user uuid,
     conv uuid,
-    last_update timestamp,
-    message text,
-    status int,
-    PRIMARY KEY (left, right)
-) WITH CLUSTERING ORDER BY (right ASC)
+    PRIMARY KEY (user, conv)
+) WITH CLUSTERING ORDER BY (conv ASC)
     AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1620,15 +1660,19 @@ CREATE TABLE brig_test.connection (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
-CREATE INDEX conn_status ON brig_test.connection (status);
 
-CREATE TABLE brig_test.password_reset (
-    key ascii PRIMARY KEY,
-    code ascii,
-    retries int,
-    timeout timestamp,
-    user uuid
-) WITH bloom_filter_fp_chance = 0.1
+CREATE KEYSPACE gundeck_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
+
+CREATE TABLE gundeck_test.push (
+    ptoken text,
+    app text,
+    transport int,
+    client text,
+    connection blob,
+    usr uuid,
+    PRIMARY KEY (ptoken, app, transport)
+) WITH CLUSTERING ORDER BY (app ASC, transport ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -1643,34 +1687,35 @@ CREATE TABLE brig_test.password_reset (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.federation_remotes (
-    domain text PRIMARY KEY,
-    search_policy int
-) WITH bloom_filter_fp_chance = 0.01
+CREATE TABLE gundeck_test.notifications (
+    user uuid,
+    id timeuuid,
+    clients set<text>,
+    payload blob,
+    PRIMARY KEY (user, id)
+) WITH CLUSTERING ORDER BY (id ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy', 'compaction_window_size': '1', 'compaction_window_unit': 'DAYS', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
     AND default_time_to_live = 0
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 0
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.invitation (
-    inviter uuid,
-    id uuid,
-    code ascii,
-    created_at timestamp,
-    email text,
-    name text,
-    phone text,
-    PRIMARY KEY (inviter, id)
-) WITH CLUSTERING ORDER BY (id ASC)
+CREATE TABLE gundeck_test.meta (
+    id int,
+    version int,
+    date timestamp,
+    descr text,
+    PRIMARY KEY (id, version)
+) WITH CLUSTERING ORDER BY (version ASC)
     AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1686,61 +1731,16 @@ CREATE TABLE brig_test.invitation (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE brig_test.activation_keys (
-    key ascii PRIMARY KEY,
-    challenge ascii,
-    code ascii,
-    key_text text,
-    key_type ascii,
-    retries int,
-    user uuid
-) WITH bloom_filter_fp_chance = 0.1
-    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
-    AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
-    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
-    AND crc_check_chance = 1.0
-    AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 0
-    AND gc_grace_seconds = 864000
-    AND max_index_interval = 2048
-    AND memtable_flush_period_in_ms = 0
-    AND min_index_interval = 128
-    AND read_repair_chance = 0.0
-    AND speculative_retry = '99PERCENTILE';
-
-CREATE TABLE brig_test.vcodes (
-    key ascii,
-    scope int,
-    account uuid,
-    email text,
-    phone text,
-    retries int,
-    value ascii,
-    PRIMARY KEY (key, scope)
-) WITH CLUSTERING ORDER BY (scope ASC)
-    AND bloom_filter_fp_chance = 0.1
-    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
-    AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
-    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
-    AND crc_check_chance = 1.0
-    AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 0
-    AND gc_grace_seconds = 0
-    AND max_index_interval = 2048
-    AND memtable_flush_period_in_ms = 0
-    AND min_index_interval = 128
-    AND read_repair_chance = 0.0
-    AND speculative_retry = '99PERCENTILE';
-
-CREATE TABLE brig_test.service_prefix (
-    prefix text,
-    name text,
-    service uuid,
-    provider uuid,
-    PRIMARY KEY (prefix, name, service)
-) WITH CLUSTERING ORDER BY (name ASC, service ASC)
+CREATE TABLE gundeck_test.user_push (
+    usr uuid,
+    ptoken text,
+    app text,
+    transport int,
+    arn text,
+    client text,
+    connection blob,
+    PRIMARY KEY (usr, ptoken, app, transport)
+) WITH CLUSTERING ORDER BY (ptoken ASC, app ASC, transport ASC)
     AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 5bb93e8be4..66502f1858 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -27,9 +27,10 @@ testCrudFederationRemotes = do
   let parseFedConns :: HasCallStack => Response -> App [Value]
       parseFedConns resp =
         -- Pick out the list of federation domain configs
-        getJSON 200 resp %. "remotes" & asList
-        -- Enforce that the values are objects and not something else
-        >>= traverse (fmap Object . asObject)
+        getJSON 200 resp %. "remotes"
+          & asList
+          -- Enforce that the values are objects and not something else
+          >>= traverse (fmap Object . asObject)
       addOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => fedConn -> [fedConn2] -> App ()
       addOnce fedConn want = do
         res <- Internal.createFedConn OwnDomain fedConn
@@ -74,7 +75,7 @@ testCrudFederationRemotes = do
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"
 
-  remote1J  <- make remote1
+  remote1J <- make remote1
   remote1J' <- make remote1'
 
   resetFedConns OwnDomain

From 98bf16afda6c0c4f41ff47954167897af26c66aa Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Sun, 4 Jun 2023 15:03:48 +0200
Subject: [PATCH 131/220] Fixup

---
 services/brig/schema/src/V77_FederationRemotes.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/brig/schema/src/V77_FederationRemotes.hs b/services/brig/schema/src/V77_FederationRemotes.hs
index c5bb9a8f2e..250164ceb8 100644
--- a/services/brig/schema/src/V77_FederationRemotes.hs
+++ b/services/brig/schema/src/V77_FederationRemotes.hs
@@ -28,7 +28,7 @@ import Text.RawString.QQ
 
 migration :: Migration
 migration =
-  Migration 76 "Table for keeping track of instances we federate with" $
+  Migration 77 "Table for keeping track of instances we federate with" $
     schema'
       [r| CREATE TABLE federation_remotes (
             domain text PRIMARY KEY,

From 6f77d21f10398fbb98b718059b8ad5e570bc2067 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 5 Jun 2023 14:00:07 +1000
Subject: [PATCH 132/220] FS-1179: Moving a delete call into Brig, and calling
 it from Galley.

Moving the deletion of one-on-one conversations into Brig, as Galley
can't access the tables where that info is stored. This added a new
internal API endpoint.

Integration tests are currently failing on trying to delete one-on-one
connections in Brig.
---
 .../wire-api/src/Wire/API/FederationUpdate.hs |  8 ++++
 .../src/Wire/API/Routes/Internal/Brig.hs      | 17 +++++++
 services/brig/src/Brig/API/Internal.hs        | 10 +++++
 services/brig/src/Brig/App.hs                 |  2 +-
 services/brig/src/Brig/Data/Connection.hs     | 12 +++++
 services/galley/galley.cabal                  |  1 -
 .../galley/src/Galley/Cassandra/Connection.hs | 20 ---------
 services/galley/src/Galley/Run.hs             | 45 ++++++++++++++-----
 .../galley/test/integration/Federation.hs     |  9 +---
 9 files changed, 82 insertions(+), 42 deletions(-)
 delete mode 100644 services/galley/src/Galley/Cassandra/Connection.hs

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 1f609baa0b..c01b06204e 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -3,6 +3,7 @@ module Wire.API.FederationUpdate
     updateFedDomains,
     getAllowedDomainsInitial,
     updateFedDomains',
+    deleteFederationRemoteGalley
   )
 where
 
@@ -20,10 +21,14 @@ import Util.Options (Endpoint (..))
 import Wire.API.Routes.FederationDomainConfig (FederationDomainConfig (domain), FederationDomainConfigs (remotes, updateInterval))
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
+import Data.Domain
 
 getFedRemotes :: ClientM FederationDomainConfigs
 getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
 
+deleteFedRemoteGalley :: Domain -> ClientM ()
+deleteFedRemoteGalley dom = namedClient @IAPI.API @"delete-federation-remote-galley" dom
+
 -- Initial function for getting the set of domains from brig, and an update interval
 getAllowedDomainsInitial :: L.Logger -> ClientEnv -> IO FederationDomainConfigs
 getAllowedDomainsInitial logger clientEnv =
@@ -47,6 +52,9 @@ getAllowedDomainsInitial logger clientEnv =
 getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
 getAllowedDomains = runClientM getFedRemotes
 
+deleteFederationRemoteGalley :: Domain -> ClientEnv -> IO (Either ClientError ())
+deleteFederationRemoteGalley dom = runClientM $ deleteFedRemoteGalley dom
+
 -- Old value -> new value -> action
 type FedUpdateCallback = FederationDomainConfigs -> FederationDomainConfigs -> IO ()
 
diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index 05435a1e05..5bca3ed97f 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -438,6 +438,23 @@ type FederationRemotesAPI =
                :> Capture "domain" Domain
                :> Delete '[JSON] ()
            )
+    -- This is nominally similar to delete-federation-remotes,
+    -- but is called from Galley to delete the one-on-one coversations.
+    -- This is needed as Galley doesn't have access to the tables
+    -- that hold these values. We don't want these deletes to happen
+    -- in delete-federation-remotes as brig might fall over and leave
+    -- some records hanging around. Galley uses a Rabbit queue to track
+    -- what is has done and can recover from a service falling over.
+    :<|> Named
+      "delete-federation-remote-galley"
+      ( Description FederationRemotesAPIDescription
+          :> Description FederationRemotesAPIDeleteDescription
+          :> "federation"
+          :> "remote"
+          :> Capture "domain" Domain
+          :> "galley"
+          :> Delete '[JSON] ()
+      )
 
 type FederationRemotesAPIDescription =
   "See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections for background. "
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index c22f25a57b..a4b22c0695 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -184,6 +184,7 @@ federationRemotesAPI =
     :<|> Named @"get-federation-remotes" getFederationRemotes
     :<|> Named @"update-federation-remotes" updateFederationRemotes
     :<|> Named @"delete-federation-remotes" deleteFederationRemotes
+    :<|> Named @"delete-federation-remote-galley" deleteFederationRemoteGalley
 
 addFederationRemote :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 addFederationRemote fedDomConf = do
@@ -300,6 +301,15 @@ deleteFederationRemotes dom = do
   lift . wrapClient . Data.deleteFederationRemote $ dom
   assertNoDomainsFromConfigFiles dom
 
+-- | Remove one-on-one conversations for the given remote domain. This is called from Galley as
+-- part of the defederation process, and should not be called duriung the initial domain removal
+-- call to brig. This is so we can ensure that domains are correctly cleaned up if a service
+-- falls over for whatever reason.
+deleteFederationRemoteGalley :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
+deleteFederationRemoteGalley dom = do
+  lift . wrapClient . Data.deleteRemoteConnectionsDomain $ dom
+  assertNoDomainsFromConfigFiles dom
+
 -- | Responds with 'Nothing' if field is NULL in existing user or user does not exist.
 getAccountConferenceCallingConfig :: UserId -> (Handler r) (ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig)
 getAccountConferenceCallingConfig uid =
diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs
index ca2d7a79ff..99e0108d61 100644
--- a/services/brig/src/Brig/App.hs
+++ b/services/brig/src/Brig/App.hs
@@ -158,7 +158,7 @@ import Wire.API.User.Identity (Email)
 import Wire.API.User.Profile (Locale)
 
 schemaVersion :: Int32
-schemaVersion = 76
+schemaVersion = 77
 
 -------------------------------------------------------------------------------
 -- Environment
diff --git a/services/brig/src/Brig/Data/Connection.hs b/services/brig/src/Brig/Data/Connection.hs
index 21f2969da4..de47134988 100644
--- a/services/brig/src/Brig/Data/Connection.hs
+++ b/services/brig/src/Brig/Data/Connection.hs
@@ -37,10 +37,12 @@ module Brig.Data.Connection
     countConnections,
     deleteConnections,
     deleteRemoteConnections,
+    deleteRemoteConnectionsDomain,
     remoteConnectionInsert,
     remoteConnectionSelect,
     remoteConnectionSelectFrom,
     remoteConnectionDelete,
+    remoteConnectionSelectFromDomain,
     remoteConnectionClear,
 
     -- * Re-exports
@@ -323,6 +325,13 @@ deleteRemoteConnections (tUntagged -> Qualified remoteUser remoteDomain) (fromRa
   pooledForConcurrentlyN_ 16 locals $ \u ->
     write remoteConnectionDelete $ params LocalQuorum (u, remoteDomain, remoteUser)
 
+deleteRemoteConnectionsDomain :: (MonadClient m, MonadUnliftIO m) => Domain -> m ()
+deleteRemoteConnectionsDomain dom = do
+  -- Select all triples for the given domain
+  triples <- retry x1 . query remoteConnectionSelectFromDomain $ params One $ pure dom
+  -- Delete them
+  pooledForConcurrentlyN_ 16 triples $ write remoteConnectionDelete . params LocalQuorum
+
 -- Queries
 
 connectionInsert :: PrepQuery W (UserId, UserId, RelationWithHistory, UTCTimeMillis, ConvId) ()
@@ -385,6 +394,9 @@ remoteConnectionUpdate = "UPDATE connection_remote set status = ?, last_update =
 remoteConnectionDelete :: PrepQuery W (UserId, Domain, UserId) ()
 remoteConnectionDelete = "DELETE FROM connection_remote where left = ? AND right_domain = ? AND right_user = ?"
 
+remoteConnectionSelectFromDomain :: PrepQuery R (Identity Domain) (UserId, Domain, UserId)
+remoteConnectionSelectFromDomain = "SELECT left, right_domain, right_user FROM connection_remote where right_domain = ?"
+
 remoteConnectionClear :: PrepQuery W (Identity UserId) ()
 remoteConnectionClear = "DELETE FROM connection_remote where left = ?"
 
diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index dd04f1902d..533dfe1369 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -70,7 +70,6 @@ library
     Galley.Cassandra.Access
     Galley.Cassandra.Client
     Galley.Cassandra.Code
-    Galley.Cassandra.Connection
     Galley.Cassandra.Conversation
     Galley.Cassandra.Conversation.Members
     Galley.Cassandra.Conversation.MLS
diff --git a/services/galley/src/Galley/Cassandra/Connection.hs b/services/galley/src/Galley/Cassandra/Connection.hs
deleted file mode 100644
index 56301aee97..0000000000
--- a/services/galley/src/Galley/Cassandra/Connection.hs
+++ /dev/null
@@ -1,20 +0,0 @@
-module Galley.Cassandra.Connection where
-
-import Cassandra (Consistency (LocalQuorum), MonadClient, PrepQuery, W, params, retry, write, x1)
-import Data.Domain
-import Galley.Cassandra.Instances ()
-import Imports
-
--- Queries targeting this table are usually in Brig, but I've put this one
--- here so that we don't have yet another network call to Brig when most
--- everything is already happening in galley
-
-deleteRemoteConnectionsByDomain ::
-  MonadClient m =>
-  Domain ->
-  m ()
-deleteRemoteConnectionsByDomain domain =
-  retry x1 . write remoteConnectionsDeleteByDomain $ params LocalQuorum $ pure domain
-
-remoteConnectionsDeleteByDomain :: PrepQuery W (Identity Domain) ()
-remoteConnectionsDeleteByDomain = "DELETE FROM connection_remote where right_domain = ?"
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index df4640cb81..d181a2473a 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -20,9 +20,7 @@ module Galley.Run
     mkApp,
     mkLogger,
     -- Exported for tests
-    deleteFederationDomainRemote,
-    deleteFederationDomainLocal,
-    deleteFederationDomainOneOnOne,
+    deleteFederationDomain
   )
 where
 
@@ -32,7 +30,7 @@ import Bilge.Request (requestIdName)
 import Cassandra (runClient, shutdown)
 import Cassandra.Schema (versionCheck)
 import qualified Control.Concurrent.Async as Async
-import Control.Exception (finally)
+import Control.Exception (finally, throwIO)
 import Control.Lens (view, (.~), (^.))
 import Control.Monad.Codensity
 import qualified Data.Aeson as Aeson
@@ -61,7 +59,6 @@ import Galley.App
 import qualified Galley.App as App
 import Galley.Aws (awsEnv)
 import Galley.Cassandra
-import Galley.Cassandra.Connection
 import Galley.Data.Conversation.Types (convMetadata)
 import qualified Galley.Effects.MemberStore as E
 import Galley.Env
@@ -83,8 +80,7 @@ import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
 import Polysemy.Error
 import Servant hiding (route)
-import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http))
-import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
+import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http), defaultMakeClientRequest)
 import qualified System.Logger as Log
 import System.Logger.Extended (mkLogger)
 import Util.Options
@@ -249,10 +245,7 @@ readRabbitMq channel mqq env = void $ AMQP.consumeMsgs channel mqq AMQP.Ack $ \(
       Log.err (env ^. applog) $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
       AMQP.nackEnv envelope
     Right dom -> do
-      runApp env $ do
-        deleteFederationDomainRemote dom
-        deleteFederationDomainLocal dom
-        deleteFederationDomainOneOnOne dom
+      runApp env $ deleteFederationDomain dom
       AMQP.ackEnv envelope
 
 mkApp :: Opts -> IORef FederationDomainConfigs -> Log.Logger -> Codensity IO (Application, Env)
@@ -342,6 +335,15 @@ collectAuthMetrics m env = do
 insertIntoMap :: (ConvId, a) -> Map ConvId (N.NonEmpty a) -> Map ConvId (N.NonEmpty a)
 insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
 
+-- Bundle all of the deletes together for easy calling
+-- Errors & exceptions are thrown to IO to stop the message being ACKed, eventually timing it
+-- out so that it can be redelivered.
+deleteFederationDomain :: Domain -> App ()
+deleteFederationDomain d = do
+  deleteFederationDomainRemote d
+  deleteFederationDomainLocal d
+  deleteFederationDomainOneOnOne d
+
 -- Remove remote members from local conversations
 deleteFederationDomainRemote :: Domain -> App ()
 deleteFederationDomainRemote dom = do
@@ -356,6 +358,7 @@ deleteFederationDomainRemote dom = do
     -- to the conversation. However we also don't want to DOS
     -- clients. Maybe suppress and send out a bulk version?
     liftIO
+      -- All errors, either exceptions or Either e, get thrown into IO
       $ evalGalleyToIO env
       $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Federation domain removal: Remove from conversation error")
         . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
@@ -400,6 +403,7 @@ deleteFederationDomainLocal dom = do
   -- Process each user.
   for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
     liftIO $
+      -- All errors, either exceptions or Either e, get thrown into IO
       evalGalleyToIO env $
         mapError @NoChanges (const (InternalErrorWithDescription "No Changes: Could not remove a local member from a remote conversation.")) $
           do
@@ -425,7 +429,24 @@ deleteFederationDomainLocal dom = do
 -- let rcnv = toRemoteUnsafe dom cnv
 -- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
 
+
+-- TODO: The DB table that this tries to update aren't available to
+-- Galley and need to be moved into brig. This will complicate the calling
+-- to delete a domain, but likely we can expose it as an internal API and
+-- eat the coverhead cost of the http call. This should also allow for the
+-- senario where galley falls over and has to redo the domain deletion so
+-- that request isn't lost.
 deleteFederationDomainOneOnOne :: Domain -> App ()
 deleteFederationDomainOneOnOne dom = do
   env <- ask
-  liftIO $ runClient (env ^. cstate) . deleteRemoteConnectionsByDomain $ dom
\ No newline at end of file
+  let c = mkClientEnv (env ^. manager) (env ^. brig)
+  liftIO (deleteFederationRemoteGalley dom c) >>= either
+    (\e -> do
+      Log.err (env ^. applog) $ Log.msg @Text "Could not delete one-on-one messages in Brig" . Log.field "error" (show e)
+      -- Throw the error into IO to match the other functions and to prevent the
+      -- message from rabbit being ACKed.
+      liftIO $ throwIO e
+    )
+    pure
+  where
+    mkClientEnv mgr (Endpoint h p) = ClientEnv mgr (BaseUrl Http (unpack h) (fromIntegral p) "") Nothing defaultMakeClientRequest
\ No newline at end of file
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index e7548d28d9..32c66f30d0 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -92,18 +92,12 @@ updateFedDomainsTest = do
 fromFedList :: FederationDomainConfigs -> Set Domain
 fromFedList = Set.fromList . fmap domain . remotes
 
--- Bundle all of the deletes together for easy calling
-deleteFederationDomain :: Domain -> App ()
-deleteFederationDomain d = do
-    deleteFederationDomainRemote d
-    deleteFederationDomainLocal d
-    deleteFederationDomainOneOnOne d
-
 deleteFederationDomains :: FederationDomainConfigs -> FederationDomainConfigs -> App ()
 deleteFederationDomains old new = do
     let prev = fromFedList old
         curr = fromFedList new
         deletedDomains = Set.difference prev curr
+    -- Call into the galley code
     for_ deletedDomains deleteFederationDomain
 
 constHandlers :: MonadIO m => [RetryStatus -> Handler m Bool]
@@ -131,7 +125,6 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
   _ <- postQualifiedMembers alice (remoteCharlie <| remoteBob :| []) qConvId
   -- Remove the remote user from the local domain
   liftIO $ runApp env $ deleteFederationDomains old new
-  liftIO $ assertBool "Fooooo" False
   -- Check that the conversation still exists.
   getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
     const 200 === statusCode

From 26aa9ef0abf21878700d1f561807a25fa32aedaf Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 5 Jun 2023 14:11:49 +1000
Subject: [PATCH 133/220] FS-1179: Fixing tests by telling cassandra to run
 queries anyway

---
 services/brig/src/Brig/Data/Connection.hs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/services/brig/src/Brig/Data/Connection.hs b/services/brig/src/Brig/Data/Connection.hs
index de47134988..487184b74b 100644
--- a/services/brig/src/Brig/Data/Connection.hs
+++ b/services/brig/src/Brig/Data/Connection.hs
@@ -327,10 +327,11 @@ deleteRemoteConnections (tUntagged -> Qualified remoteUser remoteDomain) (fromRa
 
 deleteRemoteConnectionsDomain :: (MonadClient m, MonadUnliftIO m) => Domain -> m ()
 deleteRemoteConnectionsDomain dom = do
-  -- Select all triples for the given domain
-  triples <- retry x1 . query remoteConnectionSelectFromDomain $ params One $ pure dom
-  -- Delete them
-  pooledForConcurrentlyN_ 16 triples $ write remoteConnectionDelete . params LocalQuorum
+  -- Select all triples for the given domain, and then delete them
+  runConduit $
+    paginateC remoteConnectionSelectFromDomain (paramsP LocalQuorum (pure dom) 100) x1
+      .| C.mapM_
+        (pooledMapConcurrentlyN_ 16 $ write remoteConnectionDelete . params LocalQuorum)
 
 -- Queries
 
@@ -395,7 +396,7 @@ remoteConnectionDelete :: PrepQuery W (UserId, Domain, UserId) ()
 remoteConnectionDelete = "DELETE FROM connection_remote where left = ? AND right_domain = ? AND right_user = ?"
 
 remoteConnectionSelectFromDomain :: PrepQuery R (Identity Domain) (UserId, Domain, UserId)
-remoteConnectionSelectFromDomain = "SELECT left, right_domain, right_user FROM connection_remote where right_domain = ?"
+remoteConnectionSelectFromDomain = "SELECT left, right_domain, right_user FROM connection_remote where right_domain = ? ALLOW FILTERING"
 
 remoteConnectionClear :: PrepQuery W (Identity UserId) ()
 remoteConnectionClear = "DELETE FROM connection_remote where left = ?"

From 03153e0c29cf9b99f55e62fd91ace015105019b4 Mon Sep 17 00:00:00 2001
From: fisx <mf@zerobuzz.net>
Date: Mon, 5 Jun 2023 11:33:53 +0200
Subject: [PATCH 134/220] Update docs/src/understand/configure-federation.md

Co-authored-by: Sven Tennie <sven.tennie@gmail.com>
---
 docs/src/understand/configure-federation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index 679cd24008..08c04e4148 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -448,7 +448,7 @@ the sysadmin:
    change in the future.
 
   - This end-point enjoys a comparably high amount of traffic.  If you
-   have many a large instance (say, >100 pods), *and* you set a very
+   have many pods (a large instance with say, >100 pods), *and* you set a very
    short update interval (<10s), you should monitor brig's service and
    database load closely in the beginning.
 

From 346690613282e0b12903a911be18036af33ca491 Mon Sep 17 00:00:00 2001
From: fisx <mf@zerobuzz.net>
Date: Mon, 5 Jun 2023 11:34:08 +0200
Subject: [PATCH 135/220] Update
 docs/src/developer/developer/federation-design-aspects.md

Co-authored-by: Sven Tennie <sven.tennie@gmail.com>
---
 docs/src/developer/developer/federation-design-aspects.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/developer/developer/federation-design-aspects.md b/docs/src/developer/developer/federation-design-aspects.md
index 0d2805e472..5e63c6e695 100644
--- a/docs/src/developer/developer/federation-design-aspects.md
+++ b/docs/src/developer/developer/federation-design-aspects.md
@@ -2,7 +2,7 @@
 
 (configuring-remote-connections-dev-perspective)=
 
-## keeping track of federator remotes
+## Keeping track of federator remotes
 
 **Since [PR#3260](https://github.com/wireapp/wire-server/pull/3260).**
 

From 5571ca2ef72501526329cb034d13240508921127 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 5 Jun 2023 19:56:57 +1000
Subject: [PATCH 136/220] FS-1179: Coping code from other tests to setup remote
 conversations.

Setitng up remote conversations, using code from another test. Querying
this database status requires messing around with Cql and polysemy
rather than being able to look at API requests and reponses. The test is
currently broken and I need to dig into why, and it is possible that
this is a timing issue between brig and galley, but I'm not certain.
---
 .../galley/test/integration/Federation.hs     | 187 ++++++++++++++++--
 1 file changed, 173 insertions(+), 14 deletions(-)

diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 32c66f30d0..7dbf9c1d66 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -1,9 +1,10 @@
+{-# LANGUAGE RecordWildCards #-}
 module Federation where
 
 import API.Util
 import Bilge.Assert
 import Bilge.Response
-import Control.Lens ((^.))
+import Control.Lens ((^.), view)
 import Control.Monad.Catch
 import Control.Monad.Codensity (lowerCodensity)
 import Data.Domain
@@ -25,9 +26,27 @@ import Test.Tasty.HUnit
 import Galley.API.Util
 import qualified Data.UUID as UUID
 import qualified Galley.Data.Conversation.Types as Types
-import Galley.Types.Conversations.Members (defMemberStatus, LocalMember (..), RemoteMember (..))
-import Wire.API.Conversation.Role (roleNameWireMember)
+import Galley.Types.Conversations.Members (defMemberStatus, LocalMember (..), RemoteMember (..), localMemberToOther)
+import Wire.API.Conversation.Role (roleNameWireMember, roleNameWireAdmin)
 import Wire.API.Conversation.Protocol (Protocol(..))
+import Wire.API.Federation.API.Galley (ConversationUpdate(..), GetConversationsResponse (..))
+import Wire.API.Conversation.Action
+import Wire.API.Event.Conversation
+import Wire.API.Internal.Notification
+import qualified Data.List1 as List1
+import qualified Test.Tasty.Cannon as WS
+import Data.Time (getCurrentTime)
+import Data.Singletons
+import Test.Tasty.Cannon (TimeoutUnit (..), (#))
+import Federator.MockServer
+import Galley.App
+import qualified Wire.API.Routes.MultiTablePaging as Public
+import qualified Galley.Effects.ListItems as E
+import qualified Wire.API.Conversation as Public
+import qualified Cassandra as C
+import qualified Data.ByteString as LBS
+import Wire.API.Routes.MultiTablePaging
+import Data.Range (unsafeRange)
 
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
@@ -70,8 +89,8 @@ updateFedDomainsTest = do
   (_, env) <- liftIO $ lowerCodensity $ mkApp opts r l
   -- Common variables.
   let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
-      remoteDomain = Domain "far-away.example.org"
-      remoteDomain2 = Domain "far-away-two.example.net"
+      remoteDomain = Domain "far-away.example.com"
+      remoteDomain2 = Domain "far-away-two.example.com"
   liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
   liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
   -- Setup a conversation for a known remote domain.
@@ -83,11 +102,14 @@ updateFedDomainsTest = do
   -- Adding a new federation domain, this too should be a no-op
   updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval
 
-  -- Removing a single domain
+  -- Removing a single domain: Remove a remote domain from local conversations
   updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval
 
--- Removing multiple domains
--- updateFedDomainsCallback old new
+  -- Removing a single domain: Remove a local domain from remote conversations
+  updateFedDomainRemoveLocalFromRemote env remoteDomain interval
+
+  -- Removing multiple domains
+  -- updateFedDomainsCallback old new
 
 fromFedList :: FederationDomainConfigs -> Set Domain
 fromFedList = Set.fromList . fmap domain . remotes
@@ -105,9 +127,9 @@ constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
 
 updateFedDomainRemoveRemoteFromLocal :: Env -> Domain -> Domain -> Int -> TestM ()
 updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = recovering x3 constHandlers $ const $ do
-  s <- ask
-  let opts = s ^. tsGConf
-      localDomain = opts ^. optSettings . setFederationDomain
+  -- s <- ask
+  let -- opts = s ^. tsGConf
+      -- localDomain = opts ^. optSettings . setFederationDomain
       new = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain2 FullSearch] interval
       old = new {remotes = FederationDomainConfig remoteDomain FullSearch : remotes new}
   qalice <- randomQualifiedUser
@@ -116,17 +138,16 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
   let alice = qUnqualified qalice
       remoteBob = Qualified bobId remoteDomain
       remoteCharlie = Qualified charlieId remoteDomain2
-  -- Create a conversation
+  -- Create a local conversation
   conv <- postConv alice [] (Just "remote gossip") [] Nothing Nothing
   let qConvId = decodeQualifiedConvId conv
-      convId = qUnqualified qConvId
   connectWithRemoteUser alice remoteBob
   connectWithRemoteUser alice remoteCharlie
   _ <- postQualifiedMembers alice (remoteCharlie <| remoteBob :| []) qConvId
   -- Remove the remote user from the local domain
   liftIO $ runApp env $ deleteFederationDomains old new
   -- Check that the conversation still exists.
-  getConvQualified (qUnqualified qalice) (Qualified convId localDomain) !!! do
+  getConvQualified alice qConvId !!! do
     const 200 === statusCode
     let findRemote :: Qualified UserId -> Conversation -> Maybe (Qualified UserId)
         findRemote u = find (== u) . fmap omQualifiedId . cmOthers . cnvMembers
@@ -135,6 +156,144 @@ updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = r
     const (Right $ pure remoteCharlie) === (fmap (findRemote remoteCharlie) <$> responseJsonEither)
     const (Right qalice) === (fmap (memId . cmSelf . cnvMembers) <$> responseJsonEither)
 
+updateFedDomainRemoveLocalFromRemote :: Env -> Domain -> Int -> TestM ()
+updateFedDomainRemoveLocalFromRemote env remoteDomain interval = recovering x3 constHandlers $ const $ do
+  c <- view tsCannon
+  let new = FederationDomainConfigs AllowDynamic [] interval
+      old = new {remotes = FederationDomainConfig remoteDomain FullSearch : remotes new}
+  -- Make our users
+  qalice <- randomQualifiedUser
+  qbob <- Qualified <$> randomId <*> pure remoteDomain
+  let alice = qUnqualified qalice
+      update = memberUpdate {mupHidden = Just False}
+  -- Create a remote conversation
+  -- START: code from putRemoteConvMemberOk
+  qconv <- Qualified <$> randomId <*> pure remoteDomain
+  connectWithRemoteUser alice qbob
+
+  fedGalleyClient <- view tsFedGalleyClient
+  now <- liftIO getCurrentTime
+  let cu = ConversationUpdate
+        { cuTime = now
+        , cuOrigUserId = qbob
+        , cuConvId = qUnqualified qconv
+        , cuAlreadyPresentUsers = []
+        , cuAction = SomeConversationAction (sing @'ConversationJoinTag) (ConversationJoin (pure qalice) roleNameWireMember)
+        }
+  runFedClient @"on-conversation-updated" fedGalleyClient remoteDomain cu
+  -- Expected member state
+  let memberAlice =
+        Member
+          { memId = qalice,
+            memService = Nothing,
+            memOtrMutedStatus = mupOtrMuteStatus update,
+            memOtrMutedRef = mupOtrMuteRef update,
+            memOtrArchived = Just True == mupOtrArchive update,
+            memOtrArchivedRef = mupOtrArchiveRef update,
+            memHidden = Just True == mupHidden update,
+            memHiddenRef = mupHiddenRef update,
+            memConvRoleName = roleNameWireMember
+          }
+      -- Update member state & verify push notification
+  WS.bracketR c alice $ \ws -> do
+    putMember alice update qconv !!! const 200 === statusCode
+    void . liftIO . WS.assertMatch (5 # Second) ws $ \n -> do
+      let e = List1.head (WS.unpackPayload n)
+      ntfTransient n @?= False
+      evtConv e @?= qconv
+      evtType e @?= MemberStateUpdate
+      evtFrom e @?= qalice
+      case evtData e of
+        EdMemberUpdate mis -> do
+          assertEqual "otr_muted_status" (mupOtrMuteStatus update) (misOtrMutedStatus mis)
+          assertEqual "otr_muted_ref" (mupOtrMuteRef update) (misOtrMutedRef mis)
+          assertEqual "otr_archived" (mupOtrArchive update) (misOtrArchived mis)
+          assertEqual "otr_archived_ref" (mupOtrArchiveRef update) (misOtrArchivedRef mis)
+          assertEqual "hidden" (mupHidden update) (misHidden mis)
+          assertEqual "hidden_ref" (mupHiddenRef update) (misHiddenRef mis)
+        x -> assertFailure $ "Unexpected event data: " ++ show x
+
+  -- Fetch remote conversation
+  let bobAsLocal =
+        LocalMember
+          (qUnqualified qbob)
+          defMemberStatus
+          Nothing
+          roleNameWireAdmin
+  let mockConversation =
+        mkProteusConv
+          (qUnqualified qconv)
+          (qUnqualified qbob)
+          roleNameWireMember
+          [localMemberToOther remoteDomain bobAsLocal]
+      remoteConversationResponse = GetConversationsResponse [mockConversation]
+  (rs, _) <-
+    withTempMockFederator'
+      (mockReply remoteConversationResponse)
+      $ getConvQualified alice qconv
+        <!! const 200 === statusCode
+  -- Verify new member state
+  let alice' = cmSelf . cnvMembers <$> responseJsonUnsafe rs
+  liftIO $ do
+    assertBool "user" (isJust alice')
+    let newAlice = fromJust alice'
+    assertEqual "id" (memId memberAlice) (memId newAlice)
+    assertEqual "otr_muted_status" (memOtrMutedStatus memberAlice) (memOtrMutedStatus newAlice)
+    assertEqual "otr_muted_ref" (memOtrMutedRef memberAlice) (memOtrMutedRef newAlice)
+    assertEqual "otr_archived" (memOtrArchived memberAlice) (memOtrArchived newAlice)
+    assertEqual "otr_archived_ref" (memOtrArchivedRef memberAlice) (memOtrArchivedRef newAlice)
+    assertEqual "hidden" (memHidden memberAlice) (memHidden newAlice)
+    assertEqual "hidden_ref" (memHiddenRef memberAlice) (memHiddenRef newAlice)
+  -- END: code from putRemoteConvMemberOk
+  
+  -- Remove the remote user from the local domain
+  liftIO $ runApp env $ deleteFederationDomains old new
+
+  --
+  -- Do not make any calls that would need to talk to a federation
+  -- member. This process must assume that the ex-federation member
+  -- will not accept any packets from us, so we shouldn't even try.
+  -- We can only rely on our own DB.
+  --
+
+  -- get the conversation metadata.
+  -- This shouldn't return anything as the conversation shouldn't be accessible.
+--  let rConvId = toRemoteUnsafe remoteDomain $ qUnqualified qconv
+--  meta <- liftIO $ evalGalleyToIO env
+--    $ E.getRemoteConversationStatus alice [rConvId]
+--  liftIO $ case M.lookup rConvId meta of
+--    Nothing -> assertBool "Empty meta status" False
+--    Just status -> assertBool (show status) False
+
+
+  convIds <- liftIO $ evalGalleyToIO env $
+    pageToConvIdPage Public.PagingRemotes
+    . fmap (tUntagged @'QRemote)
+    <$> E.listItems alice Nothing (unsafeRange 100)
+  case find (== qconv) $ mtpResults $ convIds of
+    Nothing -> pure ()
+    Just c' -> liftIO $ assertFailure $ "Found conversation where none was expected: " <> show c'
+
+  -- Check that the conversation still exists.
+  getConvQualified (qUnqualified qalice) qconv !!! do
+    const 200 === statusCode
+    let findRemote :: Qualified UserId -> Conversation -> Maybe (Qualified UserId)
+        findRemote u = find (== u) . fmap omQualifiedId . cmOthers . cnvMembers
+    -- Check that only one remote user was removed.
+    const (Right Nothing) === (fmap (findRemote qbob) <$> responseJsonEither)
+    -- const (Right $ pure remoteCharlie) === (fmap (findRemote remoteCharlie) <$> responseJsonEither)
+    const (Right qalice) === (fmap (memId . cmSelf . cnvMembers) <$> responseJsonEither)
+
+
+pageToConvIdPage :: Public.LocalOrRemoteTable -> C.PageWithState (Qualified ConvId) -> Public.ConvIdsPage
+pageToConvIdPage table page@C.PageWithState {..} =
+  Public.MultiTablePage
+    { mtpResults = pwsResults,
+      mtpHasMore = C.pwsHasMore page,
+      mtpPagingState = Public.ConversationPagingState table (LBS.toStrict . C.unPagingState <$> pwsState)
+    }
+
+
 updateFedDomainsAddRemote :: Env -> Domain -> Domain -> Int -> TestM ()
 updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval = do
   s <- ask

From ad83b4561f54751726b72bbdbc6e141b447bf4ab Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 6 Jun 2023 10:13:19 +0200
Subject: [PATCH 137/220] Fix cannon config map

---
 charts/cannon/templates/configmap.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/charts/cannon/templates/configmap.yaml b/charts/cannon/templates/configmap.yaml
index 940d601306..0c9c2c43cf 100644
--- a/charts/cannon/templates/configmap.yaml
+++ b/charts/cannon/templates/configmap.yaml
@@ -14,6 +14,10 @@ data:
       host: gundeck
       port: 8080
 
+    brig:
+      host: brig
+      port: 8080
+
     drainOpts:
       gracePeriodSeconds: {{ .Values.config.drainOpts.gracePeriodSeconds }}
       millisecondsBetweenBatches: {{ .Values.config.drainOpts.millisecondsBetweenBatches }}

From 0990d32ebfffcf847e5ca546f0e4736b1f4ca884 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 6 Jun 2023 10:22:35 +0200
Subject: [PATCH 138/220] whitespace

---
 integration/test/Test/Brig.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 66502f1858..a218903c11 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -31,6 +31,7 @@ testCrudFederationRemotes = do
           & asList
           -- Enforce that the values are objects and not something else
           >>= traverse (fmap Object . asObject)
+
       addOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => fedConn -> [fedConn2] -> App ()
       addOnce fedConn want = do
         res <- Internal.createFedConn OwnDomain fedConn

From d49da1a2774e166fcd37467eadf0d9948348314a Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 6 Jun 2023 18:26:04 +1000
Subject: [PATCH 139/220] FS-1179: Improving tests in galley, and adding a new
 test to brig

---
 services/brig/src/Brig/Data/Connection.hs     |  1 +
 .../brig/test/integration/API/Internal.hs     | 56 +++++++++++++++++-
 services/galley/src/Galley/API/Update.hs      |  2 +
 services/galley/src/Galley/Run.hs             |  6 +-
 .../galley/test/integration/Federation.hs     | 57 +++++--------------
 5 files changed, 76 insertions(+), 46 deletions(-)

diff --git a/services/brig/src/Brig/Data/Connection.hs b/services/brig/src/Brig/Data/Connection.hs
index 487184b74b..aa3049df93 100644
--- a/services/brig/src/Brig/Data/Connection.hs
+++ b/services/brig/src/Brig/Data/Connection.hs
@@ -44,6 +44,7 @@ module Brig.Data.Connection
     remoteConnectionDelete,
     remoteConnectionSelectFromDomain,
     remoteConnectionClear,
+    remoteConnectionsSelectUsers,
 
     -- * Re-exports
     module T,
diff --git a/services/brig/test/integration/API/Internal.hs b/services/brig/test/integration/API/Internal.hs
index 516b8934c9..5f1182ebce 100644
--- a/services/brig/test/integration/API/Internal.hs
+++ b/services/brig/test/integration/API/Internal.hs
@@ -62,6 +62,11 @@ import qualified Wire.API.Team.Feature as ApiFt
 import qualified Wire.API.Team.Member as Team
 import Wire.API.User
 import Wire.API.User.Client
+import Cassandra.Exec (x1)
+import Data.Json.Util (toUTCTimeMillis)
+import Data.Time
+import Brig.Data.Connection
+import Data.Domain
 
 tests :: Opt.Opts -> Manager -> Cass.ClientState -> Brig -> Endpoint -> Gundeck -> Galley -> IO TestTree
 tests opts mgr db brig brigep gundeck galley = do
@@ -82,9 +87,58 @@ tests opts mgr db brig brigep gundeck galley = do
               test mgr "put,put" $ testKpcPutPut brig,
               test mgr "add key package ref" $ testAddKeyPackageRef brig
             ],
-        test mgr "writetimeToInt64" $ testWritetimeRepresentation opts mgr db brig brigep galley
+        test mgr "writetimeToInt64" $ testWritetimeRepresentation opts mgr db brig brigep galley,
+        test mgr "delete-federation-remote-galley" $ testDeleteFederationRemoteGalley db brig
       ]
 
+testDeleteFederationRemoteGalley ::  forall m. TestConstraints m => Cass.ClientState -> Brig -> m ()
+testDeleteFederationRemoteGalley db brig = do
+  let remoteDomain1 = Domain "far-away.example.com"
+      remoteDomain2 = Domain "far-away-two.example.com"
+      isRemote1 = (== remoteDomain1)
+      isRemote2 = (== remoteDomain2)
+  localUser <- randomUser brig
+  let localUserId = userId localUser
+  remoteUserId1 <- randomId
+  remoteUserId2 <- randomId
+  now <- liftIO $ getCurrentTime
+  convId <- randomId
+
+  -- Write the local and remote users into 'connection_remote'
+  let params1 = (localUserId, remoteDomain1, remoteUserId1, Conn.AcceptedWithHistory, toUTCTimeMillis now, remoteDomain1, convId)
+  liftIO $ Cass.runClient db $ Cass.retry x1 $
+    Cass.write remoteConnectionInsert (Cass.params Cass.LocalQuorum params1)
+  let params2 = (localUserId, remoteDomain2, remoteUserId2, Conn.AcceptedWithHistory, toUTCTimeMillis now, remoteDomain1, convId)
+  liftIO $ Cass.runClient db $ Cass.retry x1 $
+    Cass.write remoteConnectionInsert (Cass.params Cass.LocalQuorum params2)
+  
+  -- Check that the value exists in the DB as expected.
+  -- Remote 1
+  liftIO (Cass.runClient db $ Cass.retry x1 $
+    Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)) >>=
+    liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote1 . fst)
+  -- Remote 2
+  liftIO (Cass.runClient db $ Cass.retry x1 $
+    Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)) >>=
+    liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote2 . fst)
+  
+  -- Make the API call to delete remote domain 1
+  delete
+    ( brig
+        . paths ["i", "federation", "remote", toByteString' $ domainText remoteDomain1, "galley"]
+    ) !!!
+      const 200 === statusCode
+
+  -- Check 'connection_remote' for the local user and ensure
+  -- that there are no conversations for the remote domain.
+  liftIO (Cass.runClient db $ Cass.retry x1 $
+    Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)) >>=
+    liftIO . assertBool "connection_remote entry should NOT exist for the user" . not . any (isRemote1 . fst)
+  -- But remote domain 2 is still listed
+  liftIO (Cass.runClient db $ Cass.retry x1 $
+    Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)) >>=
+    liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote2 . fst)
+
 testSuspendUser :: forall m. TestConstraints m => Cass.ClientState -> Brig -> m ()
 testSuspendUser db brig = do
   user <- randomUser brig
diff --git a/services/galley/src/Galley/API/Update.hs b/services/galley/src/Galley/API/Update.hs
index 7ea35b409c..cf199216c9 100644
--- a/services/galley/src/Galley/API/Update.hs
+++ b/services/galley/src/Galley/API/Update.hs
@@ -1661,6 +1661,8 @@ updateLocalStateOfRemoteConv requestingDomain cu = do
       SConversationMemberUpdateTag ->
         pure (Just sca, [])
       SConversationDeleteTag -> do
+        -- Present users comes from `cuAlreadyPresentUsers`, so
+        -- any users that need to be deleted have to be included in it.
         E.deleteMembersInRemoteConversation rconvId presentUsers
         pure (Just sca, [])
       SConversationRenameTag -> pure (Just sca, [])
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index d181a2473a..e61859d3c0 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -340,6 +340,8 @@ insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user
 -- out so that it can be redelivered.
 deleteFederationDomain :: Domain -> App ()
 deleteFederationDomain d = do
+  env <- ask
+  Log.err (env ^. applog) $ Log.field "domain" $ show d
   deleteFederationDomainRemote d
   deleteFederationDomainLocal d
   deleteFederationDomainOneOnOne d
@@ -397,11 +399,13 @@ deleteFederationDomainLocal :: Domain -> App ()
 deleteFederationDomainLocal dom = do
   env <- ask
   localUsers <- liftIO $ evalGalleyToIO env $ E.getLocalMembersByDomain dom
+  Log.err (env ^. applog) $ Log.field "localUsers" $ show localUsers
   -- As above, build the map so we can get all local users per conversation
   let rCnvMap = foldr insertIntoMap mempty localUsers
       localDomain = env ^. options . optSettings . setFederationDomain
   -- Process each user.
   for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
+    Log.err (env ^. applog) $ Log.field "(cnv, lUsers)" (show (cnv, lUsers))
     liftIO $
       -- All errors, either exceptions or Either e, get thrown into IO
       evalGalleyToIO env $
@@ -415,7 +419,7 @@ deleteFederationDomainLocal dom = do
                       { cuTime = now,
                         cuOrigUserId = tUntagged lUser,
                         cuConvId = cnv,
-                        cuAlreadyPresentUsers = mempty,
+                        cuAlreadyPresentUsers = [user],
                         cuAction = SomeConversationAction (sing @'ConversationDeleteTag) ()
                       }
               -- These functions are used directly rather than as part of a larger conversation
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 7dbf9c1d66..fcc1402e54 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -39,14 +39,13 @@ import Data.Time (getCurrentTime)
 import Data.Singletons
 import Test.Tasty.Cannon (TimeoutUnit (..), (#))
 import Federator.MockServer
-import Galley.App
 import qualified Wire.API.Routes.MultiTablePaging as Public
-import qualified Galley.Effects.ListItems as E
 import qualified Wire.API.Conversation as Public
 import qualified Cassandra as C
 import qualified Data.ByteString as LBS
 import Wire.API.Routes.MultiTablePaging
-import Data.Range (unsafeRange)
+import Galley.Cassandra.Queries
+import Cassandra.Exec (x1)
 
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
@@ -102,15 +101,12 @@ updateFedDomainsTest = do
   -- Adding a new federation domain, this too should be a no-op
   updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval
 
-  -- Removing a single domain: Remove a remote domain from local conversations
+  -- Remove a remote domain from local conversations
   updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval
 
-  -- Removing a single domain: Remove a local domain from remote conversations
+  -- Remove a local domain from remote conversations
   updateFedDomainRemoveLocalFromRemote env remoteDomain interval
 
-  -- Removing multiple domains
-  -- updateFedDomainsCallback old new
-
 fromFedList :: FederationDomainConfigs -> Set Domain
 fromFedList = Set.fromList . fmap domain . remotes
 
@@ -127,10 +123,7 @@ constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
 
 updateFedDomainRemoveRemoteFromLocal :: Env -> Domain -> Domain -> Int -> TestM ()
 updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval = recovering x3 constHandlers $ const $ do
-  -- s <- ask
-  let -- opts = s ^. tsGConf
-      -- localDomain = opts ^. optSettings . setFederationDomain
-      new = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain2 FullSearch] interval
+  let new = FederationDomainConfigs AllowDynamic [FederationDomainConfig remoteDomain2 FullSearch] interval
       old = new {remotes = FederationDomainConfig remoteDomain FullSearch : remotes new}
   qalice <- randomQualifiedUser
   bobId <- randomId
@@ -249,42 +242,18 @@ updateFedDomainRemoveLocalFromRemote env remoteDomain interval = recovering x3 c
   -- Remove the remote user from the local domain
   liftIO $ runApp env $ deleteFederationDomains old new
 
-  --
-  -- Do not make any calls that would need to talk to a federation
-  -- member. This process must assume that the ex-federation member
-  -- will not accept any packets from us, so we shouldn't even try.
-  -- We can only rely on our own DB.
-  --
-
-  -- get the conversation metadata.
-  -- This shouldn't return anything as the conversation shouldn't be accessible.
---  let rConvId = toRemoteUnsafe remoteDomain $ qUnqualified qconv
---  meta <- liftIO $ evalGalleyToIO env
---    $ E.getRemoteConversationStatus alice [rConvId]
---  liftIO $ case M.lookup rConvId meta of
---    Nothing -> assertBool "Empty meta status" False
---    Just status -> assertBool (show status) False
-
+  -- Get the list of remote conversations for the user.
+  -- convIds <- liftIO $ evalGalleyToIO env $
+  --   pageToConvIdPage Public.PagingRemotes
+  --   . fmap (tUntagged @'QRemote)
+  --   <$> E.listItems alice Nothing (toRange $ Proxy @1000)
 
-  convIds <- liftIO $ evalGalleyToIO env $
-    pageToConvIdPage Public.PagingRemotes
-    . fmap (tUntagged @'QRemote)
-    <$> E.listItems alice Nothing (unsafeRange 100)
-  case find (== qconv) $ mtpResults $ convIds of
+  convIds <- liftIO $ C.runClient (env ^. cstate) $ 
+      C.retry x1 $ C.query selectUserRemoteConvs (C.params C.LocalQuorum (pure alice))
+  case find (== qUnqualified qconv) $ snd <$> convIds of
     Nothing -> pure ()
     Just c' -> liftIO $ assertFailure $ "Found conversation where none was expected: " <> show c'
 
-  -- Check that the conversation still exists.
-  getConvQualified (qUnqualified qalice) qconv !!! do
-    const 200 === statusCode
-    let findRemote :: Qualified UserId -> Conversation -> Maybe (Qualified UserId)
-        findRemote u = find (== u) . fmap omQualifiedId . cmOthers . cnvMembers
-    -- Check that only one remote user was removed.
-    const (Right Nothing) === (fmap (findRemote qbob) <$> responseJsonEither)
-    -- const (Right $ pure remoteCharlie) === (fmap (findRemote remoteCharlie) <$> responseJsonEither)
-    const (Right qalice) === (fmap (memId . cmSelf . cnvMembers) <$> responseJsonEither)
-
-
 pageToConvIdPage :: Public.LocalOrRemoteTable -> C.PageWithState (Qualified ConvId) -> Public.ConvIdsPage
 pageToConvIdPage table page@C.PageWithState {..} =
   Public.MultiTablePage

From 81e1278f217591aef0201d3ee116d73dfb4b089d Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 6 Jun 2023 18:40:45 +1000
Subject: [PATCH 140/220] FS-1179: PR formatting

---
 .../wire-api/src/Wire/API/FederationUpdate.hs |   4 +-
 .../src/Wire/API/Routes/Internal/Brig.hs      |  18 +--
 services/brig/default.nix                     |   2 +
 .../brig/test/integration/API/Internal.hs     |  68 ++++++----
 services/galley/src/Galley/Run.hs             |  22 ++--
 .../galley/test/integration/Federation.hs     | 124 +++++++++---------
 services/galley/test/integration/RabbitMQ.hs  |   2 +-
 7 files changed, 132 insertions(+), 108 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index c01b06204e..5f64f1baa3 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -3,13 +3,14 @@ module Wire.API.FederationUpdate
     updateFedDomains,
     getAllowedDomainsInitial,
     updateFedDomains',
-    deleteFederationRemoteGalley
+    deleteFederationRemoteGalley,
   )
 where
 
 import Control.Concurrent.Async
 import Control.Exception (ErrorCall (ErrorCall), throwIO)
 import qualified Control.Retry as R
+import Data.Domain
 import qualified Data.Set as Set
 import Data.Text (unpack)
 import Imports
@@ -21,7 +22,6 @@ import Util.Options (Endpoint (..))
 import Wire.API.Routes.FederationDomainConfig (FederationDomainConfig (domain), FederationDomainConfigs (remotes, updateInterval))
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
-import Data.Domain
 
 getFedRemotes :: ClientM FederationDomainConfigs
 getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index 5bca3ed97f..67766b2aa7 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -446,15 +446,15 @@ type FederationRemotesAPI =
     -- some records hanging around. Galley uses a Rabbit queue to track
     -- what is has done and can recover from a service falling over.
     :<|> Named
-      "delete-federation-remote-galley"
-      ( Description FederationRemotesAPIDescription
-          :> Description FederationRemotesAPIDeleteDescription
-          :> "federation"
-          :> "remote"
-          :> Capture "domain" Domain
-          :> "galley"
-          :> Delete '[JSON] ()
-      )
+           "delete-federation-remote-galley"
+           ( Description FederationRemotesAPIDescription
+               :> Description FederationRemotesAPIDeleteDescription
+               :> "federation"
+               :> "remote"
+               :> Capture "domain" Domain
+               :> "galley"
+               :> Delete '[JSON] ()
+           )
 
 type FederationRemotesAPIDescription =
   "See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections for background. "
diff --git a/services/brig/default.nix b/services/brig/default.nix
index d679f1a80f..12302227a5 100644
--- a/services/brig/default.nix
+++ b/services/brig/default.nix
@@ -120,6 +120,7 @@
 , statistics
 , stomp-queue
 , streaming-commons
+, string-conversions
 , swagger2
 , tasty
 , tasty-cannon
@@ -259,6 +260,7 @@ mkDerivation {
     ssl-util
     statistics
     stomp-queue
+    string-conversions
     swagger2
     template
     template-haskell
diff --git a/services/brig/test/integration/API/Internal.hs b/services/brig/test/integration/API/Internal.hs
index 5f1182ebce..e090b90c17 100644
--- a/services/brig/test/integration/API/Internal.hs
+++ b/services/brig/test/integration/API/Internal.hs
@@ -28,11 +28,13 @@ import API.MLS (createClient)
 import API.MLS.Util
 import Bilge
 import Bilge.Assert
+import Brig.Data.Connection
 import Brig.Data.User (lookupFeatureConferenceCalling, lookupStatus, userExists)
 import qualified Brig.Options as Opt
 import Brig.Types.Intra
 import qualified Cassandra as C
 import qualified Cassandra as Cass
+import Cassandra.Exec (x1)
 import Cassandra.Util
 import Control.Exception (ErrorCall (ErrorCall), throwIO)
 import Control.Lens ((^.), (^?!))
@@ -41,9 +43,12 @@ import qualified Data.Aeson.Lens as Aeson
 import qualified Data.Aeson.Types as Aeson
 import Data.ByteString.Conversion (toByteString')
 import Data.Default
+import Data.Domain
 import Data.Id
+import Data.Json.Util (toUTCTimeMillis)
 import Data.Qualified (Qualified (qDomain, qUnqualified))
 import qualified Data.Set as Set
+import Data.Time
 import GHC.TypeLits (KnownSymbol)
 import Imports
 import Servant.API (ToHttpApiData (toUrlPiece))
@@ -62,11 +67,6 @@ import qualified Wire.API.Team.Feature as ApiFt
 import qualified Wire.API.Team.Member as Team
 import Wire.API.User
 import Wire.API.User.Client
-import Cassandra.Exec (x1)
-import Data.Json.Util (toUTCTimeMillis)
-import Data.Time
-import Brig.Data.Connection
-import Data.Domain
 
 tests :: Opt.Opts -> Manager -> Cass.ClientState -> Brig -> Endpoint -> Gundeck -> Galley -> IO TestTree
 tests opts mgr db brig brigep gundeck galley = do
@@ -91,7 +91,7 @@ tests opts mgr db brig brigep gundeck galley = do
         test mgr "delete-federation-remote-galley" $ testDeleteFederationRemoteGalley db brig
       ]
 
-testDeleteFederationRemoteGalley ::  forall m. TestConstraints m => Cass.ClientState -> Brig -> m ()
+testDeleteFederationRemoteGalley :: forall m. TestConstraints m => Cass.ClientState -> Brig -> m ()
 testDeleteFederationRemoteGalley db brig = do
   let remoteDomain1 = Domain "far-away.example.com"
       remoteDomain2 = Domain "far-away-two.example.com"
@@ -106,38 +106,54 @@ testDeleteFederationRemoteGalley db brig = do
 
   -- Write the local and remote users into 'connection_remote'
   let params1 = (localUserId, remoteDomain1, remoteUserId1, Conn.AcceptedWithHistory, toUTCTimeMillis now, remoteDomain1, convId)
-  liftIO $ Cass.runClient db $ Cass.retry x1 $
-    Cass.write remoteConnectionInsert (Cass.params Cass.LocalQuorum params1)
+  liftIO $
+    Cass.runClient db $
+      Cass.retry x1 $
+        Cass.write remoteConnectionInsert (Cass.params Cass.LocalQuorum params1)
   let params2 = (localUserId, remoteDomain2, remoteUserId2, Conn.AcceptedWithHistory, toUTCTimeMillis now, remoteDomain1, convId)
-  liftIO $ Cass.runClient db $ Cass.retry x1 $
-    Cass.write remoteConnectionInsert (Cass.params Cass.LocalQuorum params2)
-  
+  liftIO $
+    Cass.runClient db $
+      Cass.retry x1 $
+        Cass.write remoteConnectionInsert (Cass.params Cass.LocalQuorum params2)
+
   -- Check that the value exists in the DB as expected.
   -- Remote 1
-  liftIO (Cass.runClient db $ Cass.retry x1 $
-    Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)) >>=
-    liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote1 . fst)
+  liftIO
+    ( Cass.runClient db $
+        Cass.retry x1 $
+          Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)
+    )
+    >>= liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote1 . fst)
   -- Remote 2
-  liftIO (Cass.runClient db $ Cass.retry x1 $
-    Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)) >>=
-    liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote2 . fst)
-  
+  liftIO
+    ( Cass.runClient db $
+        Cass.retry x1 $
+          Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)
+    )
+    >>= liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote2 . fst)
+
   -- Make the API call to delete remote domain 1
   delete
     ( brig
         . paths ["i", "federation", "remote", toByteString' $ domainText remoteDomain1, "galley"]
-    ) !!!
-      const 200 === statusCode
+    )
+    !!! const 200 === statusCode
 
   -- Check 'connection_remote' for the local user and ensure
   -- that there are no conversations for the remote domain.
-  liftIO (Cass.runClient db $ Cass.retry x1 $
-    Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)) >>=
-    liftIO . assertBool "connection_remote entry should NOT exist for the user" . not . any (isRemote1 . fst)
+  liftIO
+    ( Cass.runClient db $
+        Cass.retry x1 $
+          Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)
+    )
+    >>= liftIO . assertBool "connection_remote entry should NOT exist for the user" . not . any (isRemote1 . fst)
   -- But remote domain 2 is still listed
-  liftIO (Cass.runClient db $ Cass.retry x1 $
-    Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)) >>=
-    liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote2 . fst)
+  liftIO
+    ( Cass.runClient db $
+        Cass.retry x1 $
+          Cass.query remoteConnectionsSelectUsers (Cass.params Cass.LocalQuorum $ pure localUserId)
+    )
+    >>= liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote2 . fst)
 
 testSuspendUser :: forall m. TestConstraints m => Cass.ClientState -> Brig -> m ()
 testSuspendUser db brig = do
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index e61859d3c0..782fa12837 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -20,7 +20,7 @@ module Galley.Run
     mkApp,
     mkLogger,
     -- Exported for tests
-    deleteFederationDomain
+    deleteFederationDomain,
   )
 where
 
@@ -433,7 +433,6 @@ deleteFederationDomainLocal dom = do
 -- let rcnv = toRemoteUnsafe dom cnv
 -- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
 
-
 -- TODO: The DB table that this tries to update aren't available to
 -- Galley and need to be moved into brig. This will complicate the calling
 -- to delete a domain, but likely we can expose it as an internal API and
@@ -444,13 +443,14 @@ deleteFederationDomainOneOnOne :: Domain -> App ()
 deleteFederationDomainOneOnOne dom = do
   env <- ask
   let c = mkClientEnv (env ^. manager) (env ^. brig)
-  liftIO (deleteFederationRemoteGalley dom c) >>= either
-    (\e -> do
-      Log.err (env ^. applog) $ Log.msg @Text "Could not delete one-on-one messages in Brig" . Log.field "error" (show e)
-      -- Throw the error into IO to match the other functions and to prevent the
-      -- message from rabbit being ACKed.
-      liftIO $ throwIO e
-    )
-    pure
+  liftIO (deleteFederationRemoteGalley dom c)
+    >>= either
+      ( \e -> do
+          Log.err (env ^. applog) $ Log.msg @Text "Could not delete one-on-one messages in Brig" . Log.field "error" (show e)
+          -- Throw the error into IO to match the other functions and to prevent the
+          -- message from rabbit being ACKed.
+          liftIO $ throwIO e
+      )
+      pure
   where
-    mkClientEnv mgr (Endpoint h p) = ClientEnv mgr (BaseUrl Http (unpack h) (fromIntegral p) "") Nothing defaultMakeClientRequest
\ No newline at end of file
+    mkClientEnv mgr (Endpoint h p) = ClientEnv mgr (BaseUrl Http (unpack h) (fromIntegral p) "") Nothing defaultMakeClientRequest
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index fcc1402e54..0e1d9662e4 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -1,81 +1,84 @@
 {-# LANGUAGE RecordWildCards #-}
+
 module Federation where
 
 import API.Util
 import Bilge.Assert
 import Bilge.Response
-import Control.Lens ((^.), view)
+import qualified Cassandra as C
+import Cassandra.Exec (x1)
+import Control.Lens (view, (^.))
 import Control.Monad.Catch
 import Control.Monad.Codensity (lowerCodensity)
+import qualified Data.ByteString as LBS
 import Data.Domain
 import Data.Id
 import Data.List.NonEmpty
+import qualified Data.List1 as List1
 import Data.Qualified
+import qualified Data.Set as Set
+import Data.Singletons
+import Data.Time (getCurrentTime)
+import qualified Data.UUID as UUID
+import Federator.MockServer
+import Galley.API.Util
+import Galley.Cassandra.Queries
+import qualified Galley.Data.Conversation.Types as Types
 import Galley.Env
 import Galley.Monad
 import Galley.Options
 import Galley.Run
+import Galley.Types.Conversations.Members (LocalMember (..), RemoteMember (..), defMemberStatus, localMemberToOther)
 import Imports
+import Test.Tasty.Cannon (TimeoutUnit (..), (#))
+import qualified Test.Tasty.Cannon as WS
+import Test.Tasty.HUnit
 import TestSetup
 import UnliftIO.Retry
 import Wire.API.Conversation
-import Wire.API.Routes.FederationDomainConfig
-import Wire.API.User.Search
-import qualified Data.Set as Set
-import Test.Tasty.HUnit
-import Galley.API.Util
-import qualified Data.UUID as UUID
-import qualified Galley.Data.Conversation.Types as Types
-import Galley.Types.Conversations.Members (defMemberStatus, LocalMember (..), RemoteMember (..), localMemberToOther)
-import Wire.API.Conversation.Role (roleNameWireMember, roleNameWireAdmin)
-import Wire.API.Conversation.Protocol (Protocol(..))
-import Wire.API.Federation.API.Galley (ConversationUpdate(..), GetConversationsResponse (..))
+import qualified Wire.API.Conversation as Public
 import Wire.API.Conversation.Action
+import Wire.API.Conversation.Protocol (Protocol (..))
+import Wire.API.Conversation.Role (roleNameWireAdmin, roleNameWireMember)
 import Wire.API.Event.Conversation
+import Wire.API.Federation.API.Galley (ConversationUpdate (..), GetConversationsResponse (..))
 import Wire.API.Internal.Notification
-import qualified Data.List1 as List1
-import qualified Test.Tasty.Cannon as WS
-import Data.Time (getCurrentTime)
-import Data.Singletons
-import Test.Tasty.Cannon (TimeoutUnit (..), (#))
-import Federator.MockServer
-import qualified Wire.API.Routes.MultiTablePaging as Public
-import qualified Wire.API.Conversation as Public
-import qualified Cassandra as C
-import qualified Data.ByteString as LBS
+import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.MultiTablePaging
-import Galley.Cassandra.Queries
-import Cassandra.Exec (x1)
+import qualified Wire.API.Routes.MultiTablePaging as Public
+import Wire.API.User.Search
 
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
 
 isConvMemberLTests :: TestM ()
 isConvMemberLTests = do
-    s <- ask
-    let opts = s ^. tsGConf
-        localDomain = opts ^. optSettings . setFederationDomain
-        remoteDomain = Domain "far-away.example.com"
-        convId = Id $ fromJust $ UUID.fromString "8cc34301-6949-46c5-bb93-00a72268e2f5"
-        convLocalMembers = [LocalMember userId defMemberStatus Nothing roleNameWireMember]
-        convRemoteMembers = [RemoteMember rUserId roleNameWireMember]
-        lconv = toLocalUnsafe localDomain $ Types.Conversation
+  s <- ask
+  let opts = s ^. tsGConf
+      localDomain = opts ^. optSettings . setFederationDomain
+      remoteDomain = Domain "far-away.example.com"
+      convId = Id $ fromJust $ UUID.fromString "8cc34301-6949-46c5-bb93-00a72268e2f5"
+      convLocalMembers = [LocalMember userId defMemberStatus Nothing roleNameWireMember]
+      convRemoteMembers = [RemoteMember rUserId roleNameWireMember]
+      lconv =
+        toLocalUnsafe localDomain $
+          Types.Conversation
             convId
             convLocalMembers
             convRemoteMembers
             False
             (defConversationMetadata userId)
             ProtocolProteus
-        lUserId :: Local UserId
-        lUserId = toLocalUnsafe localDomain $ Id $ fromJust $ UUID.fromString "217352c0-8b2b-4653-ac76-a88d19490dad" -- A random V4 UUID
-        userId = qUnqualified $ tUntagged lUserId
-        rUserId :: Remote UserId
-        rUserId = toRemoteUnsafe remoteDomain $ Id $ fromJust $ UUID.fromString "d87745f5-dfe7-4ff0-8772-b9c22118b372"
-    liftIO $ assertBool "UserId" $ isConvMemberL lconv userId
-    liftIO $ assertBool "Local UserId" $ isConvMemberL lconv lUserId
-    liftIO $ assertBool "Remote UserId" $ isConvMemberL lconv rUserId
-    liftIO $ assertBool "Qualified UserId (local)" $ isConvMemberL lconv $ tUntagged lUserId
-    liftIO $ assertBool "Qualified UserId (remote)" $ isConvMemberL lconv $ tUntagged rUserId
+      lUserId :: Local UserId
+      lUserId = toLocalUnsafe localDomain $ Id $ fromJust $ UUID.fromString "217352c0-8b2b-4653-ac76-a88d19490dad" -- A random V4 UUID
+      userId = qUnqualified $ tUntagged lUserId
+      rUserId :: Remote UserId
+      rUserId = toRemoteUnsafe remoteDomain $ Id $ fromJust $ UUID.fromString "d87745f5-dfe7-4ff0-8772-b9c22118b372"
+  liftIO $ assertBool "UserId" $ isConvMemberL lconv userId
+  liftIO $ assertBool "Local UserId" $ isConvMemberL lconv lUserId
+  liftIO $ assertBool "Remote UserId" $ isConvMemberL lconv rUserId
+  liftIO $ assertBool "Qualified UserId (local)" $ isConvMemberL lconv $ tUntagged lUserId
+  liftIO $ assertBool "Qualified UserId (remote)" $ isConvMemberL lconv $ tUntagged rUserId
 
 updateFedDomainsTest :: TestM ()
 updateFedDomainsTest = do
@@ -112,11 +115,11 @@ fromFedList = Set.fromList . fmap domain . remotes
 
 deleteFederationDomains :: FederationDomainConfigs -> FederationDomainConfigs -> App ()
 deleteFederationDomains old new = do
-    let prev = fromFedList old
-        curr = fromFedList new
-        deletedDomains = Set.difference prev curr
-    -- Call into the galley code
-    for_ deletedDomains deleteFederationDomain
+  let prev = fromFedList old
+      curr = fromFedList new
+      deletedDomains = Set.difference prev curr
+  -- Call into the galley code
+  for_ deletedDomains deleteFederationDomain
 
 constHandlers :: MonadIO m => [RetryStatus -> Handler m Bool]
 constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
@@ -166,13 +169,14 @@ updateFedDomainRemoveLocalFromRemote env remoteDomain interval = recovering x3 c
 
   fedGalleyClient <- view tsFedGalleyClient
   now <- liftIO getCurrentTime
-  let cu = ConversationUpdate
-        { cuTime = now
-        , cuOrigUserId = qbob
-        , cuConvId = qUnqualified qconv
-        , cuAlreadyPresentUsers = []
-        , cuAction = SomeConversationAction (sing @'ConversationJoinTag) (ConversationJoin (pure qalice) roleNameWireMember)
-        }
+  let cu =
+        ConversationUpdate
+          { cuTime = now,
+            cuOrigUserId = qbob,
+            cuConvId = qUnqualified qconv,
+            cuAlreadyPresentUsers = [],
+            cuAction = SomeConversationAction (sing @'ConversationJoinTag) (ConversationJoin (pure qalice) roleNameWireMember)
+          }
   runFedClient @"on-conversation-updated" fedGalleyClient remoteDomain cu
   -- Expected member state
   let memberAlice =
@@ -187,7 +191,7 @@ updateFedDomainRemoveLocalFromRemote env remoteDomain interval = recovering x3 c
             memHiddenRef = mupHiddenRef update,
             memConvRoleName = roleNameWireMember
           }
-      -- Update member state & verify push notification
+  -- Update member state & verify push notification
   WS.bracketR c alice $ \ws -> do
     putMember alice update qconv !!! const 200 === statusCode
     void . liftIO . WS.assertMatch (5 # Second) ws $ \n -> do
@@ -238,7 +242,7 @@ updateFedDomainRemoveLocalFromRemote env remoteDomain interval = recovering x3 c
     assertEqual "hidden" (memHidden memberAlice) (memHidden newAlice)
     assertEqual "hidden_ref" (memHiddenRef memberAlice) (memHiddenRef newAlice)
   -- END: code from putRemoteConvMemberOk
-  
+
   -- Remove the remote user from the local domain
   liftIO $ runApp env $ deleteFederationDomains old new
 
@@ -248,8 +252,11 @@ updateFedDomainRemoveLocalFromRemote env remoteDomain interval = recovering x3 c
   --   . fmap (tUntagged @'QRemote)
   --   <$> E.listItems alice Nothing (toRange $ Proxy @1000)
 
-  convIds <- liftIO $ C.runClient (env ^. cstate) $ 
-      C.retry x1 $ C.query selectUserRemoteConvs (C.params C.LocalQuorum (pure alice))
+  convIds <-
+    liftIO $
+      C.runClient (env ^. cstate) $
+        C.retry x1 $
+          C.query selectUserRemoteConvs (C.params C.LocalQuorum (pure alice))
   case find (== qUnqualified qconv) $ snd <$> convIds of
     Nothing -> pure ()
     Just c' -> liftIO $ assertFailure $ "Found conversation where none was expected: " <> show c'
@@ -262,7 +269,6 @@ pageToConvIdPage table page@C.PageWithState {..} =
       mtpPagingState = Public.ConversationPagingState table (LBS.toStrict . C.unPagingState <$> pwsState)
     }
 
-
 updateFedDomainsAddRemote :: Env -> Domain -> Domain -> Int -> TestM ()
 updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval = do
   s <- ask
diff --git a/services/galley/test/integration/RabbitMQ.hs b/services/galley/test/integration/RabbitMQ.hs
index 44dbf76a71..b83f52e233 100644
--- a/services/galley/test/integration/RabbitMQ.hs
+++ b/services/galley/test/integration/RabbitMQ.hs
@@ -1 +1 @@
-module RabbitMQ where
\ No newline at end of file
+module RabbitMQ where

From 7e56c4a231d6709be6499e16a1783b73dc9009b1 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 7 Jun 2023 18:25:58 +1000
Subject: [PATCH 141/220] Adding a round-trip test for rabbitmq.

Using the pub and sub functions from the main galley code
---
 services/galley/galley.cabal                 |  1 +
 services/galley/src/Galley/Run.hs            | 42 +++++----
 services/galley/test/integration/Main.hs     |  4 +-
 services/galley/test/integration/RabbitMQ.hs | 89 ++++++++++++++++++++
 4 files changed, 118 insertions(+), 18 deletions(-)

diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index 12f9273239..1b1a989d50 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -451,6 +451,7 @@ executable galley-integration
     , directory
     , errors
     , exceptions
+    , extended
     , extra                  >=1.3
     , federator
     , filepath
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 138a740e61..7d2f83d3b0 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -21,6 +21,9 @@ module Galley.Run
     mkLogger,
     -- Exported for tests
     deleteFederationDomain,
+    publishRabbitMsg,
+    readRabbitMq,
+    ensureQueue
   )
 where
 
@@ -175,8 +178,9 @@ complexFederationUpdate env clientEnv rmq = void $ Codensity $ Async.withAsync $
           -- This exception should _NOT_ be caught, or if it is it needs to be rethrown. This
           -- will kill the thread. This handler is then used to start a new one.
           ensureQueue channel mqq
-          writeRabbitMq clientEnv env channel mqq threadRef
-          readRabbitMq channel mqq env
+          writeRabbitMq clientEnv (env ^. applog) (env ^. fedDomains) channel mqq threadRef
+          let performDelete = runApp env . deleteFederationDomain
+          readRabbitMq channel mqq (env ^. applog) performDelete
 
           -- Keep this thread around until it is killed.
           forever $ threadDelay maxBound
@@ -204,13 +208,14 @@ ensureQueue channel mqq = do
 -- this channel thread. We don't want to leak those resources.
 writeRabbitMq ::
   ClientEnv ->
-  Env ->
+  Log.Logger ->
+  IORef FederationDomainConfigs ->
   AMQP.Channel ->
   Text ->
   IORef (Maybe (Async.Async ())) ->
   IO ()
-writeRabbitMq clientEnv env channel mqq threadRef = do
-  threadId <- updateFedDomains' ioref clientEnv (env ^. applog) $ \old new -> do
+writeRabbitMq clientEnv logger ioref channel mqq threadRef = do
+  threadId <- updateFedDomains' ioref clientEnv logger $ \old new -> do
     let fromFedList = Set.fromList . remotes
         prevDoms = fromFedList old
         currDoms = fromFedList new
@@ -220,15 +225,16 @@ writeRabbitMq clientEnv env channel mqq threadRef = do
     for_ deletedDomains $ \fedCfg -> do
       -- We're using the default exchange. This will deliver the
       -- message to the queue name used for the routing key
-      void $
-        AMQP.publishMsg channel "" mqq $
-          AMQP.newMsg
-            { AMQP.msgBody = Aeson.encode @MsgData $ domain fedCfg,
-              AMQP.msgDeliveryMode = pure AMQP.Persistent
-            }
+      void $ publishRabbitMsg channel mqq $ domain fedCfg
   atomicWriteIORef threadRef $ pure threadId
-  where
-    ioref = env ^. fedDomains
+
+-- Split out to help with integration testing
+publishRabbitMsg :: AMQP.Channel -> Text -> MsgData -> IO (Maybe Int)
+publishRabbitMsg channel mqq dom = AMQP.publishMsg channel "" mqq $
+  AMQP.newMsg
+    { AMQP.msgBody = Aeson.encode @MsgData dom
+    , AMQP.msgDeliveryMode = pure AMQP.Persistent
+    }
 
 -- Read messages from RabbitMQ, process the message, and ACK or NACK it as appropriate.
 -- This is automatically killed by `amqp`, we don't need to handle it.
@@ -238,14 +244,14 @@ writeRabbitMq clientEnv env channel mqq threadRef = do
 -- that is set when the queue is created. When the active consumer disconnects for
 -- whatever reason, rabbit will pick another of the subscribed clients to be the new
 -- active consumer.
-readRabbitMq :: AMQP.Channel -> Text -> Env -> IO ()
-readRabbitMq channel mqq env = void $ AMQP.consumeMsgs channel mqq AMQP.Ack $ \(message, envelope) ->
+readRabbitMq :: AMQP.Channel -> Text -> Log.Logger -> (Domain -> IO ()) -> IO ()
+readRabbitMq channel mqq logger go = void $ AMQP.consumeMsgs channel mqq AMQP.Ack $ \(message, envelope) ->
   case Aeson.eitherDecode @MsgData (AMQP.msgBody message) of
     Left e -> do
-      Log.err (env ^. applog) $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
+      Log.err logger $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
       AMQP.nackEnv envelope
     Right dom -> do
-      runApp env $ deleteFederationDomain dom
+      go dom
       AMQP.ackEnv envelope
 
 mkApp :: Opts -> IORef FederationDomainConfigs -> Log.Logger -> Codensity IO (Application, Env)
@@ -407,6 +413,8 @@ deleteFederationDomainLocal dom = do
   for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
     Log.err (env ^. applog) $ Log.field "(cnv, lUsers)" (show (cnv, lUsers))
     liftIO $
+      -- All errors, either exceptions or Either e, get thrown into IO
+      
       -- All errors, either exceptions or Either e, get thrown into IO
       evalGalleyToIO env $
         mapError @NoChanges (const (InternalErrorWithDescription "No Changes: Could not remove a local member from a remote conversation.")) $
diff --git a/services/galley/test/integration/Main.hs b/services/galley/test/integration/Main.hs
index 519cf856b2..8b865763f5 100644
--- a/services/galley/test/integration/Main.hs
+++ b/services/galley/test/integration/Main.hs
@@ -54,6 +54,7 @@ import Util.Options
 import Util.Options.Common
 import Util.Test
 import qualified Util.Test.SQS as SQS
+import RabbitMQ (rabbitPubSub)
 
 newtype ServiceConfigFile = ServiceConfigFile String
   deriving (Eq, Ord, Typeable)
@@ -97,7 +98,8 @@ main = withOpenSSL $ runTests go
               (pathsConsistencyCheck . treeToPaths . compile $ Galley.API.sitemap),
           API.tests setup,
           test setup "Federation Domains" updateFedDomainsTest,
-          test setup "isConvMemberL" isConvMemberLTests
+          test setup "isConvMemberL" isConvMemberLTests,
+          test setup "RabbitMQ round-trip" rabbitPubSub
         ]
     getOpts gFile iFile = do
       m <- newManager tlsManagerSettings {managerResponseTimeout = responseTimeoutMicro 300000000}
diff --git a/services/galley/test/integration/RabbitMQ.hs b/services/galley/test/integration/RabbitMQ.hs
index b83f52e233..323d44c78a 100644
--- a/services/galley/test/integration/RabbitMQ.hs
+++ b/services/galley/test/integration/RabbitMQ.hs
@@ -1 +1,90 @@
 module RabbitMQ where
+
+import qualified System.Logger as Log
+import Imports
+import TestSetup
+import Network.AMQP.Extended (RabbitMqHooks (RabbitMqHooks), openConnectionWithRetries)
+import qualified Network.AMQP.Extended as AMQP
+import Control.Lens ((^.), view)
+import Data.Domain
+import Galley.Options
+import Galley.Run (publishRabbitMsg, readRabbitMq, ensureQueue)
+import Test.Tasty.HUnit
+import Data.Text (unpack, pack)
+import System.Random
+
+-- compete and timeout are from https://wiki.haskell.org/Timing_out_computations
+compete :: [IO a] -> IO a
+compete actions = do
+    mvar <- newEmptyMVar
+    tids <- mapM (\action -> forkIO $ action >>= putMVar mvar) actions
+    result <- takeMVar mvar
+    mapM_ killThread tids
+    pure result
+
+timeout :: Int -> IO a -> IO (Maybe a)
+timeout usec action = compete [fmap Just action, threadDelay usec >> pure Nothing]
+
+-- Test the round trip to rabbit from galley
+-- and especially that we get out the domain we put in.`
+rabbitPubSub :: TestM ()
+rabbitPubSub = view (tsGConf . optRabbitmq)
+  >>= maybe (pure ()) withRabbitOpts
+
+-- Generate a simple random string
+randomString :: MonadIO m => m String
+randomString = getStdRandom $ \gen ->
+  let (count, gen') = randomR (3, 10) gen
+  -- I'm not happy with using replicate, but my hoogling didn't
+  -- show anything closer to what I wanted.
+  in foldr step ([], gen') $ replicate count ()
+  where
+    step _ (s, g) = let (c, g') = randomR ('a', 'z') g in (c:s, g')
+
+randomDomain :: MonadIO m => m Domain
+randomDomain = Domain <$> do
+  a <- randomString
+  b <- randomString
+  pure $ pack $ a <> "." <> b
+
+withRabbitOpts :: RabbitMqOpts -> TestM ()
+withRabbitOpts rabbitOpts = do
+  log' <- liftIO $ Log.new Log.defSettings
+  -- If this IORef has a Just in it, that will contain
+  -- an error message. When we successfully get our message
+  -- from Rabbit we empty it
+  ioref <- liftIO $ newIORef $ pure "Empty: Maybe we weren't able to talk to RabbitMQ yet"
+  -- Make a random string to ensure we aren't seeing leftover
+  -- messages from other tests
+  dom <- randomDomain
+  queue <- pack . ("galley-integration-testing-" <>) <$> randomString
+  -- We don't care about the result from timeout, as
+  -- the openConnect... function will loop forever. We
+  -- just need to kill it in a timely manner
+  liftIO $ void $ timeout timeout_us $
+    openConnectionWithRetries log' host port vhost $
+      RabbitMqHooks
+        { AMQP.onConnectionClose = pure ()
+        , AMQP.onChannelException = \_e -> pure ()
+        , AMQP.onNewChannel = \chan -> do
+          -- Run the tests here
+          ensureQueue chan queue
+          void $ publishRabbitMsg chan queue dom
+          liftIO $ readRabbitMq chan queue log' $ \dom' -> do
+            -- Append the random string
+            if dom == dom'
+            then do -- Happy path
+              atomicWriteIORef ioref Nothing
+            else do -- Sad path
+              atomicWriteIORef ioref $ Just $ "expected \"" <> unpack (domainText dom) <> "\" but got \"" <> unpack (domainText dom') <> "\""
+          -- Keep this around until we kill it.
+          forever $ threadDelay 1_000_000
+        }
+  liftIO $ readIORef ioref >>= maybe
+    (pure ())
+    assertFailure
+  where
+    timeout_us = 5 * 1_000_000 -- seconds scale
+    host = rabbitOpts ^. rabbitmqHost
+    port = rabbitOpts ^. rabbitmqPort
+    vhost = rabbitOpts ^. rabbitmqVHost
\ No newline at end of file

From 781cacda524164676dd8e243095735b595a7b532 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 9 Jun 2023 11:29:07 +0200
Subject: [PATCH 142/220] hi ci


From 5cd4559f64d2a36ae9b448cfb44e75d4eac99d6b Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Fri, 9 Jun 2023 11:43:07 +0200
Subject: [PATCH 143/220] docs.

---
 docs/src/understand/configure-federation.md | 8 +++++---
 docs/src/understand/searchability.md        | 2 ++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index 08c04e4148..1e5c591421 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -437,9 +437,9 @@ the sysadmin:
 * [`GET`](https://staging-nginz-https.zinfra.io/api-internal/swagger-ui/brig/#/brig/get_i_federation_remotes)
 
   - this serves an object with 3 fields:
-     - `remotes` (from cassandra): the list of remote domains with search strategy (and
+     - `remotes` (from cassandra): the list of remote domains with search policy (and
        possibly other information in the future);
-     - `strategy` (from config): one of `allowNone`, `allowDynamic`, `allowAll` (see above)
+     - `strategy` (from config): federation strategy; one of `allowNone`, `allowDynamic`, `allowAll` (see above)
      - `update_interval` (from config): the suggested update frequency with which calling
        services should refresh their information.
 
@@ -472,6 +472,7 @@ The `remotes` list looks like this:
   },
   {
     "domain": "evil.example.com"
+    "search_policy": "no_search"
   },
   ...
 ]
@@ -485,7 +486,8 @@ It serves two purposes:
 2. Independently of the federation strategy, the list provides
   information about remote backends that may change dynamically (at
   the time of writing this: search policy, see
-  {ref}`searching-users-on-another-federated-backend`)
+  {ref}`searching-users-on-another-federated-backend` and
+  {ref}`user-searchability` for more context)
 
 The search policy for a remote backend can be:
 
diff --git a/docs/src/understand/searchability.md b/docs/src/understand/searchability.md
index 8a444d5db6..b1608e0d6a 100644
--- a/docs/src/understand/searchability.md
+++ b/docs/src/understand/searchability.md
@@ -1,3 +1,5 @@
+(user-searchability)=
+
 # User Searchability
 
 You can configure how search is limited or not based on user membership in a given team.

From 61a5c87aa5511f3a0e693a6cc0a0ac3dc4f2303d Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Sun, 11 Jun 2023 12:30:49 +0200
Subject: [PATCH 144/220] docs.

---
 docs/src/understand/configure-federation.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index 1e5c591421..426db08e8f 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -495,7 +495,8 @@ The search policy for a remote backend can be:
 - `exact_handle_search`: Only users where the handle exactly matches are returned.
 - `full_search`: Additionally to `exact_handle_search`, users are found by a freetext search on handle and display name.
 
-Default is `no_search`.
+If federation strategy is `allowAll`, and there is no entry for a
+domain in the database, default is `no_search`.
 
 Also see {ref}`configuring-remote-connections-dev-perspective` for the
 developer's point of view on this topic.

From 3a35a64109c063dc8377b211830348ea48098f1b Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 15 Jun 2023 15:42:46 +1000
Subject: [PATCH 145/220] FS-1179: Add an internal endpoint for deleting a
 federation domain

---
 services/galley/galley.cabal                  |   2 -
 services/galley/src/Galley/API/Internal.hs    | 160 ++++++++++-
 services/galley/src/Galley/Run.hs             | 272 +-----------------
 .../galley/test/integration/Federation.hs     |  12 +-
 services/galley/test/integration/Main.hs      |   4 +-
 services/galley/test/integration/RabbitMQ.hs  |  90 ------
 6 files changed, 166 insertions(+), 374 deletions(-)
 delete mode 100644 services/galley/test/integration/RabbitMQ.hs

diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index 1b1a989d50..d6a46c1c03 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -372,7 +372,6 @@ executable galley-integration
     API.Util.TeamFeature
     Federation
     Main
-    RabbitMQ
     TestHelpers
     TestSetup
 
@@ -451,7 +450,6 @@ executable galley-integration
     , directory
     , errors
     , exceptions
-    , extended
     , extra                  >=1.3
     , federator
     , filepath
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index bbe817a4e2..a93fa80e1d 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -21,6 +21,8 @@ module Galley.API.Internal
     InternalAPI,
     deleteLoop,
     safeForever,
+    -- Exported for tests
+    deleteFederationDomain
   )
 where
 
@@ -70,7 +72,7 @@ import Galley.Types.Conversations.Members (RemoteMember (rmId))
 import Galley.Types.UserList
 import Imports hiding (head)
 import qualified Network.AMQP as Q
-import Network.Wai.Predicate hiding (Error, err)
+import Network.Wai.Predicate hiding (Error, err, setStatus)
 import qualified Network.Wai.Predicate as Predicate
 import Network.Wai.Routing hiding (App, route, toList)
 import Network.Wai.Utilities hiding (Error)
@@ -88,6 +90,7 @@ import Wire.API.CustomBackend
 import Wire.API.Error
 import Wire.API.Error.Galley
 import Wire.API.Event.Conversation
+import Wire.API.FederationUpdate
 import Wire.API.Federation.API
 import Wire.API.Federation.API.Galley
 import Wire.API.Federation.Error
@@ -96,10 +99,25 @@ import Wire.API.Routes.API
 import Wire.API.Routes.Internal.Galley
 import Wire.API.Routes.Internal.Galley.TeamsIntra
 import Wire.API.Routes.MultiTablePaging (mtpHasMore, mtpPagingState, mtpResults)
-import Wire.API.Team.Feature
+import Wire.API.Team.Feature hiding (setStatus)
 import Wire.API.Team.Member
 import Wire.Sem.Paging
 import Wire.Sem.Paging.Cassandra
+import Data.Domain
+import qualified Data.Map as Map
+import qualified Wire.API.Federation.API.Galley as F
+import qualified Data.List.NonEmpty as N
+import qualified Galley.Effects.MemberStore as E
+import Galley.API.Action
+import Util.Options
+import Wire.API.Conversation.Role
+import Network.HTTP.Types
+import Network.Wai
+import Data.Text (unpack)
+import Galley.Data.Conversation.Types
+import Galley.API.Federation (onConversationUpdated)
+import Servant.Client (ClientEnv (ClientEnv), BaseUrl (BaseUrl), Scheme (Http), defaultMakeClientRequest)
+import Control.Exception
 
 internalAPI :: API InternalAPI GalleyEffects
 internalAPI =
@@ -298,6 +316,10 @@ internalSitemap = unsafeCallsFed @'Galley @"on-client-removed" $ unsafeCallsFed
   delete "/i/custom-backend/by-domain/:domain" (continue CustomBackend.internalDeleteCustomBackendByDomainH) $
     capture "domain"
       .&. accept "application" "json"
+    
+  delete "/i/federation/:domain" (continue internalDeleteFederationDomainH) $
+    capture "domain"
+      .&. accept "application" "json"
 
 rmUser ::
   forall p1 p2 r.
@@ -466,3 +488,137 @@ guardLegalholdPolicyConflictsH ::
 guardLegalholdPolicyConflictsH glh = do
   mapError @LegalholdConflicts (const $ Tagged @'MissingLegalholdConsent ()) $
     guardLegalholdPolicyConflicts (glhProtectee glh) (glhUserClients glh)
+
+-- Build the map, keyed by conversations to the list of members
+insertIntoMap :: (ConvId, a) -> Map ConvId (N.NonEmpty a) -> Map ConvId (N.NonEmpty a)
+insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
+
+-- Bundle all of the deletes together for easy calling
+-- Errors & exceptions are thrown to IO to stop the message being ACKed, eventually timing it
+-- out so that it can be redelivered.
+deleteFederationDomain :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg)) r,
+          Member (Error InternalError) r, Member (Error FederationError) r, Member (Input (Local ())) r,
+          Member MemberStore r, Member ConversationStore r, Member (Embed IO) r,
+          Member CodeStore r, Member TeamStore r, Member BrigAccess r, Member GundeckAccess r, Member ExternalAccess r) => Domain -> Sem r ()
+deleteFederationDomain d = do
+  deleteFederationDomainRemote d
+  deleteFederationDomainLocal d
+  deleteFederationDomainOneOnOne d
+
+
+internalDeleteFederationDomainH :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg)) r,
+          Member (Error InternalError) r, Member (Error FederationError) r, Member (Input (Local ())) r,
+          Member MemberStore r, Member ConversationStore r, Member (Embed IO) r,
+          Member CodeStore r, Member TeamStore r, Member BrigAccess r, Member GundeckAccess r, Member ExternalAccess r) => Domain ::: JSON -> Sem r Response
+internalDeleteFederationDomainH (domain ::: _) = do
+  deleteFederationDomain domain
+  -- TODO: Do we generally also accept HTTP 204, No Content?
+  pure (empty & setStatus status200)
+
+-- Remove remote members from local conversations
+deleteFederationDomainRemote :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg)) r,
+          Member (Error InternalError) r, Member (Error FederationError) r,
+          Member MemberStore r, Member ConversationStore r,
+          Member CodeStore r, Member TeamStore r) => Domain -> Sem r ()
+deleteFederationDomainRemote dom = do
+  remoteUsers <- E.getRemoteMembersByDomain dom
+  env <- input
+  let lCnvMap = foldr insertIntoMap mempty remoteUsers
+      localDomain = env ^. Galley.App.options . optSettings . setFederationDomain
+  for_ (Map.toList lCnvMap) $ \(cnvId, rUsers) -> do
+    let lCnvId = toLocalUnsafe localDomain cnvId
+    -- This value contains an event that we might need to
+    -- send out to all of the local clients that are a party
+    -- to the conversation. However we also don't want to DOS
+    -- clients. Maybe suppress and send out a bulk version?
+    -- All errors, either exceptions or Either e, get thrown into IO
+    mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Federation domain removal: Remove from conversation error")
+      . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
+      . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription "Federation domain removal: Action denied, remove conversation member")
+      . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription "Federation domain removal: Invalid operation")
+      . mapToRuntimeError @'NotATeamMember (InternalErrorWithDescription "Federation domain removal: Not a team member")
+      . mapError @NoChanges (const (InternalErrorWithDescription "Federation domain removal: No changes"))
+      -- This is allowed to send notifications to _local_ clients.
+      -- But we are suppressing those events as we don't want to
+      -- DOS our users if a large and deeply interconnected federation
+      -- member is removed. Sending out hundreds or thousands of events
+      -- to each client isn't something we want to be doing.
+      $ do
+        conv <- getConversationWithError lCnvId
+        let lConv = toLocalUnsafe localDomain conv
+        updateLocalConversationUserUnchecked
+          @'ConversationRemoveMembersTag
+          lConv
+          undefined
+          $ tUntagged . rmId <$> rUsers -- This field can be undefined as the path for ConversationRemoveMembersTag doens't use it
+
+        -- Check if the conversation if type 2 or 3, one-on-one conversations.
+        -- If it is, then we need to remove the entire conversation as users
+        -- aren't able to delete those types of conversations themselves.
+        -- Check that we are in a type 2 or a type 3 conversation
+        when (cnvmType (convMetadata conv) `elem` [One2OneConv, ConnectConv]) $
+          -- If we are, delete it.
+          updateLocalConversationUserUnchecked
+            @'ConversationDeleteTag
+            lConv
+            undefined
+            ()
+
+-- Remove local members from remote conversations
+deleteFederationDomainLocal :: (Member (Input (Local ())) r, Member (Input Env) r,
+          Member (Error InternalError) r, Member (P.Logger (Msg -> Msg)) r,
+          Member MemberStore r, Member (Embed IO) r, Member BrigAccess r,
+          Member GundeckAccess r, Member ExternalAccess r) => Domain -> Sem r ()
+deleteFederationDomainLocal dom = do
+  localUsers <- E.getLocalMembersByDomain dom
+  env <- input
+  -- As above, build the map so we can get all local users per conversation
+  let rCnvMap = foldr insertIntoMap mempty localUsers
+      localDomain = env ^. Galley.App.options . optSettings . setFederationDomain
+  -- Process each user.
+  for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
+    -- All errors, either exceptions or Either e, get thrown into IO
+    mapError @NoChanges (const (InternalErrorWithDescription "No Changes: Could not remove a local member from a remote conversation.")) $ do
+      now <- liftIO $ getCurrentTime
+      for_ lUsers $ \user -> do
+        let lUser = toLocalUnsafe localDomain user
+            convUpdate =
+              F.ConversationUpdate
+                { cuTime = now,
+                  cuOrigUserId = tUntagged lUser,
+                  cuConvId = cnv,
+                  cuAlreadyPresentUsers = [user],
+                  cuAction = SomeConversationAction (sing @'ConversationDeleteTag) ()
+                }
+        -- These functions are used directly rather than as part of a larger conversation
+        -- delete function, as we don't have an originating user, and we can't send data
+        -- to the remote backend.
+        -- We don't need to check the conversation type here, as we can't tell the
+        -- remote federation server to delete the conversation. They will have to do a
+        -- similar processing run for removing the local domain from their federation list.
+        onConversationUpdated dom convUpdate
+
+-- let rcnv = toRemoteUnsafe dom cnv
+-- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
+
+-- TODO: The DB table that this tries to update aren't available to
+-- Galley and need to be moved into brig. This will complicate the calling
+-- to delete a domain, but likely we can expose it as an internal API and
+-- eat the coverhead cost of the http call. This should also allow for the
+-- senario where galley falls over and has to redo the domain deletion so
+-- that request isn't lost.
+deleteFederationDomainOneOnOne :: (Member (Input Env) r, Member (Embed IO) r, Member (P.Logger (Msg -> Msg)) r) => Domain -> Sem r ()
+deleteFederationDomainOneOnOne dom = do
+  env <- input
+  let c = mkClientEnv (env ^. manager) (env ^. brig)
+  liftIO (deleteFederationRemoteGalley dom c)
+    >>= either
+      ( \e -> do
+          P.err $ Log.msg @String "Could not delete one-on-one messages in Brig" . Log.field "error" (show e)
+          -- Throw the error into IO to match the other functions and to prevent the
+          -- message from rabbit being ACKed.
+          liftIO $ throwIO e
+      )
+      pure
+  where
+    mkClientEnv mgr (Endpoint h p) = ClientEnv mgr (BaseUrl Http (unpack h) (fromIntegral p) "") Nothing defaultMakeClientRequest
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 1d79c9c6dd..e3260fb644 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -18,12 +18,7 @@
 module Galley.Run
   ( run,
     mkApp,
-    mkLogger,
-    -- Exported for tests
-    deleteFederationDomain,
-    publishRabbitMsg,
-    readRabbitMq,
-    ensureQueue
+    mkLogger
   )
 where
 
@@ -33,47 +28,30 @@ import Bilge.Request (requestIdName)
 import Cassandra (runClient, shutdown)
 import Cassandra.Schema (versionCheck)
 import qualified Control.Concurrent.Async as Async
-import Control.Exception (finally, throwIO)
+import Control.Exception (finally)
 import Control.Lens (view, (.~), (^.))
 import Control.Monad.Codensity
 import qualified Data.Aeson as Aeson
 import Data.Default
-import Data.Domain (Domain)
 import Data.Id
-import qualified Data.List.NonEmpty as N
-import qualified Data.Map as Map
 import Data.Metrics (Metrics)
 import Data.Metrics.AWS (gaugeTokenRemaing)
 import qualified Data.Metrics.Middleware as M
 import Data.Metrics.Servant (servantPlusWAIPrometheusMiddleware)
 import Data.Misc (portNumber)
-import Data.Qualified
-import qualified Data.Set as Set
 import Data.Singletons
 import Data.Text (unpack)
-import Data.Time (getCurrentTime)
 import qualified Galley.API as API
-import Galley.API.Action
-import Galley.API.Error
 import Galley.API.Federation
 import Galley.API.Internal
-import Galley.API.Util (getConversationWithError)
 import Galley.App
 import qualified Galley.App as App
 import Galley.Aws (awsEnv)
 import Galley.Cassandra
-import Galley.Data.Conversation.Types (convMetadata)
-import qualified Galley.Effects.MemberStore as E
-import Galley.Env
 import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
-import Galley.Types.Conversations.Members
 import Imports
-import qualified Network.AMQP as AMQP
-import Network.AMQP.Extended (RabbitMqHooks (RabbitMqHooks), openConnectionWithRetries)
-import qualified Network.AMQP.Extended as AMQP
-import qualified Network.AMQP.Types as AMQP
 import Network.HTTP.Client (defaultManagerSettings, newManager)
 import qualified Network.HTTP.Media.RenderHeader as HTTPMedia
 import qualified Network.HTTP.Types as HTTP
@@ -81,27 +59,17 @@ import Network.Wai
 import qualified Network.Wai.Middleware.Gunzip as GZip
 import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
-import Polysemy.Error
 import Servant hiding (route)
 import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http), defaultMakeClientRequest)
 import qualified System.Logger as Log
 import System.Logger.Extended (mkLogger)
 import Util.Options
-import Wire.API.Conversation (ConvType (ConnectConv, One2OneConv), cnvmType)
-import Wire.API.Conversation.Action
-import Wire.API.Conversation.Role
-import Wire.API.Error
-import Wire.API.Error.Galley
-import qualified Wire.API.Federation.API.Galley as F
 import Wire.API.FederationUpdate
 import Wire.API.Routes.API
 import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
 
--- This type is used to tie the amqp sending and receiving message types together.
-type MsgData = Domain
-
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
   l <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
@@ -124,119 +92,11 @@ run opts = lowerCodensity $ do
   forM_ (env ^. aEnv) $ \aws ->
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
 
-  -- If rabbit is defined, we can do federation work
-  traverse_ (complexFederationUpdate env clientEnv)$ opts ^. optRabbitmq
-
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
   void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
--- Complex handling. Most of the complexity comes from interweaving both rabbit queue handling
--- and the HTTP calls out to brig for the new lists of federation domains. Rabbit handles the
--- heavey lifting of ensuring single threaded processing of the domains to be deleted.
-complexFederationUpdate ::
-  Env ->
-  ClientEnv ->
-  AMQP.RabbitMqOpts ->
-  Codensity IO ()
-complexFederationUpdate env clientEnv rmq = void $ Codensity $ Async.withAsync $ do
-  -- This ioref is needed so that we can kill the async thread that
-  -- is forked by updateFedDomains'
-  threadRef <- newIORef Nothing
-  let mqh = AMQP.host rmq
-      mqp = AMQP.port rmq
-      mqv = AMQP.vHost rmq
-      mqq = "domain-deletion-queue"
-  openConnectionWithRetries (env ^. applog) mqh mqp mqv $
-    RabbitMqHooks
-      { AMQP.onConnectionClose = do
-          Log.info (env ^. applog) $ Log.msg @Text "AMQP connection closed"
-          killForkedThread threadRef,
-        AMQP.onChannelException = \e -> do
-          Log.err (env ^. applog) $ Log.msg @Text "AMQP channel exception" . Log.field "exception" (show e)
-          killForkedThread threadRef,
-        AMQP.onNewChannel = \channel -> do
-          -- NOTE: `amqp` uses ChanThreadKilledException to signal that this channel is closed
-          -- This exception should _NOT_ be caught, or if it is it needs to be rethrown. This
-          -- will kill the thread. This handler is then used to start a new one.
-          ensureQueue channel mqq
-          writeRabbitMq clientEnv (env ^. applog) (env ^. fedDomains) channel mqq threadRef
-          let performDelete = runApp env . deleteFederationDomain
-          readRabbitMq channel mqq (env ^. applog) performDelete
-
-          -- Keep this thread around until it is killed.
-          forever $ threadDelay maxBound
-      }
-  where
-    killForkedThread ref =
-      readIORef ref
-        >>= maybe
-          (pure ())
-          ( \t -> do
-              Async.cancel t
-              atomicWriteIORef ref Nothing
-          )
-
--- Ensure that the queue exists and is single active consumer.
--- Queue declaration is idempotent
-ensureQueue :: AMQP.Channel -> Text -> IO ()
-ensureQueue channel mqq = do
-  void $ AMQP.declareQueue channel $ AMQP.newQueue {AMQP.queueName = mqq, AMQP.queueHeaders = headers}
-  where
-    headers = AMQP.FieldTable $ Map.fromList [("x-single-active-consumer", AMQP.FVBool True)]
-
--- Update federation domains, write deleted domains to rabbitmq
--- Push this thread id somewhere so we can make sure it is killed with
--- this channel thread. We don't want to leak those resources.
-writeRabbitMq ::
-  ClientEnv ->
-  Log.Logger ->
-  IORef FederationDomainConfigs ->
-  AMQP.Channel ->
-  Text ->
-  IORef (Maybe (Async.Async ())) ->
-  IO ()
-writeRabbitMq clientEnv logger ioref channel mqq threadRef = do
-  threadId <- updateFedDomains' ioref clientEnv logger $ \old new -> do
-    let fromFedList = Set.fromList . remotes
-        prevDoms = fromFedList old
-        currDoms = fromFedList new
-        deletedDomains = Set.difference prevDoms currDoms
-    -- Write to the queue
-    -- NOTE: This type must be compatible with what is being read from the queue.
-    for_ deletedDomains $ \fedCfg -> do
-      -- We're using the default exchange. This will deliver the
-      -- message to the queue name used for the routing key
-      void $ publishRabbitMsg channel mqq $ domain fedCfg
-  atomicWriteIORef threadRef $ pure threadId
-
--- Split out to help with integration testing
-publishRabbitMsg :: AMQP.Channel -> Text -> MsgData -> IO (Maybe Int)
-publishRabbitMsg channel mqq dom = AMQP.publishMsg channel "" mqq $
-  AMQP.newMsg
-    { AMQP.msgBody = Aeson.encode @MsgData dom
-    , AMQP.msgDeliveryMode = pure AMQP.Persistent
-    }
-
--- Read messages from RabbitMQ, process the message, and ACK or NACK it as appropriate.
--- This is automatically killed by `amqp`, we don't need to handle it.
---
--- We can run this on every galley instance, and rabbitmq will handle the single
--- consumer constraint for us. This is done via the x-single-active-consumer header
--- that is set when the queue is created. When the active consumer disconnects for
--- whatever reason, rabbit will pick another of the subscribed clients to be the new
--- active consumer.
-readRabbitMq :: AMQP.Channel -> Text -> Log.Logger -> (Domain -> IO ()) -> IO ()
-readRabbitMq channel mqq logger go = void $ AMQP.consumeMsgs channel mqq AMQP.Ack $ \(message, envelope) ->
-  case Aeson.eitherDecode @MsgData (AMQP.msgBody message) of
-    Left e -> do
-      Log.err logger $ Log.msg @Text "Could not decode message from RabbitMQ" . Log.field "error" (show e)
-      AMQP.nackEnv envelope
-    Right dom -> do
-      go dom
-      AMQP.ackEnv envelope
-
 mkApp :: Opts -> IORef FederationDomainConfigs -> Log.Logger -> Codensity IO (Application, Env)
 mkApp opts fedDoms logger =
   do
@@ -318,130 +178,4 @@ collectAuthMetrics m env = do
     forever $ do
       mbRemaining <- readAuthExpiration env
       gaugeTokenRemaing m mbRemaining
-      threadDelay 1_000_000
-
--- Build the map, keyed by conversations to the list of members
-insertIntoMap :: (ConvId, a) -> Map ConvId (N.NonEmpty a) -> Map ConvId (N.NonEmpty a)
-insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user)) cnvId m
-
--- Bundle all of the deletes together for easy calling
--- Errors & exceptions are thrown to IO to stop the message being ACKed, eventually timing it
--- out so that it can be redelivered.
-deleteFederationDomain :: Domain -> App ()
-deleteFederationDomain d = do
-  env <- ask
-  Log.err (env ^. applog) $ Log.field "domain" $ show d
-  deleteFederationDomainRemote d
-  deleteFederationDomainLocal d
-  deleteFederationDomainOneOnOne d
-
--- Remove remote members from local conversations
-deleteFederationDomainRemote :: Domain -> App ()
-deleteFederationDomainRemote dom = do
-  env <- ask
-  remoteUsers <- liftIO $ evalGalleyToIO env $ E.getRemoteMembersByDomain dom
-  let lCnvMap = foldr insertIntoMap mempty remoteUsers
-      localDomain = env ^. options . optSettings . setFederationDomain
-  for_ (Map.toList lCnvMap) $ \(cnvId, rUsers) -> do
-    let lCnvId = toLocalUnsafe localDomain cnvId
-    -- This value contains an event that we might need to
-    -- send out to all of the local clients that are a party
-    -- to the conversation. However we also don't want to DOS
-    -- clients. Maybe suppress and send out a bulk version?
-    liftIO
-      -- All errors, either exceptions or Either e, get thrown into IO
-      $ evalGalleyToIO env
-      $ mapToRuntimeError @F.RemoveFromConversationError (InternalErrorWithDescription "Federation domain removal: Remove from conversation error")
-        . mapToRuntimeError @'ConvNotFound (InternalErrorWithDescription "Federation domain removal: Conversation not found")
-        . mapToRuntimeError @('ActionDenied 'RemoveConversationMember) (InternalErrorWithDescription "Federation domain removal: Action denied, remove conversation member")
-        . mapToRuntimeError @'InvalidOperation (InternalErrorWithDescription "Federation domain removal: Invalid operation")
-        . mapToRuntimeError @'NotATeamMember (InternalErrorWithDescription "Federation domain removal: Not a team member")
-        . mapError @NoChanges (const (InternalErrorWithDescription "Federation domain removal: No changes"))
-      -- This is allowed to send notifications to _local_ clients.
-      -- But we are suppressing those events as we don't want to
-      -- DOS our users if a large and deeply interconnected federation
-      -- member is removed. Sending out hundreds or thousands of events
-      -- to each client isn't something we want to be doing.
-      $ do
-        conv <- getConversationWithError lCnvId
-        let lConv = toLocalUnsafe localDomain conv
-        updateLocalConversationUserUnchecked
-          @'ConversationRemoveMembersTag
-          lConv
-          undefined
-          $ tUntagged . rmId <$> rUsers -- This field can be undefined as the path for ConversationRemoveMembersTag doens't use it
-
-        -- Check if the conversation if type 2 or 3, one-on-one conversations.
-        -- If it is, then we need to remove the entire conversation as users
-        -- aren't able to delete those types of conversations themselves.
-        -- Check that we are in a type 2 or a type 3 conversation
-        when (cnvmType (convMetadata conv) `elem` [One2OneConv, ConnectConv]) $
-          -- If we are, delete it.
-          updateLocalConversationUserUnchecked
-            @'ConversationDeleteTag
-            lConv
-            undefined
-            ()
-
--- Remove local members from remote conversations
-deleteFederationDomainLocal :: Domain -> App ()
-deleteFederationDomainLocal dom = do
-  env <- ask
-  localUsers <- liftIO $ evalGalleyToIO env $ E.getLocalMembersByDomain dom
-  Log.err (env ^. applog) $ Log.field "localUsers" $ show localUsers
-  -- As above, build the map so we can get all local users per conversation
-  let rCnvMap = foldr insertIntoMap mempty localUsers
-      localDomain = env ^. options . optSettings . setFederationDomain
-  -- Process each user.
-  for_ (Map.toList rCnvMap) $ \(cnv, lUsers) -> do
-    Log.err (env ^. applog) $ Log.field "(cnv, lUsers)" (show (cnv, lUsers))
-    liftIO $
-      -- All errors, either exceptions or Either e, get thrown into IO
-      
-      -- All errors, either exceptions or Either e, get thrown into IO
-      evalGalleyToIO env $
-        mapError @NoChanges (const (InternalErrorWithDescription "No Changes: Could not remove a local member from a remote conversation.")) $
-          do
-            now <- liftIO $ getCurrentTime
-            for_ lUsers $ \user -> do
-              let lUser = toLocalUnsafe localDomain user
-                  convUpdate =
-                    F.ConversationUpdate
-                      { cuTime = now,
-                        cuOrigUserId = tUntagged lUser,
-                        cuConvId = cnv,
-                        cuAlreadyPresentUsers = [user],
-                        cuAction = SomeConversationAction (sing @'ConversationDeleteTag) ()
-                      }
-              -- These functions are used directly rather than as part of a larger conversation
-              -- delete function, as we don't have an originating user, and we can't send data
-              -- to the remote backend.
-              -- We don't need to check the conversation type here, as we can't tell the
-              -- remote federation server to delete the conversation. They will have to do a
-              -- similar processing run for removing the local domain from their federation list.
-              onConversationUpdated dom convUpdate
-
--- let rcnv = toRemoteUnsafe dom cnv
--- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
-
--- TODO: The DB table that this tries to update aren't available to
--- Galley and need to be moved into brig. This will complicate the calling
--- to delete a domain, but likely we can expose it as an internal API and
--- eat the coverhead cost of the http call. This should also allow for the
--- senario where galley falls over and has to redo the domain deletion so
--- that request isn't lost.
-deleteFederationDomainOneOnOne :: Domain -> App ()
-deleteFederationDomainOneOnOne dom = do
-  env <- ask
-  let c = mkClientEnv (env ^. manager) (env ^. brig)
-  liftIO (deleteFederationRemoteGalley dom c)
-    >>= either
-      ( \e -> do
-          Log.err (env ^. applog) $ Log.msg @Text "Could not delete one-on-one messages in Brig" . Log.field "error" (show e)
-          -- Throw the error into IO to match the other functions and to prevent the
-          -- message from rabbit being ACKed.
-          liftIO $ throwIO e
-      )
-      pure
-  where
-    mkClientEnv mgr (Endpoint h p) = ClientEnv mgr (BaseUrl Http (unpack h) (fromIntegral p) "") Nothing defaultMakeClientRequest
+      threadDelay 1_000_000
\ No newline at end of file
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 0e1d9662e4..167def740a 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -47,6 +47,8 @@ import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.MultiTablePaging
 import qualified Wire.API.Routes.MultiTablePaging as Public
 import Wire.API.User.Search
+import Galley.App
+import Galley.API.Internal
 
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
@@ -118,8 +120,9 @@ deleteFederationDomains old new = do
   let prev = fromFedList old
       curr = fromFedList new
       deletedDomains = Set.difference prev curr
+  env <- ask
   -- Call into the galley code
-  for_ deletedDomains deleteFederationDomain
+  for_ deletedDomains $ liftIO . evalGalleyToIO env . deleteFederationDomain
 
 constHandlers :: MonadIO m => [RetryStatus -> Handler m Bool]
 constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
@@ -245,13 +248,6 @@ updateFedDomainRemoveLocalFromRemote env remoteDomain interval = recovering x3 c
 
   -- Remove the remote user from the local domain
   liftIO $ runApp env $ deleteFederationDomains old new
-
-  -- Get the list of remote conversations for the user.
-  -- convIds <- liftIO $ evalGalleyToIO env $
-  --   pageToConvIdPage Public.PagingRemotes
-  --   . fmap (tUntagged @'QRemote)
-  --   <$> E.listItems alice Nothing (toRange $ Proxy @1000)
-
   convIds <-
     liftIO $
       C.runClient (env ^. cstate) $
diff --git a/services/galley/test/integration/Main.hs b/services/galley/test/integration/Main.hs
index 8b865763f5..519cf856b2 100644
--- a/services/galley/test/integration/Main.hs
+++ b/services/galley/test/integration/Main.hs
@@ -54,7 +54,6 @@ import Util.Options
 import Util.Options.Common
 import Util.Test
 import qualified Util.Test.SQS as SQS
-import RabbitMQ (rabbitPubSub)
 
 newtype ServiceConfigFile = ServiceConfigFile String
   deriving (Eq, Ord, Typeable)
@@ -98,8 +97,7 @@ main = withOpenSSL $ runTests go
               (pathsConsistencyCheck . treeToPaths . compile $ Galley.API.sitemap),
           API.tests setup,
           test setup "Federation Domains" updateFedDomainsTest,
-          test setup "isConvMemberL" isConvMemberLTests,
-          test setup "RabbitMQ round-trip" rabbitPubSub
+          test setup "isConvMemberL" isConvMemberLTests
         ]
     getOpts gFile iFile = do
       m <- newManager tlsManagerSettings {managerResponseTimeout = responseTimeoutMicro 300000000}
diff --git a/services/galley/test/integration/RabbitMQ.hs b/services/galley/test/integration/RabbitMQ.hs
deleted file mode 100644
index c14beff7bf..0000000000
--- a/services/galley/test/integration/RabbitMQ.hs
+++ /dev/null
@@ -1,90 +0,0 @@
-module RabbitMQ where
-
-import qualified System.Logger as Log
-import Imports
-import TestSetup
-import Network.AMQP.Extended (RabbitMqHooks (RabbitMqHooks), openConnectionWithRetries)
-import qualified Network.AMQP.Extended as AMQP
-import Control.Lens (view)
-import Data.Domain
-import Galley.Options
-import Galley.Run (publishRabbitMsg, readRabbitMq, ensureQueue)
-import Test.Tasty.HUnit
-import Data.Text (unpack, pack)
-import System.Random
-
--- compete and timeout are from https://wiki.haskell.org/Timing_out_computations
-compete :: [IO a] -> IO a
-compete actions = do
-    mvar <- newEmptyMVar
-    tids <- mapM (\action -> forkIO $ action >>= putMVar mvar) actions
-    result <- takeMVar mvar
-    mapM_ killThread tids
-    pure result
-
-timeout :: Int -> IO a -> IO (Maybe a)
-timeout usec action = compete [fmap Just action, threadDelay usec >> pure Nothing]
-
--- Test the round trip to rabbit from galley
--- and especially that we get out the domain we put in.`
-rabbitPubSub :: TestM ()
-rabbitPubSub = view (tsGConf . optRabbitmq)
-  >>= maybe (pure ()) withRabbitOpts
-
--- Generate a simple random string
-randomString :: MonadIO m => m String
-randomString = getStdRandom $ \gen ->
-  let (count, gen') = randomR (3, 10) gen
-  -- I'm not happy with using replicate, but my hoogling didn't
-  -- show anything closer to what I wanted.
-  in foldr step ([], gen') $ replicate count ()
-  where
-    step _ (s, g) = let (c, g') = randomR ('a', 'z') g in (c:s, g')
-
-randomDomain :: MonadIO m => m Domain
-randomDomain = Domain <$> do
-  a <- randomString
-  b <- randomString
-  pure $ pack $ a <> "." <> b
-
-withRabbitOpts :: AMQP.RabbitMqOpts -> TestM ()
-withRabbitOpts rabbitOpts = do
-  log' <- liftIO $ Log.new Log.defSettings
-  -- If this IORef has a Just in it, that will contain
-  -- an error message. When we successfully get our message
-  -- from Rabbit we empty it
-  ioref <- liftIO $ newIORef $ pure "Empty: Maybe we weren't able to talk to RabbitMQ yet"
-  -- Make a random string to ensure we aren't seeing leftover
-  -- messages from other tests
-  dom <- randomDomain
-  queue <- pack . ("galley-integration-testing-" <>) <$> randomString
-  -- We don't care about the result from timeout, as
-  -- the openConnect... function will loop forever. We
-  -- just need to kill it in a timely manner
-  liftIO $ void $ timeout timeout_us $
-    openConnectionWithRetries log' host port vhost $
-      RabbitMqHooks
-        { AMQP.onConnectionClose = pure ()
-        , AMQP.onChannelException = \_e -> pure ()
-        , AMQP.onNewChannel = \chan -> do
-          -- Run the tests here
-          ensureQueue chan queue
-          void $ publishRabbitMsg chan queue dom
-          liftIO $ readRabbitMq chan queue log' $ \dom' -> do
-            -- Append the random string
-            if dom == dom'
-            then do -- Happy path
-              atomicWriteIORef ioref Nothing
-            else do -- Sad path
-              atomicWriteIORef ioref $ Just $ "expected \"" <> unpack (domainText dom) <> "\" but got \"" <> unpack (domainText dom') <> "\""
-          -- Keep this around until we kill it.
-          forever $ threadDelay 1_000_000
-        }
-  liftIO $ readIORef ioref >>= maybe
-    (pure ())
-    assertFailure
-  where
-    timeout_us = 5 * 1_000_000 -- seconds scale
-    host =  AMQP.host rabbitOpts
-    port =  AMQP.port rabbitOpts
-    vhost = AMQP.vHost rabbitOpts
\ No newline at end of file

From 1dc2112932676640b6790f9f088f89e1dd46305d Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 15 Jun 2023 17:24:29 +1000
Subject: [PATCH 146/220] FS-1179: Adding a defederation worker to background
 worker

Adding a new worker on background worker that reads from a queue and
calls Galley to delete a federation domain.
---
 .../API/Federation/BackendNotifications.hs    | 14 ++---
 .../background-worker/background-worker.cabal |  7 +++
 .../src/Wire/BackendNotificationPusher.hs     | 52 ++++++++++++++++++-
 .../src/Wire/BackgroundWorker/Env.hs          |  7 ++-
 .../src/Wire/BackgroundWorker/Options.hs      |  1 +
 .../Wire/BackendNotificationPusherSpec.hs     |  7 ++-
 6 files changed, 76 insertions(+), 12 deletions(-)

diff --git a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
index 44693f3d11..bf1c43337b 100644
--- a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
+++ b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
@@ -81,17 +81,17 @@ enqueue :: Q.Channel -> Domain -> Domain -> Q.DeliveryMode -> FedQueueClient c (
 enqueue channel originDomain targetDomain deliveryMode (FedQueueClient action) =
   runReaderT action FedQueueEnv {..}
 
-routingKey :: Domain -> Text
-routingKey d = "backend-notifications." <> domainText d
+routingKey :: Text -> Text
+routingKey t = "backend-notifications." <> t
 
 -- | If you ever change this function and modify
 -- queue parameters, know that it will start failing in the
 -- next release! So be prepared to write migrations.
-ensureQueue :: Q.Channel -> Domain -> IO ()
-ensureQueue chan domain = do
+ensureQueue :: Q.Channel -> Text -> IO ()
+ensureQueue chan queue = do
   let opts =
         Q.QueueOpts
-          { Q.queueName = routingKey domain,
+          { Q.queueName = routingKey queue,
             Q.queuePassive = False,
             Q.queueDurable = True,
             Q.queueExclusive = False,
@@ -155,8 +155,8 @@ instance KnownComponent c => RunClient (FedQueueClient c) where
         -- Empty string means default exchange
         exchange = ""
     liftIO $ do
-      ensureQueue env.channel env.targetDomain
-      void $ Q.publishMsg env.channel exchange (routingKey env.targetDomain) msg
+      ensureQueue env.channel env.targetDomain._domainText
+      void $ Q.publishMsg env.channel exchange (routingKey env.targetDomain._domainText) msg
     pure $
       Response
         { responseHttpVersion = http20,
diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal
index 52905c5ec6..af29995f59 100644
--- a/services/background-worker/background-worker.cabal
+++ b/services/background-worker/background-worker.cabal
@@ -27,13 +27,19 @@ library
   build-depends:
       aeson
     , amqp
+    , bytestring
+    , bytestring-conversion
     , exceptions
     , extended
     , HsOpenSSL
+    , http-client
+    , http-types
     , http2-manager
     , imports
+    , lens
     , monad-control
     , retry
+    , text
     , tinylog
     , transformers-base
     , types-common
@@ -161,6 +167,7 @@ test-suite background-worker-test
     , background-worker
     , federator
     , hspec
+    , http-client
     , imports
     , QuickCheck
     , tinylog
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index b03172be2b..ac28f48245 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -13,14 +13,21 @@ import qualified System.Logger.Class as Log
 import Wire.API.Federation.BackendNotifications
 import Wire.API.Federation.Client
 import Wire.BackgroundWorker.Env
+import Network.HTTP.Client
+import Network.HTTP.Types
+import Util.Options
+import Control.Lens ((^.), to)
+import Data.ByteString.Conversion
+import qualified Data.ByteString.Lazy as L
+import Data.Text.Encoding
 
 startPushingNotifications ::
   Q.Channel ->
   Domain ->
   AppT IO Q.ConsumerTag
 startPushingNotifications chan domain = do
-  lift $ ensureQueue chan domain
-  QL.consumeMsgs chan (routingKey domain) Q.Ack (pushNotification domain)
+  lift $ ensureQueue chan domain._domainText
+  QL.consumeMsgs chan (routingKey domain._domainText) Q.Ack (pushNotification domain)
 
 -- | This class exists to help with testing, making the envelope in unit test is
 -- too difficult. So we use fake envelopes in the unit tests.
@@ -78,6 +85,46 @@ pushNotification targetDomain (msg, envelope) = do
         liftIO $ either throwM pure =<< sendNotification fcEnv notif.targetComponent notif.path notif.body
         lift $ ack envelope
 
+deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
+deleteFederationDomain chan = do
+  env <- ask
+  let manager = httpManager env
+      req :: Domain -> Request
+      req dom = defaultRequest
+        { method = methodDelete
+        , secure = False
+        , host = galley env ^. epHost . to encodeUtf8
+        , port = galley env ^. epPort . to fromIntegral
+        , path = "/i/federation/" <> toByteString' dom
+        , requestHeaders = ("Accept", "application/json") : requestHeaders defaultRequest
+        }
+  lift $ ensureQueue chan queue
+  QL.consumeMsgs chan (routingKey queue) Q.Ack $ \(msg, envelope) -> do
+    either
+      (\e -> do
+        logErr e
+        liftIO $ Q.nackEnv envelope
+      )
+      (\d -> do
+        resp <- liftIO $ httpLbs (req d) manager
+        go envelope resp
+      )
+      $ A.eitherDecode (Q.msgBody msg)
+    
+  where
+    go :: Q.Envelope -> Response L.ByteString -> AppT IO ()
+    go envelope resp = do
+      let code = statusCode $ responseStatus resp
+      if code >= 200 && code <= 299
+      then do
+        logErr $ show resp
+        liftIO $ Q.ackEnv envelope
+      else liftIO $ Q.nackEnv envelope
+    logErr err = Log.err $
+      Log.msg (Log.val "Failed delete federation domain")
+        . Log.field "error" err
+    queue = "background-worker-delete-federation"
+
 -- FUTUREWORK: Recosider using 1 channel for many consumers. It shouldn't matter
 -- for a handful of remote domains.
 startWorker :: [Domain] -> Q.Channel -> AppT IO ()
@@ -86,4 +133,5 @@ startWorker remoteDomains chan = do
   -- delivered in order.
   lift $ Q.qos chan 0 1 False
   mapM_ (startPushingNotifications chan) remoteDomains
+  void $ deleteFederationDomain chan
   forever $ threadDelay maxBound
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
index 0213e3549d..57d89524df 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
@@ -15,18 +15,23 @@ import System.Logger.Class
 import qualified System.Logger.Extended as Log
 import Util.Options
 import Wire.BackgroundWorker.Options
+import Network.HTTP.Client
 
 data Env = Env
   { http2Manager :: Http2Manager,
+    httpManager :: Manager,
     logger :: Logger,
-    federatorInternal :: Endpoint
+    federatorInternal :: Endpoint,
+    galley :: Endpoint
   }
 
 mkEnv :: Opts -> IO Env
 mkEnv opts = do
   http2Manager <- initHttp2Manager
   logger <- Log.mkLogger opts.logLevel Nothing opts.logFormat
+  httpManager <- newManager defaultManagerSettings
   let federatorInternal = opts.federatorInternal
+      galley = opts.galley
   pure Env {..}
 
 initHttp2Manager :: IO Http2Manager
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
index 8cd4cff909..db36932185 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
@@ -11,6 +11,7 @@ data Opts = Opts
   { logLevel :: !Level,
     logFormat :: !(Maybe (Last LogFormat)),
     federatorInternal :: !Endpoint,
+    galley :: !Endpoint,
     rabbitmq :: !RabbitMqOpts,
     remoteDomains :: [Domain]
   }
diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
index ae6baeee53..edcea3605a 100644
--- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
+++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
@@ -19,12 +19,15 @@ import Wire.API.Federation.BackendNotifications
 import Wire.API.RawJson
 import Wire.BackendNotificationPusher
 import Wire.BackgroundWorker.Env
+import Network.HTTP.Client
 
 runTestAppT :: AppT IO a -> Int -> IO a
-runTestAppT app port = do
+runTestAppT app federatorPort = do
   http2Manager <- initHttp2Manager
   logger <- Logger.new Logger.defSettings
-  let federatorInternal = Endpoint "localhost" (fromIntegral port)
+  httpManager <- newManager defaultManagerSettings
+  let federatorInternal = Endpoint "localhost" (fromIntegral federatorPort)
+      galley = Endpoint "localhost" 8080 -- TODO: Find the correct port
       env = Env {..}
   runAppT env app
 

From a9519ebe635e37ec7febc057306001dcd97347bf Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 15 Jun 2023 17:33:40 +1000
Subject: [PATCH 147/220] FS-1179: Some timeout config for the background
 worker.

---
 .../background-worker/background-worker.integration.yaml   | 4 ++++
 .../src/Wire/BackendNotificationPusher.hs                  | 1 +
 .../background-worker/src/Wire/BackgroundWorker/Env.hs     | 7 ++++++-
 .../background-worker/src/Wire/BackgroundWorker/Options.hs | 3 ++-
 .../test/Test/Wire/BackendNotificationPusherSpec.hs        | 3 ++-
 5 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/services/background-worker/background-worker.integration.yaml b/services/background-worker/background-worker.integration.yaml
index c2dd54bfb6..c5d532e886 100644
--- a/services/background-worker/background-worker.integration.yaml
+++ b/services/background-worker/background-worker.integration.yaml
@@ -4,6 +4,10 @@ federatorInternal:
   host: 127.0.0.1
   port: 8097
 
+galley:
+  host: 127.0.0.1
+  port: 8085
+
 rabbitmq:
   host: 127.0.0.1
   port: 5672
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index ac28f48245..ab03b67267 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -97,6 +97,7 @@ deleteFederationDomain chan = do
         , port = galley env ^. epPort . to fromIntegral
         , path = "/i/federation/" <> toByteString' dom
         , requestHeaders = ("Accept", "application/json") : requestHeaders defaultRequest
+        , responseTimeout = defederationTimeout env
         }
   lift $ ensureQueue chan queue
   QL.consumeMsgs chan (routingKey queue) Q.Ack $ \(msg, envelope) -> do
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
index 57d89524df..47dc657fb6 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
@@ -22,7 +22,8 @@ data Env = Env
     httpManager :: Manager,
     logger :: Logger,
     federatorInternal :: Endpoint,
-    galley :: Endpoint
+    galley :: Endpoint,
+    defederationTimeout :: ResponseTimeout
   }
 
 mkEnv :: Opts -> IO Env
@@ -32,6 +33,10 @@ mkEnv opts = do
   httpManager <- newManager defaultManagerSettings
   let federatorInternal = opts.federatorInternal
       galley = opts.galley
+      defederationTimeout = maybe
+        responseTimeoutNone
+        (\t -> responseTimeoutMicro $ 1000000 * t) -- seconds to microseconds
+        opts.defederationTimeout
   pure Env {..}
 
 initHttp2Manager :: IO Http2Manager
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
index db36932185..e3331dfed5 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
@@ -13,7 +13,8 @@ data Opts = Opts
     federatorInternal :: !Endpoint,
     galley :: !Endpoint,
     rabbitmq :: !RabbitMqOpts,
-    remoteDomains :: [Domain]
+    remoteDomains :: [Domain],
+    defederationTimeout :: Maybe Int -- Seconds, Nothing for no timeout
   }
   deriving (Show, Generic)
 
diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
index edcea3605a..9af9832b87 100644
--- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
+++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
@@ -27,7 +27,8 @@ runTestAppT app federatorPort = do
   logger <- Logger.new Logger.defSettings
   httpManager <- newManager defaultManagerSettings
   let federatorInternal = Endpoint "localhost" (fromIntegral federatorPort)
-      galley = Endpoint "localhost" 8080 -- TODO: Find the correct port
+      galley = Endpoint "localhost" 8085
+      defederationTimeout = responseTimeoutNone
       env = Env {..}
   runAppT env app
 

From 74caf8210fe3d95e82d55f843c642cdc54761169 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 15 Jun 2023 19:06:17 +1000
Subject: [PATCH 148/220] Keeping brig and backend worker in sync

---
 .../API/Federation/BackendNotifications.hs     |  7 +++++++
 .../src/Wire/BackendNotificationPusher.hs      |  2 +-
 services/brig/src/Brig/API/Internal.hs         | 18 ++++++++++++++++++
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
index bf1c43337b..ec43ff3db0 100644
--- a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
+++ b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
@@ -84,6 +84,13 @@ enqueue channel originDomain targetDomain deliveryMode (FedQueueClient action) =
 routingKey :: Text -> Text
 routingKey t = "backend-notifications." <> t
 
+-- Shared values for both brig and background worker so they are
+-- kept in sync about what types they are expecting and where
+-- they are stored in Rabbit.
+type DefederationDomain = Domain
+defederationQueue :: Text
+defederationQueue = "delete-federation"
+
 -- | If you ever change this function and modify
 -- queue parameters, know that it will start failing in the
 -- next release! So be prepared to write migrations.
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index ab03b67267..e1e5ffe786 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -110,7 +110,7 @@ deleteFederationDomain chan = do
         resp <- liftIO $ httpLbs (req d) manager
         go envelope resp
       )
-      $ A.eitherDecode (Q.msgBody msg)
+      $ A.eitherDecode @DefederationDomain (Q.msgBody msg)
     
   where
     go :: Q.Envelope -> Response L.ByteString -> AppT IO ()
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 2f30eba183..402bb6a4d4 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -1,3 +1,5 @@
+{-# LANGUAGE TypeApplications #-}
+
 -- This file is part of the Wire Server implementation.
 --
 -- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
@@ -105,6 +107,8 @@ import Wire.API.User.Activation
 import Wire.API.User.Client
 import Wire.API.User.Password
 import Wire.API.User.RichInfo
+import qualified Network.AMQP as Q
+import Wire.API.Federation.BackendNotifications
 
 ---------------------------------------------------------------------------
 -- Sitemap (servant)
@@ -309,6 +313,20 @@ deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
   lift . wrapClient . Data.deleteFederationRemote $ dom
   assertNoDomainsFromConfigFiles dom
+  env <- ask
+  for_ (env ^. rabbitmqChannel) $ \chan -> liftIO . withMVar chan $ \chan' -> do
+    -- ensureQueue uses routingKey internally
+    ensureQueue chan' defederationQueue
+    Q.publishMsg chan' "" queue $ Q.newMsg
+      -- Check that this message type is compatible with what
+      -- background worker is expecting
+      { Q.msgBody = encode @DefederationDomain dom
+      , Q.msgDeliveryMode = pure Q.Persistent
+      , Q.msgContentType = pure "application/json"
+      }
+  where
+    -- Ensure that this is kept in sync with background worker
+    queue = routingKey defederationQueue
 
 -- | Remove one-on-one conversations for the given remote domain. This is called from Galley as
 -- part of the defederation process, and should not be called duriung the initial domain removal

From 977fd77ee96240347b3e6b3162d43de81d86f034 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 20 Jun 2023 17:56:19 +1000
Subject: [PATCH 149/220] WIP: New tests and various changes to background
 worker

---
 .../background-worker/background-worker.cabal |  95 +++++++-
 .../src/Wire/BackendNotificationPusher.hs     |  69 +-----
 .../src/Wire/BackgroundWorker.hs              |  30 ++-
 .../src/Wire/BackgroundWorker/Util.hs         |  14 ++
 .../src/Wire/Defederation.hs                  |  69 ++++++
 services/background-worker/test/Main.hs       |  43 +++-
 .../Wire/BackendNotificationPusherSpec.hs     | 215 ++++++++----------
 .../test/Test/Wire/Defederation.hs            |  47 ++++
 .../background-worker/test/Test/Wire/Util.hs  | 105 +++++++++
 9 files changed, 497 insertions(+), 190 deletions(-)
 create mode 100644 services/background-worker/src/Wire/BackgroundWorker/Util.hs
 create mode 100644 services/background-worker/src/Wire/Defederation.hs
 create mode 100644 services/background-worker/test/Test/Wire/Defederation.hs
 create mode 100644 services/background-worker/test/Test/Wire/Util.hs

diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal
index af29995f59..097179b112 100644
--- a/services/background-worker/background-worker.cabal
+++ b/services/background-worker/background-worker.cabal
@@ -16,6 +16,8 @@ library
     Wire.BackgroundWorker
     Wire.BackgroundWorker.Env
     Wire.BackgroundWorker.Options
+    Wire.BackgroundWorker.Util
+    Wire.Defederation
 
   hs-source-dirs:     src
   default-language:   Haskell2010
@@ -27,6 +29,7 @@ library
   build-depends:
       aeson
     , amqp
+    , async
     , bytestring
     , bytestring-conversion
     , exceptions
@@ -145,6 +148,86 @@ executable background-worker
     UndecidableInstances
     ViewPatterns
 
+executable background-worker-integration
+  default-language:   Haskell2010
+  type:               exitcode-stdio-1.0
+  hs-source-dirs:     test
+  main-is:            Main.hs
+  ghc-options:
+    -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates
+    -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path
+    -funbox-strict-fields -threaded -with-rtsopts=-N
+    -Wredundant-constraints -Wunused-packages
+
+  -- cabal-fmt: expand test
+  other-modules:
+    Main
+    Test.Wire.BackendNotificationPusherSpec
+    Test.Wire.Defederation
+    Test.Wire.Util
+
+  build-depends:
+      aeson
+    , amqp
+    , background-worker
+    , base
+    , exceptions
+    , federator
+    , HsOpenSSL
+    , hspec
+    , imports
+    , optparse-applicative
+    , QuickCheck
+    , tagged
+    , tasty
+    , tasty-hunit
+    , types-common
+    , wire-api
+    , wire-api-federation
+    , yaml
+
+  default-extensions:
+    NoImplicitPrelude
+    AllowAmbiguousTypes
+    BangPatterns
+    ConstraintKinds
+    DataKinds
+    DefaultSignatures
+    DeriveFunctor
+    DeriveGeneric
+    DeriveLift
+    DeriveTraversable
+    DerivingStrategies
+    DerivingVia
+    DuplicateRecordFields
+    EmptyCase
+    FlexibleContexts
+    FlexibleInstances
+    FunctionalDependencies
+    GADTs
+    InstanceSigs
+    KindSignatures
+    LambdaCase
+    MultiParamTypeClasses
+    MultiWayIf
+    NamedFieldPuns
+    OverloadedRecordDot
+    OverloadedStrings
+    PackageImports
+    PatternSynonyms
+    PolyKinds
+    QuasiQuotes
+    RankNTypes
+    ScopedTypeVariables
+    StandaloneDeriving
+    TupleSections
+    TypeApplications
+    TypeFamilies
+    TypeFamilyDependencies
+    TypeOperators
+    UndecidableInstances
+    ViewPatterns
+
 test-suite background-worker-test
   default-language:   Haskell2010
   type:               exitcode-stdio-1.0
@@ -160,20 +243,28 @@ test-suite background-worker-test
   other-modules:
     Main
     Test.Wire.BackendNotificationPusherSpec
+    Test.Wire.Defederation
+    Test.Wire.Util
 
   build-depends:
       aeson
     , amqp
     , background-worker
+    , base
+    , exceptions
     , federator
+    , HsOpenSSL
     , hspec
-    , http-client
     , imports
+    , optparse-applicative
     , QuickCheck
-    , tinylog
+    , tagged
+    , tasty
+    , tasty-hunit
     , types-common
     , wire-api
     , wire-api-federation
+    , yaml
 
   default-extensions:
     NoImplicitPrelude
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index e1e5ffe786..a740954f19 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -13,13 +13,8 @@ import qualified System.Logger.Class as Log
 import Wire.API.Federation.BackendNotifications
 import Wire.API.Federation.Client
 import Wire.BackgroundWorker.Env
-import Network.HTTP.Client
-import Network.HTTP.Types
-import Util.Options
-import Control.Lens ((^.), to)
-import Data.ByteString.Conversion
-import qualified Data.ByteString.Lazy as L
-import Data.Text.Encoding
+import Wire.BackgroundWorker.Util
+import Control.Concurrent.Async
 
 startPushingNotifications ::
   Q.Channel ->
@@ -29,16 +24,6 @@ startPushingNotifications chan domain = do
   lift $ ensureQueue chan domain._domainText
   QL.consumeMsgs chan (routingKey domain._domainText) Q.Ack (pushNotification domain)
 
--- | This class exists to help with testing, making the envelope in unit test is
--- too difficult. So we use fake envelopes in the unit tests.
-class RabbitMQEnvelope e where
-  ack :: e -> IO ()
-  reject :: e -> Bool -> IO ()
-
-instance RabbitMQEnvelope Q.Envelope where
-  ack = Q.ackEnv
-  reject = Q.rejectEnv
-
 pushNotification :: RabbitMQEnvelope e => Domain -> (Q.Message, e) -> AppT IO ()
 pushNotification targetDomain (msg, envelope) = do
   -- Jittered exponential backoff with 10ms as starting delay and
@@ -85,54 +70,14 @@ pushNotification targetDomain (msg, envelope) = do
         liftIO $ either throwM pure =<< sendNotification fcEnv notif.targetComponent notif.path notif.body
         lift $ ack envelope
 
-deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
-deleteFederationDomain chan = do
-  env <- ask
-  let manager = httpManager env
-      req :: Domain -> Request
-      req dom = defaultRequest
-        { method = methodDelete
-        , secure = False
-        , host = galley env ^. epHost . to encodeUtf8
-        , port = galley env ^. epPort . to fromIntegral
-        , path = "/i/federation/" <> toByteString' dom
-        , requestHeaders = ("Accept", "application/json") : requestHeaders defaultRequest
-        , responseTimeout = defederationTimeout env
-        }
-  lift $ ensureQueue chan queue
-  QL.consumeMsgs chan (routingKey queue) Q.Ack $ \(msg, envelope) -> do
-    either
-      (\e -> do
-        logErr e
-        liftIO $ Q.nackEnv envelope
-      )
-      (\d -> do
-        resp <- liftIO $ httpLbs (req d) manager
-        go envelope resp
-      )
-      $ A.eitherDecode @DefederationDomain (Q.msgBody msg)
-    
-  where
-    go :: Q.Envelope -> Response L.ByteString -> AppT IO ()
-    go envelope resp = do
-      let code = statusCode $ responseStatus resp
-      if code >= 200 && code <= 299
-      then do
-        logErr $ show resp
-        liftIO $ Q.ackEnv envelope
-      else liftIO $ Q.nackEnv envelope
-    logErr err = Log.err $
-      Log.msg (Log.val "Failed delete federation domain")
-        . Log.field "error" err
-    queue = "background-worker-delete-federation"
-
 -- FUTUREWORK: Recosider using 1 channel for many consumers. It shouldn't matter
 -- for a handful of remote domains.
-startWorker :: [Domain] -> Q.Channel -> AppT IO ()
+startWorker :: [Domain] -> Q.Channel -> AppT IO (Async ())
 startWorker remoteDomains chan = do
   -- This ensures that we receive notifications 1 by 1 which ensures they are
   -- delivered in order.
   lift $ Q.qos chan 0 1 False
-  mapM_ (startPushingNotifications chan) remoteDomains
-  void $ deleteFederationDomain chan
-  forever $ threadDelay maxBound
+  env <- ask
+  liftIO $ async $ do
+    mapM_ (runAppT env . startPushingNotifications chan) remoteDomains
+    forever $ threadDelay maxBound
\ No newline at end of file
diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index fea244b963..3a23b61f05 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -7,18 +7,38 @@ import Network.AMQP.Extended
 import qualified Wire.BackendNotificationPusher as BackendNotificationPusher
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Options
+import Wire.Defederation
+import Control.Concurrent.Async
 
 -- FUTUREWORK: Start an http service with status and metrics endpoints
 run :: Opts -> IO ()
 run opts = do
   env <- mkEnv opts
-  -- FUTUREWORK: Make some way to tracking all the workers, currently there is
-  -- only one so we can just block on it.
+  threadsRef <- newIORef []
   openConnectionWithRetries env.logger opts.rabbitmq.host opts.rabbitmq.port opts.rabbitmq.vHost $
     RabbitMqHooks
-      { onNewChannel = runAppT env . BackendNotificationPusher.startWorker opts.remoteDomains,
+      { onNewChannel = \chan -> runAppT env $ do
+          -- Channels are threadsafe: https://hackage.haskell.org/package/amqp-0.22.1/docs/Network-AMQP.html
+          -- So we can async them for concurrency.
+          pushThread <- BackendNotificationPusher.startWorker opts.remoteDomains chan
+          deleteThread <- deleteWorker chan
+          let threads = [pushThread, deleteThread]
+          -- Write out the handles for the threads
+          atomicWriteIORef threadsRef threads
+          liftIO $ traverse_ wait threads
+          -- clear the threadRef if the threads finish
+          atomicWriteIORef threadsRef []
         -- FUTUREWORK: Use these for metrics
-        onChannelException = const $ pure (),
-        onConnectionClose = pure ()
+        --
+        -- When the channel dies for whatever reason, kill all of the async
+        -- threads and clean up the threadsRef state
+        , onChannelException = const $ do
+            threads <- readIORef threadsRef
+            traverse_ cancel threads
+            atomicWriteIORef threadsRef []
+        , onConnectionClose = do
+            threads <- readIORef threadsRef
+            traverse_ cancel threads
+            atomicWriteIORef threadsRef []
       }
   forever $ threadDelay maxBound
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Util.hs b/services/background-worker/src/Wire/BackgroundWorker/Util.hs
new file mode 100644
index 0000000000..6a23bcf11f
--- /dev/null
+++ b/services/background-worker/src/Wire/BackgroundWorker/Util.hs
@@ -0,0 +1,14 @@
+module Wire.BackgroundWorker.Util where
+
+import Imports
+import qualified Network.AMQP as Q
+
+-- | This class exists to help with testing, making the envelope in unit test is
+-- too difficult. So we use fake envelopes in the unit tests.
+class RabbitMQEnvelope e where
+  ack :: e -> IO ()
+  reject :: e -> Bool -> IO ()
+
+instance RabbitMQEnvelope Q.Envelope where
+  ack = Q.ackEnv
+  reject = Q.rejectEnv
\ No newline at end of file
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
new file mode 100644
index 0000000000..e6b008ec0c
--- /dev/null
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -0,0 +1,69 @@
+module Wire.Defederation where
+
+import qualified Data.Aeson as A
+import Data.Domain
+import Imports
+import qualified Network.AMQP as Q
+import qualified Network.AMQP.Lifted as QL
+import qualified System.Logger.Class as Log
+import Wire.API.Federation.BackendNotifications
+import Wire.BackgroundWorker.Env
+import Wire.BackgroundWorker.Util
+import Control.Concurrent.Async
+import Network.HTTP.Client
+import Network.HTTP.Types
+import Util.Options
+import Control.Lens ((^.), to)
+import Data.ByteString.Conversion
+import qualified Data.ByteString.Lazy as L
+import Data.Text.Encoding
+
+deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
+deleteFederationDomain chan = do
+  lift $ ensureQueue chan defederationQueue
+  QL.consumeMsgs chan (routingKey defederationQueue) Q.Ack deleteFederationDomainInner
+
+deleteFederationDomainInner :: RabbitMQEnvelope e => (Q.Message, e) -> AppT IO ()
+deleteFederationDomainInner (msg, envelope) = do
+  env <- ask
+  let manager = httpManager env
+      req :: Domain -> Request
+      req dom = defaultRequest
+        { method = methodDelete
+        , secure = False
+        , host = galley env ^. epHost . to encodeUtf8
+        , port = galley env ^. epPort . to fromIntegral
+        , path = "/i/federation/" <> toByteString' dom
+        , requestHeaders = ("Accept", "application/json") : requestHeaders defaultRequest
+        , responseTimeout = defederationTimeout env
+        }
+  either
+    (\e -> do
+      logErr e
+      liftIO $ reject envelope True -- ensure that the message is requeued
+    )
+    (\d -> do
+      resp <- liftIO (httpLbs (req d) manager)
+      go resp
+    )
+    $ A.eitherDecode @DefederationDomain (Q.msgBody msg)
+  where
+    go :: Response L.ByteString -> AppT IO ()
+    go resp = do
+      let code = statusCode $ responseStatus resp
+      if code >= 200 && code <= 299
+      then do
+        logErr $ show resp
+        liftIO $ ack envelope
+      else liftIO $ reject envelope True -- ensure that the message is requeued
+    logErr err = Log.err $
+      Log.msg (Log.val "Failed delete federation domain")
+        . Log.field "error" err
+
+deleteWorker :: Q.Channel -> AppT IO (Async ())
+deleteWorker chan = do
+  lift $ Q.qos chan 0 1 False
+  env <- ask
+  liftIO $ async $ do
+    void $ runAppT env $ deleteFederationDomain chan
+    forever $ threadDelay maxBound
\ No newline at end of file
diff --git a/services/background-worker/test/Main.hs b/services/background-worker/test/Main.hs
index a824f8c30c..5fe9e51284 100644
--- a/services/background-worker/test/Main.hs
+++ b/services/background-worker/test/Main.hs
@@ -1 +1,42 @@
-{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
+module Main
+  ( main,
+  )
+where
+
+import Imports
+import Test.Tasty.Options
+import Test.Tasty
+import Util.Test
+import OpenSSL (withOpenSSL)
+import Data.Yaml (decodeFileEither)
+import Data.Proxy
+import Test.Wire.Util
+import Test.Wire.BackendNotificationPusherSpec
+import Test.Wire.Defederation
+
+runTests :: (String -> String -> TestTree) -> IO ()
+runTests run = defaultMainWithIngredients ings $
+  askOption $
+    \(ServiceConfigFile c) ->
+      askOption $ \(IntegrationConfigFile i) -> run c i
+  where
+    ings =
+      includingOptions
+        [ Option (Proxy :: Proxy ServiceConfigFile),
+          Option (Proxy :: Proxy IntegrationConfigFile)
+        ]
+        : defaultIngredients
+
+main :: IO ()
+main = withOpenSSL $ runTests go
+  where
+    go o i = withResource (getOpts o i) releaseOpts $ \setup -> do
+      testGroup "background-worker"
+        [ spec setup
+        , deleteFederationDomainSpec setup
+        ]
+    getOpts oFile iFile = do
+      opts <- handleParseError =<< decodeFileEither oFile
+      iConf <- handleParseError =<< decodeFileEither iFile
+      pure $ TestSetup opts iConf
+    releaseOpts _ = pure ()
\ No newline at end of file
diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
index 9af9832b87..2a15ed5d24 100644
--- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
+++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
@@ -2,143 +2,118 @@
 
 module Test.Wire.BackendNotificationPusherSpec where
 
+import Test.Wire.Util
 import qualified Data.Aeson as Aeson
 import Data.Domain
 import Data.Range
 import Federator.MockServer
 import Imports
 import qualified Network.AMQP as Q
-import qualified System.Logger as Logger
 import Test.Hspec
 import Test.QuickCheck
-import Util.Options
 import Wire.API.Federation.API
 import Wire.API.Federation.API.Brig
 import Wire.API.Federation.API.Common
 import Wire.API.Federation.BackendNotifications
 import Wire.API.RawJson
+import Test.Tasty
+import Test.Tasty.HUnit
 import Wire.BackendNotificationPusher
-import Wire.BackgroundWorker.Env
-import Network.HTTP.Client
 
-runTestAppT :: AppT IO a -> Int -> IO a
-runTestAppT app federatorPort = do
-  http2Manager <- initHttp2Manager
-  logger <- Logger.new Logger.defSettings
-  httpManager <- newManager defaultManagerSettings
-  let federatorInternal = Endpoint "localhost" (fromIntegral federatorPort)
-      galley = Endpoint "localhost" 8085
-      defederationTimeout = responseTimeoutNone
-      env = Env {..}
-  runAppT env app
+spec :: IO TestSetup -> TestTree
+spec setup = testGroup "Wire.BackendNotificationPusher" 
+  [ testCase "should push notifications" $ do
+      s <- setup
+      let returnSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
+      let origDomain = Domain "origin.example.com"
+          targetDomain = Domain "target.example.com"
+      -- Just using 'arbitrary' could generate a very big list, making tests very
+      -- slow. Make me wonder if notification pusher should even try to parse the
+      -- actual content, seems like wasted compute power.
+      notifContent <- generate $ UserDeletedConnectionsNotification <$> arbitrary <*> (unsafeRange . (: []) <$> arbitrary)
+      let notif =
+            BackendNotification
+              { targetComponent = Brig,
+                ownDomain = origDomain,
+                path = "/on-user-deleted-connections",
+                body = RawJson $ Aeson.encode notifContent
+              }
+      envelope <- newFakeEnvelope
+      let msg =
+            Q.newMsg
+              { Q.msgBody = Aeson.encode notif,
+                Q.msgContentType = Just "application/json"
+              }
 
-spec :: Spec
-spec = describe "Wire.BackendNotificationPusher" $ do
-  it "should push notifications" $ do
-    let returnSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
-    let origDomain = Domain "origin.example.com"
-        targetDomain = Domain "target.example.com"
-    -- Just using 'arbitrary' could generate a very big list, making tests very
-    -- slow. Make me wonder if notification pusher should even try to parse the
-    -- actual content, seems like wasted compute power.
-    notifContent <- generate $ UserDeletedConnectionsNotification <$> arbitrary <*> (unsafeRange . (: []) <$> arbitrary)
-    let notif =
-          BackendNotification
-            { targetComponent = Brig,
-              ownDomain = origDomain,
-              path = "/on-user-deleted-connections",
-              body = RawJson $ Aeson.encode notifContent
-            }
-    envelope <- newFakeEnvelope
-    let msg =
-          Q.newMsg
-            { Q.msgBody = Aeson.encode notif,
-              Q.msgContentType = Just "application/json"
-            }
+      (_, fedReqs) <-
+        withTempMockFederator [] returnSuccess . runTestAppT s $ do
+          pushNotification targetDomain (msg, envelope)
 
-    (_, fedReqs) <-
-      withTempMockFederator [] returnSuccess . runTestAppT $ do
-        pushNotification targetDomain (msg, envelope)
+      readIORef envelope.acks `shouldReturn` 1
+      readIORef envelope.rejections `shouldReturn` []
+      fedReqs
+        `shouldBe` [ FederatedRequest
+                       { frTargetDomain = targetDomain,
+                         frOriginDomain = origDomain,
+                         frComponent = Brig,
+                         frRPC = "on-user-deleted-connections",
+                         frBody = Aeson.encode notifContent
+                       }
+                   ]
+  , testCase "should reject invalid notifications" $ do
+      s <- setup
+      let returnSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
+      envelope <- newFakeEnvelope
+      let msg =
+            Q.newMsg
+              { Q.msgBody = "unparseable notification",
+                Q.msgContentType = Just "application/json"
+              }
+      (_, fedReqs) <-
+        withTempMockFederator [] returnSuccess . runTestAppT s $
+          pushNotification (Domain "target.example.com") (msg, envelope)
 
-    readIORef envelope.acks `shouldReturn` 1
-    readIORef envelope.rejections `shouldReturn` []
-    fedReqs
-      `shouldBe` [ FederatedRequest
-                     { frTargetDomain = targetDomain,
-                       frOriginDomain = origDomain,
-                       frComponent = Brig,
-                       frRPC = "on-user-deleted-connections",
-                       frBody = Aeson.encode notifContent
-                     }
-                 ]
+      readIORef envelope.acks `shouldReturn` 0
+      readIORef envelope.rejections `shouldReturn` [False]
+      fedReqs `shouldBe` []
+  , testCase "should retry failed deliveries" $ do
+      s <- setup
+      isFirstReqRef <- newIORef True
+      let returnSuccessSecondTime _ =
+            atomicModifyIORef isFirstReqRef $ \isFirstReq ->
+              if isFirstReq
+                then (False, ("text/html", "<marquee>down for maintenance</marquee>"))
+                else (False, ("application/json", Aeson.encode EmptyResponse))
+          origDomain = Domain "origin.example.com"
+          targetDomain = Domain "target.example.com"
+      notifContent <- generate $ UserDeletedConnectionsNotification <$> arbitrary <*> (unsafeRange . (: []) <$> arbitrary)
+      let notif =
+            BackendNotification
+              { targetComponent = Brig,
+                ownDomain = origDomain,
+                path = "/on-user-deleted-connections",
+                body = RawJson $ Aeson.encode notifContent
+              }
+      envelope <- newFakeEnvelope
+      let msg =
+            Q.newMsg
+              { Q.msgBody = Aeson.encode notif,
+                Q.msgContentType = Just "application/json"
+              }
 
-  it "should reject invalid notifications" $ do
-    let returnSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
-    envelope <- newFakeEnvelope
-    let msg =
-          Q.newMsg
-            { Q.msgBody = "unparseable notification",
-              Q.msgContentType = Just "application/json"
-            }
-    (_, fedReqs) <-
-      withTempMockFederator [] returnSuccess . runTestAppT $
-        pushNotification (Domain "target.example.com") (msg, envelope)
+      (_, fedReqs) <-
+        withTempMockFederator [] returnSuccessSecondTime . runTestAppT s $ do
+          pushNotification targetDomain (msg, envelope)
 
-    readIORef envelope.acks `shouldReturn` 0
-    readIORef envelope.rejections `shouldReturn` [False]
-    fedReqs `shouldBe` []
-
-  it "should retry failed deliveries" $ do
-    isFirstReqRef <- newIORef True
-    let returnSuccessSecondTime _ =
-          atomicModifyIORef isFirstReqRef $ \isFirstReq ->
-            if isFirstReq
-              then (False, ("text/html", "<marquee>down for maintenance</marquee>"))
-              else (False, ("application/json", Aeson.encode EmptyResponse))
-        origDomain = Domain "origin.example.com"
-        targetDomain = Domain "target.example.com"
-    notifContent <- generate $ UserDeletedConnectionsNotification <$> arbitrary <*> (unsafeRange . (: []) <$> arbitrary)
-    let notif =
-          BackendNotification
-            { targetComponent = Brig,
-              ownDomain = origDomain,
-              path = "/on-user-deleted-connections",
-              body = RawJson $ Aeson.encode notifContent
-            }
-    envelope <- newFakeEnvelope
-    let msg =
-          Q.newMsg
-            { Q.msgBody = Aeson.encode notif,
-              Q.msgContentType = Just "application/json"
-            }
-
-    (_, fedReqs) <-
-      withTempMockFederator [] returnSuccessSecondTime . runTestAppT $ do
-        pushNotification targetDomain (msg, envelope)
-
-    readIORef envelope.acks `shouldReturn` 1
-    readIORef envelope.rejections `shouldReturn` []
-    let expectedReq =
-          FederatedRequest
-            { frTargetDomain = targetDomain,
-              frOriginDomain = origDomain,
-              frComponent = Brig,
-              frRPC = "on-user-deleted-connections",
-              frBody = Aeson.encode notifContent
-            }
-    fedReqs `shouldBe` [expectedReq, expectedReq]
-
-instance RabbitMQEnvelope FakeEnvelope where
-  ack e = atomicModifyIORef' e.acks $ \a -> (a + 1, ())
-  reject e requeueFlag = atomicModifyIORef' e.rejections $ \r -> (r <> [requeueFlag], ())
-
-data FakeEnvelope = FakeEnvelope
-  { rejections :: IORef [Bool],
-    acks :: IORef Int
-  }
-
-newFakeEnvelope :: IO FakeEnvelope
-newFakeEnvelope =
-  FakeEnvelope
-    <$> newIORef []
-    <*> newIORef 0
+      readIORef envelope.acks `shouldReturn` 1
+      readIORef envelope.rejections `shouldReturn` []
+      let expectedReq =
+            FederatedRequest
+              { frTargetDomain = targetDomain,
+                frOriginDomain = origDomain,
+                frComponent = Brig,
+                frRPC = "on-user-deleted-connections",
+                frBody = Aeson.encode notifContent
+              }
+      fedReqs `shouldBe` [expectedReq, expectedReq]
+  ]
\ No newline at end of file
diff --git a/services/background-worker/test/Test/Wire/Defederation.hs b/services/background-worker/test/Test/Wire/Defederation.hs
new file mode 100644
index 0000000000..c0218e8112
--- /dev/null
+++ b/services/background-worker/test/Test/Wire/Defederation.hs
@@ -0,0 +1,47 @@
+{-# LANGUAGE RecordWildCards #-}
+
+module Test.Wire.Defederation where
+
+import Test.Wire.Util
+import qualified Data.Aeson as Aeson
+import Federator.MockServer
+import Imports
+import qualified Network.AMQP as Q
+import Test.Hspec
+import Wire.API.Federation.API.Common
+import Wire.Defederation
+import Wire.API.Federation.BackendNotifications
+import Data.Domain
+import Test.Tasty
+import Test.Tasty.HUnit
+
+deleteFederationDomainSpec :: IO TestSetup -> TestTree
+deleteFederationDomainSpec setup = testGroup "Wire.BackendNotificationPusher.deleteFederationDomain"
+  [ testCase "should fail on message decoding" $ do
+      s <- setup
+      envelope <- newFakeEnvelope
+      let msg = Q.newMsg {Q.msgBody = Aeson.encode @[()] [], Q.msgContentType = Just "application/json"}
+          respSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
+      resps <- withTempMockFederator [] respSuccess .
+        runTestAppT s $ deleteFederationDomainInner (msg, envelope)
+      case resps of
+        ((), []) -> pure ()
+        _ -> assertFailure "Expected call to federation"
+      readIORef envelope.acks `shouldReturn` 0
+      readIORef envelope.rejections `shouldReturn` [True]
+  , testCase "should succeed on message decoding" $ do
+      s <- setup
+      envelope <- newFakeEnvelope
+      let msg = Q.newMsg
+            { Q.msgBody = Aeson.encode @DefederationDomain (Domain "far-away.example.com")
+            , Q.msgContentType = Just "application/json"
+            }
+          respSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
+      resps <- withTempMockFederator [] respSuccess .
+        runTestAppT s $ deleteFederationDomainInner (msg, envelope)
+      case resps of
+        ((), []) -> pure ()
+        _ -> assertFailure "Expected call to federation"
+      readIORef envelope.acks `shouldReturn` 1
+      readIORef envelope.rejections `shouldReturn` []
+  ]
\ No newline at end of file
diff --git a/services/background-worker/test/Test/Wire/Util.hs b/services/background-worker/test/Test/Wire/Util.hs
new file mode 100644
index 0000000000..923593c027
--- /dev/null
+++ b/services/background-worker/test/Test/Wire/Util.hs
@@ -0,0 +1,105 @@
+{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+
+module Test.Wire.Util where
+
+import Imports
+import Control.Monad.Catch
+import Test.Tasty.Options
+import Data.Tagged
+import Test.Tasty
+import Test.Tasty.HUnit
+import Options.Applicative
+import Data.Aeson
+import Util.Options
+import Wire.BackgroundWorker.Env hiding (federatorInternal, galley)
+import qualified Wire.BackgroundWorker.Env as E
+import Wire.BackgroundWorker.Options hiding (federatorInternal, galley)
+import Wire.BackgroundWorker.Util
+
+data IntegrationConfig = IntegrationConfig
+  { galley :: Endpoint
+  , federatorInternal :: Endpoint
+  }
+  deriving (Show, Generic)
+instance FromJSON IntegrationConfig
+
+newtype ServiceConfigFile = ServiceConfigFile String
+  deriving (Eq, Ord, Typeable)
+
+instance IsOption ServiceConfigFile where
+  defaultValue = ServiceConfigFile "/etc/wire/background-worker/conf/background-worker.yaml"
+  parseValue = fmap ServiceConfigFile . safeRead
+  optionName = pure "service-config"
+  optionHelp = pure "Service config file to read from"
+  optionCLParser =
+    fmap ServiceConfigFile $
+      strOption $
+        ( short (untag (pure 's' :: Tagged ServiceConfigFile Char))
+            <> long (untag (optionName :: Tagged ServiceConfigFile String))
+            <> help (untag (optionHelp :: Tagged ServiceConfigFile String))
+        )
+
+newtype TestM a = TestM { runTestM :: ReaderT TestSetup IO a}
+  deriving
+    ( Functor,
+      Applicative,
+      Monad,
+      MonadReader TestSetup,
+      MonadIO,
+      MonadCatch,
+      MonadThrow,
+      MonadMask,
+      MonadUnliftIO,
+      MonadFail
+    )
+
+data TestSetup = TestSetup
+  { opts :: Opts
+  , iConf :: IntegrationConfig
+  }
+
+test :: IO TestSetup -> TestName -> TestM a -> TestTree
+test s n h = testCase n runTest
+  where
+    runTest :: Assertion
+    runTest = do
+      setup <- s
+      void . flip runReaderT setup . runTestM $ h
+
+natAppT :: AppT IO a -> TestM a
+natAppT app =
+  TestM $ do
+    e <- ask
+    e' <- liftIO $ setupToEnv e
+    liftIO $ runReaderT (unAppT app) e'
+
+setupToEnv :: TestSetup -> IO Env
+setupToEnv setup = do
+  e <- mkEnv $ setup.opts
+  pure $ e
+    { E.federatorInternal = federatorInternal $ iConf $ setup
+    , E.galley = galley $ iConf $ setup
+    }
+
+runTestAppT :: MonadIO m => TestSetup -> AppT m a -> Int -> m a
+runTestAppT setup app federatorPort = do
+  env <- liftIO $ setupToEnv setup
+  runReaderT (unAppT app) $ env
+    { E.federatorInternal = (E.federatorInternal env) { _epPort  = fromIntegral federatorPort }
+    }
+
+data FakeEnvelope = FakeEnvelope
+  { rejections :: IORef [Bool],
+    acks :: IORef Int
+  }
+
+newFakeEnvelope :: IO FakeEnvelope
+newFakeEnvelope =
+  FakeEnvelope
+    <$> newIORef []
+    <*> newIORef 0
+
+instance RabbitMQEnvelope FakeEnvelope where
+  ack e = atomicModifyIORef' e.acks $ \a -> (a + 1, ())
+  reject e requeueFlag = atomicModifyIORef' e.rejections $ \r -> (r <> [requeueFlag], ())
\ No newline at end of file

From 0080e893495bf3c80f1de9c53d7f8752c7ef3a4b Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 20 Jun 2023 14:09:15 +0200
Subject: [PATCH 150/220] hi ci


From ab9c14a1371e824e1ca4bc655411794df4d9567f Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 20 Jun 2023 14:38:16 +0200
Subject: [PATCH 151/220] Tweak docs.

---
 docs/src/understand/configure-federation.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index 426db08e8f..47979b7372 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -401,6 +401,9 @@ cargohold:
 
 **Since [PR#3260](https://github.com/wireapp/wire-server/pull/3260).**
 
+Also see {ref}`configuring-remote-connections-dev-perspective` for the
+developer's point of view on this topic.
+
 You also need to define the federation strategy (whom to federate
 with), and the frequency with which the other backend services will
 refresh their cache of this configuration.
@@ -496,10 +499,9 @@ The search policy for a remote backend can be:
 - `full_search`: Additionally to `exact_handle_search`, users are found by a freetext search on handle and display name.
 
 If federation strategy is `allowAll`, and there is no entry for a
-domain in the database, default is `no_search`.
-
-Also see {ref}`configuring-remote-connections-dev-perspective` for the
-developer's point of view on this topic.
+domain in the database, default is `no_search`.  The field in
+cassandra is not nullable, ie., you always have to explicitly name a
+search policy if you create an entry.
 
 #### If your instance has been federating before
 

From 046b87bc1d63610ea2a1531240f3af1f8bd25372 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 20 Jun 2023 21:35:45 +0200
Subject: [PATCH 152/220] nit-pick.

---
 libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
index 6891d5a466..9db8fb3044 100644
--- a/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/FederationDomainConfig.hs
@@ -18,8 +18,8 @@
 module Wire.API.Routes.FederationDomainConfig
   ( FederationDomainConfig (..),
     FederationDomainConfigs (..),
-    FederationStrategy (..),
     defFederationDomainConfigs,
+    FederationStrategy (..),
   )
 where
 

From f5779982d7c03360520af1e1140b2f8be323a68f Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 20 Jun 2023 21:36:20 +0200
Subject: [PATCH 153/220] Mark test case as flaky.

---
 services/brig/test/integration/API/Search.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/brig/test/integration/API/Search.hs b/services/brig/test/integration/API/Search.hs
index 4a1111245e..0030c80f08 100644
--- a/services/brig/test/integration/API/Search.hs
+++ b/services/brig/test/integration/API/Search.hs
@@ -79,7 +79,7 @@ tests opts mgr galley brig = do
         testWithBothIndices opts mgr "size - when exact handle matches a team user" $ testSearchSize brig True,
         testWithBothIndices opts mgr "size - when exact handle matches a non team user" $ testSearchSize brig False,
         test mgr "empty query" $ testSearchEmpty brig,
-        test mgr "reindex" $ testReindex brig,
+        flakyTest mgr "reindex" $ testReindex brig,
         testWithBothIndices opts mgr "no match" $ testSearchNoMatch brig,
         testWithBothIndices opts mgr "no extra results" $ testSearchNoExtraResults brig,
         testWithBothIndices opts mgr "order-handle (prefix match)" $ testOrderHandle brig,

From 4f289418d14fa61ac66ab6737d6572efe7982026 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 20 Jun 2023 21:51:58 +0200
Subject: [PATCH 154/220] Tweak log levels.

---
 libs/wire-api/src/Wire/API/FederationUpdate.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 38cc134967..ea166c5e97 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -34,7 +34,7 @@ getAllowedDomainsInitial logger clientEnv =
         getAllowedDomains clientEnv >>= \case
           Right s -> pure $ Just s
           Left e -> do
-            L.log logger L.Debug $
+            L.log logger L.Info $
               L.msg (L.val "Could not retrieve an initial list of federation domains from Brig.")
                 L.~~ "error" L..= show e
             pure Nothing
@@ -54,7 +54,7 @@ getAllowedDomainsLoop :: L.Logger -> ClientEnv -> FedUpdateCallback -> IORef Fed
 getAllowedDomainsLoop logger clientEnv callback env = forever $ do
   getAllowedDomains clientEnv >>= \case
     Left e ->
-      L.log logger L.Fatal $
+      L.log logger L.Info $
         L.msg (L.val "Could not retrieve an updated list of federation domains from Brig; I'll keep trying!")
           L.~~ "error" L..= show e
     Right new -> do

From 5908f7a89222ddac557cf4c90ff01b7cbccd015e Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 20 Jun 2023 22:07:07 +0200
Subject: [PATCH 155/220] Tweak log msg.

---
 services/brig/src/Brig/API/Internal.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index cd2545d32b..3640ec57f3 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -263,7 +263,7 @@ getFederationRemotes = lift $ do
   unless (maybe True (> 0) mu) $
     randomRIO (0 :: Int, 1000)
       >>= \case
-        0 -> Log.warn (Log.msg (Log.val "Invalid brig configuration: setFederationDomainConfigsUpdateFreq must be > 0, using default 10 seconds."))
+        0 -> Log.warn (Log.msg (Log.val "Invalid brig configuration: setFederationDomainConfigsUpdateFreq must be > 0.  setting to 1 second."))
         _ -> pure ()
 
   defFederationDomainConfigs

From 8ea7eb63191d1e74005f80b74ad112b7b678c519 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 15:14:33 +1000
Subject: [PATCH 156/220] wip

---
 libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs  |  2 +-
 services/background-worker/src/Wire/Defederation.hs | 12 ++++++++++--
 services/galley/src/Galley/API/Internal.hs          |  3 +++
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index bcca9a3093..1e9427becd 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -513,7 +513,7 @@ type FederationRemotesAPI =
     -- some records hanging around. Galley uses a Rabbit queue to track
     -- what is has done and can recover from a service falling over.
     :<|> Named
-           "delete-federation-remote-galley"
+           "delete-federation-remote-from-galley"
            ( Description FederationRemotesAPIDescription
                :> Description FederationRemotesAPIDeleteDescription
                :> "federation"
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index e6b008ec0c..65d3435270 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -17,14 +17,23 @@ import Control.Lens ((^.), to)
 import Data.ByteString.Conversion
 import qualified Data.ByteString.Lazy as L
 import Data.Text.Encoding
+import Control.Retry
 
 deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
 deleteFederationDomain chan = do
   lift $ ensureQueue chan defederationQueue
   QL.consumeMsgs chan (routingKey defederationQueue) Q.Ack deleteFederationDomainInner
 
+-- What should we do with non-recoverable (unparsable) errors/messages?
+-- should we deadletter, or do something else?
+-- Deadlettering has a privacy implication
+--
+-- We should also throttle the messages being retried.
+--
+-- Can we ensure that messages are handled in-order, one at a time?
+-- Is this a `amqp` or rabbit thing?
 deleteFederationDomainInner :: RabbitMQEnvelope e => (Q.Message, e) -> AppT IO ()
-deleteFederationDomainInner (msg, envelope) = do
+deleteFederationDomainInner (msg, envelope) = do -- recovering _ $ do -- retry x times, NACK after that?
   env <- ask
   let manager = httpManager env
       req :: Domain -> Request
@@ -53,7 +62,6 @@ deleteFederationDomainInner (msg, envelope) = do
       let code = statusCode $ responseStatus resp
       if code >= 200 && code <= 299
       then do
-        logErr $ show resp
         liftIO $ ack envelope
       else liftIO $ reject envelope True -- ensure that the message is requeued
     logErr err = Log.err $
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index a93fa80e1d..29890b8ccf 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -501,6 +501,7 @@ deleteFederationDomain :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg))
           Member MemberStore r, Member ConversationStore r, Member (Embed IO) r,
           Member CodeStore r, Member TeamStore r, Member BrigAccess r, Member GundeckAccess r, Member ExternalAccess r) => Domain -> Sem r ()
 deleteFederationDomain d = do
+  -- TODO rename a wee bit
   deleteFederationDomainRemote d
   deleteFederationDomainLocal d
   deleteFederationDomainOneOnOne d
@@ -607,6 +608,8 @@ deleteFederationDomainLocal dom = do
 -- eat the coverhead cost of the http call. This should also allow for the
 -- senario where galley falls over and has to redo the domain deletion so
 -- that request isn't lost.
+--
+-- These need to be recoverable? TODO
 deleteFederationDomainOneOnOne :: (Member (Input Env) r, Member (Embed IO) r, Member (P.Logger (Msg -> Msg)) r) => Domain -> Sem r ()
 deleteFederationDomainOneOnOne dom = do
   env <- input

From 582b654ad09ca6c2098ab584eef9bf71d4c03007 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 16:56:54 +1000
Subject: [PATCH 157/220] Update
 libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs

Co-authored-by: fisx <mf@zerobuzz.net>
---
 libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
index 8dcf5df7fb..87819ff1e6 100644
--- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
+++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs
@@ -469,8 +469,8 @@ type AuthAPI =
            )
 
 -- | This is located in brig, not in federator, because brig has a cassandra instance.  This
--- is not ideal, but since all services have a local in-ram copy of this table and keep track
--- of changes via rabbitmq, we argue it's "fine" for federators to ask brig once on startup.
+-- is not ideal, and other services could keep their local in-ram copy of this table up to date
+-- via rabbitmq, but FUTUREWORK.
 type FederationRemotesAPI =
   Named
     "add-federation-remotes"

From a5e484093005911ac8057c80539055dd674c836d Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 16:57:18 +1000
Subject: [PATCH 158/220] Update docs/src/understand/configure-federation.md

Co-authored-by: fisx <mf@zerobuzz.net>
---
 docs/src/understand/configure-federation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md
index 47979b7372..3477a10242 100644
--- a/docs/src/understand/configure-federation.md
+++ b/docs/src/understand/configure-federation.md
@@ -424,7 +424,7 @@ The default strategy of `allowNone` effectively disables federation
 user uses in a search.  `allowDynamic` only federates with known
 remote backends listed in cassandra.
 
-The update frequence determines how often other services will refresh
+The update frequency determines how often other services will refresh
 the information about remote connections from brig.
 
 More information about individual remote connections is stored in

From bb5455c29577d47a4151941686f6d2bcb308990c Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 17:07:14 +1000
Subject: [PATCH 159/220] Update services/brig/src/Brig/API/Internal.hs

Co-authored-by: fisx <mf@zerobuzz.net>
---
 services/brig/src/Brig/API/Internal.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 3640ec57f3..8e17528b10 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -272,8 +272,8 @@ getFederationRemotes = lift $ do
     & maybe id (\v cfg -> cfg {updateInterval = min 1 v}) mu
     & pure
 
-updateFederationRemotes :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
-updateFederationRemotes dom fedcfg = do
+updateFederationRemote :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
+updateFederationRemote dom fedcfg = do
   assertDomainIsNotUpdated dom fedcfg
   assertNoDomainsFromConfigFiles dom
   (lift . wrapClient . Data.updateFederationRemote $ fedcfg) >>= \case

From ac6a5c210383102121f4b5bd38b846b14c79dc49 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 17:07:32 +1000
Subject: [PATCH 160/220] Update services/brig/src/Brig/API/Internal.hs

Co-authored-by: fisx <mf@zerobuzz.net>
---
 services/brig/src/Brig/API/Internal.hs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 8e17528b10..6f350e40d9 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -300,6 +300,9 @@ assertNoDomainsFromConfigFiles dom = do
 -- | Remove the entry from the database if present (or do nothing if not).  This responds with
 -- 533 if the entry was also present in the config file, but only *after* it has removed the
 -- entry from cassandra.
+--
+-- The ordering on this delete then check seems weird, but allows us to default all the
+-- way back to config file state for a federation domain.
 deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
 deleteFederationRemotes dom = do
   lift . wrapClient . Data.deleteFederationRemote $ dom

From f44ff72907cdaae7fefce8e21efd2a499dfd12be Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 17:07:44 +1000
Subject: [PATCH 161/220] Update services/brig/src/Brig/API/Internal.hs

Co-authored-by: fisx <mf@zerobuzz.net>
---
 services/brig/src/Brig/API/Internal.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 6f350e40d9..edd51521cb 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -303,8 +303,8 @@ assertNoDomainsFromConfigFiles dom = do
 --
 -- The ordering on this delete then check seems weird, but allows us to default all the
 -- way back to config file state for a federation domain.
-deleteFederationRemotes :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
-deleteFederationRemotes dom = do
+deleteFederationRemote :: Domain -> ExceptT Brig.API.Error.Error (AppT r) ()
+deleteFederationRemote dom = do
   lift . wrapClient . Data.deleteFederationRemote $ dom
   assertNoDomainsFromConfigFiles dom
 

From a753af50b9685b02d9fbd9f78381b41790ada84a Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 17:07:59 +1000
Subject: [PATCH 162/220] Update services/federator/src/Federator/Run.hs

Co-authored-by: fisx <mf@zerobuzz.net>
---
 services/federator/src/Federator/Run.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 9b2e6a390a..123f6fe64d 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -64,7 +64,7 @@ run opts = do
   logger <- LogExt.mkLogger (Opt.logLevel opts) (Opt.logNetStrings opts) (Opt.logFormat opts)
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res -> do
-    (ioref, updateAllowedDomainsThread) <- updateFedDomains (brig opts) logger (\_ _ -> pure ())
+    (ioref, updateFedDomainsThread) <- updateFedDomains (brig opts) logger (\_ _ -> pure ())
     bracket (newEnv opts res logger ioref) closeEnv $ \env -> do
       let externalServer = serveInward env portExternal
           internalServer = serveOutward env portInternal

From b0f24b5d7f1f83bb83ed794b289970e9e4f57c6f Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 17:08:16 +1000
Subject: [PATCH 163/220] Update services/galley/src/Galley/App.hs

Co-authored-by: fisx <mf@zerobuzz.net>
---
 services/galley/src/Galley/App.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index e341706bb9..dfd0604755 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -163,9 +163,7 @@ createEnv m o l r = do
   mgr <- initHttpManager o
   h2mgr <- initHttp2Manager
   validateOptions l o
-
-  let brigEndpoint = o ^. optBrig
-  Env def m o l mgr h2mgr (o ^. optFederator) brigEndpoint cass
+  Env def m o l mgr h2mgr (o ^. optFederator) (o ^. optBrig) cass
     <$> Q.new 16000
     <*> initExtEnv
     <*> maybe (pure Nothing) (fmap Just . Aws.mkEnv l mgr) (o ^. optJournal)

From c1588b43a12674e0fc9cac64615f728309a92ea6 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 21 Jun 2023 17:23:07 +1000
Subject: [PATCH 164/220] FS-1115: PR notes

---
 integration/test/API/BrigInternal.hs           |  6 ------
 libs/wire-api/src/Wire/API/FederationUpdate.hs | 12 ++++++++++--
 services/brig/src/Brig/API/Internal.hs         |  4 ++--
 services/federator/src/Federator/Run.hs        |  2 +-
 services/galley/src/Galley/Run.hs              | 12 +++++-------
 services/galley/test/integration/API/Util.hs   |  5 +----
 6 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 887e40b055..5364eaa586 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -64,12 +64,6 @@ instance ToJSON FedConn where
 instance MakesValue FedConn where
   make = pure . toJSON
 
-instance FromJSON FedConn where
-  parseJSON = withObject "FedConn" $ \obj -> do
-    FedConn
-      <$> obj .: fromString "domain"
-      <*> obj .: fromString "search_policy"
-
 createFedConn :: (HasCallStack, MakesValue dom, MakesValue fedConn) => dom -> fedConn -> App Response
 createFedConn dom fedConn = do
   res <- createFedConn' dom fedConn
diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index ea166c5e97..73c8c740f8 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -5,7 +5,7 @@ module Wire.API.FederationUpdate
 where
 
 import Control.Concurrent.Async
-import Control.Exception (ErrorCall (ErrorCall), throwIO)
+import Control.Exception (ErrorCall (ErrorCall), finally, throwIO)
 import qualified Control.Retry as R
 import qualified Data.Set as Set
 import Data.Text (unpack)
@@ -72,7 +72,15 @@ updateFedDomains :: Endpoint -> L.Logger -> FedUpdateCallback -> IO (IORef Feder
 updateFedDomains (Endpoint h p) log' cb = do
   clientEnv <- newManager defaultManagerSettings <&> \mgr -> ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
   ioref <- newIORef =<< getAllowedDomainsInitial log' clientEnv
-  updateDomainsThread <- async $ getAllowedDomainsLoop log' clientEnv cb ioref
+  updateDomainsThread <-
+    async $
+      let go = finally
+            (getAllowedDomainsLoop log' clientEnv cb ioref)
+            $ do
+              L.log log' L.Error $ L.msg (L.val "Federation domain sync thread died, restarting domain synchronization.")
+              go
+       in go
+
   pure (ioref, updateDomainsThread)
   where
     baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index edd51521cb..10afb03a9c 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -190,8 +190,8 @@ federationRemotesAPI :: ServerT BrigIRoutes.FederationRemotesAPI (Handler r)
 federationRemotesAPI =
   Named @"add-federation-remotes" addFederationRemote
     :<|> Named @"get-federation-remotes" getFederationRemotes
-    :<|> Named @"update-federation-remotes" updateFederationRemotes
-    :<|> Named @"delete-federation-remotes" deleteFederationRemotes
+    :<|> Named @"update-federation-remotes" updateFederationRemote
+    :<|> Named @"delete-federation-remotes" deleteFederationRemote
 
 addFederationRemote :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 addFederationRemote fedDomConf = do
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 123f6fe64d..7f3a37605f 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -71,7 +71,7 @@ run opts = do
       withMonitor logger (onNewSSLContext env) (optSettings opts) $ do
         internalServerThread <- async internalServer
         externalServerThread <- async externalServer
-        void $ waitAnyCancel [updateAllowedDomainsThread, internalServerThread, externalServerThread]
+        void $ waitAnyCancel [updateFedDomainsThread, internalServerThread, externalServerThread]
   where
     endpointInternal = federatorInternal opts
     portInternal = fromIntegral $ endpointInternal ^. epPort
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index a61fdfd692..9b8ba9ab6e 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -63,15 +63,12 @@ import System.Logger.Extended (mkLogger)
 import Util.Options
 import Wire.API.FederationUpdate
 import Wire.API.Routes.API
-import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.API.Routes.Public.Galley as GalleyAPI
 import Wire.API.Routes.Version.Wai
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
-  l <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
-  (ioref, _) <- lift $ updateFedDomains (opts ^. optBrig) l $ \_ _ -> pure ()
-  (app, env) <- mkApp opts ioref l
+  (app, env) <- mkApp opts
   settings <-
     lift $
       newSettings $
@@ -85,12 +82,13 @@ run opts = lowerCodensity $ do
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
-  void $ Codensity $ Async.withAsync $ runApp env undefined
   lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
 
-mkApp :: Opts -> IORef FederationDomainConfigs -> Log.Logger -> Codensity IO (Application, Env)
-mkApp opts fedDoms logger =
+mkApp :: Opts -> Codensity IO (Application, Env)
+mkApp opts =
   do
+    logger <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
+    (fedDoms, _) <- lift $ updateFedDomains (opts ^. optBrig) logger $ \_ _ -> pure ()
     metrics <- lift $ M.metrics
     env <- lift $ App.createEnv metrics opts logger fedDoms
     lift $ runClient (env ^. cstate) $ versionCheck schemaVersion
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index f77e8f57aa..b8ecd8ea53 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -125,7 +125,6 @@ import Wire.API.MLS.Proposal
 import Wire.API.MLS.Serialisation
 import Wire.API.Message
 import qualified Wire.API.Message.Proto as Proto
-import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.Internal.Brig.Connection
 import Wire.API.Routes.Internal.Galley.ConversationsIntra
 import qualified Wire.API.Routes.Internal.Galley.TeamFeatureNoConfigMulti as Multi
@@ -2456,9 +2455,7 @@ instance HasSettingsOverrides TestM where
     ts :: TestSetup <- ask
     let opts = f (ts ^. tsGConf)
     liftIO . lowerCodensity $ do
-      ioref <- newIORef defFederationDomainConfigs
-      logger <- lift $ Run.mkLogger (opts ^. Opts.optLogLevel) (opts ^. Opts.optLogNetStrings) (opts ^. Opts.optLogFormat)
-      (galleyApp, _env) <- Run.mkApp opts ioref logger
+      (galleyApp, _env) <- Run.mkApp opts
       port' <- withMockServer galleyApp
       liftIO $
         runReaderT

From 9494a196fefbef4c4b17dbd0cea90c4e8cc5c553 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Wed, 21 Jun 2023 14:32:46 +0000
Subject: [PATCH 165/220] try fix helm charts

---
 charts/brig/templates/configmap.yaml          | 4 ++--
 charts/brig/values.yaml                       | 2 ++
 charts/gundeck/templates/configmap.yaml       | 4 ++++
 hack/helm_vars/wire-server/values.yaml.gotmpl | 2 +-
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml
index 116141ffd2..2ed6eb6833 100644
--- a/charts/brig/templates/configmap.yaml
+++ b/charts/brig/templates/configmap.yaml
@@ -248,10 +248,10 @@ data:
       setDeleteThrottleMillis: {{ .setDeleteThrottleMillis }}
       setFederationDomain: {{ .setFederationDomain }}
       {{- if .setFederationStrategy }}
-      setFederationStrategy: {{ toYaml .setFederationStrategy | nindent 8 }}
+      setFederationStrategy: {{ .setFederationStrategy }}
       {{- end }}
       {{- if .setFederationDomainConfigsUpdateFreq }}
-      setFederationDomainConfigsUpdateFreq: {{ toYaml .setFederationDomainConfigsUpdateFreq | nindent 8 }}
+      setFederationDomainConfigsUpdateFreq: {{ .setFederationDomainConfigsUpdateFreq }}
       {{- end }}
       {{- if .setFederationDomainConfigs }}
       # 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml
index fcab57a54e..7f75695522 100644
--- a/charts/brig/values.yaml
+++ b/charts/brig/values.yaml
@@ -100,6 +100,8 @@ config:
     # Disable one ore more API versions. Please make sure the configuration value is the same in all these charts:
     # brig, cannon, cargohold, galley, gundeck, proxy, spar.
     # setDisabledAPIVersions: [ v3 ]
+    setFederationStrategy: allowNone
+    setFederationDomainConfigsUpdateFreq: 10
   smtp:
     passwordFile: /etc/wire/brig/secrets/smtp-password.txt
   proxy: {}
diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml
index 2349e68cc4..4f63a09144 100644
--- a/charts/gundeck/templates/configmap.yaml
+++ b/charts/gundeck/templates/configmap.yaml
@@ -13,6 +13,10 @@ data:
       host: 0.0.0.0
       port: {{ $.Values.service.internalPort }}
 
+    brig:
+      host: brig
+      port: 8080
+
     cassandra:
       endpoint:
         host: {{ .cassandra.host }}
diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl
index a9b70b147e..b51b67ae9f 100644
--- a/hack/helm_vars/wire-server/values.yaml.gotmpl
+++ b/hack/helm_vars/wire-server/values.yaml.gotmpl
@@ -85,7 +85,7 @@ brig:
           search_policy: full_search
         - domain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
           search_policy: full_search
-      setFederationStrategy: allowList
+      setFederationStrategy: allowDynamic
       setFederationDomainConfigsUpdateFreq: 10
       set2FACodeGenerationDelaySecs: 5
       setNonceTtlSecs: 300

From 553ee0d3b88560510c7c913c0cbcd598ed771905 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 22 Jun 2023 18:17:11 +1000
Subject: [PATCH 166/220] WIP

---
 changelog.d/5-internal/fs-1179                      |  1 +
 services/background-worker/src/Wire/Defederation.hs |  1 -
 services/brig/src/Brig/API/Internal.hs              |  2 +-
 services/galley/src/Galley/API/Internal.hs          | 12 ++++++------
 services/galley/src/Galley/Run.hs                   |  2 --
 services/galley/test/integration/Federation.hs      |  4 +---
 6 files changed, 9 insertions(+), 13 deletions(-)
 create mode 100644 changelog.d/5-internal/fs-1179

diff --git a/changelog.d/5-internal/fs-1179 b/changelog.d/5-internal/fs-1179
new file mode 100644
index 0000000000..2f702e41f9
--- /dev/null
+++ b/changelog.d/5-internal/fs-1179
@@ -0,0 +1 @@
+Adding a new internal APIs to Brig and Galley to defederate domains.
\ No newline at end of file
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index 65d3435270..04d22707b4 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -17,7 +17,6 @@ import Control.Lens ((^.), to)
 import Data.ByteString.Conversion
 import qualified Data.ByteString.Lazy as L
 import Data.Text.Encoding
-import Control.Retry
 
 deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
 deleteFederationDomain chan = do
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 0ffd70d287..8d09c2778d 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -197,7 +197,7 @@ federationRemotesAPI =
     :<|> Named @"get-federation-remotes" getFederationRemotes
     :<|> Named @"update-federation-remotes" updateFederationRemote
     :<|> Named @"delete-federation-remotes" deleteFederationRemote
-    :<|> Named @"delete-federation-remote-galley" deleteFederationRemoteGalley
+    :<|> Named @"delete-federation-remote-from-galley" deleteFederationRemoteGalley
 
 addFederationRemote :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) ()
 addFederationRemote fedDomConf = do
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index 29890b8ccf..c8263fd826 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -502,8 +502,8 @@ deleteFederationDomain :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg))
           Member CodeStore r, Member TeamStore r, Member BrigAccess r, Member GundeckAccess r, Member ExternalAccess r) => Domain -> Sem r ()
 deleteFederationDomain d = do
   -- TODO rename a wee bit
-  deleteFederationDomainRemote d
-  deleteFederationDomainLocal d
+  deleteFederationDomainRemoteUserFromLocalConversations d
+  deleteFederationDomainLocalUserFromRemoteConversation d
   deleteFederationDomainOneOnOne d
 
 
@@ -517,11 +517,11 @@ internalDeleteFederationDomainH (domain ::: _) = do
   pure (empty & setStatus status200)
 
 -- Remove remote members from local conversations
-deleteFederationDomainRemote :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg)) r,
+deleteFederationDomainRemoteUserFromLocalConversations :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg)) r,
           Member (Error InternalError) r, Member (Error FederationError) r,
           Member MemberStore r, Member ConversationStore r,
           Member CodeStore r, Member TeamStore r) => Domain -> Sem r ()
-deleteFederationDomainRemote dom = do
+deleteFederationDomainRemoteUserFromLocalConversations dom = do
   remoteUsers <- E.getRemoteMembersByDomain dom
   env <- input
   let lCnvMap = foldr insertIntoMap mempty remoteUsers
@@ -566,11 +566,11 @@ deleteFederationDomainRemote dom = do
             ()
 
 -- Remove local members from remote conversations
-deleteFederationDomainLocal :: (Member (Input (Local ())) r, Member (Input Env) r,
+deleteFederationDomainLocalUserFromRemoteConversation :: (Member (Input (Local ())) r, Member (Input Env) r,
           Member (Error InternalError) r, Member (P.Logger (Msg -> Msg)) r,
           Member MemberStore r, Member (Embed IO) r, Member BrigAccess r,
           Member GundeckAccess r, Member ExternalAccess r) => Domain -> Sem r ()
-deleteFederationDomainLocal dom = do
+deleteFederationDomainLocalUserFromRemoteConversation dom = do
   localUsers <- E.getLocalMembersByDomain dom
   env <- input
   -- As above, build the map so we can get all local users per conversation
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 4ea14910e7..f11eb95d9f 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -52,7 +52,6 @@ import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
 import Imports
-import Network.HTTP.Client (defaultManagerSettings, newManager)
 import qualified Network.HTTP.Media.RenderHeader as HTTPMedia
 import qualified Network.HTTP.Types as HTTP
 import Network.Wai
@@ -60,7 +59,6 @@ import qualified Network.Wai.Middleware.Gunzip as GZip
 import qualified Network.Wai.Middleware.Gzip as GZip
 import Network.Wai.Utilities.Server
 import Servant hiding (route)
-import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http), defaultMakeClientRequest)
 import qualified System.Logger as Log
 import System.Logger.Extended (mkLogger)
 import Util.Options
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 167def740a..7e6633c19e 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -88,9 +88,7 @@ updateFedDomainsTest = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  l <- liftIO $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
-  r <- newIORef defFederationDomainConfigs
-  (_, env) <- liftIO $ lowerCodensity $ mkApp opts r l
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
   let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
       remoteDomain = Domain "far-away.example.com"

From d649710e1a4d6f6619554ec64fef9fba8a97c74e Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Thu, 22 Jun 2023 09:25:04 +0000
Subject: [PATCH 167/220] hi ci


From 0008cf844981b30817785b85d5d405a811105e7f Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Thu, 22 Jun 2023 13:58:15 +0000
Subject: [PATCH 168/220] changed federation strategy in CI from allowDynamic
 to allowAll

---
 hack/helm_vars/wire-server/values.yaml.gotmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl
index b51b67ae9f..9f81c5ebb6 100644
--- a/hack/helm_vars/wire-server/values.yaml.gotmpl
+++ b/hack/helm_vars/wire-server/values.yaml.gotmpl
@@ -85,7 +85,7 @@ brig:
           search_policy: full_search
         - domain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
           search_policy: full_search
-      setFederationStrategy: allowDynamic
+      setFederationStrategy: allowAll
       setFederationDomainConfigsUpdateFreq: 10
       set2FACodeGenerationDelaySecs: 5
       setNonceTtlSecs: 300

From 8d8242a85a74e3f0cabaec3ce2dcdc68909b7c02 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 23 Jun 2023 10:55:07 +0000
Subject: [PATCH 169/220] correct error in case of allowDynamic

---
 services/galley/src/Galley/API/Action.hs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs
index 9ad88acf3d..5c3f34590b 100644
--- a/services/galley/src/Galley/API/Action.hs
+++ b/services/galley/src/Galley/API/Action.hs
@@ -752,7 +752,12 @@ notifyConversationAction tag quid notifyOrigDomain con lconv targets action = do
     for_ failedNotifies $ \case
       -- rethrow invalid-domain errors and mis-configured federation errors
       (_, ex@(FederationCallFailure (FederatorClientError (Wai.Error (Wai.Status 422 _) _ _ _)))) -> throw ex
+      -- FUTUREWORK: This error occurs when federation strategy is set to `allowDynamic`
+      -- and the remote domain is not in the allow list
+      -- Is it ok to throw all 400 errors?
+      (_, ex@(FederationCallFailure (FederatorClientError (Wai.Error (Wai.Status 400 _) _ _ _)))) -> throw ex
       (_, ex@(FederationCallFailure (FederatorClientHTTP2Error (FederatorClientConnectionError _)))) -> throw ex
+      -- FUTUREWORK: Do we really want to ignore any other errors?
       _ -> pure ()
     updates <-
       E.runFederatedConcurrentlyEither (toList (bmRemotes targets)) $

From 0a1b73078dc7c8101a58a7360e8700e81952d4d4 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 23 Jun 2023 11:46:02 +0000
Subject: [PATCH 170/220] default case made explicit

---
 services/galley/src/Galley/API/Action.hs | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs
index 5c3f34590b..763855d38d 100644
--- a/services/galley/src/Galley/API/Action.hs
+++ b/services/galley/src/Galley/API/Action.hs
@@ -757,8 +757,18 @@ notifyConversationAction tag quid notifyOrigDomain con lconv targets action = do
       -- Is it ok to throw all 400 errors?
       (_, ex@(FederationCallFailure (FederatorClientError (Wai.Error (Wai.Status 400 _) _ _ _)))) -> throw ex
       (_, ex@(FederationCallFailure (FederatorClientHTTP2Error (FederatorClientConnectionError _)))) -> throw ex
-      -- FUTUREWORK: Do we really want to ignore any other errors?
-      _ -> pure ()
+      -- FUTUREWORK: Default case (`_ -> pure ()`) is now explicit. Do we really want to ignore all these errors?
+      (_, FederationCallFailure (FederatorClientHTTP2Error _)) -> pure ()
+      (_, FederationCallFailure (FederatorClientError _)) -> pure ()
+      (_, FederationCallFailure FederatorClientStreamingNotSupported) -> pure ()
+      (_, FederationCallFailure (FederatorClientServantError _)) -> pure ()
+      (_, FederationCallFailure (FederatorClientVersionNegotiationError _)) -> pure ()
+      (_, FederationCallFailure FederatorClientVersionMismatch) -> pure ()
+      (_, FederationNotImplemented) -> pure ()
+      (_, FederationNotConfigured) -> pure ()
+      (_, FederationUnexpectedBody _) -> pure ()
+      (_, FederationUnexpectedError _) -> pure ()
+      (_, FederationUnreachableDomains _) -> pure ()
     updates <-
       E.runFederatedConcurrentlyEither (toList (bmRemotes targets)) $
         \ruids -> do

From 57fb696e890dac9f4d24ebbe0d730f10a9d5fe96 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 26 Jun 2023 17:29:35 +1000
Subject: [PATCH 171/220] FS-1179: Modifications to tests to help stop hangups

---
 .../galley/test/integration/Federation.hs     | 67 +++++++++++++++++--
 services/galley/test/integration/Main.hs      |  8 ++-
 2 files changed, 67 insertions(+), 8 deletions(-)

diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 7e6633c19e..04c267fcc6 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -20,6 +20,7 @@ import qualified Data.Set as Set
 import Data.Singletons
 import Data.Time (getCurrentTime)
 import qualified Data.UUID as UUID
+import Data.UUID.V4 (nextRandom)
 import Federator.MockServer
 import Galley.API.Util
 import Galley.Cassandra.Queries
@@ -56,9 +57,11 @@ x3 = limitRetries 3 <> exponentialBackoff 100000
 isConvMemberLTests :: TestM ()
 isConvMemberLTests = do
   s <- ask
-  let opts = s ^. tsGConf
+  uuid <- liftIO nextRandom
+  let uuid' = UUID.toText uuid
+      opts = s ^. tsGConf
       localDomain = opts ^. optSettings . setFederationDomain
-      remoteDomain = Domain "far-away.example.com"
+      remoteDomain = Domain $ "far-away.example.com" <> uuid'
       convId = Id $ fromJust $ UUID.fromString "8cc34301-6949-46c5-bb93-00a72268e2f5"
       convLocalMembers = [LocalMember userId defMemberStatus Nothing roleNameWireMember]
       convRemoteMembers = [RemoteMember rUserId roleNameWireMember]
@@ -82,17 +85,19 @@ isConvMemberLTests = do
   liftIO $ assertBool "Qualified UserId (local)" $ isConvMemberL lconv $ tUntagged lUserId
   liftIO $ assertBool "Qualified UserId (remote)" $ isConvMemberL lconv $ tUntagged rUserId
 
-updateFedDomainsTest :: TestM ()
-updateFedDomainsTest = do
+updateFedDomainsTestNoop' :: TestM ()
+updateFedDomainsTestNoop' = do
   s <- ask
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
   (_, env) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
-  let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
-      remoteDomain = Domain "far-away.example.com"
-      remoteDomain2 = Domain "far-away-two.example.com"
+  uuid <- liftIO nextRandom
+  let uuid' = UUID.toText uuid
+      interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain = Domain $ "far-away.example.com" <> uuid'
+      remoteDomain2 = Domain $ "far-away-two.example.com" <> uuid'
   liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
   liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
   -- Setup a conversation for a known remote domain.
@@ -101,12 +106,60 @@ updateFedDomainsTest = do
   -- working on the domain.
   updateFedDomainsTestNoop env remoteDomain interval
 
+updateFedDomainsTestAddRemote' :: TestM ()
+updateFedDomainsTestAddRemote' = do
+  s <- ask
+  let opts = s ^. tsGConf
+  -- Don't need the actual server, and we certainly don't want it running.
+  -- But this is how the env is made, so it is what we do
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  -- Common variables.
+  uuid <- liftIO nextRandom
+  let uuid' = UUID.toText uuid
+      interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain = Domain $ "far-away.example.com" <> uuid'
+      remoteDomain2 = Domain $ "far-away-two.example.com" <> uuid'
+  liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
+  liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
+
   -- Adding a new federation domain, this too should be a no-op
   updateFedDomainsAddRemote env remoteDomain remoteDomain2 interval
 
+updateFedDomainsTestRemoveRemoteFromLocal' :: TestM ()
+updateFedDomainsTestRemoveRemoteFromLocal' = do
+  s <- ask
+  let opts = s ^. tsGConf
+  -- Don't need the actual server, and we certainly don't want it running.
+  -- But this is how the env is made, so it is what we do
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  -- Common variables.
+  uuid <- liftIO nextRandom
+  let uuid' = UUID.toText uuid
+      interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain = Domain $ "far-away.example.com" <> uuid'
+      remoteDomain2 = Domain $ "far-away-two.example.com" <> uuid'
+  liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
+  liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
+  
   -- Remove a remote domain from local conversations
   updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval
 
+updateFedDomainsTestRemoveLocalFromRemote' :: TestM ()
+updateFedDomainsTestRemoveLocalFromRemote' = do
+  s <- ask
+  let opts = s ^. tsGConf
+  -- Don't need the actual server, and we certainly don't want it running.
+  -- But this is how the env is made, so it is what we do
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  -- Common variables.
+  uuid <- liftIO nextRandom
+  let uuid' = UUID.toText uuid
+      interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain = Domain $ "far-away.example.com" <> uuid'
+      remoteDomain2 = Domain $ "far-away-two.example.com" <> uuid'
+  liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
+  liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
+
   -- Remove a local domain from remote conversations
   updateFedDomainRemoveLocalFromRemote env remoteDomain interval
 
diff --git a/services/galley/test/integration/Main.hs b/services/galley/test/integration/Main.hs
index 519cf856b2..49e442c849 100644
--- a/services/galley/test/integration/Main.hs
+++ b/services/galley/test/integration/Main.hs
@@ -96,7 +96,13 @@ main = withOpenSSL $ runTests go
               mempty
               (pathsConsistencyCheck . treeToPaths . compile $ Galley.API.sitemap),
           API.tests setup,
-          test setup "Federation Domains" updateFedDomainsTest,
+          testGroup "Federation Domains"
+            [ test setup "No-Op" updateFedDomainsTestNoop'
+            , test setup "Add Remote" updateFedDomainsTestAddRemote'
+            , test setup "Remove Remote From Local" updateFedDomainsTestRemoveRemoteFromLocal'
+            , test setup "Remove Local From Remote" updateFedDomainsTestRemoveLocalFromRemote'
+            ],
+          
           test setup "isConvMemberL" isConvMemberLTests
         ]
     getOpts gFile iFile = do

From 7b8d78c8e1f228123dd3caf534a89f5e834946fd Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 26 Jun 2023 17:46:22 +1000
Subject: [PATCH 172/220] Cleaning up comments

---
 services/galley/src/Galley/API/Internal.hs | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index 13b75f5cb1..edd17dcc6a 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -501,7 +501,6 @@ deleteFederationDomain :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg))
           Member MemberStore r, Member ConversationStore r, Member (Embed IO) r,
           Member CodeStore r, Member TeamStore r, Member BrigAccess r, Member GundeckAccess r, Member ExternalAccess r) => Domain -> Sem r ()
 deleteFederationDomain d = do
-  -- TODO rename a wee bit
   deleteFederationDomainRemoteUserFromLocalConversations d
   deleteFederationDomainLocalUserFromRemoteConversation d
   deleteFederationDomainOneOnOne d
@@ -601,13 +600,6 @@ deleteFederationDomainLocalUserFromRemoteConversation dom = do
 -- let rcnv = toRemoteUnsafe dom cnv
 -- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
 
--- TODO: The DB table that this tries to update aren't available to
--- Galley and need to be moved into brig. This will complicate the calling
--- to delete a domain, but likely we can expose it as an internal API and
--- eat the coverhead cost of the http call. This should also allow for the
--- senario where galley falls over and has to redo the domain deletion so
--- that request isn't lost.
---
 -- These need to be recoverable? TODO
 deleteFederationDomainOneOnOne :: (Member (Input Env) r, Member (Embed IO) r, Member (P.Logger (Msg -> Msg)) r) => Domain -> Sem r ()
 deleteFederationDomainOneOnOne dom = do

From 22e77982d61363fe33bdbb9bff4885ca5baeb83c Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 13:14:54 +0200
Subject: [PATCH 173/220] Create use domain names in integration tests.

(this avoids hypothetical concurrency problems.)
---
 integration/test/Test/Brig.hs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index fe01c22e51..6106b95764 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -4,9 +4,12 @@ import qualified API.Brig as Public
 import qualified API.BrigInternal as Internal
 import qualified API.Common as API
 import qualified API.GalleyInternal as Internal
+import Control.Monad.IO.Class (liftIO)
 import Data.Aeson.Types
 import qualified Data.Set as Set
 import Data.String.Conversions
+import qualified Data.UUID as UUID
+import qualified Data.UUID.V4 as UUID
 import GHC.Stack
 import SetupHelpers
 import Testlib.Assertions
@@ -69,10 +72,13 @@ testCrudFederationRemotes = do
         res <- Internal.updateFedConn' OwnDomain domain fedConn
         addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
+  dom1 :: String <- (<> ".example.com") . UUID.toString <$> liftIO UUID.nextRandom
+  dom2 :: String <- (<> ".example.com") . UUID.toString <$> liftIO UUID.nextRandom
+
   let remote1, remote1', remote1'' :: Internal.FedConn
-      remote1 = Internal.FedConn (cs "good.example.com") "no_search"
+      remote1 = Internal.FedConn dom1 "no_search"
       remote1' = remote1 {Internal.searchStrategy = "full_search"}
-      remote1'' = remote1 {Internal.domain = "meh.example.com"}
+      remote1'' = remote1 {Internal.domain = dom2}
 
       cfgRemotesExpect :: Internal.FedConn
       cfgRemotesExpect = Internal.FedConn (cs "example.com") "full_search"

From 866f52b97f87aae004ae2b01590180a293b05266 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 13:17:39 +0200
Subject: [PATCH 174/220] Better errors in /integration.

---
 integration/test/API/BrigInternal.hs | 30 +++++++++++++++-------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs
index 5364eaa586..138cb5d803 100644
--- a/integration/test/API/BrigInternal.hs
+++ b/integration/test/API/BrigInternal.hs
@@ -66,20 +66,21 @@ instance MakesValue FedConn where
 
 createFedConn :: (HasCallStack, MakesValue dom, MakesValue fedConn) => dom -> fedConn -> App Response
 createFedConn dom fedConn = do
-  res <- createFedConn' dom fedConn
-  res.status `shouldMatchRange` (200, 299)
-  pure res
+  bindResponse (createFedConn' dom fedConn) $ \res -> do
+    res.status `shouldMatchRange` (200, 299)
+    pure res
 
 createFedConn' :: (HasCallStack, MakesValue dom, MakesValue fedConn) => dom -> fedConn -> App Response
 createFedConn' dom fedConn = do
   req <- rawBaseRequest dom Brig Unversioned "/i/federation/remotes"
-  make fedConn >>= \v -> submit "POST" $ req & addJSON v
+  conn <- make fedConn
+  submit "POST" $ req & addJSON conn
 
 readFedConns :: (HasCallStack, MakesValue dom) => dom -> App Response
 readFedConns dom = do
-  res <- readFedConns' dom
-  res.status `shouldMatchRange` (200, 299)
-  pure res
+  bindResponse (readFedConns' dom) $ \res -> do
+    res.status `shouldMatchRange` (200, 299)
+    pure res
 
 readFedConns' :: (HasCallStack, MakesValue dom) => dom -> App Response
 readFedConns' dom = do
@@ -88,20 +89,21 @@ readFedConns' dom = do
 
 updateFedConn :: (HasCallStack, MakesValue owndom, MakesValue fedConn) => owndom -> String -> fedConn -> App Response
 updateFedConn owndom dom fedConn = do
-  res <- updateFedConn' owndom dom fedConn
-  res.status `shouldMatchRange` (200, 299)
-  pure res
+  bindResponse (updateFedConn' owndom dom fedConn) $ \res -> do
+    res.status `shouldMatchRange` (200, 299)
+    pure res
 
 updateFedConn' :: (HasCallStack, MakesValue owndom, MakesValue fedConn) => owndom -> String -> fedConn -> App Response
 updateFedConn' owndom dom fedConn = do
   req <- rawBaseRequest owndom Brig Unversioned ("/i/federation/remotes/" <> dom)
-  make fedConn >>= \v -> submit "PUT" $ addJSON v req
+  conn <- make fedConn
+  submit "PUT" $ addJSON conn req
 
 deleteFedConn :: (HasCallStack, MakesValue owndom) => owndom -> String -> App Response
 deleteFedConn owndom dom = do
-  res <- deleteFedConn' owndom dom
-  res.status `shouldMatchRange` (200, 299)
-  pure res
+  bindResponse (deleteFedConn' owndom dom) $ \res -> do
+    res.status `shouldMatchRange` (200, 299)
+    pure res
 
 deleteFedConn' :: (HasCallStack, MakesValue owndom) => owndom -> String -> App Response
 deleteFedConn' owndom dom = do

From 372c71c7c3f8478c5364c2ee1d8150e1c39e8bc7 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 13:17:56 +0200
Subject: [PATCH 175/220] nit-picks.

---
 integration/test/Testlib/Assertions.hs | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/integration/test/Testlib/Assertions.hs b/integration/test/Testlib/Assertions.hs
index 6061342e75..05e8d98f16 100644
--- a/integration/test/Testlib/Assertions.hs
+++ b/integration/test/Testlib/Assertions.hs
@@ -92,14 +92,10 @@ shouldMatchRange ::
   (Int, Int) ->
   App ()
 shouldMatchRange a (lower, upper) = do
-  xa <- make a
-  xl <- make lower
-  xu <- make upper
-  when (xa < xl || xa > xu) $ do
+  xa :: Int <- asInt a
+  when (xa < lower || xa > upper) $ do
     pa <- prettyJSON xa
-    pu <- prettyJSON xu
-    pl <- prettyJSON xl
-    assertFailure $ "Actual:\n" <> pa <> "\nExpected:\n(" <> pl <> "," <> pu <> ")"
+    assertFailure $ "Actual:\n" <> pa <> "\nExpected:\nin range (" <> show lower <> ", " <> show upper <> ") (including bounds)"
 
 shouldMatchSet ::
   (MakesValue a, MakesValue b, HasCallStack) =>

From 88bf1958c1d176409eb8cb80bd3fe018bc9582a7 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 13:21:51 +0200
Subject: [PATCH 176/220] Better errors in /integration.

---
 integration/test/Test/Brig.hs | 36 +++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs
index 6106b95764..7f292118c6 100644
--- a/integration/test/Test/Brig.hs
+++ b/integration/test/Test/Brig.hs
@@ -38,39 +38,39 @@ testCrudFederationRemotes = do
 
       addOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => fedConn -> [fedConn2] -> App ()
       addOnce fedConn want = do
-        res <- Internal.createFedConn OwnDomain fedConn
-        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
-        res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        sort res2 `shouldMatch` sort want
+        bindResponse (Internal.createFedConn OwnDomain fedConn) $ \res -> do
+          addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
+          res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
+          sort res2 `shouldMatch` sort want
 
       addFail :: HasCallStack => MakesValue fedConn => fedConn -> App ()
       addFail fedConn = do
-        res <- Internal.createFedConn' OwnDomain fedConn
-        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
+        bindResponse (Internal.createFedConn' OwnDomain fedConn) $ \res -> do
+          addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
       deleteOnce :: (Ord fedConn, ToJSON fedConn, MakesValue fedConn) => String -> [fedConn] -> App ()
       deleteOnce domain want = do
-        res <- Internal.deleteFedConn OwnDomain domain
-        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
-        res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        sort res2 `shouldMatch` sort want
+        bindResponse (Internal.deleteFedConn OwnDomain domain) $ \res -> do
+          addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
+          res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
+          sort res2 `shouldMatch` sort want
 
       deleteFail :: HasCallStack => String -> App ()
       deleteFail del = do
-        res <- Internal.deleteFedConn' OwnDomain del
-        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
+        bindResponse (Internal.deleteFedConn' OwnDomain del) $ \res -> do
+          addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
       updateOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => String -> fedConn -> [fedConn2] -> App ()
       updateOnce domain fedConn want = do
-        res <- Internal.updateFedConn OwnDomain domain fedConn
-        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
-        res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
-        sort res2 `shouldMatch` sort want
+        bindResponse (Internal.updateFedConn OwnDomain domain fedConn) $ \res -> do
+          addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
+          res2 <- parseFedConns =<< Internal.readFedConns OwnDomain
+          sort res2 `shouldMatch` sort want
 
       updateFail :: (MakesValue fedConn, HasCallStack) => String -> fedConn -> App ()
       updateFail domain fedConn = do
-        res <- Internal.updateFedConn' OwnDomain domain fedConn
-        addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
+        bindResponse (Internal.updateFedConn' OwnDomain domain fedConn) $ \res -> do
+          addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
 
   dom1 :: String <- (<> ".example.com") . UUID.toString <$> liftIO UUID.nextRandom
   dom2 :: String <- (<> ".example.com") . UUID.toString <$> liftIO UUID.nextRandom

From 2ba8dd60943e6e88c1e7dd810123ad2d21dbf472 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 13:43:45 +0200
Subject: [PATCH 177/220] revert commit noise.

---
 services/brig/test/integration/API/User/Account.hs | 2 +-
 services/brig/test/integration/API/User/Client.hs  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/brig/test/integration/API/User/Account.hs b/services/brig/test/integration/API/User/Account.hs
index bfb77a18e6..2c1899915c 100644
--- a/services/brig/test/integration/API/User/Account.hs
+++ b/services/brig/test/integration/API/User/Account.hs
@@ -119,7 +119,7 @@ tests _ at opts p b c ch g aws userJournalWatcher =
       test p "post /activate - 200/204 + expiry" $ testActivateWithExpiry opts b at,
       test p "get /users/:uid - 404" $ testNonExistingUserUnqualified b,
       test p "get /users/<localdomain>/:uid - 404" $ testNonExistingUser b,
-      test p "get /users/:domain/:uid - 4xx" $ testUserInvalidDomain b,
+      test p "get /users/:domain/:uid - 422" $ testUserInvalidDomain b,
       test p "get /users/:uid - 200" $ testExistingUserUnqualified b,
       test p "get /users/<localdomain>/:uid - 200" $ testExistingUser b,
       test p "get /users?:id=.... - 200" $ testMultipleUsersUnqualified b,
diff --git a/services/brig/test/integration/API/User/Client.hs b/services/brig/test/integration/API/User/Client.hs
index 3207e9da9f..5dc5dfd830 100644
--- a/services/brig/test/integration/API/User/Client.hs
+++ b/services/brig/test/integration/API/User/Client.hs
@@ -91,7 +91,7 @@ tests _cl _at opts p db n b c g =
       test p "get /users/<localdomain>/:uid/clients - 200" $ testGetUserClientsQualified opts b,
       test p "get /users/:uid/prekeys - 200" $ testGetUserPrekeys b,
       test p "get /users/<localdomain>/:uid/prekeys - 200" $ testGetUserPrekeysQualified b opts,
-      test p "get /users/:domain/:uid/prekeys - 4xx" $ testGetUserPrekeysInvalidDomain b,
+      test p "get /users/:domain/:uid/prekeys - 422" $ testGetUserPrekeysInvalidDomain b,
       test p "get /users/:uid/prekeys/:client - 200" $ testGetClientPrekey b,
       test p "get /users/<localdomain>/:uid/prekeys/:client - 200" $ testGetClientPrekeyQualified b opts,
       test p "post /users/prekeys" $ testMultiUserGetPrekeys b,

From a20a21a9cc27cb2af086eb4dc48b9448af3bbedf Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 13:48:41 +0200
Subject: [PATCH 178/220] Fix `make list-flaky-tests`.

---
 Makefile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 183442150d..7e5b5f7078 100644
--- a/Makefile
+++ b/Makefile
@@ -87,7 +87,7 @@ endif
 # ci here doesn't refer to continuous integration, but to cabal-run-integration.sh
 # Usage: make ci                        - build & run all tests, excluding integration
 #        make ci package=all            - build & run all tests, including integration
-#        make ci package=brig           - build brig & run "brig-integration" 
+#        make ci package=brig           - build brig & run "brig-integration"
 #        make ci package=integration    - build & run "integration"
 #
 # You can pass environment variables to all the suites, like so
@@ -128,7 +128,8 @@ sanitize-pr:
 
 list-flaky-tests:
 	@echo -e "\n\nif you want to run these, set RUN_FLAKY_TESTS=1\n\n"
-	@git grep -Hn '\bflakyTestCase \"'
+	@git grep -Hne '\bflakyTestCase \"'
+	@git grep -Hne '[^^]\bflakyTest\b'
 
 .PHONY: cabal-fmt
 cabal-fmt:

From 354252d31a0ace02d7f31d19706d3dec772f146f Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 22:10:41 +0200
Subject: [PATCH 179/220] nit-picks, renames, minor refactorings.

---
 libs/wire-api/src/Wire/API/FederationUpdate.hs | 11 ++++-------
 services/federator/src/Federator/Env.hs        |  2 +-
 services/federator/src/Federator/Response.hs   |  2 +-
 services/federator/src/Federator/Run.hs        |  4 ++--
 services/galley/src/Galley/App.hs              |  1 -
 5 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 73c8c740f8..2c27e3c0c2 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -19,9 +19,6 @@ import Wire.API.Routes.FederationDomainConfig (FederationDomainConfig (domain),
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
 
-getFedRemotes :: ClientM FederationDomainConfigs
-getFedRemotes = namedClient @IAPI.API @"get-federation-remotes"
-
 -- Initial function for getting the set of domains from brig, and an update interval
 getAllowedDomainsInitial :: L.Logger -> ClientEnv -> IO FederationDomainConfigs
 getAllowedDomainsInitial logger clientEnv =
@@ -35,7 +32,7 @@ getAllowedDomainsInitial logger clientEnv =
           Right s -> pure $ Just s
           Left e -> do
             L.log logger L.Info $
-              L.msg (L.val "Could not retrieve an initial list of federation domains from Brig.")
+              L.msg (L.val "Failed to reach brig for federation setup, retrying...")
                 L.~~ "error" L..= show e
             pure Nothing
    in R.retrying policy (const (pure . isNothing)) (const go) >>= \case
@@ -43,12 +40,12 @@ getAllowedDomainsInitial logger clientEnv =
         Nothing -> throwIO $ ErrorCall "*** Failed to reach brig for federation setup, giving up!"
 
 getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
-getAllowedDomains = runClientM getFedRemotes
+getAllowedDomains = runClientM (namedClient @IAPI.API @"get-federation-remotes")
 
--- Old value -> new value -> action
+-- | old value -> new value -> action
 type FedUpdateCallback = FederationDomainConfigs -> FederationDomainConfigs -> IO ()
 
--- The callback takes the previous and the new values of the federation domain configs
+-- | The callback takes the previous and the new values of the federation domain configs
 -- and runs a given action. This function is not called if a new config value cannot be fetched.
 getAllowedDomainsLoop :: L.Logger -> ClientEnv -> FedUpdateCallback -> IORef FederationDomainConfigs -> IO ()
 getAllowedDomainsLoop logger clientEnv callback env = forever $ do
diff --git a/services/federator/src/Federator/Env.hs b/services/federator/src/Federator/Env.hs
index b14f4212b2..660dc7a547 100644
--- a/services/federator/src/Federator/Env.hs
+++ b/services/federator/src/Federator/Env.hs
@@ -41,7 +41,7 @@ data Env = Env
     _requestId :: RequestId,
     _dnsResolver :: Resolver,
     _runSettings :: RunSettings,
-    _allowedRemoteDomains :: IORef FederationDomainConfigs,
+    _domainConfigs :: IORef FederationDomainConfigs,
     _service :: Component -> Endpoint,
     _httpManager :: HTTP.Manager,
     _http2Manager :: IORef Http2Manager
diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs
index aaa238dd35..e606777c14 100644
--- a/services/federator/src/Federator/Response.hs
+++ b/services/federator/src/Federator/Response.hs
@@ -145,7 +145,7 @@ runFederator env =
           DiscoveryFailure
         ]
     . runInputConst env
-    . runInputSem (embed @IO (readIORef (view allowedRemoteDomains env)))
+    . runInputSem (embed @IO (readIORef (view domainConfigs env)))
     . runInputSem (embed @IO (readIORef (view http2Manager env)))
     . runInputConst (view runSettings env)
     . interpretServiceHTTP
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index 7f3a37605f..bb33d9189c 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -61,9 +61,9 @@ import qualified Wire.Network.DNS.Helper as DNS
 -- FUTUREWORK(federation): Add metrics and status endpoints
 run :: Opts -> IO ()
 run opts = do
-  logger <- LogExt.mkLogger (Opt.logLevel opts) (Opt.logNetStrings opts) (Opt.logFormat opts)
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res -> do
+    logger <- LogExt.mkLogger (Opt.logLevel opts) (Opt.logNetStrings opts) (Opt.logFormat opts)
     (ioref, updateFedDomainsThread) <- updateFedDomains (brig opts) logger (\_ _ -> pure ())
     bracket (newEnv opts res logger ioref) closeEnv $ \env -> do
       let externalServer = serveInward env portExternal
@@ -92,7 +92,7 @@ run opts = do
 -- Environment
 
 newEnv :: Opts -> DNS.Resolver -> Log.Logger -> IORef FederationDomainConfigs -> IO Env
-newEnv o _dnsResolver _applog _allowedRemoteDomains = do
+newEnv o _dnsResolver _applog _domainConfigs = do
   _metrics <- Metrics.metrics
   let _requestId = def
       _runSettings = Opt.optSettings o
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index dfd0604755..c80f993cd9 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -1,4 +1,3 @@
-{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE StrictData #-}
 
 -- This file is part of the Wire Server implementation.

From a2432df2ad201ed182f9f33e7c6884f43fa3db45 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 22:44:53 +0200
Subject: [PATCH 180/220] s/type/newtype/

---
 libs/wire-api/src/Wire/API/FederationUpdate.hs | 16 ++++++++++++----
 services/cannon/src/Cannon/Run.hs              |  2 +-
 services/federator/src/Federator/Run.hs        |  2 +-
 services/galley/src/Galley/Run.hs              |  2 +-
 services/gundeck/src/Gundeck/Run.hs            |  2 +-
 5 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 2c27e3c0c2..f7ed1006f2 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -1,5 +1,6 @@
 module Wire.API.FederationUpdate
   ( FedUpdateCallback,
+    emptyFedUpdateCallback,
     updateFedDomains,
   )
 where
@@ -42,13 +43,20 @@ getAllowedDomainsInitial logger clientEnv =
 getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
 getAllowedDomains = runClientM (namedClient @IAPI.API @"get-federation-remotes")
 
--- | old value -> new value -> action
-type FedUpdateCallback = FederationDomainConfigs -> FederationDomainConfigs -> IO ()
-
 -- | The callback takes the previous and the new values of the federation domain configs
 -- and runs a given action. This function is not called if a new config value cannot be fetched.
+newtype FedUpdateCallback = FedUpdateCallback
+  { fromFedUpdateCallback ::
+      FederationDomainConfigs -> -- old value
+      FederationDomainConfigs -> -- new value
+      IO ()
+  }
+
+emptyFedUpdateCallback :: FedUpdateCallback
+emptyFedUpdateCallback = FedUpdateCallback $ \_ _ -> pure ()
+
 getAllowedDomainsLoop :: L.Logger -> ClientEnv -> FedUpdateCallback -> IORef FederationDomainConfigs -> IO ()
-getAllowedDomainsLoop logger clientEnv callback env = forever $ do
+getAllowedDomainsLoop logger clientEnv (FedUpdateCallback callback) env = forever $ do
   getAllowedDomains clientEnv >>= \case
     Left e ->
       L.log logger L.Info $
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index 81f198ac01..451b9e9f7b 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -85,7 +85,7 @@ run o = do
   -- Get the federation domain list from Brig and start the updater loop
   let brigEndpoint = Endpoint bh bp
       Brig bh bp = o ^. brig
-  (_, updateDomainsThread) <- updateFedDomains brigEndpoint g (\_ _ -> pure ())
+  (_, updateDomainsThread) <- updateFedDomains brigEndpoint g emptyFedUpdateCallback
 
   let middleware :: Wai.Middleware
       middleware =
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index bb33d9189c..f46f3f7ea1 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -64,7 +64,7 @@ run opts = do
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res -> do
     logger <- LogExt.mkLogger (Opt.logLevel opts) (Opt.logNetStrings opts) (Opt.logFormat opts)
-    (ioref, updateFedDomainsThread) <- updateFedDomains (brig opts) logger (\_ _ -> pure ())
+    (ioref, updateFedDomainsThread) <- updateFedDomains (brig opts) logger emptyFedUpdateCallback
     bracket (newEnv opts res logger ioref) closeEnv $ \env -> do
       let externalServer = serveInward env portExternal
           internalServer = serveOutward env portInternal
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 9b8ba9ab6e..ec01af7879 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -88,7 +88,7 @@ mkApp :: Opts -> Codensity IO (Application, Env)
 mkApp opts =
   do
     logger <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
-    (fedDoms, _) <- lift $ updateFedDomains (opts ^. optBrig) logger $ \_ _ -> pure ()
+    (fedDoms, _) <- lift $ updateFedDomains (opts ^. optBrig) logger emptyFedUpdateCallback
     metrics <- lift $ M.metrics
     env <- lift $ App.createEnv metrics opts logger fedDoms
     lift $ runClient (env ^. cstate) $ versionCheck schemaVersion
diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs
index e83b11a614..97413d1f45 100644
--- a/services/gundeck/src/Gundeck/Run.hs
+++ b/services/gundeck/src/Gundeck/Run.hs
@@ -67,7 +67,7 @@ run o = do
   let throttleMillis = fromMaybe defSqsThrottleMillis $ o ^. (optSettings . setSqsThrottleMillis)
 
   -- Get the federation domain list from Brig and start the updater loop
-  (_, updateDomainsThread) <- updateFedDomains (o ^. optBrig) l (\_ _ -> pure ())
+  (_, updateDomainsThread) <- updateFedDomains (o ^. optBrig) l emptyFedUpdateCallback
 
   lst <- Async.async $ Aws.execute (e ^. awsEnv) (Aws.listen throttleMillis (runDirect e . onEvent))
   wtbs <- forM (e ^. threadBudgetState) $ \tbs -> Async.async $ runDirect e $ watchThreadBudgetState m tbs 10

From 3318c4a6b0581daa0165a205df76c22ec31cbea2 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 22:56:02 +0200
Subject: [PATCH 181/220] Polish Wire.API.FederationUpdate (names and
 declaration order).

---
 .../wire-api/src/Wire/API/FederationUpdate.hs | 82 +++++++++----------
 services/cannon/src/Cannon/Run.hs             |  2 +-
 services/federator/src/Federator/Run.hs       |  2 +-
 services/galley/src/Galley/Run.hs             |  2 +-
 services/gundeck/src/Gundeck/Run.hs           |  2 +-
 5 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index f7ed1006f2..54be7d04ec 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -1,7 +1,7 @@
 module Wire.API.FederationUpdate
-  ( FedUpdateCallback,
-    emptyFedUpdateCallback,
-    updateFedDomains,
+  ( syncFedDomainConfigs,
+    SyncFedDomainConfigsCallback (..),
+    emptySyncFedDomainConfigsCallback,
   )
 where
 
@@ -13,23 +13,41 @@ import Data.Text (unpack)
 import Imports
 import Network.HTTP.Client (defaultManagerSettings, newManager)
 import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), ClientError, Scheme (Http), runClientM)
-import Servant.Client.Internal.HttpClient (ClientM, defaultMakeClientRequest)
+import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
 import qualified System.Logger as L
 import Util.Options (Endpoint (..))
 import Wire.API.Routes.FederationDomainConfig (FederationDomainConfig (domain), FederationDomainConfigs (remotes, updateInterval))
 import qualified Wire.API.Routes.Internal.Brig as IAPI
 import Wire.API.Routes.Named (namedClient)
 
--- Initial function for getting the set of domains from brig, and an update interval
-getAllowedDomainsInitial :: L.Logger -> ClientEnv -> IO FederationDomainConfigs
-getAllowedDomainsInitial logger clientEnv =
+-- | 'FedUpdateCallback' is not called if a new settings cannot be fetched, or if they are
+-- equal to the old settings.
+syncFedDomainConfigs :: Endpoint -> L.Logger -> SyncFedDomainConfigsCallback -> IO (IORef FederationDomainConfigs, Async ())
+syncFedDomainConfigs (Endpoint h p) log' cb = do
+  let baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
+  clientEnv <- newManager defaultManagerSettings <&> \mgr -> ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
+  ioref <- newIORef =<< initialize log' clientEnv
+  updateDomainsThread <-
+    async $
+      let go = finally
+            (loop log' clientEnv cb ioref)
+            $ do
+              L.log log' L.Error $ L.msg (L.val "Federation domain sync thread died, restarting domain synchronization.")
+              go
+       in go
+
+  pure (ioref, updateDomainsThread)
+
+-- | Initial function for getting the set of domains from brig, and an update interval
+initialize :: L.Logger -> ClientEnv -> IO FederationDomainConfigs
+initialize logger clientEnv =
   let -- keep trying every 3s for one minute
       policy :: R.RetryPolicy
       policy = R.constantDelay 3_081_003 <> R.limitRetries 20
 
       go :: IO (Maybe FederationDomainConfigs)
       go = do
-        getAllowedDomains clientEnv >>= \case
+        fetch clientEnv >>= \case
           Right s -> pure $ Just s
           Left e -> do
             L.log logger L.Info $
@@ -40,24 +58,9 @@ getAllowedDomainsInitial logger clientEnv =
         Just c -> pure c
         Nothing -> throwIO $ ErrorCall "*** Failed to reach brig for federation setup, giving up!"
 
-getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
-getAllowedDomains = runClientM (namedClient @IAPI.API @"get-federation-remotes")
-
--- | The callback takes the previous and the new values of the federation domain configs
--- and runs a given action. This function is not called if a new config value cannot be fetched.
-newtype FedUpdateCallback = FedUpdateCallback
-  { fromFedUpdateCallback ::
-      FederationDomainConfigs -> -- old value
-      FederationDomainConfigs -> -- new value
-      IO ()
-  }
-
-emptyFedUpdateCallback :: FedUpdateCallback
-emptyFedUpdateCallback = FedUpdateCallback $ \_ _ -> pure ()
-
-getAllowedDomainsLoop :: L.Logger -> ClientEnv -> FedUpdateCallback -> IORef FederationDomainConfigs -> IO ()
-getAllowedDomainsLoop logger clientEnv (FedUpdateCallback callback) env = forever $ do
-  getAllowedDomains clientEnv >>= \case
+loop :: L.Logger -> ClientEnv -> SyncFedDomainConfigsCallback -> IORef FederationDomainConfigs -> IO ()
+loop logger clientEnv (SyncFedDomainConfigsCallback callback) env = forever $ do
+  fetch clientEnv >>= \case
     Left e ->
       L.log logger L.Info $
         L.msg (L.val "Could not retrieve an updated list of federation domains from Brig; I'll keep trying!")
@@ -73,19 +76,16 @@ getAllowedDomainsLoop logger clientEnv (FedUpdateCallback callback) env = foreve
       Set.fromList (domain <$> remotes o)
         == Set.fromList (domain <$> remotes n)
 
-updateFedDomains :: Endpoint -> L.Logger -> FedUpdateCallback -> IO (IORef FederationDomainConfigs, Async ())
-updateFedDomains (Endpoint h p) log' cb = do
-  clientEnv <- newManager defaultManagerSettings <&> \mgr -> ClientEnv mgr baseUrl Nothing defaultMakeClientRequest
-  ioref <- newIORef =<< getAllowedDomainsInitial log' clientEnv
-  updateDomainsThread <-
-    async $
-      let go = finally
-            (getAllowedDomainsLoop log' clientEnv cb ioref)
-            $ do
-              L.log log' L.Error $ L.msg (L.val "Federation domain sync thread died, restarting domain synchronization.")
-              go
-       in go
+fetch :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
+fetch = runClientM (namedClient @IAPI.API @"get-federation-remotes")
 
-  pure (ioref, updateDomainsThread)
-  where
-    baseUrl = BaseUrl Http (unpack h) (fromIntegral p) ""
+-- | The callback takes the previous and the new settings and runs a given action.
+newtype SyncFedDomainConfigsCallback = SyncFedDomainConfigsCallback
+  { fromFedUpdateCallback ::
+      FederationDomainConfigs -> -- old value
+      FederationDomainConfigs -> -- new value
+      IO ()
+  }
+
+emptySyncFedDomainConfigsCallback :: SyncFedDomainConfigsCallback
+emptySyncFedDomainConfigsCallback = SyncFedDomainConfigsCallback $ \_ _ -> pure ()
diff --git a/services/cannon/src/Cannon/Run.hs b/services/cannon/src/Cannon/Run.hs
index 451b9e9f7b..19338dccf3 100644
--- a/services/cannon/src/Cannon/Run.hs
+++ b/services/cannon/src/Cannon/Run.hs
@@ -85,7 +85,7 @@ run o = do
   -- Get the federation domain list from Brig and start the updater loop
   let brigEndpoint = Endpoint bh bp
       Brig bh bp = o ^. brig
-  (_, updateDomainsThread) <- updateFedDomains brigEndpoint g emptyFedUpdateCallback
+  (_, updateDomainsThread) <- syncFedDomainConfigs brigEndpoint g emptySyncFedDomainConfigsCallback
 
   let middleware :: Wai.Middleware
       middleware =
diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs
index f46f3f7ea1..9d444e3085 100644
--- a/services/federator/src/Federator/Run.hs
+++ b/services/federator/src/Federator/Run.hs
@@ -64,7 +64,7 @@ run opts = do
   let resolvConf = mkResolvConf (optSettings opts) DNS.defaultResolvConf
   DNS.withCachingResolver resolvConf $ \res -> do
     logger <- LogExt.mkLogger (Opt.logLevel opts) (Opt.logNetStrings opts) (Opt.logFormat opts)
-    (ioref, updateFedDomainsThread) <- updateFedDomains (brig opts) logger emptyFedUpdateCallback
+    (ioref, updateFedDomainsThread) <- syncFedDomainConfigs (brig opts) logger emptySyncFedDomainConfigsCallback
     bracket (newEnv opts res logger ioref) closeEnv $ \env -> do
       let externalServer = serveInward env portExternal
           internalServer = serveOutward env portInternal
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index ec01af7879..eacde238bd 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -88,7 +88,7 @@ mkApp :: Opts -> Codensity IO (Application, Env)
 mkApp opts =
   do
     logger <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
-    (fedDoms, _) <- lift $ updateFedDomains (opts ^. optBrig) logger emptyFedUpdateCallback
+    (fedDoms, _) <- lift $ syncFedDomainConfigs (opts ^. optBrig) logger emptySyncFedDomainConfigsCallback
     metrics <- lift $ M.metrics
     env <- lift $ App.createEnv metrics opts logger fedDoms
     lift $ runClient (env ^. cstate) $ versionCheck schemaVersion
diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs
index 97413d1f45..f2703fb619 100644
--- a/services/gundeck/src/Gundeck/Run.hs
+++ b/services/gundeck/src/Gundeck/Run.hs
@@ -67,7 +67,7 @@ run o = do
   let throttleMillis = fromMaybe defSqsThrottleMillis $ o ^. (optSettings . setSqsThrottleMillis)
 
   -- Get the federation domain list from Brig and start the updater loop
-  (_, updateDomainsThread) <- updateFedDomains (o ^. optBrig) l emptyFedUpdateCallback
+  (_, updateDomainsThread) <- syncFedDomainConfigs (o ^. optBrig) l emptySyncFedDomainConfigsCallback
 
   lst <- Async.async $ Aws.execute (e ^. awsEnv) (Aws.listen throttleMillis (runDirect e . onEvent))
   wtbs <- forM (e ^. threadBudgetState) $ \tbs -> Async.async $ runDirect e $ watchThreadBudgetState m tbs 10

From 47ae9099ade5d2b8600b8ce61c41578511101353 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Mon, 26 Jun 2023 23:14:46 +0200
Subject: [PATCH 182/220] Fix: Always cancel `syncFedDomainUpdateThread`.

---
 services/galley/src/Galley/Run.hs            | 15 ++++++++++-----
 services/galley/test/integration/API/Util.hs |  2 +-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index eacde238bd..30fdf47379 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -68,7 +68,7 @@ import Wire.API.Routes.Version.Wai
 
 run :: Opts -> IO ()
 run opts = lowerCodensity $ do
-  (app, env) <- mkApp opts
+  (app, env, syndFedDomainConfigsThread) <- mkApp opts
   settings <-
     lift $
       newSettings $
@@ -82,13 +82,13 @@ run opts = lowerCodensity $ do
     void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
-  lift $ finally (runSettingsWithShutdown settings app Nothing) (shutdown (env ^. cstate))
+  lift $ finally (runSettingsWithShutdown settings app Nothing) (closeApp env syndFedDomainConfigsThread)
 
-mkApp :: Opts -> Codensity IO (Application, Env)
+mkApp :: Opts -> Codensity IO (Application, Env, Async.Async ())
 mkApp opts =
   do
     logger <- lift $ mkLogger (opts ^. optLogLevel) (opts ^. optLogNetStrings) (opts ^. optLogFormat)
-    (fedDoms, _) <- lift $ syncFedDomainConfigs (opts ^. optBrig) logger emptySyncFedDomainConfigsCallback
+    (fedDoms, syndFedDomainConfigsThread) <- lift $ syncFedDomainConfigs (opts ^. optBrig) logger emptySyncFedDomainConfigsCallback
     metrics <- lift $ M.metrics
     env <- lift $ App.createEnv metrics opts logger fedDoms
     lift $ runClient (env ^. cstate) $ versionCheck schemaVersion
@@ -102,7 +102,7 @@ mkApp opts =
       Log.info logger $ Log.msg @Text "Galley application finished."
       Log.flush logger
       Log.close logger
-    pure (middlewares $ servantApp env, env)
+    pure (middlewares $ servantApp env, env, syndFedDomainConfigsThread)
   where
     rtree = compile API.sitemap
     runGalley e r k = evalGalleyToIO e (route rtree r k)
@@ -125,6 +125,11 @@ mkApp opts =
     lookupReqId :: Request -> RequestId
     lookupReqId = maybe def RequestId . lookup requestIdName . requestHeaders
 
+closeApp :: Env -> Async.Async () -> IO ()
+closeApp env syndFedDomainConfigsThread = do
+  shutdown (env ^. cstate)
+  Async.cancel syndFedDomainConfigsThread
+
 customFormatters :: Servant.ErrorFormatters
 customFormatters =
   defaultErrorFormatters
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index b8ecd8ea53..7339e15578 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -2455,7 +2455,7 @@ instance HasSettingsOverrides TestM where
     ts :: TestSetup <- ask
     let opts = f (ts ^. tsGConf)
     liftIO . lowerCodensity $ do
-      (galleyApp, _env) <- Run.mkApp opts
+      (galleyApp, _env, _thread) <- Run.mkApp opts -- FUTUREWORK: always call Run.closeApp at the end.
       port' <- withMockServer galleyApp
       liftIO $
         runReaderT

From a830cdb51dc55c54a17e939c87f2c175bced49c6 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 27 Jun 2023 00:08:23 +0200
Subject: [PATCH 183/220] hi ci


From 3931f8906cf7416f2e1c540fb060d68cccde5130 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 27 Jun 2023 14:18:19 +1000
Subject: [PATCH 184/220] Fixing things post-merge

---
 .../wire-api/src/Wire/API/FederationUpdate.hs |   9 +-
 .../background-worker/background-worker.cabal |   1 +
 .../src/Wire/BackgroundWorker.hs              |  17 +--
 .../src/Wire/Defederation.hs                  |  38 ++++--
 services/brig/brig.cabal                      |   1 +
 services/brig/schema/src/Main.hs              |   4 +-
 .../schema/src/V78_ConnectionRemoteIndex.hs   |  17 +++
 services/brig/src/Brig/Data/Connection.hs     |   2 +-
 services/galley/galley.cabal                  |   1 +
 services/galley/schema/src/Main.hs            |   4 +-
 .../schema/src/V82_RemoteDomainIndexes.hs     |  20 ++++
 services/galley/src/Galley/API/Action.hs      |  16 ---
 services/galley/src/Galley/API/Internal.hs    |   9 +-
 services/galley/src/Galley/API/Update.hs      | 113 ------------------
 .../galley/src/Galley/Cassandra/Queries.hs    |   4 +-
 .../galley/test/integration/Federation.hs     |  51 ++++----
 16 files changed, 115 insertions(+), 192 deletions(-)
 create mode 100644 services/brig/schema/src/V78_ConnectionRemoteIndex.hs
 create mode 100644 services/galley/schema/src/V82_RemoteDomainIndexes.hs

diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 582b273c34..69dd0d4811 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -2,6 +2,7 @@ module Wire.API.FederationUpdate
   ( syncFedDomainConfigs,
     SyncFedDomainConfigsCallback (..),
     emptySyncFedDomainConfigsCallback,
+    deleteFederationRemoteGalley
   )
 where
 
@@ -13,7 +14,7 @@ import qualified Data.Set as Set
 import Data.Text (unpack)
 import Imports
 import Network.HTTP.Client (defaultManagerSettings, newManager)
-import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), ClientError, Scheme (Http), runClientM)
+import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), ClientError, Scheme (Http), runClientM, ClientM)
 import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
 import qualified System.Logger as L
 import Util.Options (Endpoint (..))
@@ -61,9 +62,6 @@ initialize logger clientEnv =
         Just c -> pure c
         Nothing -> throwIO $ ErrorCall "*** Failed to reach brig for federation setup, giving up!"
 
-getAllowedDomains :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
-getAllowedDomains = runClientM getFedRemotes
-
 deleteFederationRemoteGalley :: Domain -> ClientEnv -> IO (Either ClientError ())
 deleteFederationRemoteGalley dom = runClientM $ deleteFedRemoteGalley dom
 
@@ -88,9 +86,6 @@ loop logger clientEnv (SyncFedDomainConfigsCallback callback) env = forever $ do
 fetch :: ClientEnv -> IO (Either ClientError FederationDomainConfigs)
 fetch = runClientM (namedClient @IAPI.API @"get-federation-remotes")
 
-updateFedDomains' :: IORef FederationDomainConfigs -> ClientEnv -> L.Logger -> FedUpdateCallback -> IO (Async ())
-updateFedDomains' ioref clientEnv log' cb = async $ getAllowedDomainsLoop log' clientEnv cb ioref
-
 -- | The callback takes the previous and the new settings and runs a given action.
 newtype SyncFedDomainConfigsCallback = SyncFedDomainConfigsCallback
   { fromFedUpdateCallback ::
diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal
index 097179b112..91cb05aab3 100644
--- a/services/background-worker/background-worker.cabal
+++ b/services/background-worker/background-worker.cabal
@@ -30,6 +30,7 @@ library
       aeson
     , amqp
     , async
+    , bilge
     , bytestring
     , bytestring-conversion
     , exceptions
diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index 3a23b61f05..d8049743c0 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -15,6 +15,11 @@ run :: Opts -> IO ()
 run opts = do
   env <- mkEnv opts
   threadsRef <- newIORef []
+  let cancelThreads = do
+        -- Kill all of the threads and clean up the IORef
+        threads <- readIORef threadsRef
+        traverse_ cancel threads
+        atomicWriteIORef threadsRef []
   openConnectionWithRetries env.logger opts.rabbitmq.host opts.rabbitmq.port opts.rabbitmq.vHost $
     RabbitMqHooks
       { onNewChannel = \chan -> runAppT env $ do
@@ -25,6 +30,8 @@ run opts = do
           let threads = [pushThread, deleteThread]
           -- Write out the handles for the threads
           atomicWriteIORef threadsRef threads
+          -- Wait for all the threads. This shouldn't occure
+          -- as the threads all have `forever $ threadDelay ...`
           liftIO $ traverse_ wait threads
           -- clear the threadRef if the threads finish
           atomicWriteIORef threadsRef []
@@ -32,13 +39,7 @@ run opts = do
         --
         -- When the channel dies for whatever reason, kill all of the async
         -- threads and clean up the threadsRef state
-        , onChannelException = const $ do
-            threads <- readIORef threadsRef
-            traverse_ cancel threads
-            atomicWriteIORef threadsRef []
-        , onConnectionClose = do
-            threads <- readIORef threadsRef
-            traverse_ cancel threads
-            atomicWriteIORef threadsRef []
+        , onChannelException = const cancelThreads
+        , onConnectionClose = cancelThreads
       }
   forever $ threadDelay maxBound
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index 04d22707b4..1d72a343ab 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -17,22 +17,23 @@ import Control.Lens ((^.), to)
 import Data.ByteString.Conversion
 import qualified Data.ByteString.Lazy as L
 import Data.Text.Encoding
+import Control.Retry
+import Bilge.Retry
+import Control.Monad.Catch
 
 deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
 deleteFederationDomain chan = do
   lift $ ensureQueue chan defederationQueue
   QL.consumeMsgs chan (routingKey defederationQueue) Q.Ack deleteFederationDomainInner
 
+x3 :: RetryPolicy
+x3 = limitRetries 3 <> exponentialBackoff 100000
+
 -- What should we do with non-recoverable (unparsable) errors/messages?
 -- should we deadletter, or do something else?
--- Deadlettering has a privacy implication
---
--- We should also throttle the messages being retried.
---
--- Can we ensure that messages are handled in-order, one at a time?
--- Is this a `amqp` or rabbit thing?
+-- Deadlettering has a privacy implication -- FUTUREWORK.
 deleteFederationDomainInner :: RabbitMQEnvelope e => (Q.Message, e) -> AppT IO ()
-deleteFederationDomainInner (msg, envelope) = do -- recovering _ $ do -- retry x times, NACK after that?
+deleteFederationDomainInner (msg, envelope) = do
   env <- ask
   let manager = httpManager env
       req :: Domain -> Request
@@ -48,11 +49,20 @@ deleteFederationDomainInner (msg, envelope) = do -- recovering _ $ do -- retry x
   either
     (\e -> do
       logErr e
-      liftIO $ reject envelope True -- ensure that the message is requeued
+      -- ensure that the message is _NOT_ requeued
+      -- This means that we won't process this message again
+      -- as it is unparsable.
+      liftIO $ reject envelope False
     )
     (\d -> do
-      resp <- liftIO (httpLbs (req d) manager)
-      go resp
+      -- Retry the request a couple of times. If the final one fails, catch the exception\
+      -- so that we can NACK the message and requeue it.
+      resp <- try $ recovering x3 httpHandlers $ \_ -> liftIO $ httpLbs (req d) manager
+      either
+        -- Requeue the exception and rethrow the exception
+        (\(e :: SomeException) -> liftIO (reject envelope True) >> throwM e)
+        go
+        resp
     )
     $ A.eitherDecode @DefederationDomain (Q.msgBody msg)
   where
@@ -62,9 +72,13 @@ deleteFederationDomainInner (msg, envelope) = do -- recovering _ $ do -- retry x
       if code >= 200 && code <= 299
       then do
         liftIO $ ack envelope
-      else liftIO $ reject envelope True -- ensure that the message is requeued
+      else
+         -- ensure that the message is requeued
+         -- This message was able to be parsed but something
+         -- else in our stack failed and we should try again.
+        liftIO $ reject envelope True
     logErr err = Log.err $
-      Log.msg (Log.val "Failed delete federation domain")
+      Log.msg (Log.val "Failed to delete federation domain")
         . Log.field "error" err
 
 deleteWorker :: Q.Channel -> AppT IO (Async ())
diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 4cd6a01aa2..5704a0f811 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -663,6 +663,7 @@ executable brig-schema
     V75_AddOAuthCodeChallenge
     V76_AddSupportedProtocols
     V77_FederationRemotes
+    V78_ConnectionRemoteIndex
     V_FUTUREWORK
 
   hs-source-dirs:     schema/src
diff --git a/services/brig/schema/src/Main.hs b/services/brig/schema/src/Main.hs
index 8e1b05598a..706e017b95 100644
--- a/services/brig/schema/src/Main.hs
+++ b/services/brig/schema/src/Main.hs
@@ -57,6 +57,7 @@ import qualified V74_AddOAuthTables
 import qualified V75_AddOAuthCodeChallenge
 import qualified V76_AddSupportedProtocols
 import qualified V77_FederationRemotes
+import qualified V78_ConnectionRemoteIndex
 
 main :: IO ()
 main = do
@@ -101,7 +102,8 @@ main = do
       V74_AddOAuthTables.migration,
       V75_AddOAuthCodeChallenge.migration,
       V76_AddSupportedProtocols.migration,
-      V77_FederationRemotes.migration
+      V77_FederationRemotes.migration,
+      V78_ConnectionRemoteIndex.migration
       -- When adding migrations here, don't forget to update
       -- 'schemaVersion' in Brig.App
 
diff --git a/services/brig/schema/src/V78_ConnectionRemoteIndex.hs b/services/brig/schema/src/V78_ConnectionRemoteIndex.hs
new file mode 100644
index 0000000000..7be602f7c6
--- /dev/null
+++ b/services/brig/schema/src/V78_ConnectionRemoteIndex.hs
@@ -0,0 +1,17 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module V78_ConnectionRemoteIndex
+  ( migration,
+  )
+where
+
+import Cassandra.Schema
+import Imports
+import Text.RawString.QQ
+
+migration :: Migration
+migration = Migration 78 "Add a secondary index for federated (remote) connections" $ do
+  schema'
+    [r| CREATE INDEX on connection_remote (right_domain)
+      |]
diff --git a/services/brig/src/Brig/Data/Connection.hs b/services/brig/src/Brig/Data/Connection.hs
index aa3049df93..fcd91b4f51 100644
--- a/services/brig/src/Brig/Data/Connection.hs
+++ b/services/brig/src/Brig/Data/Connection.hs
@@ -397,7 +397,7 @@ remoteConnectionDelete :: PrepQuery W (UserId, Domain, UserId) ()
 remoteConnectionDelete = "DELETE FROM connection_remote where left = ? AND right_domain = ? AND right_user = ?"
 
 remoteConnectionSelectFromDomain :: PrepQuery R (Identity Domain) (UserId, Domain, UserId)
-remoteConnectionSelectFromDomain = "SELECT left, right_domain, right_user FROM connection_remote where right_domain = ? ALLOW FILTERING"
+remoteConnectionSelectFromDomain = "SELECT left, right_domain, right_user FROM connection_remote where right_domain = ?"
 
 remoteConnectionClear :: PrepQuery W (Identity UserId) ()
 remoteConnectionClear = "DELETE FROM connection_remote where left = ?"
diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index f32a0a05cb..a2b213e9f7 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -667,6 +667,7 @@ executable galley-schema
     V79_TeamFeatureMlsE2EId
     V80_AddConversationCodePassword
     V81_TeamFeatureMlsE2EIdUpdate
+    V82_RemoteDomainIndexes
 
   hs-source-dirs:     schema/src
   default-extensions:
diff --git a/services/galley/schema/src/Main.hs b/services/galley/schema/src/Main.hs
index f94b736ee3..a5b321e6af 100644
--- a/services/galley/schema/src/Main.hs
+++ b/services/galley/schema/src/Main.hs
@@ -84,6 +84,7 @@ import qualified V78_TeamFeatureOutlookCalIntegration
 import qualified V79_TeamFeatureMlsE2EId
 import qualified V80_AddConversationCodePassword
 import qualified V81_TeamFeatureMlsE2EIdUpdate
+import qualified V82_RemoteDomainIndexes
 
 main :: IO ()
 main = do
@@ -153,7 +154,8 @@ main = do
       V78_TeamFeatureOutlookCalIntegration.migration,
       V79_TeamFeatureMlsE2EId.migration,
       V80_AddConversationCodePassword.migration,
-      V81_TeamFeatureMlsE2EIdUpdate.migration
+      V81_TeamFeatureMlsE2EIdUpdate.migration,
+      V82_RemoteDomainIndexes.migration
       -- When adding migrations here, don't forget to update
       -- 'schemaVersion' in Galley.Cassandra
       -- (see also docs/developer/cassandra-interaction.md)
diff --git a/services/galley/schema/src/V82_RemoteDomainIndexes.hs b/services/galley/schema/src/V82_RemoteDomainIndexes.hs
new file mode 100644
index 0000000000..b2739db9fe
--- /dev/null
+++ b/services/galley/schema/src/V82_RemoteDomainIndexes.hs
@@ -0,0 +1,20 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module V82_RemoteDomainIndexes
+  ( migration,
+  )
+where
+
+import Cassandra.Schema
+import Imports
+import Text.RawString.QQ
+
+migration :: Migration
+migration = Migration 82 "Add a secondary index for remote domains on local conversations, and remote conversations with local membership" $ do
+  schema'
+    [r| CREATE INDEX on member_remote_user (user_remote_domain)
+      |]
+  schema'
+    [r| CREATE INDEX on user_remote_conv (conv_remote_domain)
+      |]
\ No newline at end of file
diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs
index 996c1f93f1..64388e8718 100644
--- a/services/galley/src/Galley/API/Action.hs
+++ b/services/galley/src/Galley/API/Action.hs
@@ -88,7 +88,6 @@ import Polysemy.Error
 import Polysemy.Input
 import Polysemy.TinyLog
 import qualified Polysemy.TinyLog as P
-import qualified Polysemy.TinyLog as TinyLog
 import qualified System.Logger as Log
 import Wire.API.Connection (Relation (Accepted))
 import Wire.API.Conversation hiding (Conversation, Member)
@@ -340,21 +339,6 @@ performAction tag origUser lconv action = do
       pure (mempty, action)
     SConversationRemoveMembersTag -> do
       let presentVictims = filter (isConvMemberL lconv) (toList action)
-      -- _ <-
-      --   error $
-      --     "-----------------------------\n\n\n"
-      --       <> "lconv = "
-      --       <> show lconv
-      --       <> "\n\n\n"
-      --       <> "action = "
-      --       <> show action
-      --       <> "\n\n\n"
-      --       <> "presentVictims = "
-      --       <> show presentVictims
-      --       <> "\n\n\n"
-      --       <> "-----------------------------"
-      TinyLog.err $ Log.msg ("action" :: String) . Log.field "values" (show action)
-      TinyLog.err $ Log.msg ("presentVictims" :: String) . Log.field "values" (show presentVictims)
       when (null presentVictims) noChanges
       E.deleteMembers (tUnqualified lcnv) (toUserList lconv presentVictims)
       pure (mempty, action) -- FUTUREWORK: should we return the filtered action here?
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index edd17dcc6a..77ea655f3d 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -512,7 +512,6 @@ internalDeleteFederationDomainH :: (Member (Input Env) r, Member (P.Logger (Msg
           Member CodeStore r, Member TeamStore r, Member BrigAccess r, Member GundeckAccess r, Member ExternalAccess r) => Domain ::: JSON -> Sem r Response
 internalDeleteFederationDomainH (domain ::: _) = do
   deleteFederationDomain domain
-  -- TODO: Do we generally also accept HTTP 204, No Content?
   pure (empty & setStatus status200)
 
 -- Remove remote members from local conversations
@@ -600,7 +599,13 @@ deleteFederationDomainLocalUserFromRemoteConversation dom = do
 -- let rcnv = toRemoteUnsafe dom cnv
 -- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
 
--- These need to be recoverable? TODO
+-- These need to be recoverable?
+-- This is recoverable with the following flow conditions.
+-- 1) Deletion calls to the Brig endpoint `delete-federation-remote-from-galley` are idempotent for a given domain.
+-- 2) This call is made from a function that is backed by a RabbitMQ queue.
+--    The calling function needs to catch thrown exceptions and NACK the deletion
+--    message. This will allow Rabbit to redeliver the message and give us a second
+--    go at performing the deletion.
 deleteFederationDomainOneOnOne :: (Member (Input Env) r, Member (Embed IO) r, Member (P.Logger (Msg -> Msg)) r) => Domain -> Sem r ()
 deleteFederationDomainOneOnOne dom = do
   env <- input
diff --git a/services/galley/src/Galley/API/Update.hs b/services/galley/src/Galley/API/Update.hs
index d9b32f1455..67f32c959c 100644
--- a/services/galley/src/Galley/API/Update.hs
+++ b/services/galley/src/Galley/API/Update.hs
@@ -370,7 +370,6 @@ updateRemoteConversation rcnv lusr conn action = getUpdateResult $ do
     ConversationUpdateResponseError err' -> rethrowErrors @(HasConversationActionGalleyErrors tag) err'
     ConversationUpdateResponseUpdate convUpdate _failedToProcess -> pure convUpdate
   updateLocalStateOfRemoteConv (qualifyAs rcnv convUpdate) (Just conn) >>= note NoChanges
-  notifyRemoteConversationAction lusr (qualifyAs rcnv convUpdate) (Just conn) -- TODO still needed?
 
 updateConversationReceiptModeUnqualified ::
   ( Member BrigAccess r,
@@ -1598,118 +1597,6 @@ rmBot lusr zcon b = do
         E.deliverAsync (bots `zip` repeat e)
         pure $ Updated e
 
-<<<<<<< HEAD
--- | Update the local database with information on conversation members joining
--- or leaving. Finally, push out notifications to local users.
-updateLocalStateOfRemoteConv ::
-  ( Member BrigAccess r,
-    Member GundeckAccess r,
-    Member ExternalAccess r,
-    Member (Input (Local ())) r,
-    Member MemberStore r,
-    Member P.TinyLog r
-  ) =>
-  Domain ->
-  F.ConversationUpdate ->
-  Sem r ()
-updateLocalStateOfRemoteConv requestingDomain cu = do
-  loc <- qualifyLocal ()
-  let rconvId = toRemoteUnsafe requestingDomain (F.cuConvId cu)
-      qconvId = tUntagged rconvId
-
-  -- Note: we generally do not send notifications to users that are not part of
-  -- the conversation (from our point of view), to prevent spam from the remote
-  -- backend. See also the comment below.
-  (presentUsers, allUsersArePresent) <-
-    E.selectRemoteMembers (F.cuAlreadyPresentUsers cu) rconvId
-
-  -- Perform action, and determine extra notification targets.
-  --
-  -- When new users are being added to the conversation, we consider them as
-  -- notification targets. Since we check connections before letting
-  -- people being added, this is safe against spam. However, if users that
-  -- are not in the conversations are being removed or have their membership state
-  -- updated, we do **not** add them to the list of targets, because we have no
-  -- way to make sure that they are actually supposed to receive that notification.
-
-  (mActualAction :: Maybe SomeConversationAction, extraTargets :: [UserId]) <- case F.cuAction cu of
-    sca@(SomeConversationAction singTag action) -> case singTag of
-      SConversationJoinTag -> do
-        let ConversationJoin toAdd role = action
-        let (localUsers, remoteUsers) = partitionQualified loc toAdd
-        addedLocalUsers <- Set.toList <$> addLocalUsersToRemoteConv rconvId (F.cuOrigUserId cu) localUsers
-        let allAddedUsers = map (tUntagged . qualifyAs loc) addedLocalUsers <> map tUntagged remoteUsers
-        case allAddedUsers of
-          [] -> pure (Nothing, []) -- If no users get added, its like no action was performed.
-          (u : us) -> pure (Just (SomeConversationAction (sing @'ConversationJoinTag) (ConversationJoin (u :| us) role)), addedLocalUsers)
-      SConversationLeaveTag -> do
-        let users = foldQualified loc (pure . tUnqualified) (const []) (F.cuOrigUserId cu)
-        E.deleteMembersInRemoteConversation rconvId users
-        pure (Just sca, [])
-      SConversationRemoveMembersTag -> do
-        let localUsers = getLocalUsers (tDomain loc) action
-        E.deleteMembersInRemoteConversation rconvId localUsers
-        pure (Just sca, [])
-      SConversationMemberUpdateTag ->
-        pure (Just sca, [])
-      SConversationDeleteTag -> do
-        -- Present users comes from `cuAlreadyPresentUsers`, so
-        -- any users that need to be deleted have to be included in it.
-        E.deleteMembersInRemoteConversation rconvId presentUsers
-        pure (Just sca, [])
-      SConversationRenameTag -> pure (Just sca, [])
-      SConversationMessageTimerUpdateTag -> pure (Just sca, [])
-      SConversationReceiptModeUpdateTag -> pure (Just sca, [])
-      SConversationAccessDataTag -> pure (Just sca, [])
-
-  unless allUsersArePresent $
-    P.warn $
-      Log.field "conversation" (toByteString' (F.cuConvId cu))
-        . Log.field "domain" (toByteString' requestingDomain)
-        . Log.msg
-          ( "Attempt to send notification about conversation update \
-            \to users not in the conversation" ::
-              ByteString
-          )
-
-  -- Send notifications
-  for_ mActualAction $ \(SomeConversationAction tag action) -> do
-    let event = conversationActionToEvent tag (F.cuTime cu) (F.cuOrigUserId cu) qconvId Nothing action
-        targets = nubOrd $ presentUsers <> extraTargets
-    -- FUTUREWORK: support bots?
-    pushConversationEvent Nothing event (qualifyAs loc targets) []
-
-addLocalUsersToRemoteConv ::
-  ( Member BrigAccess r,
-    Member MemberStore r,
-    Member P.TinyLog r
-  ) =>
-  Remote ConvId ->
-  Qualified UserId ->
-  [UserId] ->
-  Sem r (Set UserId)
-addLocalUsersToRemoteConv remoteConvId qAdder localUsers = do
-  connStatus <- E.getConnections localUsers (Just [qAdder]) (Just Accepted)
-  let localUserIdsSet = Set.fromList localUsers
-      connected = Set.fromList $ fmap csv2From connStatus
-      unconnected = Set.difference localUserIdsSet connected
-      connectedList = Set.toList connected
-
-  -- FUTUREWORK: Consider handling the discrepancy between the views of the
-  -- conversation-owning backend and the local backend
-  unless (Set.null unconnected) $
-    P.warn $
-      Log.msg ("A remote user is trying to add unconnected local users to a remote conversation" :: Text)
-        . Log.field "remote_user" (show qAdder)
-        . Log.field "local_unconnected_users" (show unconnected)
-
-  -- Update the local view of the remote conversation by adding only those local
-  -- users that are connected to the adder
-  E.createMembersInRemoteConversation remoteConvId connectedList
-  pure connected
-
-=======
->>>>>>> lepsa/FS-1115
 -------------------------------------------------------------------------------
 -- Helpers
 
diff --git a/services/galley/src/Galley/Cassandra/Queries.hs b/services/galley/src/Galley/Cassandra/Queries.hs
index da287e71b2..8d53bf938d 100644
--- a/services/galley/src/Galley/Cassandra/Queries.hs
+++ b/services/galley/src/Galley/Cassandra/Queries.hs
@@ -372,7 +372,7 @@ updateRemoteMemberConvRoleName = "update member_remote_user set conversation_rol
 -- Used when removing a federation domain, so that we can quickly list all of the affected remote users and conversations
 -- This returns local conversation IDs and remote users
 selectRemoteMembersByDomain :: PrepQuery R (Identity Domain) (ConvId, UserId, RoleName)
-selectRemoteMembersByDomain = "select conv, user_remote_id, conversation_role from member_remote_user where user_remote_domain = ? ALLOW FILTERING"
+selectRemoteMembersByDomain = "select conv, user_remote_id, conversation_role from member_remote_user where user_remote_domain = ?"
 
 -- local user with remote conversations
 
@@ -394,7 +394,7 @@ deleteUserRemoteConv = "delete from user_remote_conv where user = ? and conv_rem
 -- Used when removing a federation domain, so that we can quickly list all of the affected local users and conversations
 -- This returns remote conversation IDs and local users
 selectLocalMembersByDomain :: PrepQuery R (Identity Domain) (ConvId, UserId)
-selectLocalMembersByDomain = "select conv_remote_id, user from user_remote_conv where conv_remote_domain = ? ALLOW FILTERING"
+selectLocalMembersByDomain = "select conv_remote_id, user from user_remote_conv where conv_remote_domain = ?"
 
 -- remote conversation status for local user
 
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 04c267fcc6..39b48b1246 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -1,4 +1,5 @@
 {-# LANGUAGE RecordWildCards #-}
+{-# OPTIONS_GHC -Wno-unused-matches #-}
 
 module Federation where
 
@@ -20,7 +21,6 @@ import qualified Data.Set as Set
 import Data.Singletons
 import Data.Time (getCurrentTime)
 import qualified Data.UUID as UUID
-import Data.UUID.V4 (nextRandom)
 import Federator.MockServer
 import Galley.API.Util
 import Galley.Cassandra.Queries
@@ -50,6 +50,7 @@ import qualified Wire.API.Routes.MultiTablePaging as Public
 import Wire.API.User.Search
 import Galley.App
 import Galley.API.Internal
+-- import Control.Concurrent.Async
 
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
@@ -57,11 +58,9 @@ x3 = limitRetries 3 <> exponentialBackoff 100000
 isConvMemberLTests :: TestM ()
 isConvMemberLTests = do
   s <- ask
-  uuid <- liftIO nextRandom
-  let uuid' = UUID.toText uuid
-      opts = s ^. tsGConf
+  let opts = s ^. tsGConf
       localDomain = opts ^. optSettings . setFederationDomain
-      remoteDomain = Domain $ "far-away.example.com" <> uuid'
+      remoteDomain = Domain "far-away.example.com"
       convId = Id $ fromJust $ UUID.fromString "8cc34301-6949-46c5-bb93-00a72268e2f5"
       convLocalMembers = [LocalMember userId defMemberStatus Nothing roleNameWireMember]
       convRemoteMembers = [RemoteMember rUserId roleNameWireMember]
@@ -91,13 +90,13 @@ updateFedDomainsTestNoop' = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  (_, env, _) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
-  uuid <- liftIO nextRandom
-  let uuid' = UUID.toText uuid
-      interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
-      remoteDomain = Domain $ "far-away.example.com" <> uuid'
-      remoteDomain2 = Domain $ "far-away-two.example.com" <> uuid'
+  -- FUTUREWORK, NEWTICKET: These uuid strings side step issues with the tests hanging.
+  -- FUTUREWORK, NEWTICKET: Figure out the underlying issue as to why these tests occasionally hang.
+  let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain = Domain "far-away.example.com"
+      remoteDomain2 = Domain "far-away-two.example.com"
   liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
   liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
   -- Setup a conversation for a known remote domain.
@@ -112,13 +111,11 @@ updateFedDomainsTestAddRemote' = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  (_, env, _) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
-  uuid <- liftIO nextRandom
-  let uuid' = UUID.toText uuid
-      interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
-      remoteDomain = Domain $ "far-away.example.com" <> uuid'
-      remoteDomain2 = Domain $ "far-away-two.example.com" <> uuid'
+  let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain = Domain "far-away.example.com"
+      remoteDomain2 = Domain "far-away-two.example.com"
   liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
   liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
 
@@ -131,13 +128,11 @@ updateFedDomainsTestRemoveRemoteFromLocal' = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  (_, env, _) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
-  uuid <- liftIO nextRandom
-  let uuid' = UUID.toText uuid
-      interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
-      remoteDomain = Domain $ "far-away.example.com" <> uuid'
-      remoteDomain2 = Domain $ "far-away-two.example.com" <> uuid'
+  let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain = Domain "far-away.example.com"
+      remoteDomain2 = Domain "far-away-two.example.com"
   liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
   liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
   
@@ -150,13 +145,11 @@ updateFedDomainsTestRemoveLocalFromRemote' = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
+  (_, env, _) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
-  uuid <- liftIO nextRandom
-  let uuid' = UUID.toText uuid
-      interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
-      remoteDomain = Domain $ "far-away.example.com" <> uuid'
-      remoteDomain2 = Domain $ "far-away-two.example.com" <> uuid'
+  let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
+      remoteDomain = Domain "far-away.example.com"
+      remoteDomain2 = Domain "far-away-two.example.com"
   liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
   liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
 

From 5d8c001c7779818e8058cbb4b022477fbcf67eb7 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 27 Jun 2023 15:09:39 +1000
Subject: [PATCH 185/220] Adding a changelog entry

---
 changelog.d/6-federation/fs-1179 | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/6-federation/fs-1179

diff --git a/changelog.d/6-federation/fs-1179 b/changelog.d/6-federation/fs-1179
new file mode 100644
index 0000000000..589f66fccf
--- /dev/null
+++ b/changelog.d/6-federation/fs-1179
@@ -0,0 +1 @@
+Removing a federation domain will now remove all conversations for that domain.
\ No newline at end of file

From 4ed00ced374867cd2b1fc150a676f8a7ee9e431d Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 27 Jun 2023 17:22:40 +1000
Subject: [PATCH 186/220] FS-1179: Brig now deletes the notification queue for
 deleted domains

---
 .../API/Federation/BackendNotifications.hs    |   1 +
 .../wire-api/src/Wire/API/FederationUpdate.hs |   4 +-
 .../background-worker/background-worker.cabal |   4 +
 .../background-worker.integration.yaml        |   9 +-
 services/background-worker/default.nix        |  56 ++++-
 .../src/Wire/BackendNotificationPusher.hs     |   4 +-
 .../src/Wire/BackgroundWorker.hs              |  35 +++-
 .../src/Wire/BackgroundWorker/Env.hs          |  34 +++-
 .../src/Wire/BackgroundWorker/Options.hs      |   3 +-
 .../src/Wire/BackgroundWorker/Util.hs         |   2 +-
 .../src/Wire/Defederation.hs                  |  93 ++++-----
 services/background-worker/test/Main.hs       |  21 +-
 .../Wire/BackendNotificationPusherSpec.hs     | 192 +++++++++---------
 .../test/Test/Wire/Defederation.hs            |  78 +++----
 .../background-worker/test/Test/Wire/Util.hs  |  49 +++--
 services/brig/src/Brig/API/Internal.hs        |  29 ++-
 .../schema/src/V82_RemoteDomainIndexes.hs     |   2 +-
 services/galley/src/Galley/API/Internal.hs    | 120 +++++++----
 services/galley/src/Galley/Options.hs         |   2 +-
 services/galley/src/Galley/Run.hs             |   4 +-
 .../galley/test/integration/Federation.hs     |   7 +-
 services/galley/test/integration/Main.hs      |  12 +-
 22 files changed, 459 insertions(+), 302 deletions(-)

diff --git a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
index ec43ff3db0..1d2b9cb5af 100644
--- a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
+++ b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
@@ -88,6 +88,7 @@ routingKey t = "backend-notifications." <> t
 -- kept in sync about what types they are expecting and where
 -- they are stored in Rabbit.
 type DefederationDomain = Domain
+
 defederationQueue :: Text
 defederationQueue = "delete-federation"
 
diff --git a/libs/wire-api/src/Wire/API/FederationUpdate.hs b/libs/wire-api/src/Wire/API/FederationUpdate.hs
index 69dd0d4811..bd75555d36 100644
--- a/libs/wire-api/src/Wire/API/FederationUpdate.hs
+++ b/libs/wire-api/src/Wire/API/FederationUpdate.hs
@@ -2,7 +2,7 @@ module Wire.API.FederationUpdate
   ( syncFedDomainConfigs,
     SyncFedDomainConfigsCallback (..),
     emptySyncFedDomainConfigsCallback,
-    deleteFederationRemoteGalley
+    deleteFederationRemoteGalley,
   )
 where
 
@@ -14,7 +14,7 @@ import qualified Data.Set as Set
 import Data.Text (unpack)
 import Imports
 import Network.HTTP.Client (defaultManagerSettings, newManager)
-import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), ClientError, Scheme (Http), runClientM, ClientM)
+import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), ClientError, ClientM, Scheme (Http), runClientM)
 import Servant.Client.Internal.HttpClient (defaultMakeClientRequest)
 import qualified System.Logger as L
 import Util.Options (Endpoint (..))
diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal
index 91cb05aab3..57289aad2e 100644
--- a/services/background-worker/background-worker.cabal
+++ b/services/background-worker/background-worker.cabal
@@ -30,6 +30,7 @@ library
       aeson
     , amqp
     , async
+    , base
     , bilge
     , bytestring
     , bytestring-conversion
@@ -47,6 +48,7 @@ library
     , tinylog
     , transformers-base
     , types-common
+    , wire-api
     , wire-api-federation
 
   default-extensions:
@@ -170,6 +172,7 @@ executable background-worker-integration
   build-depends:
       aeson
     , amqp
+    , async
     , background-worker
     , base
     , exceptions
@@ -250,6 +253,7 @@ test-suite background-worker-test
   build-depends:
       aeson
     , amqp
+    , async
     , background-worker
     , base
     , exceptions
diff --git a/services/background-worker/background-worker.integration.yaml b/services/background-worker/background-worker.integration.yaml
index c5d532e886..6a77eed0a1 100644
--- a/services/background-worker/background-worker.integration.yaml
+++ b/services/background-worker/background-worker.integration.yaml
@@ -8,10 +8,11 @@ galley:
   host: 127.0.0.1
   port: 8085
 
+brig:
+  host: 127.0.0.1
+  port: 8082
+
 rabbitmq:
   host: 127.0.0.1
   port: 5672
-  vHost: /
-
-remoteDomains:
-  - b.example.com
+  vHost: /
\ No newline at end of file
diff --git a/services/background-worker/default.nix b/services/background-worker/default.nix
index c737582585..0e4392c4bb 100644
--- a/services/background-worker/default.nix
+++ b/services/background-worker/default.nix
@@ -5,23 +5,37 @@
 { mkDerivation
 , aeson
 , amqp
+, async
+, base
+, bilge
+, bytestring
+, bytestring-conversion
 , exceptions
 , extended
 , federator
 , gitignoreSource
 , HsOpenSSL
 , hspec
+, http-client
+, http-types
 , http2-manager
 , imports
+, lens
 , lib
 , monad-control
+, optparse-applicative
 , QuickCheck
 , retry
+, tagged
+, tasty
+, tasty-hunit
+, text
 , tinylog
 , transformers-base
 , types-common
 , wire-api
 , wire-api-federation
+, yaml
 }:
 mkDerivation {
   pname = "background-worker";
@@ -32,32 +46,68 @@ mkDerivation {
   libraryHaskellDepends = [
     aeson
     amqp
+    async
+    base
+    bilge
+    bytestring
+    bytestring-conversion
     exceptions
     extended
     HsOpenSSL
+    http-client
+    http-types
     http2-manager
     imports
+    lens
     monad-control
     retry
+    text
     tinylog
     transformers-base
     types-common
+    wire-api
+    wire-api-federation
+  ];
+  executableHaskellDepends = [
+    aeson
+    amqp
+    async
+    base
+    exceptions
+    federator
+    HsOpenSSL
+    hspec
+    imports
+    optparse-applicative
+    QuickCheck
+    tagged
+    tasty
+    tasty-hunit
+    types-common
+    wire-api
     wire-api-federation
+    yaml
   ];
-  executableHaskellDepends = [ HsOpenSSL imports types-common ];
   testHaskellDepends = [
     aeson
     amqp
+    async
+    base
+    exceptions
     federator
+    HsOpenSSL
     hspec
     imports
+    optparse-applicative
     QuickCheck
-    tinylog
+    tagged
+    tasty
+    tasty-hunit
     types-common
     wire-api
     wire-api-federation
+    yaml
   ];
   description = "Runs background work";
   license = lib.licenses.agpl3Only;
-  mainProgram = "background-worker";
 }
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index a740954f19..8d520f87ba 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -2,6 +2,7 @@
 
 module Wire.BackendNotificationPusher where
 
+import Control.Concurrent.Async
 import Control.Monad.Catch
 import Control.Retry
 import qualified Data.Aeson as A
@@ -14,7 +15,6 @@ import Wire.API.Federation.BackendNotifications
 import Wire.API.Federation.Client
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
-import Control.Concurrent.Async
 
 startPushingNotifications ::
   Q.Channel ->
@@ -80,4 +80,4 @@ startWorker remoteDomains chan = do
   env <- ask
   liftIO $ async $ do
     mapM_ (runAppT env . startPushingNotifications chan) remoteDomains
-    forever $ threadDelay maxBound
\ No newline at end of file
+    forever $ threadDelay maxBound
diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index d8049743c0..c20d72c842 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -2,18 +2,20 @@
 
 module Wire.BackgroundWorker where
 
+import Control.Concurrent.Async
+import Control.Concurrent.Chan
 import Imports
 import Network.AMQP.Extended
+import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.BackendNotificationPusher as BackendNotificationPusher
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Options
 import Wire.Defederation
-import Control.Concurrent.Async
 
 -- FUTUREWORK: Start an http service with status and metrics endpoints
 run :: Opts -> IO ()
 run opts = do
-  env <- mkEnv opts
+  (env, syncThread) <- mkEnv opts
   threadsRef <- newIORef []
   let cancelThreads = do
         -- Kill all of the threads and clean up the IORef
@@ -25,8 +27,26 @@ run opts = do
       { onNewChannel = \chan -> runAppT env $ do
           -- Channels are threadsafe: https://hackage.haskell.org/package/amqp-0.22.1/docs/Network-AMQP.html
           -- So we can async them for concurrency.
-          pushThread <- BackendNotificationPusher.startWorker opts.remoteDomains chan
           deleteThread <- deleteWorker chan
+
+          -- Since this is feeding off a Rabbit queue, it should
+          -- be safe to kill and start these threads. At worst, we
+          -- will double deliver some messages
+          pushThread <- do
+            -- get an initial list of domains from the IORef
+            initRemotes <- liftIO $ readIORef env.remoteDomains
+            -- Start the notification pusher using the initial domains
+            thread <- BackendNotificationPusher.startWorker (domain <$> initRemotes.remotes) chan
+            let go asyncThread = do
+                  -- Wait for a new set of domains from the Chan
+                  remotes <- liftIO $ readChan $ env.remoteDomainsChan
+                  -- When we have new domains, kill the previous pusher thread
+                  liftIO $ cancel asyncThread
+                  -- Start a new pusher thread and then wait for new domains
+                  -- TODO: There is a nicer way of doing this using
+                  go =<< BackendNotificationPusher.startWorker (domain <$> remotes.remotes) chan
+            go thread
+
           let threads = [pushThread, deleteThread]
           -- Write out the handles for the threads
           atomicWriteIORef threadsRef threads
@@ -34,12 +54,13 @@ run opts = do
           -- as the threads all have `forever $ threadDelay ...`
           liftIO $ traverse_ wait threads
           -- clear the threadRef if the threads finish
-          atomicWriteIORef threadsRef []
+          atomicWriteIORef threadsRef [],
         -- FUTUREWORK: Use these for metrics
         --
         -- When the channel dies for whatever reason, kill all of the async
         -- threads and clean up the threadsRef state
-        , onChannelException = const cancelThreads
-        , onConnectionClose = cancelThreads
+        onChannelException = const cancelThreads,
+        onConnectionClose = cancelThreads
       }
-  forever $ threadDelay maxBound
+  void $ forever $ threadDelay maxBound
+  liftIO $ cancel syncThread
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
index 47dc657fb6..8da1cde4cc 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
@@ -3,19 +3,23 @@
 
 module Wire.BackgroundWorker.Env where
 
+import Control.Concurrent.Async
+import Control.Concurrent.Chan
 import Control.Monad.Base
 import Control.Monad.Catch
 import Control.Monad.Trans.Control
 import HTTP2.Client.Manager
 import Imports
+import Network.HTTP.Client
 import OpenSSL.Session (SSLOption (..))
 import qualified OpenSSL.Session as SSL
 import qualified System.Logger as Log
-import System.Logger.Class
+import System.Logger.Class (Logger, MonadLogger (..))
 import qualified System.Logger.Extended as Log
 import Util.Options
+import Wire.API.FederationUpdate
+import Wire.API.Routes.FederationDomainConfig
 import Wire.BackgroundWorker.Options
-import Network.HTTP.Client
 
 data Env = Env
   { http2Manager :: Http2Manager,
@@ -23,21 +27,33 @@ data Env = Env
     logger :: Logger,
     federatorInternal :: Endpoint,
     galley :: Endpoint,
-    defederationTimeout :: ResponseTimeout
+    brig :: Endpoint,
+    defederationTimeout :: ResponseTimeout,
+    remoteDomains :: IORef FederationDomainConfigs,
+    remoteDomainsChan :: Chan FederationDomainConfigs
   }
 
-mkEnv :: Opts -> IO Env
+mkEnv :: Opts -> IO (Env, Async ())
 mkEnv opts = do
   http2Manager <- initHttp2Manager
   logger <- Log.mkLogger opts.logLevel Nothing opts.logFormat
   httpManager <- newManager defaultManagerSettings
+  remoteDomainsChan <- newChan
   let federatorInternal = opts.federatorInternal
       galley = opts.galley
-      defederationTimeout = maybe
-        responseTimeoutNone
-        (\t -> responseTimeoutMicro $ 1000000 * t) -- seconds to microseconds
-        opts.defederationTimeout
-  pure Env {..}
+      defederationTimeout =
+        maybe
+          responseTimeoutNone
+          (\t -> responseTimeoutMicro $ 1000000 * t) -- seconds to microseconds
+          opts.defederationTimeout
+      brig = opts.brig
+      callback =
+        SyncFedDomainConfigsCallback
+          { fromFedUpdateCallback = \_old new -> do
+              writeChan remoteDomainsChan new
+          }
+  (remoteDomains, syncThread) <- syncFedDomainConfigs brig logger callback
+  pure (Env {..}, syncThread)
 
 initHttp2Manager :: IO Http2Manager
 initHttp2Manager = do
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
index e3331dfed5..6faa47c971 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
@@ -1,7 +1,6 @@
 module Wire.BackgroundWorker.Options where
 
 import Data.Aeson
-import Data.Domain
 import Imports
 import Network.AMQP.Extended
 import System.Logger.Extended
@@ -12,8 +11,8 @@ data Opts = Opts
     logFormat :: !(Maybe (Last LogFormat)),
     federatorInternal :: !Endpoint,
     galley :: !Endpoint,
+    brig :: !Endpoint,
     rabbitmq :: !RabbitMqOpts,
-    remoteDomains :: [Domain],
     defederationTimeout :: Maybe Int -- Seconds, Nothing for no timeout
   }
   deriving (Show, Generic)
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Util.hs b/services/background-worker/src/Wire/BackgroundWorker/Util.hs
index 6a23bcf11f..0bb39999a8 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Util.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Util.hs
@@ -11,4 +11,4 @@ class RabbitMQEnvelope e where
 
 instance RabbitMQEnvelope Q.Envelope where
   ack = Q.ackEnv
-  reject = Q.rejectEnv
\ No newline at end of file
+  reject = Q.rejectEnv
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index 1d72a343ab..a538da3df5 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -1,25 +1,25 @@
 module Wire.Defederation where
 
+import Bilge.Retry
+import Control.Concurrent.Async
+import Control.Lens (to, (^.))
+import Control.Monad.Catch
+import Control.Retry
 import qualified Data.Aeson as A
+import Data.ByteString.Conversion
+import qualified Data.ByteString.Lazy as L
 import Data.Domain
+import Data.Text.Encoding
 import Imports
 import qualified Network.AMQP as Q
 import qualified Network.AMQP.Lifted as QL
+import Network.HTTP.Client
+import Network.HTTP.Types
 import qualified System.Logger.Class as Log
+import Util.Options
 import Wire.API.Federation.BackendNotifications
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
-import Control.Concurrent.Async
-import Network.HTTP.Client
-import Network.HTTP.Types
-import Util.Options
-import Control.Lens ((^.), to)
-import Data.ByteString.Conversion
-import qualified Data.ByteString.Lazy as L
-import Data.Text.Encoding
-import Control.Retry
-import Bilge.Retry
-import Control.Monad.Catch
 
 deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
 deleteFederationDomain chan = do
@@ -37,32 +37,33 @@ deleteFederationDomainInner (msg, envelope) = do
   env <- ask
   let manager = httpManager env
       req :: Domain -> Request
-      req dom = defaultRequest
-        { method = methodDelete
-        , secure = False
-        , host = galley env ^. epHost . to encodeUtf8
-        , port = galley env ^. epPort . to fromIntegral
-        , path = "/i/federation/" <> toByteString' dom
-        , requestHeaders = ("Accept", "application/json") : requestHeaders defaultRequest
-        , responseTimeout = defederationTimeout env
-        }
+      req dom =
+        defaultRequest
+          { method = methodDelete,
+            secure = False,
+            host = galley env ^. epHost . to encodeUtf8,
+            port = galley env ^. epPort . to fromIntegral,
+            path = "/i/federation/" <> toByteString' dom,
+            requestHeaders = ("Accept", "application/json") : requestHeaders defaultRequest,
+            responseTimeout = defederationTimeout env
+          }
   either
-    (\e -> do
-      logErr e
-      -- ensure that the message is _NOT_ requeued
-      -- This means that we won't process this message again
-      -- as it is unparsable.
-      liftIO $ reject envelope False
+    ( \e -> do
+        logErr e
+        -- ensure that the message is _NOT_ requeued
+        -- This means that we won't process this message again
+        -- as it is unparsable.
+        liftIO $ reject envelope False
     )
-    (\d -> do
-      -- Retry the request a couple of times. If the final one fails, catch the exception\
-      -- so that we can NACK the message and requeue it.
-      resp <- try $ recovering x3 httpHandlers $ \_ -> liftIO $ httpLbs (req d) manager
-      either
-        -- Requeue the exception and rethrow the exception
-        (\(e :: SomeException) -> liftIO (reject envelope True) >> throwM e)
-        go
-        resp
+    ( \d -> do
+        -- Retry the request a couple of times. If the final one fails, catch the exception\
+        -- so that we can NACK the message and requeue it.
+        resp <- try $ recovering x3 httpHandlers $ \_ -> liftIO $ httpLbs (req d) manager
+        either
+          -- Requeue the exception and rethrow the exception
+          (\(e :: SomeException) -> liftIO (reject envelope True) >> throwM e)
+          go
+          resp
     )
     $ A.eitherDecode @DefederationDomain (Q.msgBody msg)
   where
@@ -70,16 +71,16 @@ deleteFederationDomainInner (msg, envelope) = do
     go resp = do
       let code = statusCode $ responseStatus resp
       if code >= 200 && code <= 299
-      then do
-        liftIO $ ack envelope
-      else
-         -- ensure that the message is requeued
-         -- This message was able to be parsed but something
-         -- else in our stack failed and we should try again.
-        liftIO $ reject envelope True
-    logErr err = Log.err $
-      Log.msg (Log.val "Failed to delete federation domain")
-        . Log.field "error" err
+        then do
+          liftIO $ ack envelope
+        else -- ensure that the message is requeued
+        -- This message was able to be parsed but something
+        -- else in our stack failed and we should try again.
+          liftIO $ reject envelope True
+    logErr err =
+      Log.err $
+        Log.msg (Log.val "Failed to delete federation domain")
+          . Log.field "error" err
 
 deleteWorker :: Q.Channel -> AppT IO (Async ())
 deleteWorker chan = do
@@ -87,4 +88,4 @@ deleteWorker chan = do
   env <- ask
   liftIO $ async $ do
     void $ runAppT env $ deleteFederationDomain chan
-    forever $ threadDelay maxBound
\ No newline at end of file
+    forever $ threadDelay maxBound
diff --git a/services/background-worker/test/Main.hs b/services/background-worker/test/Main.hs
index 5fe9e51284..934c673a37 100644
--- a/services/background-worker/test/Main.hs
+++ b/services/background-worker/test/Main.hs
@@ -3,16 +3,16 @@ module Main
   )
 where
 
+import Data.Proxy
+import Data.Yaml (decodeFileEither)
 import Imports
-import Test.Tasty.Options
-import Test.Tasty
-import Util.Test
 import OpenSSL (withOpenSSL)
-import Data.Yaml (decodeFileEither)
-import Data.Proxy
-import Test.Wire.Util
+import Test.Tasty
+import Test.Tasty.Options
 import Test.Wire.BackendNotificationPusherSpec
 import Test.Wire.Defederation
+import Test.Wire.Util
+import Util.Test
 
 runTests :: (String -> String -> TestTree) -> IO ()
 runTests run = defaultMainWithIngredients ings $
@@ -31,12 +31,13 @@ main :: IO ()
 main = withOpenSSL $ runTests go
   where
     go o i = withResource (getOpts o i) releaseOpts $ \setup -> do
-      testGroup "background-worker"
-        [ spec setup
-        , deleteFederationDomainSpec setup
+      testGroup
+        "background-worker"
+        [ spec setup,
+          deleteFederationDomainSpec setup
         ]
     getOpts oFile iFile = do
       opts <- handleParseError =<< decodeFileEither oFile
       iConf <- handleParseError =<< decodeFileEither iFile
       pure $ TestSetup opts iConf
-    releaseOpts _ = pure ()
\ No newline at end of file
+    releaseOpts _ = pure ()
diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
index 2a15ed5d24..ab5544e20f 100644
--- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
+++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
@@ -1,8 +1,5 @@
-{-# LANGUAGE RecordWildCards #-}
-
 module Test.Wire.BackendNotificationPusherSpec where
 
-import Test.Wire.Util
 import qualified Data.Aeson as Aeson
 import Data.Domain
 import Data.Range
@@ -11,109 +8,112 @@ import Imports
 import qualified Network.AMQP as Q
 import Test.Hspec
 import Test.QuickCheck
+import Test.Tasty
+import Test.Tasty.HUnit
+import Test.Wire.Util
 import Wire.API.Federation.API
 import Wire.API.Federation.API.Brig
 import Wire.API.Federation.API.Common
 import Wire.API.Federation.BackendNotifications
 import Wire.API.RawJson
-import Test.Tasty
-import Test.Tasty.HUnit
 import Wire.BackendNotificationPusher
 
 spec :: IO TestSetup -> TestTree
-spec setup = testGroup "Wire.BackendNotificationPusher" 
-  [ testCase "should push notifications" $ do
-      s <- setup
-      let returnSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
-      let origDomain = Domain "origin.example.com"
-          targetDomain = Domain "target.example.com"
-      -- Just using 'arbitrary' could generate a very big list, making tests very
-      -- slow. Make me wonder if notification pusher should even try to parse the
-      -- actual content, seems like wasted compute power.
-      notifContent <- generate $ UserDeletedConnectionsNotification <$> arbitrary <*> (unsafeRange . (: []) <$> arbitrary)
-      let notif =
-            BackendNotification
-              { targetComponent = Brig,
-                ownDomain = origDomain,
-                path = "/on-user-deleted-connections",
-                body = RawJson $ Aeson.encode notifContent
-              }
-      envelope <- newFakeEnvelope
-      let msg =
-            Q.newMsg
-              { Q.msgBody = Aeson.encode notif,
-                Q.msgContentType = Just "application/json"
-              }
+spec setup =
+  testGroup
+    "Wire.BackendNotificationPusher"
+    [ testCase "should push notifications" $ do
+        s <- setup
+        let returnSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
+        let origDomain = Domain "origin.example.com"
+            targetDomain = Domain "target.example.com"
+        -- Just using 'arbitrary' could generate a very big list, making tests very
+        -- slow. Make me wonder if notification pusher should even try to parse the
+        -- actual content, seems like wasted compute power.
+        notifContent <- generate $ UserDeletedConnectionsNotification <$> arbitrary <*> (unsafeRange . (: []) <$> arbitrary)
+        let notif =
+              BackendNotification
+                { targetComponent = Brig,
+                  ownDomain = origDomain,
+                  path = "/on-user-deleted-connections",
+                  body = RawJson $ Aeson.encode notifContent
+                }
+        envelope <- newFakeEnvelope
+        let msg =
+              Q.newMsg
+                { Q.msgBody = Aeson.encode notif,
+                  Q.msgContentType = Just "application/json"
+                }
 
-      (_, fedReqs) <-
-        withTempMockFederator [] returnSuccess . runTestAppT s $ do
-          pushNotification targetDomain (msg, envelope)
+        (_, fedReqs) <-
+          withTempMockFederator [] returnSuccess . runTestAppT s $ do
+            pushNotification targetDomain (msg, envelope)
 
-      readIORef envelope.acks `shouldReturn` 1
-      readIORef envelope.rejections `shouldReturn` []
-      fedReqs
-        `shouldBe` [ FederatedRequest
-                       { frTargetDomain = targetDomain,
-                         frOriginDomain = origDomain,
-                         frComponent = Brig,
-                         frRPC = "on-user-deleted-connections",
-                         frBody = Aeson.encode notifContent
-                       }
-                   ]
-  , testCase "should reject invalid notifications" $ do
-      s <- setup
-      let returnSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
-      envelope <- newFakeEnvelope
-      let msg =
-            Q.newMsg
-              { Q.msgBody = "unparseable notification",
-                Q.msgContentType = Just "application/json"
-              }
-      (_, fedReqs) <-
-        withTempMockFederator [] returnSuccess . runTestAppT s $
-          pushNotification (Domain "target.example.com") (msg, envelope)
+        readIORef envelope.acks `shouldReturn` 1
+        readIORef envelope.rejections `shouldReturn` []
+        fedReqs
+          `shouldBe` [ FederatedRequest
+                         { frTargetDomain = targetDomain,
+                           frOriginDomain = origDomain,
+                           frComponent = Brig,
+                           frRPC = "on-user-deleted-connections",
+                           frBody = Aeson.encode notifContent
+                         }
+                     ],
+      testCase "should reject invalid notifications" $ do
+        s <- setup
+        let returnSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
+        envelope <- newFakeEnvelope
+        let msg =
+              Q.newMsg
+                { Q.msgBody = "unparseable notification",
+                  Q.msgContentType = Just "application/json"
+                }
+        (_, fedReqs) <-
+          withTempMockFederator [] returnSuccess . runTestAppT s $
+            pushNotification (Domain "target.example.com") (msg, envelope)
 
-      readIORef envelope.acks `shouldReturn` 0
-      readIORef envelope.rejections `shouldReturn` [False]
-      fedReqs `shouldBe` []
-  , testCase "should retry failed deliveries" $ do
-      s <- setup
-      isFirstReqRef <- newIORef True
-      let returnSuccessSecondTime _ =
-            atomicModifyIORef isFirstReqRef $ \isFirstReq ->
-              if isFirstReq
-                then (False, ("text/html", "<marquee>down for maintenance</marquee>"))
-                else (False, ("application/json", Aeson.encode EmptyResponse))
-          origDomain = Domain "origin.example.com"
-          targetDomain = Domain "target.example.com"
-      notifContent <- generate $ UserDeletedConnectionsNotification <$> arbitrary <*> (unsafeRange . (: []) <$> arbitrary)
-      let notif =
-            BackendNotification
-              { targetComponent = Brig,
-                ownDomain = origDomain,
-                path = "/on-user-deleted-connections",
-                body = RawJson $ Aeson.encode notifContent
-              }
-      envelope <- newFakeEnvelope
-      let msg =
-            Q.newMsg
-              { Q.msgBody = Aeson.encode notif,
-                Q.msgContentType = Just "application/json"
-              }
+        readIORef envelope.acks `shouldReturn` 0
+        readIORef envelope.rejections `shouldReturn` [False]
+        fedReqs `shouldBe` [],
+      testCase "should retry failed deliveries" $ do
+        s <- setup
+        isFirstReqRef <- newIORef True
+        let returnSuccessSecondTime _ =
+              atomicModifyIORef isFirstReqRef $ \isFirstReq ->
+                if isFirstReq
+                  then (False, ("text/html", "<marquee>down for maintenance</marquee>"))
+                  else (False, ("application/json", Aeson.encode EmptyResponse))
+            origDomain = Domain "origin.example.com"
+            targetDomain = Domain "target.example.com"
+        notifContent <- generate $ UserDeletedConnectionsNotification <$> arbitrary <*> (unsafeRange . (: []) <$> arbitrary)
+        let notif =
+              BackendNotification
+                { targetComponent = Brig,
+                  ownDomain = origDomain,
+                  path = "/on-user-deleted-connections",
+                  body = RawJson $ Aeson.encode notifContent
+                }
+        envelope <- newFakeEnvelope
+        let msg =
+              Q.newMsg
+                { Q.msgBody = Aeson.encode notif,
+                  Q.msgContentType = Just "application/json"
+                }
 
-      (_, fedReqs) <-
-        withTempMockFederator [] returnSuccessSecondTime . runTestAppT s $ do
-          pushNotification targetDomain (msg, envelope)
+        (_, fedReqs) <-
+          withTempMockFederator [] returnSuccessSecondTime . runTestAppT s $ do
+            pushNotification targetDomain (msg, envelope)
 
-      readIORef envelope.acks `shouldReturn` 1
-      readIORef envelope.rejections `shouldReturn` []
-      let expectedReq =
-            FederatedRequest
-              { frTargetDomain = targetDomain,
-                frOriginDomain = origDomain,
-                frComponent = Brig,
-                frRPC = "on-user-deleted-connections",
-                frBody = Aeson.encode notifContent
-              }
-      fedReqs `shouldBe` [expectedReq, expectedReq]
-  ]
\ No newline at end of file
+        readIORef envelope.acks `shouldReturn` 1
+        readIORef envelope.rejections `shouldReturn` []
+        let expectedReq =
+              FederatedRequest
+                { frTargetDomain = targetDomain,
+                  frOriginDomain = origDomain,
+                  frComponent = Brig,
+                  frRPC = "on-user-deleted-connections",
+                  frBody = Aeson.encode notifContent
+                }
+        fedReqs `shouldBe` [expectedReq, expectedReq]
+    ]
diff --git a/services/background-worker/test/Test/Wire/Defederation.hs b/services/background-worker/test/Test/Wire/Defederation.hs
index c0218e8112..48ab12045c 100644
--- a/services/background-worker/test/Test/Wire/Defederation.hs
+++ b/services/background-worker/test/Test/Wire/Defederation.hs
@@ -1,47 +1,53 @@
-{-# LANGUAGE RecordWildCards #-}
-
 module Test.Wire.Defederation where
 
-import Test.Wire.Util
 import qualified Data.Aeson as Aeson
+import Data.Domain
 import Federator.MockServer
 import Imports
 import qualified Network.AMQP as Q
 import Test.Hspec
-import Wire.API.Federation.API.Common
-import Wire.Defederation
-import Wire.API.Federation.BackendNotifications
-import Data.Domain
 import Test.Tasty
 import Test.Tasty.HUnit
+import Test.Wire.Util
+import Wire.API.Federation.API.Common
+import Wire.API.Federation.BackendNotifications
+import Wire.Defederation
 
 deleteFederationDomainSpec :: IO TestSetup -> TestTree
-deleteFederationDomainSpec setup = testGroup "Wire.BackendNotificationPusher.deleteFederationDomain"
-  [ testCase "should fail on message decoding" $ do
-      s <- setup
-      envelope <- newFakeEnvelope
-      let msg = Q.newMsg {Q.msgBody = Aeson.encode @[()] [], Q.msgContentType = Just "application/json"}
-          respSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
-      resps <- withTempMockFederator [] respSuccess .
-        runTestAppT s $ deleteFederationDomainInner (msg, envelope)
-      case resps of
-        ((), []) -> pure ()
-        _ -> assertFailure "Expected call to federation"
-      readIORef envelope.acks `shouldReturn` 0
-      readIORef envelope.rejections `shouldReturn` [True]
-  , testCase "should succeed on message decoding" $ do
-      s <- setup
-      envelope <- newFakeEnvelope
-      let msg = Q.newMsg
-            { Q.msgBody = Aeson.encode @DefederationDomain (Domain "far-away.example.com")
-            , Q.msgContentType = Just "application/json"
-            }
-          respSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
-      resps <- withTempMockFederator [] respSuccess .
-        runTestAppT s $ deleteFederationDomainInner (msg, envelope)
-      case resps of
-        ((), []) -> pure ()
-        _ -> assertFailure "Expected call to federation"
-      readIORef envelope.acks `shouldReturn` 1
-      readIORef envelope.rejections `shouldReturn` []
-  ]
\ No newline at end of file
+deleteFederationDomainSpec setup =
+  testGroup
+    "Wire.BackendNotificationPusher.deleteFederationDomain"
+    [ testCase "should fail on message decoding" $ do
+        s <- setup
+        envelope <- newFakeEnvelope
+        let msg = Q.newMsg {Q.msgBody = Aeson.encode @[()] [], Q.msgContentType = Just "application/json"}
+            respSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
+        resps <-
+          withTempMockFederator [] respSuccess
+            . runTestAppT s
+            $ deleteFederationDomainInner (msg, envelope)
+        case resps of
+          ((), []) -> pure ()
+          _ -> assertFailure "Expected call to federation"
+        readIORef envelope.acks `shouldReturn` 0
+        -- Fail to decode should not be requeued
+        readIORef envelope.rejections `shouldReturn` [False],
+      testCase "should succeed on message decoding" $ do
+        s <- setup
+        envelope <- newFakeEnvelope
+        let msg =
+              Q.newMsg
+                { Q.msgBody = Aeson.encode @DefederationDomain (Domain "far-away.example.com"),
+                  Q.msgContentType = Just "application/json"
+                }
+            respSuccess _ = pure ("application/json", Aeson.encode EmptyResponse)
+        resps <-
+          withTempMockFederator [] respSuccess
+            . runTestAppT s
+            $ deleteFederationDomainInner (msg, envelope)
+        case resps of
+          ((), []) -> pure ()
+          _ -> assertFailure "Expected call to federation"
+        readIORef envelope.acks `shouldReturn` 1
+        readIORef envelope.rejections `shouldReturn` []
+    ]
diff --git a/services/background-worker/test/Test/Wire/Util.hs b/services/background-worker/test/Test/Wire/Util.hs
index 923593c027..c51ef869e2 100644
--- a/services/background-worker/test/Test/Wire/Util.hs
+++ b/services/background-worker/test/Test/Wire/Util.hs
@@ -1,16 +1,16 @@
-{-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
 
 module Test.Wire.Util where
 
-import Imports
+import Control.Concurrent.Async
 import Control.Monad.Catch
-import Test.Tasty.Options
+import Data.Aeson
 import Data.Tagged
+import Imports
+import Options.Applicative
 import Test.Tasty
 import Test.Tasty.HUnit
-import Options.Applicative
-import Data.Aeson
+import Test.Tasty.Options
 import Util.Options
 import Wire.BackgroundWorker.Env hiding (federatorInternal, galley)
 import qualified Wire.BackgroundWorker.Env as E
@@ -18,10 +18,11 @@ import Wire.BackgroundWorker.Options hiding (federatorInternal, galley)
 import Wire.BackgroundWorker.Util
 
 data IntegrationConfig = IntegrationConfig
-  { galley :: Endpoint
-  , federatorInternal :: Endpoint
+  { galley :: Endpoint,
+    federatorInternal :: Endpoint
   }
   deriving (Show, Generic)
+
 instance FromJSON IntegrationConfig
 
 newtype ServiceConfigFile = ServiceConfigFile String
@@ -40,7 +41,7 @@ instance IsOption ServiceConfigFile where
             <> help (untag (optionHelp :: Tagged ServiceConfigFile String))
         )
 
-newtype TestM a = TestM { runTestM :: ReaderT TestSetup IO a}
+newtype TestM a = TestM {runTestM :: ReaderT TestSetup IO a}
   deriving
     ( Functor,
       Applicative,
@@ -55,8 +56,8 @@ newtype TestM a = TestM { runTestM :: ReaderT TestSetup IO a}
     )
 
 data TestSetup = TestSetup
-  { opts :: Opts
-  , iConf :: IntegrationConfig
+  { opts :: Opts,
+    iConf :: IntegrationConfig
   }
 
 test :: IO TestSetup -> TestName -> TestM a -> TestTree
@@ -71,23 +72,27 @@ natAppT :: AppT IO a -> TestM a
 natAppT app =
   TestM $ do
     e <- ask
-    e' <- liftIO $ setupToEnv e
+    (e', _) <- liftIO $ setupToEnv e
     liftIO $ runReaderT (unAppT app) e'
 
-setupToEnv :: TestSetup -> IO Env
+setupToEnv :: TestSetup -> IO (Env, Async ())
 setupToEnv setup = do
-  e <- mkEnv $ setup.opts
-  pure $ e
-    { E.federatorInternal = federatorInternal $ iConf $ setup
-    , E.galley = galley $ iConf $ setup
-    }
+  (e, thread) <- mkEnv $ setup.opts
+  pure
+    ( e
+        { E.federatorInternal = federatorInternal $ iConf $ setup,
+          E.galley = galley $ iConf $ setup
+        },
+      thread
+    )
 
 runTestAppT :: MonadIO m => TestSetup -> AppT m a -> Int -> m a
 runTestAppT setup app federatorPort = do
-  env <- liftIO $ setupToEnv setup
-  runReaderT (unAppT app) $ env
-    { E.federatorInternal = (E.federatorInternal env) { _epPort  = fromIntegral federatorPort }
-    }
+  (env, _) <- liftIO $ setupToEnv setup
+  runReaderT (unAppT app) $
+    env
+      { E.federatorInternal = (E.federatorInternal env) {_epPort = fromIntegral federatorPort}
+      }
 
 data FakeEnvelope = FakeEnvelope
   { rejections :: IORef [Bool],
@@ -102,4 +107,4 @@ newFakeEnvelope =
 
 instance RabbitMQEnvelope FakeEnvelope where
   ack e = atomicModifyIORef' e.acks $ \a -> (a + 1, ())
-  reject e requeueFlag = atomicModifyIORef' e.rejections $ \r -> (r <> [requeueFlag], ())
\ No newline at end of file
+  reject e requeueFlag = atomicModifyIORef' e.rejections $ \r -> (r <> [requeueFlag], ())
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 8d09c2778d..48726e6540 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -68,7 +68,7 @@ import Control.Lens (view, (^.))
 import Data.Aeson hiding (json)
 import Data.ByteString.Conversion
 import qualified Data.ByteString.Conversion as List
-import Data.Domain (Domain)
+import Data.Domain (Domain, domainText)
 import Data.Handle
 import Data.Id as Id
 import qualified Data.Map.Strict as Map
@@ -76,6 +76,7 @@ import Data.Qualified
 import qualified Data.Set as Set
 import Data.String.Conversions (cs)
 import Imports hiding (cs, head)
+import qualified Network.AMQP as Q
 import Network.HTTP.Types.Status
 import Network.Wai (Response)
 import Network.Wai.Predicate hiding (result, setStatus)
@@ -85,6 +86,7 @@ import Network.Wai.Utilities.ZAuth (zauthConnId)
 import Polysemy
 import Servant hiding (Handler, JSON, addHeader, respond)
 import Servant.Swagger.Internal.Orphans ()
+import qualified System.Logger as Lg
 import qualified System.Logger.Class as Log
 import System.Random (randomRIO)
 import UnliftIO.Async
@@ -92,6 +94,7 @@ import Wire.API.Connection
 import Wire.API.Error
 import qualified Wire.API.Error.Brig as E
 import Wire.API.Federation.API
+import Wire.API.Federation.BackendNotifications
 import Wire.API.Federation.Error (FederationError (..))
 import Wire.API.MLS.Credential
 import Wire.API.MLS.KeyPackage
@@ -107,8 +110,6 @@ import Wire.API.User.Activation
 import Wire.API.User.Client
 import Wire.API.User.Password
 import Wire.API.User.RichInfo
-import qualified Network.AMQP as Q
-import Wire.API.Federation.BackendNotifications
 
 ---------------------------------------------------------------------------
 -- Sitemap (servant)
@@ -317,13 +318,21 @@ deleteFederationRemote dom = do
   for_ (env ^. rabbitmqChannel) $ \chan -> liftIO . withMVar chan $ \chan' -> do
     -- ensureQueue uses routingKey internally
     ensureQueue chan' defederationQueue
-    Q.publishMsg chan' "" queue $ Q.newMsg
-      -- Check that this message type is compatible with what
-      -- background worker is expecting
-      { Q.msgBody = encode @DefederationDomain dom
-      , Q.msgDeliveryMode = pure Q.Persistent
-      , Q.msgContentType = pure "application/json"
-      }
+    void $
+      Q.publishMsg chan' "" queue $
+        Q.newMsg
+          { -- Check that this message type is compatible with what
+            -- background worker is expecting
+            Q.msgBody = encode @DefederationDomain dom,
+            Q.msgDeliveryMode = pure Q.Persistent,
+            Q.msgContentType = pure "application/json"
+          }
+    -- Drop the notification queue for the domain.
+    -- This will also drop all of the messages in the queue
+    -- as we will no longer be able to communicate with this
+    -- domain.
+    num <- Q.deleteQueue chan' . routingKey $ domainText dom
+    Lg.info (env ^. applog) $ Log.msg @String "Dropped Notifications" . Log.field "domain" (domainText dom) . Log.field "count" (show num)
   where
     -- Ensure that this is kept in sync with background worker
     queue = routingKey defederationQueue
diff --git a/services/galley/schema/src/V82_RemoteDomainIndexes.hs b/services/galley/schema/src/V82_RemoteDomainIndexes.hs
index b2739db9fe..fcc5fca612 100644
--- a/services/galley/schema/src/V82_RemoteDomainIndexes.hs
+++ b/services/galley/schema/src/V82_RemoteDomainIndexes.hs
@@ -17,4 +17,4 @@ migration = Migration 82 "Add a secondary index for remote domains on local conv
       |]
   schema'
     [r| CREATE INDEX on user_remote_conv (conv_remote_domain)
-      |]
\ No newline at end of file
+      |]
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index 77ea655f3d..3448577aba 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -22,22 +22,29 @@ module Galley.API.Internal
     deleteLoop,
     safeForever,
     -- Exported for tests
-    deleteFederationDomain
+    deleteFederationDomain,
   )
 where
 
+import Control.Exception
 import Control.Exception.Safe (catchAny)
 import Control.Lens hiding (Getter, Setter, (.=))
+import Data.Domain
 import Data.Id as Id
+import qualified Data.List.NonEmpty as N
 import Data.List1 (maybeList1)
+import qualified Data.Map as Map
 import Data.Qualified
 import Data.Range
 import Data.Singletons
+import Data.Text (unpack)
 import Data.Time
+import Galley.API.Action
 import qualified Galley.API.Clients as Clients
 import qualified Galley.API.Create as Create
 import qualified Galley.API.CustomBackend as CustomBackend
 import Galley.API.Error
+import Galley.API.Federation (onConversationUpdated)
 import Galley.API.LegalHold (unsetTeamLegalholdWhitelistedH)
 import Galley.API.LegalHold.Conflicts
 import Galley.API.MLS.Removal
@@ -52,6 +59,7 @@ import qualified Galley.API.Update as Update
 import Galley.API.Util
 import Galley.App
 import qualified Galley.Data.Conversation as Data
+import Galley.Data.Conversation.Types
 import Galley.Effects
 import Galley.Effects.BackendNotificationQueueAccess
 import Galley.Effects.ClientStore
@@ -60,6 +68,7 @@ import Galley.Effects.FederatorAccess
 import Galley.Effects.GundeckAccess
 import Galley.Effects.LegalHoldStore as LegalHoldStore
 import Galley.Effects.MemberStore
+import qualified Galley.Effects.MemberStore as E
 import Galley.Effects.ProposalStore
 import Galley.Effects.TeamStore
 import qualified Galley.Intra.Push as Intra
@@ -72,6 +81,8 @@ import Galley.Types.Conversations.Members (RemoteMember (rmId))
 import Galley.Types.UserList
 import Imports hiding (head)
 import qualified Network.AMQP as Q
+import Network.HTTP.Types
+import Network.Wai
 import Network.Wai.Predicate hiding (Error, err, setStatus)
 import qualified Network.Wai.Predicate as Predicate
 import Network.Wai.Routing hiding (App, route, toList)
@@ -82,18 +93,22 @@ import Polysemy.Error
 import Polysemy.Input
 import qualified Polysemy.TinyLog as P
 import Servant hiding (JSON, WithStatus)
+import Servant.Client (BaseUrl (BaseUrl), ClientEnv (ClientEnv), Scheme (Http), defaultMakeClientRequest)
 import System.Logger.Class hiding (Path, name)
 import qualified System.Logger.Class as Log
+import Util.Options
 import Wire.API.Conversation hiding (Member)
 import Wire.API.Conversation.Action
+import Wire.API.Conversation.Role
 import Wire.API.CustomBackend
 import Wire.API.Error
 import Wire.API.Error.Galley
 import Wire.API.Event.Conversation
-import Wire.API.FederationUpdate
 import Wire.API.Federation.API
 import Wire.API.Federation.API.Galley
+import qualified Wire.API.Federation.API.Galley as F
 import Wire.API.Federation.Error
+import Wire.API.FederationUpdate
 import Wire.API.Provider.Service hiding (Service)
 import Wire.API.Routes.API
 import Wire.API.Routes.Internal.Galley
@@ -103,21 +118,6 @@ import Wire.API.Team.Feature hiding (setStatus)
 import Wire.API.Team.Member
 import Wire.Sem.Paging
 import Wire.Sem.Paging.Cassandra
-import Data.Domain
-import qualified Data.Map as Map
-import qualified Wire.API.Federation.API.Galley as F
-import qualified Data.List.NonEmpty as N
-import qualified Galley.Effects.MemberStore as E
-import Galley.API.Action
-import Util.Options
-import Wire.API.Conversation.Role
-import Network.HTTP.Types
-import Network.Wai
-import Data.Text (unpack)
-import Galley.Data.Conversation.Types
-import Galley.API.Federation (onConversationUpdated)
-import Servant.Client (ClientEnv (ClientEnv), BaseUrl (BaseUrl), Scheme (Http), defaultMakeClientRequest)
-import Control.Exception
 
 internalAPI :: API InternalAPI GalleyEffects
 internalAPI =
@@ -316,7 +316,7 @@ internalSitemap = unsafeCallsFed @'Galley @"on-client-removed" $ unsafeCallsFed
   delete "/i/custom-backend/by-domain/:domain" (continue CustomBackend.internalDeleteCustomBackendByDomainH) $
     capture "domain"
       .&. accept "application" "json"
-    
+
   delete "/i/federation/:domain" (continue internalDeleteFederationDomainH) $
     capture "domain"
       .&. accept "application" "json"
@@ -496,29 +496,62 @@ insertIntoMap (cnvId, user) m = Map.alter (pure . maybe (pure user) (N.cons user
 -- Bundle all of the deletes together for easy calling
 -- Errors & exceptions are thrown to IO to stop the message being ACKed, eventually timing it
 -- out so that it can be redelivered.
-deleteFederationDomain :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg)) r,
-          Member (Error InternalError) r, Member (Error FederationError) r, Member (Input (Local ())) r,
-          Member MemberStore r, Member ConversationStore r, Member (Embed IO) r,
-          Member CodeStore r, Member TeamStore r, Member BrigAccess r, Member GundeckAccess r, Member ExternalAccess r) => Domain -> Sem r ()
+deleteFederationDomain ::
+  ( Member (Input Env) r,
+    Member (P.Logger (Msg -> Msg)) r,
+    Member (Error InternalError) r,
+    Member (Error FederationError) r,
+    Member (Input (Local ())) r,
+    Member MemberStore r,
+    Member ConversationStore r,
+    Member (Embed IO) r,
+    Member CodeStore r,
+    Member TeamStore r,
+    Member BrigAccess r,
+    Member GundeckAccess r,
+    Member ExternalAccess r
+  ) =>
+  Domain ->
+  Sem r ()
 deleteFederationDomain d = do
   deleteFederationDomainRemoteUserFromLocalConversations d
   deleteFederationDomainLocalUserFromRemoteConversation d
   deleteFederationDomainOneOnOne d
 
-
-internalDeleteFederationDomainH :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg)) r,
-          Member (Error InternalError) r, Member (Error FederationError) r, Member (Input (Local ())) r,
-          Member MemberStore r, Member ConversationStore r, Member (Embed IO) r,
-          Member CodeStore r, Member TeamStore r, Member BrigAccess r, Member GundeckAccess r, Member ExternalAccess r) => Domain ::: JSON -> Sem r Response
+internalDeleteFederationDomainH ::
+  ( Member (Input Env) r,
+    Member (P.Logger (Msg -> Msg)) r,
+    Member (Error InternalError) r,
+    Member (Error FederationError) r,
+    Member (Input (Local ())) r,
+    Member MemberStore r,
+    Member ConversationStore r,
+    Member (Embed IO) r,
+    Member CodeStore r,
+    Member TeamStore r,
+    Member BrigAccess r,
+    Member GundeckAccess r,
+    Member ExternalAccess r
+  ) =>
+  Domain ::: JSON ->
+  Sem r Response
 internalDeleteFederationDomainH (domain ::: _) = do
   deleteFederationDomain domain
   pure (empty & setStatus status200)
 
 -- Remove remote members from local conversations
-deleteFederationDomainRemoteUserFromLocalConversations :: (Member (Input Env) r, Member (P.Logger (Msg -> Msg)) r,
-          Member (Error InternalError) r, Member (Error FederationError) r,
-          Member MemberStore r, Member ConversationStore r,
-          Member CodeStore r, Member TeamStore r) => Domain -> Sem r ()
+deleteFederationDomainRemoteUserFromLocalConversations ::
+  ( Member (Input Env) r,
+    Member (P.Logger (Msg -> Msg)) r,
+    Member (Error InternalError) r,
+    Member (Error FederationError) r,
+    Member MemberStore r,
+    Member ConversationStore r,
+    Member CodeStore r,
+    Member TeamStore r
+  ) =>
+  Domain ->
+  Sem r ()
 deleteFederationDomainRemoteUserFromLocalConversations dom = do
   remoteUsers <- E.getRemoteMembersByDomain dom
   env <- input
@@ -550,10 +583,10 @@ deleteFederationDomainRemoteUserFromLocalConversations dom = do
           lConv
           undefined
           $ tUntagged . rmId <$> rUsers -- This field can be undefined as the path for ConversationRemoveMembersTag doens't use it
-        -- Check if the conversation if type 2 or 3, one-on-one conversations.
-        -- If it is, then we need to remove the entire conversation as users
-        -- aren't able to delete those types of conversations themselves.
-        -- Check that we are in a type 2 or a type 3 conversation
+          -- Check if the conversation if type 2 or 3, one-on-one conversations.
+          -- If it is, then we need to remove the entire conversation as users
+          -- aren't able to delete those types of conversations themselves.
+          -- Check that we are in a type 2 or a type 3 conversation
         when (cnvmType (convMetadata conv) `elem` [One2OneConv, ConnectConv]) $
           -- If we are, delete it.
           updateLocalConversationUserUnchecked
@@ -563,10 +596,19 @@ deleteFederationDomainRemoteUserFromLocalConversations dom = do
             ()
 
 -- Remove local members from remote conversations
-deleteFederationDomainLocalUserFromRemoteConversation :: (Member (Input (Local ())) r, Member (Input Env) r,
-          Member (Error InternalError) r, Member (P.Logger (Msg -> Msg)) r,
-          Member MemberStore r, Member (Embed IO) r, Member BrigAccess r,
-          Member GundeckAccess r, Member ExternalAccess r) => Domain -> Sem r ()
+deleteFederationDomainLocalUserFromRemoteConversation ::
+  ( Member (Input (Local ())) r,
+    Member (Input Env) r,
+    Member (Error InternalError) r,
+    Member (P.Logger (Msg -> Msg)) r,
+    Member MemberStore r,
+    Member (Embed IO) r,
+    Member BrigAccess r,
+    Member GundeckAccess r,
+    Member ExternalAccess r
+  ) =>
+  Domain ->
+  Sem r ()
 deleteFederationDomainLocalUserFromRemoteConversation dom = do
   localUsers <- E.getLocalMembersByDomain dom
   env <- input
diff --git a/services/galley/src/Galley/Options.hs b/services/galley/src/Galley/Options.hs
index fbcdef1ab4..5f8996834b 100644
--- a/services/galley/src/Galley/Options.hs
+++ b/services/galley/src/Galley/Options.hs
@@ -52,7 +52,7 @@ module Galley.Options
     optJournal,
     optLogLevel,
     optLogNetStrings,
-    optLogFormat
+    optLogFormat,
   )
 where
 
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index 123389401c..e5ef56827a 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -18,7 +18,7 @@
 module Galley.Run
   ( run,
     mkApp,
-    mkLogger
+    mkLogger,
   )
 where
 
@@ -174,4 +174,4 @@ collectAuthMetrics m env = do
     forever $ do
       mbRemaining <- readAuthExpiration env
       gaugeTokenRemaing m mbRemaining
-      threadDelay 1_000_000
\ No newline at end of file
+      threadDelay 1_000_000
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 39b48b1246..5c3345f293 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -22,7 +22,9 @@ import Data.Singletons
 import Data.Time (getCurrentTime)
 import qualified Data.UUID as UUID
 import Federator.MockServer
+import Galley.API.Internal
 import Galley.API.Util
+import Galley.App
 import Galley.Cassandra.Queries
 import qualified Galley.Data.Conversation.Types as Types
 import Galley.Env
@@ -48,8 +50,7 @@ import Wire.API.Routes.FederationDomainConfig
 import Wire.API.Routes.MultiTablePaging
 import qualified Wire.API.Routes.MultiTablePaging as Public
 import Wire.API.User.Search
-import Galley.App
-import Galley.API.Internal
+
 -- import Control.Concurrent.Async
 
 x3 :: RetryPolicy
@@ -135,7 +136,7 @@ updateFedDomainsTestRemoveRemoteFromLocal' = do
       remoteDomain2 = Domain "far-away-two.example.com"
   liftIO $ assertBool "remoteDomain is different to local domain" $ remoteDomain /= opts ^. optSettings . setFederationDomain
   liftIO $ assertBool "remoteDomain2 is different to local domain" $ remoteDomain2 /= opts ^. optSettings . setFederationDomain
-  
+
   -- Remove a remote domain from local conversations
   updateFedDomainRemoveRemoteFromLocal env remoteDomain remoteDomain2 interval
 
diff --git a/services/galley/test/integration/Main.hs b/services/galley/test/integration/Main.hs
index 49e442c849..974c3ee59d 100644
--- a/services/galley/test/integration/Main.hs
+++ b/services/galley/test/integration/Main.hs
@@ -96,13 +96,13 @@ main = withOpenSSL $ runTests go
               mempty
               (pathsConsistencyCheck . treeToPaths . compile $ Galley.API.sitemap),
           API.tests setup,
-          testGroup "Federation Domains"
-            [ test setup "No-Op" updateFedDomainsTestNoop'
-            , test setup "Add Remote" updateFedDomainsTestAddRemote'
-            , test setup "Remove Remote From Local" updateFedDomainsTestRemoveRemoteFromLocal'
-            , test setup "Remove Local From Remote" updateFedDomainsTestRemoveLocalFromRemote'
+          testGroup
+            "Federation Domains"
+            [ test setup "No-Op" updateFedDomainsTestNoop',
+              test setup "Add Remote" updateFedDomainsTestAddRemote',
+              test setup "Remove Remote From Local" updateFedDomainsTestRemoveRemoteFromLocal',
+              test setup "Remove Local From Remote" updateFedDomainsTestRemoveLocalFromRemote'
             ],
-          
           test setup "isConvMemberL" isConvMemberLTests
         ]
     getOpts gFile iFile = do

From 4e58cbb874f9f6477404e8396a73d5dbac991616 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 27 Jun 2023 17:40:37 +1000
Subject: [PATCH 187/220] FS-1179: Removing and errant import

---
 services/galley/test/integration/Federation.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 5c3345f293..a59f9e5c5d 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -27,7 +27,6 @@ import Galley.API.Util
 import Galley.App
 import Galley.Cassandra.Queries
 import qualified Galley.Data.Conversation.Types as Types
-import Galley.Env
 import Galley.Monad
 import Galley.Options
 import Galley.Run

From 0f83de8058f7fdf9664172aee8cea33a0d4f3105 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 27 Jun 2023 17:49:09 +1000
Subject: [PATCH 188/220] FS-1179: PR sanitisation.

---
 cassandra-schema.cql | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cassandra-schema.cql b/cassandra-schema.cql
index 8ca8c9b35e..96e434a856 100644
--- a/cassandra-schema.cql
+++ b/cassandra-schema.cql
@@ -862,6 +862,7 @@ CREATE TABLE brig_test.connection_remote (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
+CREATE INDEX connection_remote_right_domain_idx ON brig_test.connection_remote (right_domain);
 
 CREATE TABLE brig_test.users_pending_activation (
     user uuid PRIMARY KEY,
@@ -1286,6 +1287,7 @@ CREATE TABLE galley_test.user_remote_conv (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
+CREATE INDEX user_remote_conv_conv_remote_domain_idx ON galley_test.user_remote_conv (conv_remote_domain);
 
 CREATE TABLE galley_test.legalhold_whitelisted (
     team uuid PRIMARY KEY
@@ -1325,6 +1327,7 @@ CREATE TABLE galley_test.member_remote_user (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
+CREATE INDEX member_remote_user_user_remote_domain_idx ON galley_test.member_remote_user (user_remote_domain);
 
 CREATE TABLE galley_test.team_member (
     team uuid,

From ae581304f3afaef1079a1aebecbccdfa4b25a9bc Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 28 Jun 2023 13:10:05 +1000
Subject: [PATCH 189/220] FS-1179: Updating a TODO

---
 services/background-worker/src/Wire/BackgroundWorker.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index c20d72c842..e17f6df120 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -43,7 +43,7 @@ run opts = do
                   -- When we have new domains, kill the previous pusher thread
                   liftIO $ cancel asyncThread
                   -- Start a new pusher thread and then wait for new domains
-                  -- TODO: There is a nicer way of doing this using
+                  -- TODO: There is a nicer way of doing this, but I'm failing to see it.
                   go =<< BackendNotificationPusher.startWorker (domain <$> remotes.remotes) chan
             go thread
 

From 88b1e43dd0a39cafc05d5f9e83c81e8862181450 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 28 Jun 2023 17:16:33 +1000
Subject: [PATCH 190/220] FS-1179: Writing cleanup code so that we don't have
 dangling threads

---
 .../src/Wire/BackendNotificationPusher.hs     | 34 ++++++++++++++++---
 .../src/Wire/BackgroundWorker.hs              | 22 ++++--------
 .../src/Wire/Defederation.hs                  | 12 +++++--
 3 files changed, 45 insertions(+), 23 deletions(-)

diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index 8d520f87ba..bb7b72f072 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -15,6 +15,9 @@ import Wire.API.Federation.BackendNotifications
 import Wire.API.Federation.Client
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
+import Network.AMQP (cancelConsumer, ConsumerTag)
+import Control.Concurrent
+import Wire.API.Routes.FederationDomainConfig
 
 startPushingNotifications ::
   Q.Channel ->
@@ -72,12 +75,33 @@ pushNotification targetDomain (msg, envelope) = do
 
 -- FUTUREWORK: Recosider using 1 channel for many consumers. It shouldn't matter
 -- for a handful of remote domains.
-startWorker :: [Domain] -> Q.Channel -> AppT IO (Async ())
-startWorker remoteDomains chan = do
+startWorker :: Q.Channel -> AppT IO (Async ())
+startWorker chan = do
   -- This ensures that we receive notifications 1 by 1 which ensures they are
   -- delivered in order.
   lift $ Q.qos chan 0 1 False
   env <- ask
-  liftIO $ async $ do
-    mapM_ (runAppT env . startPushingNotifications chan) remoteDomains
-    forever $ threadDelay maxBound
+  let go :: [ConsumerTag] -> IO ()
+      go consumers = do
+        -- Wait for a new set of domains
+        chanRemotes <- readChan $ env.remoteDomainsChan
+        -- Cancel all of the existing consumers
+        traverse_ (cancelConsumer chan) consumers
+        -- Make new consumers for the new domains
+        consumers' <- traverse (runAppT env . startPushingNotifications chan) $ domain <$> chanRemotes.remotes
+        -- Repeat
+        go consumers'
+  initRemotes <- liftIO $ readIORef env.remoteDomains
+  consumersRef <- newIORef []
+  let cleanup :: AsyncCancelled -> IO ()
+      cleanup e = do
+        consumers <-  readIORef consumersRef
+        traverse_ (cancelConsumer chan) consumers
+        throwM e
+  -- If this thread is cancelled, catch the exception, kill the consumers, and carry on.
+  liftIO $ async $ handle cleanup $ do
+    -- Get an initial set of consumers for the domains pulled from the IORef
+    consumers <- traverse (runAppT env . startPushingNotifications chan) $ domain <$> initRemotes.remotes
+    atomicWriteIORef consumersRef consumers
+    -- Loop on waiting for new domains, tearing down consumers, and building new ones
+    go consumers
diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index e17f6df120..14853825ec 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -3,22 +3,24 @@
 module Wire.BackgroundWorker where
 
 import Control.Concurrent.Async
-import Control.Concurrent.Chan
 import Imports
 import Network.AMQP.Extended
-import Wire.API.Routes.FederationDomainConfig
 import qualified Wire.BackendNotificationPusher as BackendNotificationPusher
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Options
 import Wire.Defederation
 
 -- FUTUREWORK: Start an http service with status and metrics endpoints
+-- NOTE: Use atomic IORef writes to impose an ordering barrier on
+--       reads and writes. This stops the CPU from being too clever
+--       with its memory model and what it thinks it can get away with.
 run :: Opts -> IO ()
 run opts = do
   (env, syncThread) <- mkEnv opts
   threadsRef <- newIORef []
   let cancelThreads = do
         -- Kill all of the threads and clean up the IORef
+        -- The threads should handle the cleanup of their AMQP consumers.
         threads <- readIORef threadsRef
         traverse_ cancel threads
         atomicWriteIORef threadsRef []
@@ -32,20 +34,7 @@ run opts = do
           -- Since this is feeding off a Rabbit queue, it should
           -- be safe to kill and start these threads. At worst, we
           -- will double deliver some messages
-          pushThread <- do
-            -- get an initial list of domains from the IORef
-            initRemotes <- liftIO $ readIORef env.remoteDomains
-            -- Start the notification pusher using the initial domains
-            thread <- BackendNotificationPusher.startWorker (domain <$> initRemotes.remotes) chan
-            let go asyncThread = do
-                  -- Wait for a new set of domains from the Chan
-                  remotes <- liftIO $ readChan $ env.remoteDomainsChan
-                  -- When we have new domains, kill the previous pusher thread
-                  liftIO $ cancel asyncThread
-                  -- Start a new pusher thread and then wait for new domains
-                  -- TODO: There is a nicer way of doing this, but I'm failing to see it.
-                  go =<< BackendNotificationPusher.startWorker (domain <$> remotes.remotes) chan
-            go thread
+          pushThread <- BackendNotificationPusher.startWorker chan
 
           let threads = [pushThread, deleteThread]
           -- Write out the handles for the threads
@@ -54,6 +43,7 @@ run opts = do
           -- as the threads all have `forever $ threadDelay ...`
           liftIO $ traverse_ wait threads
           -- clear the threadRef if the threads finish
+          -- This should never happen, but there is no harm in preventative cleanup
           atomicWriteIORef threadsRef [],
         -- FUTUREWORK: Use these for metrics
         --
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index a538da3df5..e3f160b623 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -20,6 +20,7 @@ import Util.Options
 import Wire.API.Federation.BackendNotifications
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
+import Network.AMQP (cancelConsumer)
 
 deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
 deleteFederationDomain chan = do
@@ -86,6 +87,13 @@ deleteWorker :: Q.Channel -> AppT IO (Async ())
 deleteWorker chan = do
   lift $ Q.qos chan 0 1 False
   env <- ask
-  liftIO $ async $ do
-    void $ runAppT env $ deleteFederationDomain chan
+  consumerRef <- newIORef Nothing
+  let cleanup :: AsyncCancelled -> IO ()
+      cleanup e = do
+        consumer <- readIORef consumerRef
+        traverse_ (cancelConsumer chan) consumer
+        throwM e
+  liftIO $ async $ handle cleanup $ do
+    consumer <- runAppT env $ deleteFederationDomain chan
+    atomicWriteIORef consumerRef $ pure consumer
     forever $ threadDelay maxBound

From 92b777be414ed7291bb28958a7a82e2ee803f705 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 28 Jun 2023 17:48:55 +1000
Subject: [PATCH 191/220] FS-1179: PR formatting

---
 .../src/Wire/BackendNotificationPusher.hs                 | 8 ++++----
 services/background-worker/src/Wire/Defederation.hs       | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index bb7b72f072..a25fd633eb 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -2,22 +2,22 @@
 
 module Wire.BackendNotificationPusher where
 
+import Control.Concurrent
 import Control.Concurrent.Async
 import Control.Monad.Catch
 import Control.Retry
 import qualified Data.Aeson as A
 import Data.Domain
 import Imports
+import Network.AMQP (ConsumerTag, cancelConsumer)
 import qualified Network.AMQP as Q
 import qualified Network.AMQP.Lifted as QL
 import qualified System.Logger.Class as Log
 import Wire.API.Federation.BackendNotifications
 import Wire.API.Federation.Client
+import Wire.API.Routes.FederationDomainConfig
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
-import Network.AMQP (cancelConsumer, ConsumerTag)
-import Control.Concurrent
-import Wire.API.Routes.FederationDomainConfig
 
 startPushingNotifications ::
   Q.Channel ->
@@ -95,7 +95,7 @@ startWorker chan = do
   consumersRef <- newIORef []
   let cleanup :: AsyncCancelled -> IO ()
       cleanup e = do
-        consumers <-  readIORef consumersRef
+        consumers <- readIORef consumersRef
         traverse_ (cancelConsumer chan) consumers
         throwM e
   -- If this thread is cancelled, catch the exception, kill the consumers, and carry on.
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index e3f160b623..d220b2c4a6 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -11,6 +11,7 @@ import qualified Data.ByteString.Lazy as L
 import Data.Domain
 import Data.Text.Encoding
 import Imports
+import Network.AMQP (cancelConsumer)
 import qualified Network.AMQP as Q
 import qualified Network.AMQP.Lifted as QL
 import Network.HTTP.Client
@@ -20,7 +21,6 @@ import Util.Options
 import Wire.API.Federation.BackendNotifications
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
-import Network.AMQP (cancelConsumer)
 
 deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
 deleteFederationDomain chan = do

From 0513524511bf74d2224c8ee045f09013e7b7fbc9 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 29 Jun 2023 17:06:04 +1000
Subject: [PATCH 192/220] PR formatting

---
 .../background-worker/background-worker.cabal |  4 +-
 services/background-worker/default.nix        | 38 +++++++++++--------
 .../src/Wire/BackendNotificationPusher.hs     |  2 +-
 .../src/Wire/BackgroundWorker.hs              |  2 +-
 .../src/Wire/BackgroundWorker/Env.hs          |  4 +-
 .../src/Wire/BackgroundWorker/Options.hs      |  2 +-
 services/background-worker/test/Main.hs       |  2 +-
 .../Wire/BackendNotificationPusherSpec.hs     | 16 ++++----
 .../{Defederation.hs => DefederationSpec.hs}  | 15 ++++----
 .../background-worker/test/Test/Wire/Util.hs  |  7 ++--
 10 files changed, 49 insertions(+), 43 deletions(-)
 rename services/background-worker/test/Test/Wire/{Defederation.hs => DefederationSpec.hs} (88%)

diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal
index dbadde626c..b461d1dc31 100644
--- a/services/background-worker/background-worker.cabal
+++ b/services/background-worker/background-worker.cabal
@@ -168,7 +168,7 @@ executable background-worker-integration
   other-modules:
     Main
     Test.Wire.BackendNotificationPusherSpec
-    Test.Wire.Defederation
+    Test.Wire.DefederationSpec
     Test.Wire.Util
 
   build-depends:
@@ -257,7 +257,7 @@ test-suite background-worker-test
   other-modules:
     Main
     Test.Wire.BackendNotificationPusherSpec
-    Test.Wire.Defederation
+    Test.Wire.DefederationSpec
     Test.Wire.Util
 
   build-depends:
diff --git a/services/background-worker/default.nix b/services/background-worker/default.nix
index 3e37f6f407..b11ce68388 100644
--- a/services/background-worker/default.nix
+++ b/services/background-worker/default.nix
@@ -21,16 +21,13 @@
 , http-media
 , http-types
 , http2-manager
+, HUnit
 , imports
 , lens
 , lib
 , monad-control
-, optparse-applicative
 , QuickCheck
 , retry
-, tagged
-, tasty
-, tasty-hunit
 , servant
 , servant-client
 , servant-client-core
@@ -44,7 +41,6 @@
 , wai
 , wire-api
 , wire-api-federation
-, yaml
 }:
 mkDerivation {
   pname = "background-worker";
@@ -76,9 +72,10 @@ mkDerivation {
     tinylog
     transformers-base
     types-common
+    wire-api
     wire-api-federation
   ];
-  testHaskellDepends = [
+  executableHaskellDepends = [
     aeson
     amqp
     base
@@ -86,9 +83,12 @@ mkDerivation {
     containers
     extended
     federator
+    HsOpenSSL
     hspec
+    http-client
     http-media
     http-types
+    HUnit
     imports
     QuickCheck
     servant
@@ -104,25 +104,33 @@ mkDerivation {
     wire-api
     wire-api-federation
   ];
-  executableHaskellDepends = [
+  testHaskellDepends = [
     aeson
     amqp
-    async
     base
-    exceptions
+    bytestring
+    containers
+    extended
     federator
-    HsOpenSSL
     hspec
+    http-client
+    http-media
+    http-types
+    HUnit
     imports
-    optparse-applicative
     QuickCheck
-    tagged
-    tasty
-    tasty-hunit
+    servant
+    servant-client
+    servant-client-core
+    servant-server
+    text
+    tinylog
+    transformers
     types-common
+    unliftio
+    wai
     wire-api
     wire-api-federation
-    yaml
   ];
   description = "Runs background work";
   license = lib.licenses.agpl3Only;
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index d67c6f4eeb..dcd3d43c9e 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -9,6 +9,7 @@ import Control.Retry
 import qualified Data.Aeson as A
 import Data.Domain
 import qualified Data.Map.Strict as Map
+import qualified Data.Set as Set
 import qualified Data.Text as Text
 import Imports
 import Network.AMQP (cancelConsumer)
@@ -21,7 +22,6 @@ import Wire.API.Federation.Client
 import Wire.API.Routes.FederationDomainConfig
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
-import qualified Data.Set as Set
 
 startPushingNotifications ::
   Q.Channel ->
diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index 1b947fbf02..6976302670 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -5,11 +5,11 @@ module Wire.BackgroundWorker where
 import Control.Concurrent.Async
 import Control.Monad.Catch
 import Imports
+import Network.AMQP.Extended
 import qualified Wire.BackendNotificationPusher as BackendNotificationPusher
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Options
 import Wire.Defederation
-import Network.AMQP.Extended
 
 -- FUTUREWORK: Start an http service with status and metrics endpoints
 -- NOTE: Use atomic IORef writes to impose an ordering barrier on
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
index 7c07263cc7..079dcd7924 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
@@ -10,8 +10,8 @@ import Control.Monad.Catch
 import Control.Monad.Trans.Control
 import HTTP2.Client.Manager
 import Imports
-import Network.HTTP.Client
 import Network.AMQP.Extended
+import Network.HTTP.Client
 import qualified Network.RabbitMqAdmin as RabbitMqAdmin
 import OpenSSL.Session (SSLOption (..))
 import qualified OpenSSL.Session as SSL
@@ -26,7 +26,6 @@ import Wire.BackgroundWorker.Options
 
 data Env = Env
   { http2Manager :: Http2Manager,
-
     httpManager :: Manager,
     logger :: Logger,
     federatorInternal :: Endpoint,
@@ -35,7 +34,6 @@ data Env = Env
     defederationTimeout :: ResponseTimeout,
     remoteDomains :: IORef FederationDomainConfigs,
     remoteDomainsChan :: Chan FederationDomainConfigs,
-
     rabbitmqAdminClient :: RabbitMqAdmin.AdminAPI (Servant.AsClientT IO),
     rabbitmqVHost :: Text
   }
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
index 2fffb5abf5..7420c2dfff 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
@@ -17,4 +17,4 @@ data Opts = Opts
   }
   deriving (Show, Generic)
 
-instance FromJSON Opts
\ No newline at end of file
+instance FromJSON Opts
diff --git a/services/background-worker/test/Main.hs b/services/background-worker/test/Main.hs
index 52ef578fca..a824f8c30c 100644
--- a/services/background-worker/test/Main.hs
+++ b/services/background-worker/test/Main.hs
@@ -1 +1 @@
-{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
\ No newline at end of file
+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
index 19a812779b..35575f5d89 100644
--- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
+++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
@@ -4,6 +4,7 @@
 
 module Test.Wire.BackendNotificationPusherSpec where
 
+import Control.Concurrent.Chan
 import Control.Exception
 import Control.Monad.Trans.Except
 import qualified Data.Aeson as Aeson
@@ -17,34 +18,33 @@ import qualified Data.Text.Encoding as Text
 import Federator.MockServer
 import Imports
 import qualified Network.AMQP as Q
-import Test.Hspec
-import Test.QuickCheck
-import Test.Wire.Util
+import Network.HTTP.Client (defaultManagerSettings, newManager, responseTimeoutNone)
 import Network.HTTP.Media
 import Network.HTTP.Types
 import Network.RabbitMqAdmin
 import qualified Network.Wai as Wai
 import qualified Network.Wai.Internal as Wai
-import Util.Options
 import Servant hiding (respond)
 import Servant.Client
 import Servant.Client.Core
 import Servant.Client.Internal.HttpClient (mkFailureResponse)
 import Servant.Server.Generic
 import Servant.Types.SourceT
+import qualified System.Logger.Class as Logger
+import Test.Hspec
+import Test.QuickCheck
+import Test.Wire.Util
 import UnliftIO.Async
+import Util.Options
 import Wire.API.Federation.API
 import Wire.API.Federation.API.Brig
 import Wire.API.Federation.API.Common
 import Wire.API.Federation.BackendNotifications
 import Wire.API.RawJson
-import qualified System.Logger.Class as Logger
+import Wire.API.Routes.FederationDomainConfig
 import Wire.BackendNotificationPusher
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
-import Network.HTTP.Client (newManager, defaultManagerSettings, responseTimeoutNone)
-import Control.Concurrent.Chan
-import Wire.API.Routes.FederationDomainConfig
 
 spec :: Spec
 spec = do
diff --git a/services/background-worker/test/Test/Wire/Defederation.hs b/services/background-worker/test/Test/Wire/DefederationSpec.hs
similarity index 88%
rename from services/background-worker/test/Test/Wire/Defederation.hs
rename to services/background-worker/test/Test/Wire/DefederationSpec.hs
index 73495d32f7..1e2326dfd2 100644
--- a/services/background-worker/test/Test/Wire/Defederation.hs
+++ b/services/background-worker/test/Test/Wire/DefederationSpec.hs
@@ -1,21 +1,22 @@
-module Test.Wire.Defederation where
+module Test.Wire.DefederationSpec where
 
 import qualified Data.Aeson as Aeson
 import Data.Domain
 import Federator.MockServer
 import Imports
 import qualified Network.AMQP as Q
-import Test.Wire.Util
-import Test.Hspec
 import Test.HUnit.Lang
+import Test.Hspec
+import Test.Wire.Util
 import Wire.API.Federation.API.Common
 import Wire.API.Federation.BackendNotifications
 import Wire.Defederation
 
-deleteFederationDomainSpec :: Spec
-deleteFederationDomainSpec = do
+spec :: Spec
+spec = do
   describe
-    "Wire.BackendNotificationPusher.deleteFederationDomain" $ do
+    "Wire.BackendNotificationPusher.deleteFederationDomain"
+    $ do
       it "should fail on message decoding" $ do
         envelope <- newFakeEnvelope
         let msg = Q.newMsg {Q.msgBody = Aeson.encode @[()] [], Q.msgContentType = Just "application/json"}
@@ -46,4 +47,4 @@ deleteFederationDomainSpec = do
           ((), []) -> pure ()
           _ -> assertFailure "Expected call to federation"
         readIORef envelope.acks `shouldReturn` 1
-        readIORef envelope.rejections `shouldReturn` []
\ No newline at end of file
+        readIORef envelope.rejections `shouldReturn` []
diff --git a/services/background-worker/test/Test/Wire/Util.hs b/services/background-worker/test/Test/Wire/Util.hs
index 3e82e0cd27..3e1f37ae70 100644
--- a/services/background-worker/test/Test/Wire/Util.hs
+++ b/services/background-worker/test/Test/Wire/Util.hs
@@ -1,17 +1,16 @@
-{-# LANGUAGE GeneralizedNewtypeDeriving #-}
 {-# LANGUAGE RecordWildCards #-}
 
 module Test.Wire.Util where
 
+import Control.Concurrent.Chan
 import Imports
+import Network.HTTP.Client
 import qualified System.Logger.Class as Logger
 import Util.Options
+import Wire.API.Routes.FederationDomainConfig
 import Wire.BackgroundWorker.Env hiding (federatorInternal, galley)
 import qualified Wire.BackgroundWorker.Env as E
 import Wire.BackgroundWorker.Util
-import Network.HTTP.Client
-import Wire.API.Routes.FederationDomainConfig
-import Control.Concurrent.Chan
 
 runTestAppT :: AppT IO a -> Int -> IO a
 runTestAppT app port = do

From 28ceb0814fdff8d4d0a42ff940269dbb8ddbe222 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 29 Jun 2023 19:46:01 +1000
Subject: [PATCH 193/220] More comments and exception handling

---
 .../API/Federation/BackendNotifications.hs    |  6 ++++
 .../src/Wire/BackendNotificationPusher.hs     | 12 +++++--
 .../src/Wire/BackgroundWorker.hs              |  7 ++--
 .../src/Wire/Defederation.hs                  | 36 +++++++++++--------
 .../background-worker/test/Test/Wire/Util.hs  |  4 +--
 5 files changed, 42 insertions(+), 23 deletions(-)

diff --git a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
index 1d2b9cb5af..3e37702eef 100644
--- a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
+++ b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
@@ -107,6 +107,12 @@ ensureQueue chan queue = do
             Q.queueHeaders =
               Q.FieldTable $
                 Map.fromList
+                  -- single-active-consumer is used because it is order
+                  -- preserving, especially into databases and to remote servers,
+                  -- exactly what we are doing here!
+                  -- Without single active consumer, messages will be delivered
+                  -- round-robbin to all consumers, but then we lose effect-ordering
+                  -- due to processing and network times.
                   [ ("x-single-active-consumer", Q.FVBool True),
                     ("x-queue-type", Q.FVString "quorum")
                   ]
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index dcd3d43c9e..203917ea84 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -87,13 +87,20 @@ startWorker chan = do
   env <- ask
   consumersRef <- newIORef mempty
   -- Make sure threads aren't dangling if/when this async thread is killed
-  let cleanup :: AsyncCancelled -> IO ()
+  let cleanup :: Exception e => e -> IO ()
       cleanup e = do
         consumers <- readIORef consumersRef
         traverse_ (cancelConsumer chan) $ Map.elems consumers
         throwM e
+
   -- If this thread is cancelled, catch the exception, kill the consumers, and carry on.
-  liftIO $ async $ handle cleanup $ runAppT env $ do
+  -- FUTUREWORK?:
+  -- If this throws an exception on the Chan / in the forever loop, the exception will
+  -- bubble all the way up and kill the pod. Kubernetes should restart the pod automatically.
+  liftIO $ async $ flip catches
+    [ Handler $ cleanup @SomeException
+    , Handler $ cleanup @SomeAsyncException
+    ] $ runAppT env $ do
     -- Get an initial set of domains from the sync thread
     -- The Chan that we will be waiting on isn't initialised with a
     -- value until the domain update loop runs the callback for the
@@ -139,6 +146,7 @@ ensureConsumer consumers chan domain = do
   unless consumerExists $ do
     Log.info $ Log.msg (Log.val "Starting consumer") . Log.field "domain" (domainText domain)
     tag <- startPushingNotifications chan domain
+    -- TODO: Check if the map is spine strict. This strict call might not be needed.
     -- The ' version of atomicModifyIORef is strict in the function update and is useful
     -- for not leaking memory.
     oldTag <- atomicModifyIORef' consumers $ \c -> (Map.insert domain tag c, Map.lookup domain c)
diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index 6976302670..fbefc1ba72 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -31,6 +31,8 @@ run opts = do
       env.logger
       (demoteOpts opts.rabbitmq)
       $ RabbitMqHooks
+        -- If the function in onNewChannel throws an exception it will bubble up the stack as this is OUTSIDE of the
+        -- connection and channel error handling. This will kill the pod, which should be restarted by kubernetes.
         { onNewChannel = \chan -> runAppT env $ do
             -- Channels are threadsafe: https://hackage.haskell.org/package/amqp-0.22.1/docs/Network-AMQP.html
             -- So we can async them for concurrency.
@@ -46,10 +48,7 @@ run opts = do
             atomicWriteIORef threadsRef threads
             -- Wait for all the threads. This shouldn't occure
             -- as the threads all have `forever $ threadDelay ...`
-            liftIO $ traverse_ wait threads
-            -- clear the threadRef if the threads finish
-            -- This should never happen, but there is no harm in preventative cleanup
-            atomicWriteIORef threadsRef [],
+            liftIO $ traverse_ wait threads,
           -- FUTUREWORK: Use these for metrics
           --
           -- When the channel dies for whatever reason, kill all of the async
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index d220b2c4a6..3550041358 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -30,6 +30,25 @@ deleteFederationDomain chan = do
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
 
+-- Exposed for testing purposes so we can decode without further processing
+deleteFederationDomainInner' :: RabbitMQEnvelope e => (DefederationDomain -> AppT IO ()) -> (Q.Message, e) -> AppT IO ()
+deleteFederationDomainInner' go (msg, envelope) = do
+  either
+    ( \e -> do
+        void $ logErr e
+        -- ensure that the message is _NOT_ requeued
+        -- This means that we won't process this message again
+        -- as it is unparsable.
+        liftIO $ reject envelope False
+    )
+    go
+    $ A.eitherDecode @DefederationDomain (Q.msgBody msg)
+  where
+    logErr err =
+      Log.err $
+        Log.msg (Log.val "Failed to delete federation domain")
+          . Log.field "error" err
+
 -- What should we do with non-recoverable (unparsable) errors/messages?
 -- should we deadletter, or do something else?
 -- Deadlettering has a privacy implication -- FUTUREWORK.
@@ -48,15 +67,7 @@ deleteFederationDomainInner (msg, envelope) = do
             requestHeaders = ("Accept", "application/json") : requestHeaders defaultRequest,
             responseTimeout = defederationTimeout env
           }
-  either
-    ( \e -> do
-        logErr e
-        -- ensure that the message is _NOT_ requeued
-        -- This means that we won't process this message again
-        -- as it is unparsable.
-        liftIO $ reject envelope False
-    )
-    ( \d -> do
+  let callGalley d = do
         -- Retry the request a couple of times. If the final one fails, catch the exception\
         -- so that we can NACK the message and requeue it.
         resp <- try $ recovering x3 httpHandlers $ \_ -> liftIO $ httpLbs (req d) manager
@@ -65,8 +76,7 @@ deleteFederationDomainInner (msg, envelope) = do
           (\(e :: SomeException) -> liftIO (reject envelope True) >> throwM e)
           go
           resp
-    )
-    $ A.eitherDecode @DefederationDomain (Q.msgBody msg)
+  deleteFederationDomainInner' callGalley (msg, envelope)
   where
     go :: Response L.ByteString -> AppT IO ()
     go resp = do
@@ -78,10 +88,6 @@ deleteFederationDomainInner (msg, envelope) = do
         -- This message was able to be parsed but something
         -- else in our stack failed and we should try again.
           liftIO $ reject envelope True
-    logErr err =
-      Log.err $
-        Log.msg (Log.val "Failed to delete federation domain")
-          . Log.field "error" err
 
 deleteWorker :: Q.Channel -> AppT IO (Async ())
 deleteWorker chan = do
diff --git a/services/background-worker/test/Test/Wire/Util.hs b/services/background-worker/test/Test/Wire/Util.hs
index 3e1f37ae70..9dd8ce5c34 100644
--- a/services/background-worker/test/Test/Wire/Util.hs
+++ b/services/background-worker/test/Test/Wire/Util.hs
@@ -20,8 +20,8 @@ runTestAppT app port = do
   remoteDomains <- newIORef defFederationDomainConfigs
   remoteDomainsChan <- newChan
   let federatorInternal = Endpoint "localhost" (fromIntegral port)
-      galley = Endpoint "localhost" undefined -- TODO
-      brig = Endpoint "localhost" undefined -- TODO
+      galley = Endpoint "localhost" 8085
+      brig = Endpoint "localhost" 8082
       defederationTimeout = responseTimeoutNone
       rabbitmqAdminClient = undefined
       rabbitmqVHost = undefined

From e397f76723cfdbda966e76fbf4c1b1e352cc99a7 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 29 Jun 2023 19:55:03 +1000
Subject: [PATCH 194/220] Fixing tests now that they have moved to hspec

---
 .../src/Wire/BackendNotificationPusher.hs     | 51 ++++++++++---------
 .../src/Wire/BackgroundWorker.hs              |  6 +--
 .../src/Wire/Defederation.hs                  |  8 +--
 .../test/Test/Wire/DefederationSpec.hs        |  5 +-
 4 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index 203917ea84..ff0c8a5295 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -97,29 +97,34 @@ startWorker chan = do
   -- FUTUREWORK?:
   -- If this throws an exception on the Chan / in the forever loop, the exception will
   -- bubble all the way up and kill the pod. Kubernetes should restart the pod automatically.
-  liftIO $ async $ flip catches
-    [ Handler $ cleanup @SomeException
-    , Handler $ cleanup @SomeAsyncException
-    ] $ runAppT env $ do
-    -- Get an initial set of domains from the sync thread
-    -- The Chan that we will be waiting on isn't initialised with a
-    -- value until the domain update loop runs the callback for the
-    -- first time.
-    initRemotes <- liftIO $ readIORef env.remoteDomains
-    -- Get an initial set of consumers for the domains pulled from the IORef
-    -- so that we aren't just sitting around not doing anything for a bit at
-    -- the start.
-    ensureConsumers consumersRef chan $ domain <$> initRemotes.remotes
-    -- Wait for updates to the domains, this is where the bulk of the action
-    -- is going to take place
-    forever $ do
-      -- Wait for a new set of domains. This is a blocking action
-      -- so we will only move past here when we get a new set of domains.
-      -- It is a bit nicer than having another timeout value, as Brig is
-      -- already providing one in the domain update message.
-      chanRemotes <- liftIO $ readChan env.remoteDomainsChan
-      -- Make new consumers for the new domains, clean up old ones from the consumer map.
-      ensureConsumers consumersRef chan $ domain <$> chanRemotes.remotes
+  liftIO
+    $ async
+    $ flip
+      catches
+      [ Handler $ cleanup @SomeException,
+        Handler $ cleanup @SomeAsyncException
+      ]
+    $ runAppT env
+    $ do
+      -- Get an initial set of domains from the sync thread
+      -- The Chan that we will be waiting on isn't initialised with a
+      -- value until the domain update loop runs the callback for the
+      -- first time.
+      initRemotes <- liftIO $ readIORef env.remoteDomains
+      -- Get an initial set of consumers for the domains pulled from the IORef
+      -- so that we aren't just sitting around not doing anything for a bit at
+      -- the start.
+      ensureConsumers consumersRef chan $ domain <$> initRemotes.remotes
+      -- Wait for updates to the domains, this is where the bulk of the action
+      -- is going to take place
+      forever $ do
+        -- Wait for a new set of domains. This is a blocking action
+        -- so we will only move past here when we get a new set of domains.
+        -- It is a bit nicer than having another timeout value, as Brig is
+        -- already providing one in the domain update message.
+        chanRemotes <- liftIO $ readChan env.remoteDomainsChan
+        -- Make new consumers for the new domains, clean up old ones from the consumer map.
+        ensureConsumers consumersRef chan $ domain <$> chanRemotes.remotes
 
 ensureConsumers :: IORef (Map Domain Q.ConsumerTag) -> Q.Channel -> [Domain] -> AppT IO ()
 ensureConsumers consumers chan domains = do
diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index fbefc1ba72..edbd0677e1 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -31,9 +31,9 @@ run opts = do
       env.logger
       (demoteOpts opts.rabbitmq)
       $ RabbitMqHooks
-        -- If the function in onNewChannel throws an exception it will bubble up the stack as this is OUTSIDE of the
-        -- connection and channel error handling. This will kill the pod, which should be restarted by kubernetes.
-        { onNewChannel = \chan -> runAppT env $ do
+        { -- If the function in onNewChannel throws an exception it will bubble up the stack as this is OUTSIDE of the
+          -- connection and channel error handling. This will kill the pod, which should be restarted by kubernetes.
+          onNewChannel = \chan -> runAppT env $ do
             -- Channels are threadsafe: https://hackage.haskell.org/package/amqp-0.22.1/docs/Network-AMQP.html
             -- So we can async them for concurrency.
             deleteThread <- deleteWorker chan
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index 3550041358..c923f203c5 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -30,8 +30,8 @@ deleteFederationDomain chan = do
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
 
--- Exposed for testing purposes so we can decode without further processing
-deleteFederationDomainInner' :: RabbitMQEnvelope e => (DefederationDomain -> AppT IO ()) -> (Q.Message, e) -> AppT IO ()
+-- Exposed for testing purposes so we can decode without further processing the message.
+deleteFederationDomainInner' :: RabbitMQEnvelope e => (e -> DefederationDomain -> AppT IO ()) -> (Q.Message, e) -> AppT IO ()
 deleteFederationDomainInner' go (msg, envelope) = do
   either
     ( \e -> do
@@ -41,7 +41,7 @@ deleteFederationDomainInner' go (msg, envelope) = do
         -- as it is unparsable.
         liftIO $ reject envelope False
     )
-    go
+    (go envelope)
     $ A.eitherDecode @DefederationDomain (Q.msgBody msg)
   where
     logErr err =
@@ -76,7 +76,7 @@ deleteFederationDomainInner (msg, envelope) = do
           (\(e :: SomeException) -> liftIO (reject envelope True) >> throwM e)
           go
           resp
-  deleteFederationDomainInner' callGalley (msg, envelope)
+  deleteFederationDomainInner' (const callGalley) (msg, envelope)
   where
     go :: Response L.ByteString -> AppT IO ()
     go resp = do
diff --git a/services/background-worker/test/Test/Wire/DefederationSpec.hs b/services/background-worker/test/Test/Wire/DefederationSpec.hs
index 1e2326dfd2..664b55e21e 100644
--- a/services/background-worker/test/Test/Wire/DefederationSpec.hs
+++ b/services/background-worker/test/Test/Wire/DefederationSpec.hs
@@ -11,6 +11,7 @@ import Test.Wire.Util
 import Wire.API.Federation.API.Common
 import Wire.API.Federation.BackendNotifications
 import Wire.Defederation
+import Wire.BackgroundWorker.Util
 
 spec :: Spec
 spec = do
@@ -24,7 +25,7 @@ spec = do
         resps <-
           withTempMockFederator [] respSuccess
             . runTestAppT
-            $ deleteFederationDomainInner (msg, envelope)
+            $ deleteFederationDomainInner' (\e _ -> liftIO $ ack e) (msg, envelope)
         case resps of
           ((), []) -> pure ()
           _ -> assertFailure "Expected call to federation"
@@ -42,7 +43,7 @@ spec = do
         resps <-
           withTempMockFederator [] respSuccess
             . runTestAppT
-            $ deleteFederationDomainInner (msg, envelope)
+            $ deleteFederationDomainInner' (\e _ -> liftIO $ ack e) (msg, envelope)
         case resps of
           ((), []) -> pure ()
           _ -> assertFailure "Expected call to federation"

From 642af644fe97a879ef3e05da3b6ce926e76c339a Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 29 Jun 2023 20:02:04 +1000
Subject: [PATCH 195/220] Formatting

---
 services/background-worker/test/Test/Wire/DefederationSpec.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/background-worker/test/Test/Wire/DefederationSpec.hs b/services/background-worker/test/Test/Wire/DefederationSpec.hs
index 664b55e21e..6b4f3dc0ba 100644
--- a/services/background-worker/test/Test/Wire/DefederationSpec.hs
+++ b/services/background-worker/test/Test/Wire/DefederationSpec.hs
@@ -10,8 +10,8 @@ import Test.Hspec
 import Test.Wire.Util
 import Wire.API.Federation.API.Common
 import Wire.API.Federation.BackendNotifications
-import Wire.Defederation
 import Wire.BackgroundWorker.Util
+import Wire.Defederation
 
 spec :: Spec
 spec = do

From c90889c62f9abf846ba93b949d430f3c6ddb89a2 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Thu, 29 Jun 2023 22:48:55 +0200
Subject: [PATCH 196/220] Fix background-worker integration tests.

---
 .../background-worker/background-worker.cabal |  2 ++
 services/background-worker/test/Main.hs       | 23 ++++++++++++++++++-
 services/background-worker/test/Spec.hs       |  1 +
 3 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 services/background-worker/test/Spec.hs

diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal
index b461d1dc31..eed2063265 100644
--- a/services/background-worker/background-worker.cabal
+++ b/services/background-worker/background-worker.cabal
@@ -167,6 +167,7 @@ executable background-worker-integration
   -- cabal-fmt: expand test
   other-modules:
     Main
+    Spec
     Test.Wire.BackendNotificationPusherSpec
     Test.Wire.DefederationSpec
     Test.Wire.Util
@@ -256,6 +257,7 @@ test-suite background-worker-test
   -- cabal-fmt: expand test
   other-modules:
     Main
+    Spec
     Test.Wire.BackendNotificationPusherSpec
     Test.Wire.DefederationSpec
     Test.Wire.Util
diff --git a/services/background-worker/test/Main.hs b/services/background-worker/test/Main.hs
index a824f8c30c..be0d00983c 100644
--- a/services/background-worker/test/Main.hs
+++ b/services/background-worker/test/Main.hs
@@ -1 +1,22 @@
-{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
+module Main where
+
+import Imports
+import qualified Spec
+import System.Environment (withArgs)
+import Test.Hspec.Runner
+
+-- See https://hspec.github.io/hspec-discover.html#using-a-custom-main-function and
+-- /services/spar/test-integration/Main.hs
+main :: IO ()
+main = do
+  -- (we don't need wire args, appearently.  but if we ever should, here we have them.)
+  (_wireArgs, hspecArgs) <- partitionArgs <$> getArgs
+  withArgs hspecArgs . hspec $ Spec.spec
+
+partitionArgs :: [String] -> ([String], [String])
+partitionArgs = go [] []
+  where
+    go wireArgs hspecArgs ("-s" : x : xs) = go (wireArgs <> ["-s", x]) hspecArgs xs
+    go wireArgs hspecArgs ("-i" : x : xs) = go (wireArgs <> ["-i", x]) hspecArgs xs
+    go wireArgs hspecArgs (x : xs) = go wireArgs (hspecArgs <> [x]) xs
+    go wireArgs hspecArgs [] = (wireArgs, hspecArgs)
diff --git a/services/background-worker/test/Spec.hs b/services/background-worker/test/Spec.hs
new file mode 100644
index 0000000000..5416ef6a86
--- /dev/null
+++ b/services/background-worker/test/Spec.hs
@@ -0,0 +1 @@
+{-# OPTIONS_GHC -F -pgmF hspec-discover -optF --module-name=Spec #-}

From 3123028ab3ce2c6293d4eb24aeb7a106e94ecee4 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Fri, 30 Jun 2023 12:55:39 +1000
Subject: [PATCH 197/220] Updating the internal notes

---
 changelog.d/5-internal/fs-1179 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/changelog.d/5-internal/fs-1179 b/changelog.d/5-internal/fs-1179
index 2f702e41f9..9bb4d3a62c 100644
--- a/changelog.d/5-internal/fs-1179
+++ b/changelog.d/5-internal/fs-1179
@@ -1 +1,2 @@
-Adding a new internal APIs to Brig and Galley to defederate domains.
\ No newline at end of file
+Adding a new internal APIs to Brig and Galley to defederate domains.
+Background-Worker has been reworked to seperate AMQP channel handling from processing. This was done to allow a defederation worker to share the same connection management process with notification pusher.
\ No newline at end of file

From c352e1be631b0e962c8410d4f33f51a707c4bc30 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Fri, 30 Jun 2023 19:25:34 +1000
Subject: [PATCH 198/220] WPB-240: Initial implementation of notifications
 sending for defederation

---
 .../wire-api/src/Wire/API/Event/Federation.hs | 54 +++++++++++++++++++
 libs/wire-api/wire-api.cabal                  |  1 +
 services/galley/galley.cabal                  |  1 +
 services/galley/src/Galley/API/Internal.hs    | 42 +++++++++++++--
 .../Galley/Cassandra/Conversation/Members.hs  |  8 +++
 .../galley/src/Galley/Cassandra/Queries.hs    |  3 ++
 .../src/Galley/Effects/ExternalAccess.hs      |  3 +-
 .../galley/src/Galley/Effects/MemberStore.hs  |  2 +
 services/galley/src/Galley/External.hs        |  9 ++--
 .../galley/src/Galley/Intra/Push/Internal.hs  |  3 ++
 10 files changed, 117 insertions(+), 9 deletions(-)
 create mode 100644 libs/wire-api/src/Wire/API/Event/Federation.hs

diff --git a/libs/wire-api/src/Wire/API/Event/Federation.hs b/libs/wire-api/src/Wire/API/Event/Federation.hs
new file mode 100644
index 0000000000..4e8f2ef089
--- /dev/null
+++ b/libs/wire-api/src/Wire/API/Event/Federation.hs
@@ -0,0 +1,54 @@
+module Wire.API.Event.Federation
+  ( Event (..)
+  , EventType (..)
+  ) where
+
+import qualified Data.Aeson as A
+import qualified Data.Aeson.KeyMap as KeyMap
+import Data.Json.Util (ToJSONObject (toJSONObject))
+import Data.Schema
+import qualified Data.Swagger as S
+import Imports
+import Wire.Arbitrary
+import Data.Domain
+
+data Event = Event
+  { _eventType :: EventType
+  , _eventDomains :: [Domain]
+  }
+  deriving (Eq, Ord)
+
+instance Arbitrary Event where
+  arbitrary = Event
+    <$> arbitrary
+    <*> arbitrary
+
+data EventType
+  = FederationDelete
+  deriving (Eq, Ord, Generic)
+  deriving (Arbitrary) via (GenericUniform EventType)
+  deriving (A.FromJSON, A.ToJSON, S.ToSchema) via Schema EventType
+
+instance ToSchema EventType where
+  schema =
+    enum @Text "EventType" $
+      mconcat
+        [ element "federation.delete" FederationDelete
+        ]
+
+eventObjectSchema :: ObjectSchema SwaggerDoc Event
+eventObjectSchema = Event
+  <$> _eventType .= field "type" schema
+  <*> _eventDomains .= field "domains" (array schema)
+
+instance ToSchema Event where
+  schema = object "Event" eventObjectSchema
+
+instance ToJSONObject Event where
+  toJSONObject =
+    KeyMap.fromList
+      . fromMaybe []
+      . schemaOut eventObjectSchema
+
+instance S.ToSchema Event where
+  declareNamedSchema = schemaToSwagger
\ No newline at end of file
diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal
index 5f117f934e..e01ba4f5af 100644
--- a/libs/wire-api/wire-api.cabal
+++ b/libs/wire-api/wire-api.cabal
@@ -37,6 +37,7 @@ library
     Wire.API.Error.Gundeck
     Wire.API.Event.Conversation
     Wire.API.Event.FeatureConfig
+    Wire.API.Event.Federation
     Wire.API.Event.Team
     Wire.API.FederationStatus
     Wire.API.FederationUpdate
diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index a84ad86c7d..d3d3ee2cfe 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -214,6 +214,7 @@ library
     , cereal                 >=0.4
     , comonad
     , containers             >=0.5
+    , cql-io
     , cryptonite
     , currency-codes         >=2.0
     , data-default           >=0.5
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index 9eb2ce5f7c..f0d18eed4d 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -84,8 +84,8 @@ import Imports hiding (head)
 import qualified Network.AMQP as Q
 import Network.HTTP.Types
 import Network.Wai
-import Network.Wai.Predicate hiding (Error, err, setStatus)
-import qualified Network.Wai.Predicate as Predicate
+import Network.Wai.Predicate hiding (result, Error, err, setStatus)
+import qualified Network.Wai.Predicate as Predicate hiding (result)
 import Network.Wai.Routing hiding (App, route, toList)
 import Network.Wai.Utilities hiding (Error)
 import Network.Wai.Utilities.ZAuth
@@ -105,6 +105,7 @@ import Wire.API.CustomBackend
 import Wire.API.Error
 import Wire.API.Error.Galley
 import Wire.API.Event.Conversation
+import qualified Wire.API.Event.Federation as Federation
 import Wire.API.Federation.API
 import Wire.API.Federation.API.Galley
 import qualified Wire.API.Federation.API.Galley as F
@@ -119,6 +120,13 @@ import Wire.API.Team.Feature hiding (setStatus)
 import Wire.API.Team.Member
 import Wire.Sem.Paging
 import Wire.Sem.Paging.Cassandra
+import Galley.Effects.ExternalAccess
+import Galley.Intra.Push.Internal (pushEventJson)
+import Galley.Cassandra.Store (embedClient)
+import Cassandra (Consistency(LocalQuorum), Page (..), paramsP, ClientState)
+import Database.CQL.IO (paginate)
+import Galley.Cassandra.Queries
+import Galley.Cassandra.Conversation.Members
 
 internalAPI :: API InternalAPI GalleyEffects
 internalAPI =
@@ -323,7 +331,7 @@ internalSitemap = unsafeCallsFed @'Galley @"on-client-removed" $ unsafeCallsFed
     capture "domain"
       .&. accept "application" "json"
 
-  delete "/i/federation/:domain" (continue internalDeleteFederationDomainH) $
+  delete "/i/federation/:domain" (continue (internalDeleteFederationDomainH (toRange (Proxy @1000)))) $
     capture "domain"
       .&. accept "application" "json"
 
@@ -525,6 +533,7 @@ deleteFederationDomain d = do
   deleteFederationDomainOneOnOne d
 
 internalDeleteFederationDomainH ::
+  forall r.
   ( Member (Input Env) r,
     Member (P.Logger (Msg -> Msg)) r,
     Member (Error InternalError) r,
@@ -533,17 +542,42 @@ internalDeleteFederationDomainH ::
     Member MemberStore r,
     Member ConversationStore r,
     Member (Embed IO) r,
+    Member (Input ClientState) r,
     Member CodeStore r,
     Member TeamStore r,
     Member BrigAccess r,
     Member GundeckAccess r,
     Member ExternalAccess r
   ) =>
+  Range 1 1000 Int32 -> -- TODO what values should go here?
   Domain ::: JSON ->
   Sem r Response
-internalDeleteFederationDomainH (domain ::: _) = do
+internalDeleteFederationDomainH (fromRange -> maxPage) (domain ::: _) = do
+  -- We have to send the same event twice.
+  -- Once before and once after defederation work.
+  -- https://wearezeta.atlassian.net/wiki/spaces/ENGINEERIN/pages/809238539/Use+case+Stopping+to+federate+with+a+domain
+  void sendNotifications
   deleteFederationDomain domain
+  void sendNotifications
   pure (empty & setStatus status200)
+  where
+    sendNotifications = do
+      page <- embedClient $ paginate selectAllMembers (paramsP LocalQuorum () maxPage)
+      sendNotificationPage page
+    pushEvents results = do
+      let (bots, mems) = localBotsAndUsers results
+          recipients = Intra.recipient <$> mems
+          event = Intra.FederationEvent $ Federation.Event Federation.FederationDelete [domain]
+      for_ (Intra.newPush ListComplete Nothing event recipients) $ \p ->
+        push1 $ p & Intra.pushRoute .~ Intra.RouteDirect
+      deliverAsync (bots `zip` repeat (pushEventJson event))
+    sendNotificationPage page = do
+      let res = result page
+          mems = mapMaybe toMember res
+      pushEvents mems
+      when (hasMore page) $ do
+        page' <- embedClient $ nextPage page
+        sendNotificationPage page'
 
 -- Remove remote members from local conversations
 deleteFederationDomainRemoteUserFromLocalConversations ::
diff --git a/services/galley/src/Galley/Cassandra/Conversation/Members.hs b/services/galley/src/Galley/Cassandra/Conversation/Members.hs
index 018574e8e5..07bc45f2c3 100644
--- a/services/galley/src/Galley/Cassandra/Conversation/Members.hs
+++ b/services/galley/src/Galley/Cassandra/Conversation/Members.hs
@@ -18,6 +18,8 @@
 module Galley.Cassandra.Conversation.Members
   ( addMembers,
     members,
+    allMembers,
+    toMember,
     lookupRemoteMembers,
     removeMembersFromLocalConv,
     toMemberStatus,
@@ -122,6 +124,11 @@ members conv =
   fmap (mapMaybe toMember) . retry x1 $
     query Cql.selectMembers (params LocalQuorum (Identity conv))
 
+allMembers :: Client [LocalMember]
+allMembers =
+  fmap (mapMaybe toMember) . retry x1 $
+    query Cql.selectAllMembers (params LocalQuorum ())
+
 toMemberStatus ::
   ( -- otr muted
     Maybe MutedStatus,
@@ -386,6 +393,7 @@ interpretMemberStoreToCassandra = interpret $ \case
   CreateBotMember sr bid cid -> embedClient $ addBotMember sr bid cid
   GetLocalMember cid uid -> embedClient $ member cid uid
   GetLocalMembers cid -> embedClient $ members cid
+  GetAllLocalMembers -> embedClient allMembers
   GetRemoteMember cid uid -> embedClient $ lookupRemoteMember cid (tDomain uid) (tUnqualified uid)
   GetRemoteMembers rcid -> embedClient $ lookupRemoteMembers rcid
   CheckLocalMemberRemoteConv uid rcnv -> fmap (not . null) $ embedClient $ lookupLocalMemberRemoteConv uid rcnv
diff --git a/services/galley/src/Galley/Cassandra/Queries.hs b/services/galley/src/Galley/Cassandra/Queries.hs
index 8d53bf938d..3a0f2faa78 100644
--- a/services/galley/src/Galley/Cassandra/Queries.hs
+++ b/services/galley/src/Galley/Cassandra/Queries.hs
@@ -330,6 +330,9 @@ selectMember = "select user, service, provider, status, otr_muted_status, otr_mu
 selectMembers :: PrepQuery R (Identity ConvId) (UserId, Maybe ServiceId, Maybe ProviderId, Maybe MemberStatus, Maybe MutedStatus, Maybe Text, Maybe Bool, Maybe Text, Maybe Bool, Maybe Text, Maybe RoleName)
 selectMembers = "select user, service, provider, status, otr_muted_status, otr_muted_ref, otr_archived, otr_archived_ref, hidden, hidden_ref, conversation_role from member where conv = ?"
 
+selectAllMembers :: PrepQuery R () (UserId, Maybe ServiceId, Maybe ProviderId, Maybe MemberStatus, Maybe MutedStatus, Maybe Text, Maybe Bool, Maybe Text, Maybe Bool, Maybe Text, Maybe RoleName)
+selectAllMembers = "select user, service, provider, status, otr_muted_status, otr_muted_ref, otr_archived, otr_archived_ref, hidden, hidden_ref, conversation_role from member"
+
 insertMember :: PrepQuery W (ConvId, UserId, Maybe ServiceId, Maybe ProviderId, RoleName) ()
 insertMember = "insert into member (conv, user, service, provider, status, conversation_role) values (?, ?, ?, ?, 0, ?)"
 
diff --git a/services/galley/src/Galley/Effects/ExternalAccess.hs b/services/galley/src/Galley/Effects/ExternalAccess.hs
index 936f0da5c8..301eeed4ed 100644
--- a/services/galley/src/Galley/Effects/ExternalAccess.hs
+++ b/services/galley/src/Galley/Effects/ExternalAccess.hs
@@ -31,10 +31,11 @@ import Galley.Data.Services
 import Imports
 import Polysemy
 import Wire.API.Event.Conversation
+import Data.Aeson
 
 data ExternalAccess m a where
   Deliver :: Foldable f => f (BotMember, Event) -> ExternalAccess m [BotMember]
-  DeliverAsync :: Foldable f => f (BotMember, Event) -> ExternalAccess m ()
+  DeliverAsync :: (ToJSON e, Foldable f) => f (BotMember, e) -> ExternalAccess m ()
   DeliverAndDeleteAsync :: Foldable f => ConvId -> f (BotMember, Event) -> ExternalAccess m ()
 
 makeSem ''ExternalAccess
diff --git a/services/galley/src/Galley/Effects/MemberStore.hs b/services/galley/src/Galley/Effects/MemberStore.hs
index e8e6e2e34c..311ebaec90 100644
--- a/services/galley/src/Galley/Effects/MemberStore.hs
+++ b/services/galley/src/Galley/Effects/MemberStore.hs
@@ -30,6 +30,7 @@ module Galley.Effects.MemberStore
     -- * Read members
     getLocalMember,
     getLocalMembers,
+    getAllLocalMembers,
     getRemoteMember,
     getRemoteMembers,
     checkLocalMemberRemoteConv,
@@ -70,6 +71,7 @@ data MemberStore m a where
   CreateBotMember :: ServiceRef -> BotId -> ConvId -> MemberStore m BotMember
   GetLocalMember :: ConvId -> UserId -> MemberStore m (Maybe LocalMember)
   GetLocalMembers :: ConvId -> MemberStore m [LocalMember]
+  GetAllLocalMembers :: MemberStore m [LocalMember]
   GetRemoteMember :: ConvId -> Remote UserId -> MemberStore m (Maybe RemoteMember)
   GetRemoteMembers :: ConvId -> MemberStore m [RemoteMember]
   CheckLocalMemberRemoteConv :: UserId -> Remote ConvId -> MemberStore m Bool
diff --git a/services/galley/src/Galley/External.hs b/services/galley/src/Galley/External.hs
index e265c15a9e..17927a8c28 100644
--- a/services/galley/src/Galley/External.hs
+++ b/services/galley/src/Galley/External.hs
@@ -45,6 +45,7 @@ import URI.ByteString
 import UnliftIO (Async, async, waitCatch)
 import Wire.API.Event.Conversation (Event)
 import Wire.API.Provider.Service (serviceRefId, serviceRefProvider)
+import Data.Aeson (ToJSON)
 
 interpretExternalAccess ::
   ( Member (Embed IO) r,
@@ -60,7 +61,7 @@ interpretExternalAccess = interpret $ \case
 -- | Like deliver, but ignore orphaned bots and return immediately.
 --
 -- FUTUREWORK: Check if this can be removed.
-deliverAsync :: [(BotMember, Event)] -> App ()
+deliverAsync :: ToJSON e => [(BotMember, e)] -> App ()
 deliverAsync = void . forkIO . void . deliver
 
 -- | Like deliver, but remove orphaned bots and return immediately.
@@ -69,10 +70,10 @@ deliverAndDeleteAsync cnv pushes = void . forkIO $ do
   gone <- deliver pushes
   mapM_ (deleteBot cnv . botMemId) gone
 
-deliver :: [(BotMember, Event)] -> App [BotMember]
+deliver :: forall e. ToJSON e => [(BotMember, e)] -> App [BotMember]
 deliver pp = mapM (async . exec) pp >>= foldM eval [] . zip (map fst pp)
   where
-    exec :: (BotMember, Event) -> App Bool
+    exec :: (BotMember, e) -> App Bool
     exec (b, e) =
       lookupService (botMemService b) >>= \case
         Nothing -> pure False
@@ -118,7 +119,7 @@ deliver pp = mapM (async . exec) pp >>= foldM eval [] . zip (map fst pp)
 
 -- Internal -------------------------------------------------------------------
 
-deliver1 :: Service -> BotMember -> Event -> App ()
+deliver1 :: ToJSON e => Service -> BotMember -> e -> App ()
 deliver1 s bm e
   | s ^. serviceEnabled = do
       let t = toByteString' (s ^. serviceToken)
diff --git a/services/galley/src/Galley/Intra/Push/Internal.hs b/services/galley/src/Galley/Intra/Push/Internal.hs
index 5232489a3c..a09c7f4732 100644
--- a/services/galley/src/Galley/Intra/Push/Internal.hs
+++ b/services/galley/src/Galley/Intra/Push/Internal.hs
@@ -41,17 +41,20 @@ import UnliftIO.Async (mapConcurrently_)
 import Wire.API.Event.Conversation (Event (evtFrom))
 import qualified Wire.API.Event.FeatureConfig as FeatureConfig
 import qualified Wire.API.Event.Team as Teams
+import qualified Wire.API.Event.Federation as Federation
 import Wire.API.Team.Member
 
 data PushEvent
   = ConvEvent Event
   | TeamEvent Teams.Event
   | FeatureConfigEvent FeatureConfig.Event
+  | FederationEvent Federation.Event
 
 pushEventJson :: PushEvent -> Object
 pushEventJson (ConvEvent e) = toJSONObject e
 pushEventJson (TeamEvent e) = toJSONObject e
 pushEventJson (FeatureConfigEvent e) = toJSONObject e
+pushEventJson (FederationEvent e) = toJSONObject e
 
 data RecipientBy user = Recipient
   { _recipientUserId :: user,

From cbbf5356de6cce7c9317a14a1a77f48892ea81b6 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 3 Jul 2023 13:58:34 +1000
Subject: [PATCH 199/220] Removing a redundant language extension

---
 services/brig/src/Brig/API/Internal.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 7261ea58a6..89930bc134 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -1,5 +1,3 @@
-{-# LANGUAGE TypeApplications #-}
-
 -- This file is part of the Wire Server implementation.
 --
 -- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>

From 42b04b4b72fa83821c20a9436e7e22682dcd475d Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 3 Jul 2023 14:01:03 +1000
Subject: [PATCH 200/220] Removing a redundant language extension

---
 services/brig/src/Brig/API/Internal.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 7261ea58a6..89930bc134 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -1,5 +1,3 @@
-{-# LANGUAGE TypeApplications #-}
-
 -- This file is part of the Wire Server implementation.
 --
 -- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>

From 020bf6ce8d9835cd2e4a017850e82589d23eb8fc Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 4 Jul 2023 15:25:55 +1000
Subject: [PATCH 201/220] WPB-240: Matching event fanout count to the limit in
 options.

Adding tests for event fanout.
---
 .../wire-api/src/Wire/API/Event/Federation.hs | 13 +++-
 services/galley/src/Galley/API/Internal.hs    | 15 +++--
 services/galley/test/integration/API.hs       | 63 ++++++++++++++++++-
 services/galley/test/integration/API/Util.hs  | 29 +++++++++
 4 files changed, 111 insertions(+), 9 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Event/Federation.hs b/libs/wire-api/src/Wire/API/Event/Federation.hs
index 4e8f2ef089..52aac5595f 100644
--- a/libs/wire-api/src/Wire/API/Event/Federation.hs
+++ b/libs/wire-api/src/Wire/API/Event/Federation.hs
@@ -11,12 +11,13 @@ import qualified Data.Swagger as S
 import Imports
 import Wire.Arbitrary
 import Data.Domain
+import Data.Aeson (ToJSON, FromJSON)
 
 data Event = Event
   { _eventType :: EventType
   , _eventDomains :: [Domain]
   }
-  deriving (Eq, Ord)
+  deriving (Eq, Show, Ord, Generic)
 
 instance Arbitrary Event where
   arbitrary = Event
@@ -25,7 +26,7 @@ instance Arbitrary Event where
 
 data EventType
   = FederationDelete
-  deriving (Eq, Ord, Generic)
+  deriving (Eq, Show, Ord, Generic)
   deriving (Arbitrary) via (GenericUniform EventType)
   deriving (A.FromJSON, A.ToJSON, S.ToSchema) via Schema EventType
 
@@ -51,4 +52,10 @@ instance ToJSONObject Event where
       . schemaOut eventObjectSchema
 
 instance S.ToSchema Event where
-  declareNamedSchema = schemaToSwagger
\ No newline at end of file
+  declareNamedSchema = schemaToSwagger
+
+instance FromJSON Event where
+  parseJSON = schemaParseJSON
+
+instance ToJSON Event where
+  toJSON = schemaToJSON
\ No newline at end of file
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index f0d18eed4d..9ac290d3c0 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -127,6 +127,7 @@ import Cassandra (Consistency(LocalQuorum), Page (..), paramsP, ClientState)
 import Database.CQL.IO (paginate)
 import Galley.Cassandra.Queries
 import Galley.Cassandra.Conversation.Members
+import Galley.Env (currentFanoutLimit, _options)
 
 internalAPI :: API InternalAPI GalleyEffects
 internalAPI =
@@ -331,7 +332,7 @@ internalSitemap = unsafeCallsFed @'Galley @"on-client-removed" $ unsafeCallsFed
     capture "domain"
       .&. accept "application" "json"
 
-  delete "/i/federation/:domain" (continue (internalDeleteFederationDomainH (toRange (Proxy @1000)))) $
+  delete "/i/federation/:domain" (continue internalDeleteFederationDomainH) $
     capture "domain"
       .&. accept "application" "json"
 
@@ -549,10 +550,9 @@ internalDeleteFederationDomainH ::
     Member GundeckAccess r,
     Member ExternalAccess r
   ) =>
-  Range 1 1000 Int32 -> -- TODO what values should go here?
   Domain ::: JSON ->
   Sem r Response
-internalDeleteFederationDomainH (fromRange -> maxPage) (domain ::: _) = do
+internalDeleteFederationDomainH (domain ::: _) = do
   -- We have to send the same event twice.
   -- Once before and once after defederation work.
   -- https://wearezeta.atlassian.net/wiki/spaces/ENGINEERIN/pages/809238539/Use+case+Stopping+to+federate+with+a+domain
@@ -562,14 +562,19 @@ internalDeleteFederationDomainH (fromRange -> maxPage) (domain ::: _) = do
   pure (empty & setStatus status200)
   where
     sendNotifications = do
+      maxPage <- inputs $ fromRange . currentFanoutLimit . _options -- This is based on the limits in removeIfLargeFanout
       page <- embedClient $ paginate selectAllMembers (paramsP LocalQuorum () maxPage)
       sendNotificationPage page
     pushEvents results = do
       let (bots, mems) = localBotsAndUsers results
           recipients = Intra.recipient <$> mems
           event = Intra.FederationEvent $ Federation.Event Federation.FederationDelete [domain]
-      for_ (Intra.newPush ListComplete Nothing event recipients) $ \p ->
-        push1 $ p & Intra.pushRoute .~ Intra.RouteDirect
+      for_ (Intra.newPush ListComplete Nothing event recipients) $ \p -> do
+        -- TODO: Transient or not?
+        -- RouteAny is used as it will wake up mobile clients
+        -- and notify them of the changes to federation state.
+        push1 $ p & Intra.pushRoute .~ Intra.RouteAny
+                  -- & Intra.pushTransient .~ True
       deliverAsync (bots `zip` repeat (pushEventJson event))
     sendNotificationPage page = do
       let res = result page
diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs
index 95cced3baf..59a1f48fe1 100644
--- a/services/galley/test/integration/API.hs
+++ b/services/galley/test/integration/API.hs
@@ -265,7 +265,8 @@ tests s =
             [ test s "send typing indicators" postTypingIndicators,
               test s "send typing indicators without domain" postTypingIndicatorsV2,
               test s "send typing indicators with invalid pyaload" postTypingIndicatorsHandlesNonsense
-            ]
+            ],
+          test s "delete federation notifications" testDefederationNotifications
         ]
     rb1, rb2, rb3 :: Remote Backend
     rb1 =
@@ -4538,3 +4539,63 @@ testOne2OneConversationRequest shouldBeLocal actor desired = do
             pure $ statusCode resp == 200
           liftIO $ found @?= ((actor, desired) == (LocalActor, Included))
       )
+
+-- Testing defederation notifications. The important thing to note for all
+-- of this is that when defederating from a remote domain only _2_ notifications
+-- are sent, and both are identical. One notification is at the start of
+-- defederation, and one is sent at the end of defederation. No other
+-- notifications about users being removed from conversations, or conversations
+-- being deleted are sent. We are do not want to DOS either our local clients,
+-- nor our own services.
+testDefederationNotifications :: TestM ()
+testDefederationNotifications = do
+  -- alice, bob are in a team
+  (tid, alice, [bob]) <- createBindingTeamWithQualifiedMembers 2
+
+  -- charlie is a local guest
+  charlie <- randomQualifiedUser
+  connectUsers (qUnqualified alice) (pure (qUnqualified charlie))
+
+  -- dee is a remote guest
+  let remoteDomain = Domain "far-away.example.com"
+  dee <- Qualified <$> randomId <*> pure remoteDomain
+
+  connectWithRemoteUser (qUnqualified alice) dee
+
+  -- they are all in a local conversation
+  conv <-
+    responseJsonError
+      =<< postConvWithRemoteUsers
+        (qUnqualified alice)
+        Nothing
+        defNewProteusConv
+          { newConvQualifiedUsers = [bob, charlie, dee],
+            newConvTeam = Just (ConvTeamInfo tid)
+          }
+        <!! const 201 === statusCode
+
+  c <- view tsCannon
+  WS.bracketRN c (map qUnqualified [alice, bob, charlie, dee]) $ \[wsA, wsB, wsC, wsD] -> do
+    -- conversation access role changes to team only
+    (_, reqs) <- withTempMockFederator' (mockReply ()) $ do
+      -- Delete the domain that Dee lives on
+      deleteFederation remoteDomain !!! const 200 === statusCode
+      -- First notification to local clients
+      WS.assertMatchN_ (5 # Second) [wsA, wsB, wsC] $
+        wsAssertFederationDeleted remoteDomain
+      -- Second notification to local clients
+      WS.assertMatchN_ (5 # Second) [wsA, wsB, wsC] $
+        wsAssertFederationDeleted remoteDomain
+      -- dee's remote doesn't receive a notification
+      WS.assertNoEvent (5 # Second) [wsD]
+    -- There should be not requests out to the federtaion domain
+    liftIO $ reqs @?= []
+
+  -- only alice, bob, and charlie remain
+  conv2 <-
+    responseJsonError
+      =<< getConvQualified (qUnqualified alice) (cnvQualifiedId conv)
+        <!! const 200 === statusCode
+  liftIO $ sort (omQualifiedId <$> cmOthers (cnvMembers conv2)) @?= sort [bob, charlie]
+
+-- @END
\ No newline at end of file
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index 7339e15578..ef0bd9ba32 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -143,6 +143,7 @@ import Wire.API.User.Auth hiding (Access)
 import Wire.API.User.Client
 import qualified Wire.API.User.Client as Client
 import Wire.API.User.Client.Prekey
+import qualified Wire.API.Event.Federation as Fed
 
 -------------------------------------------------------------------------------
 -- API Operations
@@ -1373,6 +1374,15 @@ postJoinCodeConv' mPw u j = do
       -- `json (JoinConversationByCode j Nothing)` and `json j` are equivalent, using the latter to test backwards compatibility
       . (if isJust mPw then json (JoinConversationByCode j mPw) else json j)
 
+deleteFederation ::
+  (MonadHttp m, HasGalley m, MonadIO m) =>
+  Domain ->
+  m ResponseLBS
+deleteFederation dom = do
+  g <- viewGalley
+  delete $
+    g . paths ["/i/federation", toByteString' dom]
+
 putQualifiedAccessUpdate ::
   (MonadHttp m, HasGalley m, MonadIO m) =>
   UserId ->
@@ -1737,6 +1747,25 @@ assertJoinEvent conv usr new role e = do
   evtFrom e @?= usr
   fmap (sort . mMembers) (evtData e ^? _EdMembersJoin) @?= Just (sort (fmap (`SimpleMember` role) new))
 
+wsAssertFederationDeleted ::
+  HasCallStack =>
+  Domain ->
+  Notification ->
+  IO ()
+wsAssertFederationDeleted dom n = do
+  -- TODO: Does it matter if the defederation notifications
+  -- are transient? The docs for the ticket imply that they are
+  -- ntfTransient n @?= False
+  assertFederationDeletedEvent dom $ List1.head (WS.unpackPayload n)
+
+assertFederationDeletedEvent ::
+  Domain ->
+  Fed.Event ->
+  IO ()
+assertFederationDeletedEvent dom e = do
+  Fed._eventType e @?= Fed.FederationDelete
+  Fed._eventDomains e @?= [dom]
+
 -- FUTUREWORK: See if this one can be implemented in terms of:
 --
 -- checkConvMemberLeaveEvent :: HasCallStack => Qualified ConvId -> Qualified UserId -> WS.WebSocket -> TestM ()

From 8be415ae8b4ca136fe9e0d77c326a91e934fa471 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 5 Jul 2023 12:11:46 +1000
Subject: [PATCH 202/220] WPB-240: Comments and code in tests for quickly
 filling the DB

---
 services/galley/test/integration/API.hs      | 25 ++++++++++++++++----
 services/galley/test/integration/API/Util.hs |  4 +---
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs
index 59a1f48fe1..6d77fb0389 100644
--- a/services/galley/test/integration/API.hs
+++ b/services/galley/test/integration/API.hs
@@ -4556,8 +4556,25 @@ testDefederationNotifications = do
   charlie <- randomQualifiedUser
   connectUsers (qUnqualified alice) (pure (qUnqualified charlie))
 
-  -- dee is a remote guest
+
   let remoteDomain = Domain "far-away.example.com"
+      -- This variable should be commented out if the below
+      -- section is used to insert users to the database.
+      users = []
+  -- This section of code is useful to massively increase
+  -- the amount of users in the testing database. This is
+  -- useful for checking that notifications are being fanned
+  -- out correctly, and that all users are sent a
+  -- notification. If the database already has a large
+  -- amount of users then this can be left out and will also
+  -- allow this test to run faster.
+  --     count = 10000
+  -- users <- replicateM count randomQualifiedUser
+  -- replicateM_ count $ do
+  --   connectWithRemoteUser (qUnqualified alice) =<<
+  --     Qualified <$> randomId <*> pure remoteDomain
+
+  -- dee is a remote guest
   dee <- Qualified <$> randomId <*> pure remoteDomain
 
   connectWithRemoteUser (qUnqualified alice) dee
@@ -4575,16 +4592,16 @@ testDefederationNotifications = do
         <!! const 201 === statusCode
 
   c <- view tsCannon
-  WS.bracketRN c (map qUnqualified [alice, bob, charlie, dee]) $ \[wsA, wsB, wsC, wsD] -> do
+  WS.bracketRN c (map qUnqualified $ [alice, bob, charlie, dee] <> users) $ \(wsA:wsB:wsC:wsD:wsUsers) -> do
     -- conversation access role changes to team only
     (_, reqs) <- withTempMockFederator' (mockReply ()) $ do
       -- Delete the domain that Dee lives on
       deleteFederation remoteDomain !!! const 200 === statusCode
       -- First notification to local clients
-      WS.assertMatchN_ (5 # Second) [wsA, wsB, wsC] $
+      WS.assertMatchN_ (5 # Second) ([wsA, wsB, wsC] <> wsUsers) $
         wsAssertFederationDeleted remoteDomain
       -- Second notification to local clients
-      WS.assertMatchN_ (5 # Second) [wsA, wsB, wsC] $
+      WS.assertMatchN_ (5 # Second) ([wsA, wsB, wsC] <> wsUsers) $
         wsAssertFederationDeleted remoteDomain
       -- dee's remote doesn't receive a notification
       WS.assertNoEvent (5 # Second) [wsD]
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index ef0bd9ba32..335252ac72 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -1753,9 +1753,7 @@ wsAssertFederationDeleted ::
   Notification ->
   IO ()
 wsAssertFederationDeleted dom n = do
-  -- TODO: Does it matter if the defederation notifications
-  -- are transient? The docs for the ticket imply that they are
-  -- ntfTransient n @?= False
+  ntfTransient n @?= False
   assertFederationDeletedEvent dom $ List1.head (WS.unpackPayload n)
 
 assertFederationDeletedEvent ::

From 37cbed97bee79960c15ff70e5735b39c0a5d3fe4 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 5 Jul 2023 16:08:29 +1000
Subject: [PATCH 203/220] FS-1179: Reworking after a discussion with Akshay
 about how to use AMQP.

Workers are going to have their own connection management to RabbitMQ
which will keep things simpler and help reduce potential blocks in
processing. It also helps us not step on each other's toes when writing
this service.
---
 services/background-worker/src/Wire/BackgroundWorker.hs     | 3 ---
 services/background-worker/src/Wire/BackgroundWorker/Env.hs | 6 +++---
 .../background-worker/src/Wire/BackgroundWorker/Options.hs  | 4 ++--
 3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs
index 6c1e877a17..2fac9f5468 100644
--- a/services/background-worker/src/Wire/BackgroundWorker.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker.hs
@@ -15,9 +15,6 @@ import Wire.BackgroundWorker.Options
 import Wire.Defederation as Defederation
 
 -- FUTUREWORK: Start an http service with status and metrics endpoints
--- NOTE: Use atomic IORef writes to impose an ordering barrier on
---       reads and writes. This stops the CPU from being too clever
---       with its memory model and what it thinks it can get away with.
 run :: Opts -> IO ()
 run opts = do
   (env, syncThread) <- mkEnv opts
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
index 33a984ffdb..c7a5757cf7 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
@@ -37,17 +37,17 @@ data Worker
 
 data Env = Env
   { http2Manager :: Http2Manager,
-    httpManager :: Manager,
+    rabbitmqAdminClient :: RabbitMqAdmin.AdminAPI (Servant.AsClientT IO),
+    rabbitmqVHost :: Text,
     logger :: Logger,
     metrics :: Metrics.Metrics,
     federatorInternal :: Endpoint,
+    httpManager :: Manager,
     galley :: Endpoint,
     brig :: Endpoint,
     defederationTimeout :: ResponseTimeout,
     remoteDomains :: IORef FederationDomainConfigs,
     remoteDomainsChan :: Chan FederationDomainConfigs,
-    rabbitmqAdminClient :: RabbitMqAdmin.AdminAPI (Servant.AsClientT IO),
-    rabbitmqVHost :: Text,
     backendNotificationMetrics :: BackendNotificationMetrics,
     statuses :: IORef (Map Worker IsWorking)
   }
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
index 66b5539bde..1778dcf905 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs
@@ -11,10 +11,10 @@ data Opts = Opts
     logFormat :: !(Maybe (Last LogFormat)),
     backgroundWorker :: !Endpoint,
     federatorInternal :: !Endpoint,
+    rabbitmq :: !RabbitMqAdminOpts,
     galley :: !Endpoint,
     brig :: !Endpoint,
-    defederationTimeout :: Maybe Int, -- Seconds, Nothing for no timeout
-    rabbitmq :: !RabbitMqAdminOpts
+    defederationTimeout :: Maybe Int -- Seconds, Nothing for no timeout
   }
   deriving (Show, Generic)
 

From 92da98099469ec3be56963852f49eae2b10c1106 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 5 Jul 2023 17:33:03 +1000
Subject: [PATCH 204/220] PR formatting

---
 services/background-worker/default.nix                 |  5 +++--
 .../src/Wire/BackendNotificationPusher.hs              |  7 ++++---
 .../background-worker/src/Wire/BackgroundWorker/Env.hs | 10 ++++++----
 services/background-worker/src/Wire/Defederation.hs    |  4 ++--
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/services/background-worker/default.nix b/services/background-worker/default.nix
index cda4b03422..ec0475b3c2 100644
--- a/services/background-worker/default.nix
+++ b/services/background-worker/default.nix
@@ -80,6 +80,8 @@ mkDerivation {
     tinylog
     transformers-base
     types-common
+    unliftio
+    wai-utilities
     wire-api
     wire-api-federation
   ];
@@ -98,6 +100,7 @@ mkDerivation {
     http-types
     HUnit
     imports
+    prometheus-client
     QuickCheck
     servant
     servant-client
@@ -110,8 +113,6 @@ mkDerivation {
     unliftio
     wai
     wire-api
-    unliftio
-    wai-utilities
     wire-api-federation
   ];
   testHaskellDepends = [
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index d5a4f7ecb2..b00d70cff9 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -13,6 +13,7 @@ import qualified Data.Text as Text
 import Imports
 import Network.AMQP (cancelConsumer)
 import qualified Network.AMQP as Q
+import Network.AMQP.Extended
 import qualified Network.AMQP.Lifted as QL
 import Network.RabbitMqAdmin
 import Prometheus
@@ -22,7 +23,6 @@ import Wire.API.Federation.Client
 import Wire.API.Routes.FederationDomainConfig
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
-import Network.AMQP.Extended
 
 startPushingNotifications ::
   Q.Channel ->
@@ -105,7 +105,8 @@ startPusher chan = do
   -- FUTUREWORK?:
   -- If this throws an exception on the Chan / in the forever loop, the exception will
   -- bubble all the way up and kill the pod. Kubernetes should restart the pod automatically.
-  flip catches
+  flip
+    catches
     [ Handler $ cleanup @SomeException,
       Handler $ cleanup @SomeAsyncException
     ]
@@ -202,4 +203,4 @@ startWorker rabbitmqOpts = do
           runAppT env $ markAsNotWorking BackendNotificationPusher,
         onConnectionClose =
           runAppT env $ markAsNotWorking BackendNotificationPusher
-      }
\ No newline at end of file
+      }
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
index c7a5757cf7..f258cd4eeb 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
@@ -87,10 +87,12 @@ mkEnv opts = do
           }
   (remoteDomains, syncThread) <- syncFedDomainConfigs brig logger callback
   rabbitmqAdminClient <- mkRabbitMqAdminClientEnv opts.rabbitmq
-  statuses <- newIORef $ Map.fromList
-    [ (BackendNotificationPusher, False)
-    , (DefederationWorker, False)
-    ]
+  statuses <-
+    newIORef $
+      Map.fromList
+        [ (BackendNotificationPusher, False),
+          (DefederationWorker, False)
+        ]
   metrics <- Metrics.metrics
   backendNotificationMetrics <- mkBackendNotificationMetrics
   pure (Env {..}, syncThread)
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index 561b47b4a5..7e9b905408 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -13,6 +13,7 @@ import Data.Text.Encoding
 import Imports
 import Network.AMQP (cancelConsumer)
 import qualified Network.AMQP as Q
+import Network.AMQP.Extended
 import qualified Network.AMQP.Lifted as QL
 import Network.HTTP.Client
 import Network.HTTP.Types
@@ -21,7 +22,6 @@ import Util.Options
 import Wire.API.Federation.BackendNotifications
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Util
-import Network.AMQP.Extended
 
 deleteFederationDomain :: Q.Channel -> AppT IO Q.ConsumerTag
 deleteFederationDomain chan = do
@@ -124,4 +124,4 @@ startWorker rabbitmqOpts = do
           runAppT env . startDefederator,
         onChannelException = const onChanClose,
         onConnectionClose = onChanClose
-      }
\ No newline at end of file
+      }

From 2af54f5fa596f970474949b66a977094917fb1e8 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 5 Jul 2023 18:19:55 +1000
Subject: [PATCH 205/220] Hi CI


From 456235a7bc73ff4245b73cc44163f7849ec99feb Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 5 Jul 2023 18:54:45 +1000
Subject: [PATCH 206/220] FS-1179: Adding galley and brig to
 background-worker's configmap

---
 charts/background-worker/templates/configmap.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/charts/background-worker/templates/configmap.yaml b/charts/background-worker/templates/configmap.yaml
index 13202e9963..7e3612252d 100644
--- a/charts/background-worker/templates/configmap.yaml
+++ b/charts/background-worker/templates/configmap.yaml
@@ -20,6 +20,15 @@ data:
     federatorInternal:
       host: federator
       port: 8080
+
+    galley:
+      host: galley
+      port: 8080
+
+    brig:
+      host: brig
+      port: 8080
+
     rabbitmq:
 {{toYaml .rabbitmq | indent 6 }}
     backendNotificationPusher:

From 32b5e05b3bf6b1e51c47b45a2914f53adca88419 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 10 Jul 2023 18:43:03 +1000
Subject: [PATCH 207/220] FS-1179: Removing the confusing "integration" tests
 for background-worker

They were never actual integration tests, I was just confused about why
`make ci` wasn't picking up the existing tests, so I made it run them
and that was a mistake and caused confusion.
---
 .../background-worker/background-worker.cabal | 92 -------------------
 services/background-worker/test/Main.hs       | 23 +----
 services/background-worker/test/Spec.hs       |  1 -
 3 files changed, 1 insertion(+), 115 deletions(-)
 delete mode 100644 services/background-worker/test/Spec.hs

diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal
index 829ec7be22..5603baccaa 100644
--- a/services/background-worker/background-worker.cabal
+++ b/services/background-worker/background-worker.cabal
@@ -161,97 +161,6 @@ executable background-worker
     UndecidableInstances
     ViewPatterns
 
-executable background-worker-integration
-  default-language:   Haskell2010
-  type:               exitcode-stdio-1.0
-  hs-source-dirs:     test
-  main-is:            Main.hs
-  ghc-options:
-    -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates
-    -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path
-    -funbox-strict-fields -threaded -with-rtsopts=-N
-    -Wredundant-constraints -Wunused-packages
-
-  -- cabal-fmt: expand test
-  other-modules:
-    Main
-    Spec
-    Test.Wire.BackendNotificationPusherSpec
-    Test.Wire.DefederationSpec
-    Test.Wire.Util
-
-  build-depends:
-      aeson
-    , amqp
-    , background-worker
-    , base
-    , bytestring
-    , containers
-    , extended
-    , federator
-    , hspec
-    , http-client
-    , http-media
-    , http-types
-    , HUnit
-    , imports
-    , prometheus-client
-    , QuickCheck
-    , servant
-    , servant-client
-    , servant-client-core
-    , servant-server
-    , text
-    , tinylog
-    , transformers
-    , types-common
-    , unliftio
-    , wai
-    , wire-api
-    , wire-api-federation
-
-  default-extensions:
-    NoImplicitPrelude
-    AllowAmbiguousTypes
-    BangPatterns
-    ConstraintKinds
-    DataKinds
-    DefaultSignatures
-    DeriveFunctor
-    DeriveGeneric
-    DeriveLift
-    DeriveTraversable
-    DerivingStrategies
-    DerivingVia
-    DuplicateRecordFields
-    EmptyCase
-    FlexibleContexts
-    FlexibleInstances
-    FunctionalDependencies
-    GADTs
-    InstanceSigs
-    KindSignatures
-    LambdaCase
-    MultiParamTypeClasses
-    MultiWayIf
-    NamedFieldPuns
-    OverloadedRecordDot
-    OverloadedStrings
-    PackageImports
-    PatternSynonyms
-    PolyKinds
-    QuasiQuotes
-    RankNTypes
-    ScopedTypeVariables
-    StandaloneDeriving
-    TupleSections
-    TypeApplications
-    TypeFamilies
-    TypeFamilyDependencies
-    TypeOperators
-    UndecidableInstances
-    ViewPatterns
-
 test-suite background-worker-test
   default-language:   Haskell2010
   type:               exitcode-stdio-1.0
@@ -266,7 +175,6 @@ test-suite background-worker-test
   -- cabal-fmt: expand test
   other-modules:
     Main
-    Spec
     Test.Wire.BackendNotificationPusherSpec
     Test.Wire.DefederationSpec
     Test.Wire.Util
diff --git a/services/background-worker/test/Main.hs b/services/background-worker/test/Main.hs
index be0d00983c..52ef578fca 100644
--- a/services/background-worker/test/Main.hs
+++ b/services/background-worker/test/Main.hs
@@ -1,22 +1 @@
-module Main where
-
-import Imports
-import qualified Spec
-import System.Environment (withArgs)
-import Test.Hspec.Runner
-
--- See https://hspec.github.io/hspec-discover.html#using-a-custom-main-function and
--- /services/spar/test-integration/Main.hs
-main :: IO ()
-main = do
-  -- (we don't need wire args, appearently.  but if we ever should, here we have them.)
-  (_wireArgs, hspecArgs) <- partitionArgs <$> getArgs
-  withArgs hspecArgs . hspec $ Spec.spec
-
-partitionArgs :: [String] -> ([String], [String])
-partitionArgs = go [] []
-  where
-    go wireArgs hspecArgs ("-s" : x : xs) = go (wireArgs <> ["-s", x]) hspecArgs xs
-    go wireArgs hspecArgs ("-i" : x : xs) = go (wireArgs <> ["-i", x]) hspecArgs xs
-    go wireArgs hspecArgs (x : xs) = go wireArgs (hspecArgs <> [x]) xs
-    go wireArgs hspecArgs [] = (wireArgs, hspecArgs)
+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
\ No newline at end of file
diff --git a/services/background-worker/test/Spec.hs b/services/background-worker/test/Spec.hs
deleted file mode 100644
index 5416ef6a86..0000000000
--- a/services/background-worker/test/Spec.hs
+++ /dev/null
@@ -1 +0,0 @@
-{-# OPTIONS_GHC -F -pgmF hspec-discover -optF --module-name=Spec #-}

From b0355f763b178a8776253ef502cb15c0f9dc2147 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Mon, 10 Jul 2023 18:49:50 +1000
Subject: [PATCH 208/220] PR notes

---
 changelog.d/6-federation/fs-1179 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/6-federation/fs-1179 b/changelog.d/6-federation/fs-1179
index 589f66fccf..87ba118288 100644
--- a/changelog.d/6-federation/fs-1179
+++ b/changelog.d/6-federation/fs-1179
@@ -1 +1 @@
-Removing a federation domain will now remove all conversations for that domain.
\ No newline at end of file
+Removing a federation domain will now remove all conversations and users for that domain from the local database.
\ No newline at end of file

From c4c28aa9da91022b73e048aa422b87fb11748a73 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 11 Jul 2023 12:40:25 +0200
Subject: [PATCH 209/220] `make sanitize-pr`.

---
 .../API/Federation/BackendNotifications.hs    |  4 +--
 .../src/Wire/BackendNotificationPusher.hs     |  2 +-
 .../src/Wire/BackgroundWorker/Env.hs          | 10 +++---
 .../src/Wire/Defederation.hs                  |  4 +--
 .../Wire/BackendNotificationPusherSpec.hs     |  6 ++--
 services/brig/src/Brig/API/Internal.hs        | 18 +++++------
 services/brig/src/Brig/Data/Connection.hs     | 32 +++++++++----------
 .../brig/test/integration/API/Internal.hs     | 22 ++++++-------
 .../federator/src/Federator/Validation.hs     |  6 ++--
 services/galley/src/Galley/API/Action.hs      | 10 +++---
 services/galley/src/Galley/API/Internal.hs    |  2 +-
 services/galley/src/Galley/API/Update.hs      | 12 +++----
 .../Galley/Cassandra/Conversation/Members.hs  |  2 +-
 .../galley/src/Galley/Effects/MemberStore.hs  |  4 +--
 services/galley/src/Galley/Run.hs             |  2 +-
 .../galley/test/integration/Federation.hs     |  2 +-
 16 files changed, 69 insertions(+), 69 deletions(-)

diff --git a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
index 3e37702eef..91d0f32349 100644
--- a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
+++ b/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs
@@ -72,7 +72,7 @@ sendNotification env component path body =
     withoutFirstSlash (Text.stripPrefix "/" -> Just t) = t
     withoutFirstSlash t = t
 
-    go :: forall c. KnownComponent c => IO (Either FederatorClientError ())
+    go :: forall c. (KnownComponent c) => IO (Either FederatorClientError ())
     go =
       runFederatorClient env . void $
         clientIn (Proxy @BackendNotificationAPI) (Proxy @(FederatorClient c)) (withoutFirstSlash path) body
@@ -142,7 +142,7 @@ data EnqueueError = EnqueueError String
 
 instance Exception EnqueueError
 
-instance KnownComponent c => RunClient (FedQueueClient c) where
+instance (KnownComponent c) => RunClient (FedQueueClient c) where
   runRequestAcceptStatus :: Maybe [Status] -> Request -> FedQueueClient c Response
   runRequestAcceptStatus _ req = do
     env <- ask
diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
index b00d70cff9..10cf0bdefa 100644
--- a/services/background-worker/src/Wire/BackendNotificationPusher.hs
+++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -32,7 +32,7 @@ startPushingNotifications chan domain = do
   lift $ ensureQueue chan domain._domainText
   QL.consumeMsgs chan (routingKey domain._domainText) Q.Ack (pushNotification domain)
 
-pushNotification :: RabbitMQEnvelope e => Domain -> (Q.Message, e) -> AppT IO ()
+pushNotification :: (RabbitMQEnvelope e) => Domain -> (Q.Message, e) -> AppT IO ()
 pushNotification targetDomain (msg, envelope) = do
   -- Jittered exponential backoff with 10ms as starting delay and 300s as max
   -- delay. When 300s is reached, every retry will happen after 300s.
diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
index f258cd4eeb..30d165538d 100644
--- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs
+++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs
@@ -123,11 +123,11 @@ newtype AppT m a = AppT {unAppT :: ReaderT Env m a}
       MonadMonitor
     )
 
-deriving newtype instance MonadBase b m => MonadBase b (AppT m)
+deriving newtype instance (MonadBase b m) => MonadBase b (AppT m)
 
-deriving newtype instance MonadBaseControl b m => MonadBaseControl b (AppT m)
+deriving newtype instance (MonadBaseControl b m) => MonadBaseControl b (AppT m)
 
-instance MonadIO m => MonadLogger (AppT m) where
+instance (MonadIO m) => MonadLogger (AppT m) where
   log lvl m = do
     l <- asks logger
     Log.log l lvl m
@@ -135,10 +135,10 @@ instance MonadIO m => MonadLogger (AppT m) where
 runAppT :: Env -> AppT m a -> m a
 runAppT env app = runReaderT (unAppT app) env
 
-markAsWorking :: MonadIO m => Worker -> AppT m ()
+markAsWorking :: (MonadIO m) => Worker -> AppT m ()
 markAsWorking worker =
   flip modifyIORef (Map.insert worker True) =<< asks statuses
 
-markAsNotWorking :: MonadIO m => Worker -> AppT m ()
+markAsNotWorking :: (MonadIO m) => Worker -> AppT m ()
 markAsNotWorking worker =
   flip modifyIORef (Map.insert worker False) =<< asks statuses
diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs
index 7e9b905408..e4d08addbb 100644
--- a/services/background-worker/src/Wire/Defederation.hs
+++ b/services/background-worker/src/Wire/Defederation.hs
@@ -32,7 +32,7 @@ x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
 
 -- Exposed for testing purposes so we can decode without further processing the message.
-deleteFederationDomainInner' :: RabbitMQEnvelope e => (e -> DefederationDomain -> AppT IO ()) -> (Q.Message, e) -> AppT IO ()
+deleteFederationDomainInner' :: (RabbitMQEnvelope e) => (e -> DefederationDomain -> AppT IO ()) -> (Q.Message, e) -> AppT IO ()
 deleteFederationDomainInner' go (msg, envelope) = do
   either
     ( \e -> do
@@ -53,7 +53,7 @@ deleteFederationDomainInner' go (msg, envelope) = do
 -- What should we do with non-recoverable (unparsable) errors/messages?
 -- should we deadletter, or do something else?
 -- Deadlettering has a privacy implication -- FUTUREWORK.
-deleteFederationDomainInner :: RabbitMQEnvelope e => (Q.Message, e) -> AppT IO ()
+deleteFederationDomainInner :: (RabbitMQEnvelope e) => (Q.Message, e) -> AppT IO ()
 deleteFederationDomainInner (msg, envelope) = do
   env <- ask
   let manager = httpManager env
diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
index 88f29f1dc1..e5dd96a433 100644
--- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
+++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs
@@ -231,7 +231,7 @@ spec = do
       calls `shouldSatisfy` (\c -> length c >= 2)
       mapM_ (\vhost -> vhost `shouldBe` rabbitmqVHost) calls
 
-untilM :: Monad m => m Bool -> m ()
+untilM :: (Monad m) => m Bool -> m ()
 untilM action = do
   b <- action
   unless b $ untilM action
@@ -283,7 +283,7 @@ mockRabbitMqAdminClient :: forall api. (api ~ ToServant AdminAPI AsApi) => MockR
 mockRabbitMqAdminClient mockAdmin = fromServant $ hoistClient (Proxy @api) (flip runReaderT (mockRabbitMqAdminApp mockAdmin) . runWaiClient) (waiClient @api)
 
 -- | Create servant client for an API, this can be run using 'hoistClient'.
-waiClient :: forall api. HasClient WaiClient api => Client WaiClient api
+waiClient :: forall api. (HasClient WaiClient api) => Client WaiClient api
 waiClient = clientIn (Proxy @api) (Proxy @WaiClient)
 
 -- | Runs a servant client by directly calling a wai application, instead of
@@ -337,7 +337,7 @@ instance RunClient WaiClient where
   throwClientError :: ClientError -> WaiClient a
   throwClientError = liftIO . throwIO
 
-waiResponseToServant :: MonadIO m => Wai.Response -> m Response
+waiResponseToServant :: (MonadIO m) => Wai.Response -> m Response
 waiResponseToServant res = do
   let (status, hdrs, contBody) = Wai.responseToStream res
   body <- liftIO $ contBody $ \streamingBody -> do
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 89930bc134..5ba0fad3bd 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -137,7 +137,7 @@ istatusAPI :: forall r. ServerT BrigIRoutes.IStatusAPI (Handler r)
 istatusAPI = Named @"get-status" (pure NoContent)
 
 ejpdAPI ::
-  Member GalleyProvider r =>
+  (Member GalleyProvider r) =>
   ServerT BrigIRoutes.EJPD_API (Handler r)
 ejpdAPI =
   Brig.User.EJPD.ejpdRequest
@@ -692,14 +692,14 @@ deleteUserNoAuthH uid = do
     AccountAlreadyDeleted -> pure UserResponseAccountAlreadyDeleted
     AccountDeleted -> pure UserResponseAccountDeleted
 
-changeSelfEmailMaybeSendH :: Member BlacklistStore r => UserId -> EmailUpdate -> Maybe Bool -> (Handler r) ChangeEmailResponse
+changeSelfEmailMaybeSendH :: (Member BlacklistStore r) => UserId -> EmailUpdate -> Maybe Bool -> (Handler r) ChangeEmailResponse
 changeSelfEmailMaybeSendH u body (fromMaybe False -> validate) = do
   let email = euEmail body
   changeSelfEmailMaybeSend u (if validate then ActuallySendEmail else DoNotSendEmail) email API.AllowSCIMUpdates
 
 data MaybeSendEmail = ActuallySendEmail | DoNotSendEmail
 
-changeSelfEmailMaybeSend :: Member BlacklistStore r => UserId -> MaybeSendEmail -> Email -> API.AllowSCIMUpdates -> (Handler r) ChangeEmailResponse
+changeSelfEmailMaybeSend :: (Member BlacklistStore r) => UserId -> MaybeSendEmail -> Email -> API.AllowSCIMUpdates -> (Handler r) ChangeEmailResponse
 changeSelfEmailMaybeSend u ActuallySendEmail email allowScim = do
   API.changeSelfEmail u email allowScim
 changeSelfEmailMaybeSend u DoNotSendEmail email allowScim = do
@@ -833,24 +833,24 @@ updateConnectionInternalH (_ ::: req) = do
   API.updateConnectionInternal updateConn !>> connError
   pure $ setStatus status200 empty
 
-checkBlacklistH :: Member BlacklistStore r => Either Email Phone -> (Handler r) Response
+checkBlacklistH :: (Member BlacklistStore r) => Either Email Phone -> (Handler r) Response
 checkBlacklistH emailOrPhone = do
   bl <- lift $ API.isBlacklisted emailOrPhone
   pure $ setStatus (bool status404 status200 bl) empty
 
-deleteFromBlacklistH :: Member BlacklistStore r => Either Email Phone -> (Handler r) Response
+deleteFromBlacklistH :: (Member BlacklistStore r) => Either Email Phone -> (Handler r) Response
 deleteFromBlacklistH emailOrPhone = do
   void . lift $ API.blacklistDelete emailOrPhone
   pure empty
 
-addBlacklistH :: Member BlacklistStore r => Either Email Phone -> (Handler r) Response
+addBlacklistH :: (Member BlacklistStore r) => Either Email Phone -> (Handler r) Response
 addBlacklistH emailOrPhone = do
   void . lift $ API.blacklistInsert emailOrPhone
   pure empty
 
 -- | Get any matching prefixes. Also try for shorter prefix matches,
 -- i.e. checking for +123456 also checks for +12345, +1234, ...
-getPhonePrefixesH :: Member BlacklistPhonePrefixStore r => PhonePrefix -> (Handler r) Response
+getPhonePrefixesH :: (Member BlacklistPhonePrefixStore r) => PhonePrefix -> (Handler r) Response
 getPhonePrefixesH prefix = do
   results <- lift $ API.phonePrefixGet prefix
   pure $ case results of
@@ -858,12 +858,12 @@ getPhonePrefixesH prefix = do
     _ -> json results
 
 -- | Delete a phone prefix entry (must be an exact match)
-deleteFromPhonePrefixH :: Member BlacklistPhonePrefixStore r => PhonePrefix -> (Handler r) Response
+deleteFromPhonePrefixH :: (Member BlacklistPhonePrefixStore r) => PhonePrefix -> (Handler r) Response
 deleteFromPhonePrefixH prefix = do
   void . lift $ API.phonePrefixDelete prefix
   pure empty
 
-addPhonePrefixH :: Member BlacklistPhonePrefixStore r => JSON ::: JsonRequest ExcludedPrefix -> (Handler r) Response
+addPhonePrefixH :: (Member BlacklistPhonePrefixStore r) => JSON ::: JsonRequest ExcludedPrefix -> (Handler r) Response
 addPhonePrefixH (_ ::: req) = do
   prefix :: ExcludedPrefix <- parseJsonBody req
   void . lift $ API.phonePrefixInsert prefix
diff --git a/services/brig/src/Brig/Data/Connection.hs b/services/brig/src/Brig/Data/Connection.hs
index fcd91b4f51..3631e309ea 100644
--- a/services/brig/src/Brig/Data/Connection.hs
+++ b/services/brig/src/Brig/Data/Connection.hs
@@ -71,7 +71,7 @@ import Wire.API.Connection
 import Wire.API.Routes.Internal.Brig.Connection
 
 insertConnection ::
-  MonadClient m =>
+  (MonadClient m) =>
   Local UserId ->
   Qualified UserId ->
   RelationWithHistory ->
@@ -105,7 +105,7 @@ updateConnection c status = do
         ucLastUpdate = now
       }
 
-updateConnectionStatus :: MonadClient m => Local UserId -> Qualified UserId -> RelationWithHistory -> m UTCTimeMillis
+updateConnectionStatus :: (MonadClient m) => Local UserId -> Qualified UserId -> RelationWithHistory -> m UTCTimeMillis
 updateConnectionStatus self target status = do
   now <- toUTCTimeMillis <$> liftIO getCurrentTime
   let local (tUnqualified -> ltarget) =
@@ -118,7 +118,7 @@ updateConnectionStatus self target status = do
   pure now
 
 -- | Lookup the connection from a user 'A' to a user 'B' (A -> B).
-lookupConnection :: MonadClient m => Local UserId -> Qualified UserId -> m (Maybe UserConnection)
+lookupConnection :: (MonadClient m) => Local UserId -> Qualified UserId -> m (Maybe UserConnection)
 lookupConnection self target = runMaybeT $ do
   let local (tUnqualified -> ltarget) = do
         (_, _, rel, time, mcnv) <-
@@ -142,7 +142,7 @@ lookupConnection self target = runMaybeT $ do
 
 -- | 'lookupConnection' with more 'Relation' info.
 lookupRelationWithHistory ::
-  MonadClient m =>
+  (MonadClient m) =>
   -- | User 'A'
   Local UserId ->
   -- | User 'B'
@@ -155,7 +155,7 @@ lookupRelationWithHistory self target = do
         query1 remoteRelationSelect (params LocalQuorum (tUnqualified self, domain, rtarget))
   runIdentity <$$> retry x1 (foldQualified self local remote target)
 
-lookupRelation :: MonadClient m => Local UserId -> Qualified UserId -> m Relation
+lookupRelation :: (MonadClient m) => Local UserId -> Qualified UserId -> m Relation
 lookupRelation self target =
   lookupRelationWithHistory self target <&> \case
     Nothing -> Cancelled
@@ -163,7 +163,7 @@ lookupRelation self target =
 
 -- | For a given user 'A', lookup their outgoing connections (A -> X) to other users.
 lookupLocalConnections ::
-  MonadClient m =>
+  (MonadClient m) =>
   Local UserId ->
   Maybe UserId ->
   Range 1 500 Int32 ->
@@ -205,13 +205,13 @@ lookupRemoteConnectionsPage self pagingState size =
       (paramsPagingState LocalQuorum (Identity (tUnqualified self)) size pagingState)
 
 -- | Lookup all relations between two sets of users (cartesian product).
-lookupConnectionStatus :: MonadClient m => [UserId] -> [UserId] -> m [ConnectionStatus]
+lookupConnectionStatus :: (MonadClient m) => [UserId] -> [UserId] -> m [ConnectionStatus]
 lookupConnectionStatus from to =
   map toConnectionStatus
     <$> retry x1 (query connectionStatusSelect (params LocalQuorum (from, to)))
 
 -- | Lookup all relations between two sets of users (cartesian product).
-lookupConnectionStatus' :: MonadClient m => [UserId] -> m [ConnectionStatus]
+lookupConnectionStatus' :: (MonadClient m) => [UserId] -> m [ConnectionStatus]
 lookupConnectionStatus' from =
   map toConnectionStatus
     <$> retry x1 (query connectionStatusSelect' (params LocalQuorum (Identity from)))
@@ -226,7 +226,7 @@ lookupLocalConnectionStatuses ::
 lookupLocalConnectionStatuses froms tos = do
   concat <$> pooledMapConcurrentlyN 16 lookupStatuses froms
   where
-    lookupStatuses :: MonadClient m => UserId -> m [ConnectionStatusV2]
+    lookupStatuses :: (MonadClient m) => UserId -> m [ConnectionStatusV2]
     lookupStatuses from =
       map (uncurry $ toConnectionStatusV2 from (tDomain tos))
         <$> retry x1 (query relationsSelect (params LocalQuorum (from, tUnqualified tos)))
@@ -241,7 +241,7 @@ lookupRemoteConnectionStatuses ::
 lookupRemoteConnectionStatuses froms tos = do
   concat <$> pooledMapConcurrentlyN 16 lookupStatuses froms
   where
-    lookupStatuses :: MonadClient m => UserId -> m [ConnectionStatusV2]
+    lookupStatuses :: (MonadClient m) => UserId -> m [ConnectionStatusV2]
     lookupStatuses from =
       map (uncurry $ toConnectionStatusV2 from (tDomain tos))
         <$> retry x1 (query remoteRelationsSelect (params LocalQuorum (from, tDomain tos, tUnqualified tos)))
@@ -256,14 +256,14 @@ lookupAllStatuses lfroms = do
   let froms = tUnqualified lfroms
   concat <$> pooledMapConcurrentlyN 16 lookupAndCombine froms
   where
-    lookupAndCombine :: MonadClient m => UserId -> m [ConnectionStatusV2]
+    lookupAndCombine :: (MonadClient m) => UserId -> m [ConnectionStatusV2]
     lookupAndCombine u = (<>) <$> lookupLocalStatuses u <*> lookupRemoteStatuses u
 
-    lookupLocalStatuses :: MonadClient m => UserId -> m [ConnectionStatusV2]
+    lookupLocalStatuses :: (MonadClient m) => UserId -> m [ConnectionStatusV2]
     lookupLocalStatuses from =
       map (uncurry $ toConnectionStatusV2 from (tDomain lfroms))
         <$> retry x1 (query relationsSelectAll (params LocalQuorum (Identity from)))
-    lookupRemoteStatuses :: MonadClient m => UserId -> m [ConnectionStatusV2]
+    lookupRemoteStatuses :: (MonadClient m) => UserId -> m [ConnectionStatusV2]
     lookupRemoteStatuses from =
       map (\(d, u, r) -> toConnectionStatusV2 from d u r)
         <$> retry x1 (query remoteRelationsSelectAll (params LocalQuorum (Identity from)))
@@ -274,20 +274,20 @@ lookupRemoteConnectedUsersC u maxResults =
     .| C.map (map (uncurry toRemoteUnsafe))
 
 -- | See 'lookupContactListWithRelation'.
-lookupContactList :: MonadClient m => UserId -> m [UserId]
+lookupContactList :: (MonadClient m) => UserId -> m [UserId]
 lookupContactList u =
   fst <$$> (filter ((== AcceptedWithHistory) . snd) <$> lookupContactListWithRelation u)
 
 -- | For a given user 'A', lookup the list of users that form his contact list,
 -- i.e. the users to whom 'A' has an outgoing 'Accepted' relation (A -> B).
-lookupContactListWithRelation :: MonadClient m => UserId -> m [(UserId, RelationWithHistory)]
+lookupContactListWithRelation :: (MonadClient m) => UserId -> m [(UserId, RelationWithHistory)]
 lookupContactListWithRelation u =
   retry x1 (query contactsSelect (params LocalQuorum (Identity u)))
 
 -- | Count the number of connections a user has in a specific relation status.
 -- (If you want to distinguish 'RelationWithHistory', write a new function.)
 -- Note: The count is eventually consistent.
-countConnections :: MonadClient m => Local UserId -> [Relation] -> m Int64
+countConnections :: (MonadClient m) => Local UserId -> [Relation] -> m Int64
 countConnections u r = do
   rels <- retry x1 . query selectStatus $ params One (Identity (tUnqualified u))
   relsRemote <- retry x1 . query selectStatusRemote $ params One (Identity (tUnqualified u))
diff --git a/services/brig/test/integration/API/Internal.hs b/services/brig/test/integration/API/Internal.hs
index 9805fe5ea9..ec68cc1adf 100644
--- a/services/brig/test/integration/API/Internal.hs
+++ b/services/brig/test/integration/API/Internal.hs
@@ -90,7 +90,7 @@ tests opts mgr db brig brigep gundeck galley = do
         test mgr "delete-federation-remote-galley" $ testDeleteFederationRemoteGalley db brig
       ]
 
-testDeleteFederationRemoteGalley :: forall m. TestConstraints m => Cass.ClientState -> Brig -> m ()
+testDeleteFederationRemoteGalley :: forall m. (TestConstraints m) => Cass.ClientState -> Brig -> m ()
 testDeleteFederationRemoteGalley db brig = do
   let remoteDomain1 = Domain "far-away.example.com"
       remoteDomain2 = Domain "far-away-two.example.com"
@@ -154,7 +154,7 @@ testDeleteFederationRemoteGalley db brig = do
     )
     >>= liftIO . assertBool "connection_remote entry should exist for the user" . any (isRemote2 . fst)
 
-testSuspendUser :: forall m. TestConstraints m => Cass.ClientState -> Brig -> m ()
+testSuspendUser :: forall m. (TestConstraints m) => Cass.ClientState -> Brig -> m ()
 testSuspendUser db brig = do
   user <- randomUser brig
   let checkAccountStatus s = do
@@ -166,7 +166,7 @@ testSuspendUser db brig = do
   setAccountStatus brig (userId user) Active !!! const 200 === statusCode
   checkAccountStatus Active
 
-testSuspendNonExistingUser :: forall m. TestConstraints m => Cass.ClientState -> Brig -> m ()
+testSuspendNonExistingUser :: forall m. (TestConstraints m) => Cass.ClientState -> Brig -> m ()
 testSuspendNonExistingUser db brig = do
   nonExistingUserId <- randomId
   setAccountStatus brig nonExistingUserId Suspended !!! const 404 === statusCode
@@ -182,7 +182,7 @@ setAccountStatus brig u s =
         . json (AccountStatusUpdate s)
     )
 
-testEJPDRequest :: TestConstraints m => Manager -> Brig -> Endpoint -> Gundeck -> m ()
+testEJPDRequest :: (TestConstraints m) => Manager -> Brig -> Endpoint -> Gundeck -> m ()
 testEJPDRequest mgr brig brigep gundeck = do
   (handle1, mkUsr1, handle2, mkUsr2, mkUsr3) <- scaffolding brig gundeck
 
@@ -230,9 +230,9 @@ testEJPDRequest mgr brig brigep gundeck = do
     have <- ejpdRequestClient brigep mgr (Just True) req
     liftIO $ assertEqual "" want have
 
-testFeatureConferenceCallingByAccount :: forall m. TestConstraints m => Opt.Opts -> Manager -> Cass.ClientState -> Brig -> Endpoint -> Galley -> m ()
+testFeatureConferenceCallingByAccount :: forall m. (TestConstraints m) => Opt.Opts -> Manager -> Cass.ClientState -> Brig -> Endpoint -> Galley -> m ()
 testFeatureConferenceCallingByAccount (Opt.optSettings -> settings) mgr db brig brigep galley = do
-  let check :: HasCallStack => ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig -> m ()
+  let check :: (HasCallStack) => ApiFt.WithStatusNoLock ApiFt.ConferenceCallingConfig -> m ()
       check status = do
         uid <- userId <$> createUser "joe" brig
         _ <-
@@ -289,7 +289,7 @@ testFeatureConferenceCallingByAccount (Opt.optSettings -> settings) mgr db brig
         featureConfigsConfCalling <- getFeatureConfig @ApiFt.ConferenceCallingConfig galley uid
         liftIO $ assertEqual "GET /feature-configs/conferenceCalling" defaultIfNull (responseJsonUnsafe featureConfigsConfCalling)
 
-      readFeatureConfigs :: HasCallStack => ResponseLBS -> ApiFt.WithStatus ApiFt.ConferenceCallingConfig
+      readFeatureConfigs :: (HasCallStack) => ResponseLBS -> ApiFt.WithStatus ApiFt.ConferenceCallingConfig
       readFeatureConfigs =
         either (error . show) id
           . Aeson.parseEither Aeson.parseJSON
@@ -325,7 +325,7 @@ testGetMlsClients brig = do
         )
   liftIO $ toList cs1 @?= [ClientInfo c True]
 
-keyPackageCreate :: HasCallStack => Brig -> Http KeyPackageRef
+keyPackageCreate :: (HasCallStack) => Brig -> Http KeyPackageRef
 keyPackageCreate brig = do
   uid <- userQualifiedId <$> randomUser brig
   clid <- createClient brig uid 0
@@ -354,7 +354,7 @@ keyPackageCreate brig = do
       [] -> liftIO $ assertFailure "Claim response held no bundles"
       (h : _) -> pure $ kpbeRef h
 
-kpcPut :: HasCallStack => Brig -> KeyPackageRef -> Qualified ConvId -> Http ()
+kpcPut :: (HasCallStack) => Brig -> KeyPackageRef -> Qualified ConvId -> Http ()
 kpcPut brig ref qConv = do
   resp <-
     put
@@ -365,7 +365,7 @@ kpcPut brig ref qConv = do
       )
   liftIO $ assertEqual "PUT i/mls/key-packages/:ref/conversation failed" 204 (statusCode resp)
 
-kpcGet :: HasCallStack => Brig -> KeyPackageRef -> Http (Maybe (Qualified ConvId))
+kpcGet :: (HasCallStack) => Brig -> KeyPackageRef -> Http (Maybe (Qualified ConvId))
 kpcGet brig ref = do
   resp <-
     get (brig . paths ["i", "mls", "key-packages", toByteString' $ toUrlPiece ref, "conversation"])
@@ -445,7 +445,7 @@ getAllFeatureConfigs :: (MonadHttp m, HasCallStack) => (Request -> Request) -> U
 getAllFeatureConfigs galley uid = do
   get $ galley . paths ["feature-configs"] . zUser uid
 
-testWritetimeRepresentation :: forall m. TestConstraints m => Opt.Opts -> Manager -> Cass.ClientState -> Brig -> Endpoint -> Galley -> m ()
+testWritetimeRepresentation :: forall m. (TestConstraints m) => Opt.Opts -> Manager -> Cass.ClientState -> Brig -> Endpoint -> Galley -> m ()
 testWritetimeRepresentation _ _mgr db brig _brigep _galley = do
   quid <- userQualifiedId <$> randomUser brig
   let uid = qUnqualified quid
diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs
index 2f1cf8bbe0..325292d42d 100644
--- a/services/federator/src/Federator/Validation.hs
+++ b/services/federator/src/Federator/Validation.hs
@@ -107,7 +107,7 @@ ensureCanFederateWith targetDomain = do
         throw (FederationDenied targetDomain)
 
 decodeCertificate ::
-  Member (Error String) r =>
+  (Member (Error String) r) =>
   ByteString ->
   Sem r X509.Certificate
 decodeCertificate =
@@ -124,12 +124,12 @@ decodeCertificate =
     expectOne _ [x] = pure x
     expectOne label _ = Left $ "found multiple " <> label <> "s"
 
-parseDomain :: Member (Error ValidationError) r => ByteString -> Sem r Domain
+parseDomain :: (Member (Error ValidationError) r) => ByteString -> Sem r Domain
 parseDomain domain =
   note (DomainParseError (Text.decodeUtf8With Text.lenientDecode domain)) $
     fromByteString domain
 
-parseDomainText :: Member (Error ValidationError) r => Text -> Sem r Domain
+parseDomainText :: (Member (Error ValidationError) r) => Text -> Sem r Domain
 parseDomainText domain =
   mapError @String (const (DomainParseError domain))
     . fromEither
diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs
index 64388e8718..1e3dbbb7ab 100644
--- a/services/galley/src/Galley/API/Action.hs
+++ b/services/galley/src/Galley/API/Action.hs
@@ -257,7 +257,7 @@ type family HasConversationActionGalleyErrors (tag :: ConversationActionTag) ::
        ErrorS 'ConvNotFound
      ]
 
-noChanges :: Member (Error NoChanges) r => Sem r a
+noChanges :: (Member (Error NoChanges) r) => Sem r a
 noChanges = throw NoChanges
 
 ensureAllowed ::
@@ -538,7 +538,7 @@ performConversationAccessData qusr lconv action = do
         then pure bm
         else pure $ bm {bmBots = mempty}
 
-    maybeRemoveGuests :: Member BrigAccess r => BotsAndMembers -> Sem r BotsAndMembers
+    maybeRemoveGuests :: (Member BrigAccess r) => BotsAndMembers -> Sem r BotsAndMembers
     maybeRemoveGuests bm =
       if Set.member GuestAccessRole (cupAccessRoles action)
         then pure bm
@@ -547,7 +547,7 @@ performConversationAccessData qusr lconv action = do
           -- FUTUREWORK: should we also remove non-activated remote users?
           pure $ bm {bmLocals = Set.fromList activated}
 
-    maybeRemoveNonTeamMembers :: Member TeamStore r => BotsAndMembers -> Sem r BotsAndMembers
+    maybeRemoveNonTeamMembers :: (Member TeamStore r) => BotsAndMembers -> Sem r BotsAndMembers
     maybeRemoveNonTeamMembers bm =
       if Set.member NonTeamMemberAccessRole (cupAccessRoles action)
         then pure bm
@@ -557,7 +557,7 @@ performConversationAccessData qusr lconv action = do
             pure $ bm {bmLocals = Set.fromList onlyTeamUsers, bmRemotes = mempty}
           Nothing -> pure bm
 
-    maybeRemoveTeamMembers :: Member TeamStore r => BotsAndMembers -> Sem r BotsAndMembers
+    maybeRemoveTeamMembers :: (Member TeamStore r) => BotsAndMembers -> Sem r BotsAndMembers
     maybeRemoveTeamMembers bm =
       if Set.member TeamMemberAccessRole (cupAccessRoles action)
         then pure bm
@@ -834,7 +834,7 @@ notifyConversationAction tag quid notifyOrigDomain con lconv targets action = do
   where
     qualifiedFails :: [(QualifiedWithTag t [a], b)] -> [Qualified a]
     qualifiedFails = foldMap (sequenceA . tUntagged . fst)
-    logError :: Show a => String -> String -> (a, FederationError) -> Sem r ()
+    logError :: (Show a) => String -> String -> (a, FederationError) -> Sem r ()
     logError field msg e =
       P.warn $
         Log.field "federation call" field . Log.msg (msg <> show e)
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index 9eb2ce5f7c..5e791fb9d6 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -243,7 +243,7 @@ internalSitemap = unsafeCallsFed @'Galley @"on-client-removed" $ unsafeCallsFed
   -- Conversation API (internal) ----------------------------------------
   put "/i/conversations/:cnv/channel" (continue $ const (pure empty)) $
     zauthUserId
-      .&. (capture "cnv" :: HasCaptures r => Predicate r Predicate.Error ConvId)
+      .&. (capture "cnv" :: (HasCaptures r) => Predicate r Predicate.Error ConvId)
       .&. request
 
   get "/i/conversations/:cnv/members/:usr" (continue Query.internalGetMemberH) $
diff --git a/services/galley/src/Galley/API/Update.hs b/services/galley/src/Galley/API/Update.hs
index 67f32c959c..177f201d55 100644
--- a/services/galley/src/Galley/API/Update.hs
+++ b/services/galley/src/Galley/API/Update.hs
@@ -898,14 +898,14 @@ updateSelfMember lusr zcon qcnv update = do
   pushConversationEvent (Just zcon) e (fmap pure lusr) []
   where
     checkLocalMembership ::
-      Member MemberStore r =>
+      (Member MemberStore r) =>
       Local ConvId ->
       Sem r Bool
     checkLocalMembership lcnv =
       isMember (tUnqualified lusr)
         <$> E.getLocalMembers (tUnqualified lcnv)
     checkRemoteMembership ::
-      Member ConversationStore r =>
+      (Member ConversationStore r) =>
       Remote ConvId ->
       Sem r Bool
     checkRemoteMembership rcnv =
@@ -1019,7 +1019,7 @@ updateOtherMember lusr zcon qcnv qvictim update = do
   doUpdate qcnv lusr zcon qvictim update
 
 updateOtherMemberRemoteConv ::
-  Member (Error FederationError) r =>
+  (Member (Error FederationError) r) =>
   Remote ConvId ->
   Local UserId ->
   ConnId ->
@@ -1113,7 +1113,7 @@ removeMemberFromRemoteConv cnv lusr victim
     handleError RemoveFromConversationErrorNotFound = throwS @'ConvNotFound
     handleError RemoveFromConversationErrorUnchanged = pure Nothing
 
-    handleSuccess :: Member (Input UTCTime) r => () -> Sem r (Maybe Event)
+    handleSuccess :: (Member (Input UTCTime) r) => () -> Sem r (Maybe Event)
     handleSuccess _ = do
       t <- input
       pure . Just $
@@ -1202,7 +1202,7 @@ postProteusBroadcast ::
 postProteusBroadcast sender zcon = postBroadcast sender (Just zcon)
 
 unqualifyEndpoint ::
-  Functor f =>
+  (Functor f) =>
   Local x ->
   (QualifiedNewOtrMessage -> f (PostOtrResponse MessageSendingStatus)) ->
   Maybe IgnoreMissing ->
@@ -1600,6 +1600,6 @@ rmBot lusr zcon b = do
 -------------------------------------------------------------------------------
 -- Helpers
 
-ensureConvMember :: Member (ErrorS 'ConvNotFound) r => [LocalMember] -> UserId -> Sem r ()
+ensureConvMember :: (Member (ErrorS 'ConvNotFound) r) => [LocalMember] -> UserId -> Sem r ()
 ensureConvMember users usr =
   unless (usr `isMember` users) $ throwS @'ConvNotFound
diff --git a/services/galley/src/Galley/Cassandra/Conversation/Members.hs b/services/galley/src/Galley/Cassandra/Conversation/Members.hs
index 018574e8e5..5942729edf 100644
--- a/services/galley/src/Galley/Cassandra/Conversation/Members.hs
+++ b/services/galley/src/Galley/Cassandra/Conversation/Members.hs
@@ -56,7 +56,7 @@ import Wire.API.Provider.Service
 -- When the role is not specified, it defaults to admin.
 -- Please make sure the conversation doesn't exceed the maximum size!
 addMembers ::
-  ToUserRole a =>
+  (ToUserRole a) =>
   ConvId ->
   UserList a ->
   Client ([LocalMember], [RemoteMember])
diff --git a/services/galley/src/Galley/Effects/MemberStore.hs b/services/galley/src/Galley/Effects/MemberStore.hs
index e8e6e2e34c..5003c7b477 100644
--- a/services/galley/src/Galley/Effects/MemberStore.hs
+++ b/services/galley/src/Galley/Effects/MemberStore.hs
@@ -65,7 +65,7 @@ import Wire.API.MLS.KeyPackage
 import Wire.API.Provider.Service
 
 data MemberStore m a where
-  CreateMembers :: ToUserRole u => ConvId -> UserList u -> MemberStore m ([LocalMember], [RemoteMember])
+  CreateMembers :: (ToUserRole u) => ConvId -> UserList u -> MemberStore m ([LocalMember], [RemoteMember])
   CreateMembersInRemoteConversation :: Remote ConvId -> [UserId] -> MemberStore m ()
   CreateBotMember :: ServiceRef -> BotId -> ConvId -> MemberStore m BotMember
   GetLocalMember :: ConvId -> UserId -> MemberStore m (Maybe LocalMember)
@@ -89,5 +89,5 @@ data MemberStore m a where
 makeSem ''MemberStore
 
 -- | Add a member to a local conversation, as an admin.
-createMember :: Member MemberStore r => Local ConvId -> Local UserId -> Sem r [LocalMember]
+createMember :: (Member MemberStore r) => Local ConvId -> Local UserId -> Sem r [LocalMember]
 createMember c u = fst <$> createMembers (tUnqualified c) (UserList [tUnqualified u] [])
diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs
index ad83b9a5c3..5da88d634f 100644
--- a/services/galley/src/Galley/Run.hs
+++ b/services/galley/src/Galley/Run.hs
@@ -165,7 +165,7 @@ refreshMetrics = do
     M.gaugeSet (fromIntegral n) (M.path "galley.deletequeue.len") m
     threadDelay 1000000
 
-collectAuthMetrics :: MonadIO m => Metrics -> AWS.Env -> m ()
+collectAuthMetrics :: (MonadIO m) => Metrics -> AWS.Env -> m ()
 collectAuthMetrics m env = do
   liftIO $
     forever $ do
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index a59f9e5c5d..dbec4f3229 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -168,7 +168,7 @@ deleteFederationDomains old new = do
   -- Call into the galley code
   for_ deletedDomains $ liftIO . evalGalleyToIO env . deleteFederationDomain
 
-constHandlers :: MonadIO m => [RetryStatus -> Handler m Bool]
+constHandlers :: (MonadIO m) => [RetryStatus -> Handler m Bool]
 constHandlers = [const $ Handler $ (\(_ :: SomeException) -> pure True)]
 
 updateFedDomainRemoveRemoteFromLocal :: Env -> Domain -> Domain -> Int -> TestM ()

From 48cb312a3e5dbfa8fdc9d2beda90eb4de83d4137 Mon Sep 17 00:00:00 2001
From: Matthias Fischmann <mf@zerobuzz.net>
Date: Tue, 11 Jul 2023 12:46:01 +0200
Subject: [PATCH 210/220] Fixup

---
 services/galley/test/integration/Federation.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index dbec4f3229..0a7b7d5796 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -90,7 +90,7 @@ updateFedDomainsTestNoop' = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  (_, env, _) <- liftIO $ lowerCodensity $ mkApp opts
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
   -- FUTUREWORK, NEWTICKET: These uuid strings side step issues with the tests hanging.
   -- FUTUREWORK, NEWTICKET: Figure out the underlying issue as to why these tests occasionally hang.
@@ -111,7 +111,7 @@ updateFedDomainsTestAddRemote' = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  (_, env, _) <- liftIO $ lowerCodensity $ mkApp opts
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
   let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
       remoteDomain = Domain "far-away.example.com"
@@ -128,7 +128,7 @@ updateFedDomainsTestRemoveRemoteFromLocal' = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  (_, env, _) <- liftIO $ lowerCodensity $ mkApp opts
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
   let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
       remoteDomain = Domain "far-away.example.com"
@@ -145,7 +145,7 @@ updateFedDomainsTestRemoveLocalFromRemote' = do
   let opts = s ^. tsGConf
   -- Don't need the actual server, and we certainly don't want it running.
   -- But this is how the env is made, so it is what we do
-  (_, env, _) <- liftIO $ lowerCodensity $ mkApp opts
+  (_, env) <- liftIO $ lowerCodensity $ mkApp opts
   -- Common variables.
   let interval = (maxBound :: Int) `div` 2 -- Very large values so that we don't have to worry about automatic updates
       remoteDomain = Domain "far-away.example.com"

From 4494d069fb15025e927799f7a1d4d4dc1f817639 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 12 Jul 2023 19:03:54 +1000
Subject: [PATCH 211/220] Changing the format of federation.delete.

---
 libs/wire-api/src/Wire/API/Event/Federation.hs | 4 ++--
 services/galley/src/Galley/API/Internal.hs     | 2 +-
 services/galley/test/integration/API/Util.hs   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Event/Federation.hs b/libs/wire-api/src/Wire/API/Event/Federation.hs
index 52aac5595f..77db5ae63d 100644
--- a/libs/wire-api/src/Wire/API/Event/Federation.hs
+++ b/libs/wire-api/src/Wire/API/Event/Federation.hs
@@ -15,7 +15,7 @@ import Data.Aeson (ToJSON, FromJSON)
 
 data Event = Event
   { _eventType :: EventType
-  , _eventDomains :: [Domain]
+  , _eventDomain :: Domain
   }
   deriving (Eq, Show, Ord, Generic)
 
@@ -40,7 +40,7 @@ instance ToSchema EventType where
 eventObjectSchema :: ObjectSchema SwaggerDoc Event
 eventObjectSchema = Event
   <$> _eventType .= field "type" schema
-  <*> _eventDomains .= field "domains" (array schema)
+  <*> _eventDomain .= field "domain" schema
 
 instance ToSchema Event where
   schema = object "Event" eventObjectSchema
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index 9ac290d3c0..ba9a2c3027 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -568,7 +568,7 @@ internalDeleteFederationDomainH (domain ::: _) = do
     pushEvents results = do
       let (bots, mems) = localBotsAndUsers results
           recipients = Intra.recipient <$> mems
-          event = Intra.FederationEvent $ Federation.Event Federation.FederationDelete [domain]
+          event = Intra.FederationEvent $ Federation.Event Federation.FederationDelete domain
       for_ (Intra.newPush ListComplete Nothing event recipients) $ \p -> do
         -- TODO: Transient or not?
         -- RouteAny is used as it will wake up mobile clients
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index 1980a486d5..e6682aa218 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -1774,7 +1774,7 @@ assertFederationDeletedEvent ::
   IO ()
 assertFederationDeletedEvent dom e = do
   Fed._eventType e @?= Fed.FederationDelete
-  Fed._eventDomains e @?= [dom]
+  Fed._eventDomain e @?= dom
 
 -- FUTUREWORK: See if this one can be implemented in terms of:
 --

From d0cabf2d6a19f7b1f30b430392c7efaa186481b7 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Wed, 12 Jul 2023 19:42:57 +1000
Subject: [PATCH 212/220] Sanitizing code

---
 services/background-worker/default.nix  | 32 ++-----------------------
 services/background-worker/test/Main.hs |  2 +-
 2 files changed, 3 insertions(+), 31 deletions(-)

diff --git a/services/background-worker/default.nix b/services/background-worker/default.nix
index ec0475b3c2..62453b669b 100644
--- a/services/background-worker/default.nix
+++ b/services/background-worker/default.nix
@@ -85,36 +85,7 @@ mkDerivation {
     wire-api
     wire-api-federation
   ];
-  executableHaskellDepends = [
-    aeson
-    amqp
-    base
-    bytestring
-    containers
-    extended
-    federator
-    HsOpenSSL
-    hspec
-    http-client
-    http-media
-    http-types
-    HUnit
-    imports
-    prometheus-client
-    QuickCheck
-    servant
-    servant-client
-    servant-client-core
-    servant-server
-    text
-    tinylog
-    transformers
-    types-common
-    unliftio
-    wai
-    wire-api
-    wire-api-federation
-  ];
+  executableHaskellDepends = [ HsOpenSSL imports types-common ];
   testHaskellDepends = [
     aeson
     amqp
@@ -146,4 +117,5 @@ mkDerivation {
   ];
   description = "Runs background work";
   license = lib.licenses.agpl3Only;
+  mainProgram = "background-worker";
 }
diff --git a/services/background-worker/test/Main.hs b/services/background-worker/test/Main.hs
index 52ef578fca..a824f8c30c 100644
--- a/services/background-worker/test/Main.hs
+++ b/services/background-worker/test/Main.hs
@@ -1 +1 @@
-{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
\ No newline at end of file
+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}

From bebc4ad8845b33f57351788df41b20496bb9a2f8 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 13 Jul 2023 17:26:49 +1000
Subject: [PATCH 213/220] FS-1179: Removing dead code and bumping schema
 migrations

---
 services/brig/brig.cabal                                      | 1 -
 services/brig/schema/src/Main.hs                              | 2 +-
 ..._ConnectionRemoteIndex.hs => V79_ConnectionRemoteIndex.hs} | 4 ++--
 services/brig/src/Brig/API/Internal.hs                        | 1 -
 services/galley/src/Galley/API/Internal.hs                    | 3 ---
 services/galley/test/integration/Federation.hs                | 1 -
 6 files changed, 3 insertions(+), 9 deletions(-)
 rename services/brig/schema/src/{V78_ConnectionRemoteIndex.hs => V79_ConnectionRemoteIndex.hs} (74%)

diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 7b28af12b5..1cfa51f4de 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -281,7 +281,6 @@ library
     , ssl-util
     , statistics                 >=0.13
     , stomp-queue                >=0.3
-    , string-conversions
     , swagger2
     , template                   >=0.2
     , template-haskell
diff --git a/services/brig/schema/src/Main.hs b/services/brig/schema/src/Main.hs
index 135a4843ad..cab330b2c9 100644
--- a/services/brig/schema/src/Main.hs
+++ b/services/brig/schema/src/Main.hs
@@ -105,7 +105,7 @@ main = do
       V76_AddSupportedProtocols.migration,
       V77_FederationRemotes.migration,
       V78_ClientLastActive.migration,
-      V78_ConnectionRemoteIndex.migration
+      V79_ConnectionRemoteIndex.migration
       -- When adding migrations here, don't forget to update
       -- 'schemaVersion' in Brig.App
 
diff --git a/services/brig/schema/src/V78_ConnectionRemoteIndex.hs b/services/brig/schema/src/V79_ConnectionRemoteIndex.hs
similarity index 74%
rename from services/brig/schema/src/V78_ConnectionRemoteIndex.hs
rename to services/brig/schema/src/V79_ConnectionRemoteIndex.hs
index 7be602f7c6..729e81f72b 100644
--- a/services/brig/schema/src/V78_ConnectionRemoteIndex.hs
+++ b/services/brig/schema/src/V79_ConnectionRemoteIndex.hs
@@ -1,7 +1,7 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE QuasiQuotes #-}
 
-module V78_ConnectionRemoteIndex
+module V79_ConnectionRemoteIndex
   ( migration,
   )
 where
@@ -11,7 +11,7 @@ import Imports
 import Text.RawString.QQ
 
 migration :: Migration
-migration = Migration 78 "Add a secondary index for federated (remote) connections" $ do
+migration = Migration 79 "Add a secondary index for federated (remote) connections" $ do
   schema'
     [r| CREATE INDEX on connection_remote (right_domain)
       |]
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index 0b13656ae8..a1dd189595 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -76,7 +76,6 @@ import qualified Data.Set as Set
 import Imports hiding (head)
 import qualified Network.AMQP as Q
 import Data.Time.Clock.System
-import Imports hiding (head)
 import Network.HTTP.Types.Status
 import Network.Wai (Response)
 import Network.Wai.Predicate hiding (result, setStatus)
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index 5e791fb9d6..3541162560 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -644,9 +644,6 @@ deleteFederationDomainLocalUserFromRemoteConversation dom = do
         -- similar processing run for removing the local domain from their federation list.
         onConversationUpdated dom convUpdate
 
--- let rcnv = toRemoteUnsafe dom cnv
--- notifyRemoteConversationAction lUser (qualifyAs rcnv convUpdate) Nothing
-
 -- These need to be recoverable?
 -- This is recoverable with the following flow conditions.
 -- 1) Deletion calls to the Brig endpoint `delete-federation-remote-from-galley` are idempotent for a given domain.
diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index 0a7b7d5796..debcfd4f29 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -1,5 +1,4 @@
 {-# LANGUAGE RecordWildCards #-}
-{-# OPTIONS_GHC -Wno-unused-matches #-}
 
 module Federation where
 

From 7a67199b057e9248a6f1693644dcaabfe86ac8cb Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Thu, 13 Jul 2023 17:40:37 +1000
Subject: [PATCH 214/220] FS-1179: Removing more dead code

---
 services/galley/test/integration/Federation.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/services/galley/test/integration/Federation.hs b/services/galley/test/integration/Federation.hs
index debcfd4f29..ca3b43c9bd 100644
--- a/services/galley/test/integration/Federation.hs
+++ b/services/galley/test/integration/Federation.hs
@@ -49,8 +49,6 @@ import Wire.API.Routes.MultiTablePaging
 import qualified Wire.API.Routes.MultiTablePaging as Public
 import Wire.API.User.Search
 
--- import Control.Concurrent.Async
-
 x3 :: RetryPolicy
 x3 = limitRetries 3 <> exponentialBackoff 100000
 

From d5952e7f375576bd6d6301759ff365d7ee8748d7 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Fri, 14 Jul 2023 11:10:57 +1000
Subject: [PATCH 215/220] PR formatting

---
 services/brig/default.nix              | 2 --
 services/brig/src/Brig/API/Internal.hs | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/services/brig/default.nix b/services/brig/default.nix
index b630d299a5..e5859de928 100644
--- a/services/brig/default.nix
+++ b/services/brig/default.nix
@@ -120,7 +120,6 @@
 , statistics
 , stomp-queue
 , streaming-commons
-, string-conversions
 , swagger2
 , tasty
 , tasty-cannon
@@ -260,7 +259,6 @@ mkDerivation {
     ssl-util
     statistics
     stomp-queue
-    string-conversions
     swagger2
     template
     template-haskell
diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs
index a1dd189595..1780d4975e 100644
--- a/services/brig/src/Brig/API/Internal.hs
+++ b/services/brig/src/Brig/API/Internal.hs
@@ -73,9 +73,9 @@ import Data.Id as Id
 import qualified Data.Map.Strict as Map
 import Data.Qualified
 import qualified Data.Set as Set
+import Data.Time.Clock.System
 import Imports hiding (head)
 import qualified Network.AMQP as Q
-import Data.Time.Clock.System
 import Network.HTTP.Types.Status
 import Network.Wai (Response)
 import Network.Wai.Predicate hiding (result, setStatus)

From 0b33f11dc864bc3d8bd1b34e987e4413647a92e5 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Fri, 14 Jul 2023 14:55:49 +1000
Subject: [PATCH 216/220] Hi CI


From 5a5ed57fc5365dc627eb8594422f73487504947a Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 18 Jul 2023 12:10:09 +1000
Subject: [PATCH 217/220] WPB-240: Formatting code

---
 .../wire-api/src/Wire/API/Event/Federation.hs | 31 ++++++++++---------
 services/galley/default.nix                   |  2 ++
 services/galley/src/Galley/API/Internal.hs    | 20 ++++++------
 .../src/Galley/Effects/ExternalAccess.hs      |  2 +-
 services/galley/src/Galley/External.hs        |  2 +-
 .../galley/src/Galley/Intra/Push/Internal.hs  |  2 +-
 services/galley/test/integration/API.hs       |  5 ++-
 services/galley/test/integration/API/Util.hs  |  2 +-
 8 files changed, 35 insertions(+), 31 deletions(-)

diff --git a/libs/wire-api/src/Wire/API/Event/Federation.hs b/libs/wire-api/src/Wire/API/Event/Federation.hs
index 77db5ae63d..0665197590 100644
--- a/libs/wire-api/src/Wire/API/Event/Federation.hs
+++ b/libs/wire-api/src/Wire/API/Event/Federation.hs
@@ -1,28 +1,30 @@
 module Wire.API.Event.Federation
-  ( Event (..)
-  , EventType (..)
-  ) where
+  ( Event (..),
+    EventType (..),
+  )
+where
 
+import Data.Aeson (FromJSON, ToJSON)
 import qualified Data.Aeson as A
 import qualified Data.Aeson.KeyMap as KeyMap
+import Data.Domain
 import Data.Json.Util (ToJSONObject (toJSONObject))
 import Data.Schema
 import qualified Data.Swagger as S
 import Imports
 import Wire.Arbitrary
-import Data.Domain
-import Data.Aeson (ToJSON, FromJSON)
 
 data Event = Event
-  { _eventType :: EventType
-  , _eventDomain :: Domain
+  { _eventType :: EventType,
+    _eventDomain :: Domain
   }
   deriving (Eq, Show, Ord, Generic)
 
 instance Arbitrary Event where
-  arbitrary = Event
-    <$> arbitrary
-    <*> arbitrary
+  arbitrary =
+    Event
+      <$> arbitrary
+      <*> arbitrary
 
 data EventType
   = FederationDelete
@@ -38,9 +40,10 @@ instance ToSchema EventType where
         ]
 
 eventObjectSchema :: ObjectSchema SwaggerDoc Event
-eventObjectSchema = Event
-  <$> _eventType .= field "type" schema
-  <*> _eventDomain .= field "domain" schema
+eventObjectSchema =
+  Event
+    <$> _eventType .= field "type" schema
+    <*> _eventDomain .= field "domain" schema
 
 instance ToSchema Event where
   schema = object "Event" eventObjectSchema
@@ -58,4 +61,4 @@ instance FromJSON Event where
   parseJSON = schemaParseJSON
 
 instance ToJSON Event where
-  toJSON = schemaToJSON
\ No newline at end of file
+  toJSON = schemaToJSON
diff --git a/services/galley/default.nix b/services/galley/default.nix
index 6f114b7648..9d9a0af56a 100644
--- a/services/galley/default.nix
+++ b/services/galley/default.nix
@@ -26,6 +26,7 @@
 , conduit
 , containers
 , cookie
+, cql-io
 , cryptonite
 , currency-codes
 , data-default
@@ -149,6 +150,7 @@ mkDerivation {
     cereal
     comonad
     containers
+    cql-io
     cryptonite
     currency-codes
     data-default
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index d9f0d456b3..6eadf4ba04 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -26,6 +26,7 @@ module Galley.API.Internal
   )
 where
 
+import Cassandra (ClientState, Consistency (LocalQuorum), Page (..), paramsP)
 import Control.Exception
 import Control.Exception.Safe (catchAny)
 import Control.Lens hiding (Getter, Setter, (.=))
@@ -39,6 +40,7 @@ import Data.Range
 import Data.Singletons
 import Data.Text (unpack)
 import Data.Time
+import Database.CQL.IO (paginate)
 import Galley.API.Action
 import qualified Galley.API.Clients as Clients
 import qualified Galley.API.Create as Create
@@ -59,12 +61,16 @@ import Galley.API.Teams.Features
 import qualified Galley.API.Update as Update
 import Galley.API.Util
 import Galley.App
+import Galley.Cassandra.Conversation.Members
+import Galley.Cassandra.Queries
+import Galley.Cassandra.Store (embedClient)
 import qualified Galley.Data.Conversation as Data
 import Galley.Data.Conversation.Types
 import Galley.Effects
 import Galley.Effects.BackendNotificationQueueAccess
 import Galley.Effects.ClientStore
 import Galley.Effects.ConversationStore
+import Galley.Effects.ExternalAccess
 import Galley.Effects.FederatorAccess
 import Galley.Effects.GundeckAccess
 import Galley.Effects.LegalHoldStore as LegalHoldStore
@@ -73,7 +79,9 @@ import qualified Galley.Effects.MemberStore as E
 import Galley.Effects.ProposalStore
 import Galley.Effects.TeamStore
 import qualified Galley.Effects.TeamStore as E
+import Galley.Env (currentFanoutLimit, _options)
 import qualified Galley.Intra.Push as Intra
+import Galley.Intra.Push.Internal (pushEventJson)
 import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
@@ -85,7 +93,7 @@ import Imports hiding (head)
 import qualified Network.AMQP as Q
 import Network.HTTP.Types
 import Network.Wai
-import Network.Wai.Predicate hiding (result, Error, err, setStatus)
+import Network.Wai.Predicate hiding (Error, err, result, setStatus)
 import qualified Network.Wai.Predicate as Predicate hiding (result)
 import Network.Wai.Routing hiding (App, route, toList)
 import Network.Wai.Utilities hiding (Error)
@@ -121,14 +129,6 @@ import Wire.API.Team.Feature hiding (setStatus)
 import Wire.API.Team.Member
 import Wire.Sem.Paging
 import Wire.Sem.Paging.Cassandra
-import Galley.Effects.ExternalAccess
-import Galley.Intra.Push.Internal (pushEventJson)
-import Galley.Cassandra.Store (embedClient)
-import Cassandra (Consistency(LocalQuorum), Page (..), paramsP, ClientState)
-import Database.CQL.IO (paginate)
-import Galley.Cassandra.Queries
-import Galley.Cassandra.Conversation.Members
-import Galley.Env (currentFanoutLimit, _options)
 
 internalAPI :: API InternalAPI GalleyEffects
 internalAPI =
@@ -574,7 +574,7 @@ internalDeleteFederationDomainH (domain ::: _) = do
         -- RouteAny is used as it will wake up mobile clients
         -- and notify them of the changes to federation state.
         push1 $ p & Intra.pushRoute .~ Intra.RouteAny
-                  -- & Intra.pushTransient .~ True
+      -- & Intra.pushTransient .~ True
       deliverAsync (bots `zip` repeat (pushEventJson event))
     sendNotificationPage page = do
       let res = result page
diff --git a/services/galley/src/Galley/Effects/ExternalAccess.hs b/services/galley/src/Galley/Effects/ExternalAccess.hs
index 301eeed4ed..7f6f3c4f0f 100644
--- a/services/galley/src/Galley/Effects/ExternalAccess.hs
+++ b/services/galley/src/Galley/Effects/ExternalAccess.hs
@@ -26,12 +26,12 @@ module Galley.Effects.ExternalAccess
   )
 where
 
+import Data.Aeson
 import Data.Id
 import Galley.Data.Services
 import Imports
 import Polysemy
 import Wire.API.Event.Conversation
-import Data.Aeson
 
 data ExternalAccess m a where
   Deliver :: Foldable f => f (BotMember, Event) -> ExternalAccess m [BotMember]
diff --git a/services/galley/src/Galley/External.hs b/services/galley/src/Galley/External.hs
index 17927a8c28..7d42ace136 100644
--- a/services/galley/src/Galley/External.hs
+++ b/services/galley/src/Galley/External.hs
@@ -21,6 +21,7 @@ import Bilge.Request
 import Bilge.Retry (httpHandlers)
 import Control.Lens
 import Control.Retry
+import Data.Aeson (ToJSON)
 import Data.ByteString.Conversion.To
 import Data.Id
 import Data.Misc
@@ -45,7 +46,6 @@ import URI.ByteString
 import UnliftIO (Async, async, waitCatch)
 import Wire.API.Event.Conversation (Event)
 import Wire.API.Provider.Service (serviceRefId, serviceRefProvider)
-import Data.Aeson (ToJSON)
 
 interpretExternalAccess ::
   ( Member (Embed IO) r,
diff --git a/services/galley/src/Galley/Intra/Push/Internal.hs b/services/galley/src/Galley/Intra/Push/Internal.hs
index a09c7f4732..4cbdd47889 100644
--- a/services/galley/src/Galley/Intra/Push/Internal.hs
+++ b/services/galley/src/Galley/Intra/Push/Internal.hs
@@ -40,8 +40,8 @@ import Imports hiding (forkIO)
 import UnliftIO.Async (mapConcurrently_)
 import Wire.API.Event.Conversation (Event (evtFrom))
 import qualified Wire.API.Event.FeatureConfig as FeatureConfig
-import qualified Wire.API.Event.Team as Teams
 import qualified Wire.API.Event.Federation as Federation
+import qualified Wire.API.Event.Team as Teams
 import Wire.API.Team.Member
 
 data PushEvent
diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs
index 3ec33efece..a7eda0513f 100644
--- a/services/galley/test/integration/API.hs
+++ b/services/galley/test/integration/API.hs
@@ -4333,7 +4333,6 @@ testDefederationNotifications = do
   charlie <- randomQualifiedUser
   connectUsers (qUnqualified alice) (pure (qUnqualified charlie))
 
-
   let remoteDomain = Domain "far-away.example.com"
       -- This variable should be commented out if the below
       -- section is used to insert users to the database.
@@ -4369,7 +4368,7 @@ testDefederationNotifications = do
         <!! const 201 === statusCode
 
   c <- view tsCannon
-  WS.bracketRN c (map qUnqualified $ [alice, bob, charlie, dee] <> users) $ \(wsA:wsB:wsC:wsD:wsUsers) -> do
+  WS.bracketRN c (map qUnqualified $ [alice, bob, charlie, dee] <> users) $ \(wsA : wsB : wsC : wsD : wsUsers) -> do
     -- conversation access role changes to team only
     (_, reqs) <- withTempMockFederator' (mockReply ()) $ do
       -- Delete the domain that Dee lives on
@@ -4392,4 +4391,4 @@ testDefederationNotifications = do
         <!! const 200 === statusCode
   liftIO $ sort (omQualifiedId <$> cmOthers (cnvMembers conv2)) @?= sort [bob, charlie]
 
--- @END
\ No newline at end of file
+-- @END
diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs
index 38620d160d..060638f0e4 100644
--- a/services/galley/test/integration/API/Util.hs
+++ b/services/galley/test/integration/API/Util.hs
@@ -113,6 +113,7 @@ import Wire.API.Conversation.Role
 import Wire.API.Conversation.Typing
 import Wire.API.Event.Conversation
 import qualified Wire.API.Event.Conversation as Conv
+import qualified Wire.API.Event.Federation as Fed
 import Wire.API.Event.Team
 import qualified Wire.API.Event.Team as TE
 import Wire.API.Federation.API
@@ -145,7 +146,6 @@ import Wire.API.User.Auth hiding (Access)
 import Wire.API.User.Client
 import qualified Wire.API.User.Client as Client
 import Wire.API.User.Client.Prekey
-import qualified Wire.API.Event.Federation as Fed
 
 -------------------------------------------------------------------------------
 -- API Operations

From 2f0b50011013906ee629d1b3c03ef7ab39fb665d Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 18 Jul 2023 15:55:03 +1000
Subject: [PATCH 218/220] WPB-240: Changelog entries

---
 changelog.d/1-api-changes/WPB-240 | 3 +++
 changelog.d/6-federation/WPB-240  | 1 +
 2 files changed, 4 insertions(+)
 create mode 100644 changelog.d/1-api-changes/WPB-240
 create mode 100644 changelog.d/6-federation/WPB-240

diff --git a/changelog.d/1-api-changes/WPB-240 b/changelog.d/1-api-changes/WPB-240
new file mode 100644
index 0000000000..cea66f7f0f
--- /dev/null
+++ b/changelog.d/1-api-changes/WPB-240
@@ -0,0 +1,3 @@
+Added a new notification event type, "federation.delete".
+This event contains a single domain for a remote server that the local server is de-federating from.
+This notification is sent twice during de-federation. Once before and once after cleaning up and removing references to the remote server from the local database.
\ No newline at end of file
diff --git a/changelog.d/6-federation/WPB-240 b/changelog.d/6-federation/WPB-240
new file mode 100644
index 0000000000..6c9deb4916
--- /dev/null
+++ b/changelog.d/6-federation/WPB-240
@@ -0,0 +1 @@
+De-federating from a remote server sends a pair of notifications to clients, announcing which server will no longer be federated with.
\ No newline at end of file

From 9e0349a399bcc03a97f5b7cea7137739267295fb Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 18 Jul 2023 16:08:16 +1000
Subject: [PATCH 219/220] WPB-240: Removing dead code

---
 services/galley/src/Galley/API/Internal.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index 6eadf4ba04..f44b5cd59c 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -574,7 +574,6 @@ internalDeleteFederationDomainH (domain ::: _) = do
         -- RouteAny is used as it will wake up mobile clients
         -- and notify them of the changes to federation state.
         push1 $ p & Intra.pushRoute .~ Intra.RouteAny
-      -- & Intra.pushTransient .~ True
       deliverAsync (bots `zip` repeat (pushEventJson event))
     sendNotificationPage page = do
       let res = result page

From a41a3716ce81f763b76e609a6931494573787597 Mon Sep 17 00:00:00 2001
From: Owen Harvey <owenlharvey@gmail.com>
Date: Tue, 18 Jul 2023 17:24:59 +1000
Subject: [PATCH 220/220] WPB-240: Moving notification code into an effect, as
 mentioned in PR.

---
 services/galley/default.nix                   |  2 -
 services/galley/galley.cabal                  |  2 +-
 services/galley/src/Galley/API/Internal.hs    | 40 ++--------------
 services/galley/src/Galley/App.hs             |  1 +
 services/galley/src/Galley/Effects.hs         |  2 +
 .../Effects/DefederationNotifications.hs      | 14 ++++++
 services/galley/src/Galley/Intra/Effects.hs   | 48 ++++++++++++++++++-
 7 files changed, 70 insertions(+), 39 deletions(-)
 create mode 100644 services/galley/src/Galley/Effects/DefederationNotifications.hs

diff --git a/services/galley/default.nix b/services/galley/default.nix
index 9d9a0af56a..6f114b7648 100644
--- a/services/galley/default.nix
+++ b/services/galley/default.nix
@@ -26,7 +26,6 @@
 , conduit
 , containers
 , cookie
-, cql-io
 , cryptonite
 , currency-codes
 , data-default
@@ -150,7 +149,6 @@ mkDerivation {
     cereal
     comonad
     containers
-    cql-io
     cryptonite
     currency-codes
     data-default
diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal
index 4dec95af1e..b7c61258c1 100644
--- a/services/galley/galley.cabal
+++ b/services/galley/galley.cabal
@@ -99,6 +99,7 @@ library
     Galley.Effects.CodeStore
     Galley.Effects.ConversationStore
     Galley.Effects.CustomBackendStore
+    Galley.Effects.DefederationNotifications
     Galley.Effects.ExternalAccess
     Galley.Effects.FederatorAccess
     Galley.Effects.FireAndForget
@@ -214,7 +215,6 @@ library
     , cereal                 >=0.4
     , comonad
     , containers             >=0.5
-    , cql-io
     , cryptonite
     , currency-codes         >=2.0
     , data-default           >=0.5
diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs
index f44b5cd59c..b97fdac2da 100644
--- a/services/galley/src/Galley/API/Internal.hs
+++ b/services/galley/src/Galley/API/Internal.hs
@@ -26,7 +26,6 @@ module Galley.API.Internal
   )
 where
 
-import Cassandra (ClientState, Consistency (LocalQuorum), Page (..), paramsP)
 import Control.Exception
 import Control.Exception.Safe (catchAny)
 import Control.Lens hiding (Getter, Setter, (.=))
@@ -40,7 +39,6 @@ import Data.Range
 import Data.Singletons
 import Data.Text (unpack)
 import Data.Time
-import Database.CQL.IO (paginate)
 import Galley.API.Action
 import qualified Galley.API.Clients as Clients
 import qualified Galley.API.Create as Create
@@ -61,16 +59,13 @@ import Galley.API.Teams.Features
 import qualified Galley.API.Update as Update
 import Galley.API.Util
 import Galley.App
-import Galley.Cassandra.Conversation.Members
-import Galley.Cassandra.Queries
-import Galley.Cassandra.Store (embedClient)
 import qualified Galley.Data.Conversation as Data
 import Galley.Data.Conversation.Types
 import Galley.Effects
 import Galley.Effects.BackendNotificationQueueAccess
 import Galley.Effects.ClientStore
 import Galley.Effects.ConversationStore
-import Galley.Effects.ExternalAccess
+import Galley.Effects.DefederationNotifications (DefederationNotifications, sendDefederationNotifications)
 import Galley.Effects.FederatorAccess
 import Galley.Effects.GundeckAccess
 import Galley.Effects.LegalHoldStore as LegalHoldStore
@@ -79,9 +74,7 @@ import qualified Galley.Effects.MemberStore as E
 import Galley.Effects.ProposalStore
 import Galley.Effects.TeamStore
 import qualified Galley.Effects.TeamStore as E
-import Galley.Env (currentFanoutLimit, _options)
 import qualified Galley.Intra.Push as Intra
-import Galley.Intra.Push.Internal (pushEventJson)
 import Galley.Monad
 import Galley.Options
 import qualified Galley.Queue as Q
@@ -114,7 +107,6 @@ import Wire.API.CustomBackend
 import Wire.API.Error
 import Wire.API.Error.Galley
 import Wire.API.Event.Conversation
-import qualified Wire.API.Event.Federation as Federation
 import Wire.API.Federation.API
 import Wire.API.Federation.API.Galley
 import qualified Wire.API.Federation.API.Galley as F
@@ -543,12 +535,12 @@ internalDeleteFederationDomainH ::
     Member MemberStore r,
     Member ConversationStore r,
     Member (Embed IO) r,
-    Member (Input ClientState) r,
     Member CodeStore r,
     Member TeamStore r,
     Member BrigAccess r,
     Member GundeckAccess r,
-    Member ExternalAccess r
+    Member ExternalAccess r,
+    Member DefederationNotifications r
   ) =>
   Domain ::: JSON ->
   Sem r Response
@@ -556,32 +548,10 @@ internalDeleteFederationDomainH (domain ::: _) = do
   -- We have to send the same event twice.
   -- Once before and once after defederation work.
   -- https://wearezeta.atlassian.net/wiki/spaces/ENGINEERIN/pages/809238539/Use+case+Stopping+to+federate+with+a+domain
-  void sendNotifications
+  sendDefederationNotifications domain
   deleteFederationDomain domain
-  void sendNotifications
+  sendDefederationNotifications domain
   pure (empty & setStatus status200)
-  where
-    sendNotifications = do
-      maxPage <- inputs $ fromRange . currentFanoutLimit . _options -- This is based on the limits in removeIfLargeFanout
-      page <- embedClient $ paginate selectAllMembers (paramsP LocalQuorum () maxPage)
-      sendNotificationPage page
-    pushEvents results = do
-      let (bots, mems) = localBotsAndUsers results
-          recipients = Intra.recipient <$> mems
-          event = Intra.FederationEvent $ Federation.Event Federation.FederationDelete domain
-      for_ (Intra.newPush ListComplete Nothing event recipients) $ \p -> do
-        -- TODO: Transient or not?
-        -- RouteAny is used as it will wake up mobile clients
-        -- and notify them of the changes to federation state.
-        push1 $ p & Intra.pushRoute .~ Intra.RouteAny
-      deliverAsync (bots `zip` repeat (pushEventJson event))
-    sendNotificationPage page = do
-      let res = result page
-          mems = mapMaybe toMember res
-      pushEvents mems
-      when (hasMore page) $ do
-        page' <- embedClient $ nextPage page
-        sendNotificationPage page'
 
 -- Remove remote members from local conversations
 deleteFederationDomainRemoteUserFromLocalConversations ::
diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs
index 1b109134a1..515c558254 100644
--- a/services/galley/src/Galley/App.hs
+++ b/services/galley/src/Galley/App.hs
@@ -272,6 +272,7 @@ evalGalley e =
     . interpretFederatorAccess
     . interpretExternalAccess
     . interpretGundeckAccess
+    . interpretDefederationNotifications
     . interpretSparAccess
     . interpretBrigAccess
   where
diff --git a/services/galley/src/Galley/Effects.hs b/services/galley/src/Galley/Effects.hs
index a8dc2a5198..fc8f406bec 100644
--- a/services/galley/src/Galley/Effects.hs
+++ b/services/galley/src/Galley/Effects.hs
@@ -69,6 +69,7 @@ import Galley.Effects.ClientStore
 import Galley.Effects.CodeStore
 import Galley.Effects.ConversationStore
 import Galley.Effects.CustomBackendStore
+import Galley.Effects.DefederationNotifications
 import Galley.Effects.ExternalAccess
 import Galley.Effects.FederatorAccess
 import Galley.Effects.FireAndForget
@@ -99,6 +100,7 @@ import Wire.Sem.Paging.Cassandra
 type GalleyEffects1 =
   '[ BrigAccess,
      SparAccess,
+     DefederationNotifications,
      GundeckAccess,
      ExternalAccess,
      FederatorAccess,
diff --git a/services/galley/src/Galley/Effects/DefederationNotifications.hs b/services/galley/src/Galley/Effects/DefederationNotifications.hs
new file mode 100644
index 0000000000..db1fc30119
--- /dev/null
+++ b/services/galley/src/Galley/Effects/DefederationNotifications.hs
@@ -0,0 +1,14 @@
+{-# LANGUAGE TemplateHaskell #-}
+
+module Galley.Effects.DefederationNotifications 
+  ( DefederationNotifications (..),
+    sendDefederationNotifications
+  ) where
+
+import Polysemy
+import Data.Domain (Domain)
+
+data DefederationNotifications m a where
+  SendDefederationNotifications :: Domain -> DefederationNotifications m ()
+
+makeSem ''DefederationNotifications
\ No newline at end of file
diff --git a/services/galley/src/Galley/Intra/Effects.hs b/services/galley/src/Galley/Intra/Effects.hs
index 87c019c755..85f37c86f8 100644
--- a/services/galley/src/Galley/Intra/Effects.hs
+++ b/services/galley/src/Galley/Intra/Effects.hs
@@ -20,16 +20,27 @@ module Galley.Intra.Effects
     interpretSparAccess,
     interpretBotAccess,
     interpretGundeckAccess,
+    interpretDefederationNotifications,
   )
 where
 
+import Cassandra (ClientState, Consistency (LocalQuorum), Page (hasMore, nextPage, result), paginate, paramsP)
+import Control.Lens ((.~))
+import Data.Range (Range (fromRange))
 import Galley.API.Error
+import Galley.API.Util (localBotsAndUsers)
+import Galley.Cassandra.Conversation.Members (toMember)
+import Galley.Cassandra.Queries (selectAllMembers)
+import Galley.Cassandra.Store (embedClient)
 import Galley.Effects.BotAccess (BotAccess (..))
 import Galley.Effects.BrigAccess (BrigAccess (..))
-import Galley.Effects.GundeckAccess (GundeckAccess (..))
+import Galley.Effects.DefederationNotifications (DefederationNotifications (..))
+import Galley.Effects.ExternalAccess (ExternalAccess, deliverAsync)
+import Galley.Effects.GundeckAccess (GundeckAccess (..), push1)
 import Galley.Effects.SparAccess (SparAccess (..))
 import Galley.Env
 import Galley.Intra.Client
+import qualified Galley.Intra.Push as Intra
 import qualified Galley.Intra.Push.Internal as G
 import Galley.Intra.Spar
 import Galley.Intra.Team
@@ -41,6 +52,8 @@ import Polysemy.Error
 import Polysemy.Input
 import qualified Polysemy.TinyLog as P
 import qualified UnliftIO
+import qualified Wire.API.Event.Federation as Federation
+import Wire.API.Team.Member (ListType (ListComplete))
 
 interpretBrigAccess ::
   ( Member (Embed IO) r,
@@ -123,3 +136,36 @@ interpretGundeckAccess ::
 interpretGundeckAccess = interpret $ \case
   Push ps -> embedApp $ G.push ps
   PushSlowly ps -> embedApp $ G.pushSlowly ps
+
+interpretDefederationNotifications ::
+  ( Member (Embed IO) r,
+    Member (Input Env) r,
+    Member (Input ClientState) r,
+    Member GundeckAccess r,
+    Member ExternalAccess r
+  ) =>
+  Sem (DefederationNotifications ': r) a ->
+  Sem r a
+interpretDefederationNotifications = interpret $ \case
+  SendDefederationNotifications domain -> do
+    maxPage <- inputs $ fromRange . currentFanoutLimit . _options -- This is based on the limits in removeIfLargeFanout
+    page <- embedClient $ paginate selectAllMembers (paramsP LocalQuorum () maxPage)
+    void $ sendNotificationPage page
+    where
+      pushEvents results = do
+        let (bots, mems) = localBotsAndUsers results
+            recipients = Intra.recipient <$> mems
+            event = Intra.FederationEvent $ Federation.Event Federation.FederationDelete domain
+        for_ (Intra.newPush ListComplete Nothing event recipients) $ \p -> do
+          -- Futurework: Transient or not?
+          -- RouteAny is used as it will wake up mobile clients
+          -- and notify them of the changes to federation state.
+          push1 $ p & Intra.pushRoute .~ Intra.RouteAny
+        deliverAsync (bots `zip` repeat (G.pushEventJson event))
+      sendNotificationPage page = do
+        let res = result page
+            mems = mapMaybe toMember res
+        pushEvents mems
+        when (hasMore page) $ do
+          page' <- embedClient $ nextPage page
+          sendNotificationPage page'