[WPB 3796] Fix background worker for federation policy allowAll

Makefile

@@ -95,7 +95,7 @@ endif
 #
 # If you want to pass arguments to the test-suite call cabal-run-integration.sh directly.
 .PHONY: ci-fast
-ci-fast: c db-migrate
+ci-fast: c db-migrate cqlsh_clear_federation_remotes
 ifeq ("$(package)", "all")
 	./hack/bin/cabal-run-integration.sh all
 	./hack/bin/cabal-run-integration.sh integration
@@ -270,6 +270,18 @@ cqlsh:
 	@echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!"
 	docker exec -it $(CASSANDRA_CONTAINER) /usr/bin/cqlsh
 
+.PHONY: cqlsh_clear_federation_remotes
+cqlsh_clear_federation_remotes:
+	$(eval CASSANDRA_CONTAINER := $(shell docker ps | grep '/cassandra:' | perl -ne '/^(\S+)\s/ && print $$1'))
+	@echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!"
+	for ts in brig_test{,2} brig_dyn1_test brig_dyn2_test brig_dyn3_test; do echo "truncate $ts.federation_remotes if exists" ; done
+	echo "truncate brig_test.federation_remotes; \
+	      truncate brig_test2.federation_remotes; \
+	      truncate brig_test_dyn_1.federation_remotes; \
+	      truncate brig_test_dyn_2.federation_remotes; \
+	      truncate brig_test_dyn_3.federation_remotes;" | \
+	  docker exec -i $(CASSANDRA_CONTAINER) /usr/bin/cqlsh
+
 .PHONY: db-reset-package
 db-reset-package:
 	@echo "Deprecated! Please use 'db-reset' instead"

changelog.d/0-release-notes/WPB-3796

@@ -0,0 +1,15 @@
+New config value
+`background-worker.backendNotificationPusher.remotesRefreshIntervalMs`.
+If in doubt, set to 30000 ms.  (This controls the delay between
+backend-to-backend notification queue update pulls from rabbitMQ to
+background-worker.)
+
+If you have enabled federation and therefore rabbitMQ before this
+upgrade, you should remove all queues called
+`backend-notifications.delete-federation` manually.
+
+Remote federation domains in the config file are no longer supported.
+Before deploying this release, make sure you have completed the
+migration steps from [the CHANGELOG of Chart Release 4.36.0 (search
+for "remote connections can be configured via an internal REST
+API")](https://github.com/wireapp/wire-server/releases/tag/v2023-08-11).

changelog.d/3-bug-fixes/WPB-3796

@@ -0,0 +1 @@
+[background-worker] Get federation remote domains from rabbitMQ, not from brig (this fixes a bug in federation policy `allowAll`)

charts/background-worker/values.yaml

@@ -26,6 +26,7 @@ config:
   backendNotificationPusher:
     pushBackoffMinWait: 10000 # in microseconds, so 10ms
     pushBackoffMaxWait: 300000000 # microseconds, so 300s
+    remotesRefreshInterval: 30000000 # microseconds, so 3s
 
 serviceAccount:
   # When setting this to 'false', either make sure that a service account named

charts/brig/templates/configmap.yaml

@@ -253,12 +253,6 @@ data:
       {{- if .setFederationDomainConfigsUpdateFreq }}
       setFederationDomainConfigsUpdateFreq: {{ .setFederationDomainConfigsUpdateFreq }}
       {{- end }}
-      {{- if .setFederationDomainConfigs }}
-      # 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
-      # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
-      # for details.
-      setFederationDomainConfigs: {{ toYaml .setFederationDomainConfigs | nindent 8 }}
-      {{- end }}
       {{- if .setSearchSameTeamOnly }}
       setSearchSameTeamOnly: {{ .setSearchSameTeamOnly }}
       {{- end }}

charts/brig/templates/tests/configmap.yaml

@@ -58,6 +58,8 @@ data:
       cert: /etc/wire/integration-secrets/provider-publiccert.pem
       botHost: https://brig-integration
 
+    originDomain: 612c6a18-4e3e-11ee-a205-97507f092421.example.com
+
     backendTwo:
       brig:
         host: brig.{{ .Release.Namespace }}-fed2.svc.cluster.local
@@ -87,3 +89,5 @@ data:
       federatorExternal:
         host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local
         port: 8081
+
+      originDomain: 6da818ea-4e26-11ee-9422-9b252c5bbf2b.example.com

charts/integration/templates/integration-integration.yaml

@@ -207,3 +207,7 @@ spec:
         secretKeyRef:
           name: brig
           key: rabbitmqPassword
+    - name: TEST_INCLUDE
+      value: ""
+    - name: TASTY_PATTERN
+      value: "(!/turn/&&!/user.auth.cookies.limit/)&&/search users on remote backend/"

docs/src/developer/developer/how-to.md

@@ -108,9 +108,7 @@ make kube-integration-setup
 
 #### Deploy your local code to a kind cluster
 
-This can be useful to get quicker feedback while working on multi-backend code or configuration (e.g. helm charts) than to wait an hour for CI. This allows you to test code without uploading it to github and waiting an hour for CI.
-
-FUTUREWORK: this process is in development (update this section after it's confirmed to work):
+This can be useful to get quicker feedback while working on multi-backend code or configuration (e.g. helm charts), or integration tests. It saves you uploading everything to github and waiting an hour for CI.
 
 ##### Run tests in kind
 
@@ -119,11 +117,33 @@ FUTUREWORK: this process is in development (update this section after it's confi
 
    *Note:* First time all the images need to be uploaded. When working on one
    service it can be selectively uploaded using `make kind-upload-image-<name>`
-   (e.g. `make kind-upload-image-brig`).
+   (e.g. `make kind-upload-image-{brig,integration}`).
 2. Install wire-server using `make kind-integration-setup`.
 3. Run tests using `make kind-integration-test`.
 4. Run end2end integration tests: `make kind-integration-e2e`.
 
+If you want to run only a particular test suite, [edit this line](https://github.com/wireapp/wire-server/blob/2dfc268f233cc7b42a75243f2fd0071b81aef65b/hack/bin/integration-test.sh#L14).
+
+If you want to run only a particular test *case*, you can add env variables like `TASTY_PATTERN` or `TEST_INCLUDE` to [this block](https://github.com/wireapp/wire-server/blob/2dfc268f233cc7b42a75243f2fd0071b81aef65b/charts/integration/templates/integration-integration.yaml#L192-L209).
+
+To shut everything down (sometimes RAM / CPU use stays high indefinitely otherwise):
+
+1. `make kind-delete`
+
+----------------------------------------------------------------------
+
+make kind-integration-setup
+make kind-integration-test
+
+
+
+[needs kind] make ci-safe package=integration TEST_INCLUDE=Federation.testNotificationsForOfflineBackends
+
+
+[fixed?] make ci-safe package=brig TASTY_PATTERN='(!/turn/&&!/user.auth.cookies.limit/)&&/search users on remote backend/'
+[fixed?] make ci-safe package=brig TASTY_PATTERN='(!/turn/&&!/user.auth.cookies.limit/)&&/lookup user by qualified handle on remote backend/'
+
+
 
 
 #### Deploy your local code to a kubernetes cluster

docs/src/understand/configure-federation.md

@@ -506,9 +506,11 @@ search policy if you create an entry.
 #### If your instance has been federating before
 
 You only need to read this section if your instance has been
-federating with other instances prior to
-[PR#3260](https://github.com/wireapp/wire-server/pull/3260), and you
-are upgrading to the release containing that PR.
+federating with other instances prior to [Chart Release
+4.36.0](https://github.com/wireapp/wire-server/releases/tag/v2023-08-11),
+and you are upgrading to that PR.
+
+Ignore this section if your release is more recent than 4.38.0..
 
 From now on the federation policy set in the federator config under
 `federationStrategy` is ignored.  Instead, the federation strategy is

hack/bin/integration-test.sh

@@ -11,7 +11,7 @@ UPLOAD_LOGS=${UPLOAD_LOGS:-0}
 echo "Running integration tests on wire-server with parallelism=${HELM_PARALLELISM} ..."
 
 CHART=wire-server
-tests=(integration stern galley cargohold gundeck federator spar brig)
+tests=(integration) # brig
 
 cleanup() {
     if (( CLEANUP_LOCAL_FILES > 0 )); then

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -77,21 +77,6 @@ brig:
       setMaxConvSize: 16
       # See helmfile for the real value
       setFederationDomain: integration.example.com
-      setFederationDomainConfigs:
-        # 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260.  See
-        # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections
-        # for details.
-        - domain: integration.example.com
-          search_policy: full_search
-        - domain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
-          search_policy: full_search
-        # Remove these after fixing https://wearezeta.atlassian.net/browse/WPB-3796
-        - domain: dyn-backend-1
-          search_policy: full_search
-        - domain: dyn-backend-2
-          search_policy: full_search
-        - domain: dyn-backend-3
-          search_policy: full_search
       setFederationStrategy: allowAll
       setFederationDomainConfigsUpdateFreq: 10
       set2FACodeGenerationDelaySecs: 5
@@ -321,6 +306,7 @@ background-worker:
     backendNotificationPusher:
       pushBackoffMinWait: 1000 # 1ms
       pushBackoffMaxWait: 500000 # 0.5s
+      remotesRefreshInterval: 500000 # 0.5s
   secrets:
     rabbitmq:
       username: {{ .Values.rabbitmqUsername }}

hack/helmfile.yaml

@@ -127,14 +127,6 @@ releases:
         value: {{ .Values.federationDomain1 }}
       - name: cargohold.config.settings.federationDomain
         value: {{ .Values.federationDomain1 }}
-      - name: brig.config.optSettings.setFederationDomainConfigs[0].domain
-        value: {{ .Values.federationDomain2 }}
-      - name: brig.config.optSettings.setFederationDomainConfigs[2].domain
-        value: {{ .Values.dynBackendDomain1 }}
-      - name: brig.config.optSettings.setFederationDomainConfigs[3].domain
-        value: {{ .Values.dynBackendDomain2 }}
-      - name: brig.config.optSettings.setFederationDomainConfigs[4].domain
-        value: {{ .Values.dynBackendDomain3 }}
     needs:
       - 'databases-ephemeral'
 
@@ -151,13 +143,5 @@ releases:
         value: {{ .Values.federationDomain2 }}
       - name: cargohold.config.settings.federationDomain
         value: {{ .Values.federationDomain2 }}
-      - name: brig.config.optSettings.setFederationDomainConfigs[0].domain
-        value: {{ .Values.federationDomain1 }}
-      - name: brig.config.optSettings.setFederationDomainConfigs[2].domain
-        value: {{ .Values.dynBackendDomain1 }}
-      - name: brig.config.optSettings.setFederationDomainConfigs[3].domain
-        value: {{ .Values.dynBackendDomain2 }}
-      - name: brig.config.optSettings.setFederationDomainConfigs[4].domain
-        value: {{ .Values.dynBackendDomain3 }}
     needs:
       - 'databases-ephemeral'

integration/test/SetupHelpers.hs

@@ -9,7 +9,6 @@ import Data.Aeson hiding ((.=))
 import Data.Aeson.Types qualified as Aeson
 import Data.Default
 import Data.Function
-import Data.List qualified as List
 import Data.UUID.V1 (nextUUID)
 import Data.UUID.V4 (nextRandom)
 import GHC.Stack
@@ -81,7 +80,7 @@ resetFedConns owndom = do
     rdoms :: [String] <- do
       rawlist <- resp.json %. "remotes" & asList
       (asString . (%. "domain")) `mapM` rawlist
-    Internal.deleteFedConn' owndom `mapM_` rdoms
+    Internal.deleteFedConn owndom `mapM_` rdoms
 
 randomId :: HasCallStack => App String
 randomId = liftIO (show <$> nextRandom)
@@ -95,32 +94,10 @@ randomUserId domain = do
   uid <- randomId
   pure $ object ["id" .= uid, "domain" .= d]
 
-addFullSearchFor :: [String] -> Value -> App Value
-addFullSearchFor domains val =
-  modifyField
-    "optSettings.setFederationDomainConfigs"
-    ( \configs -> do
-        cfg <- assertJust "" configs
-        xs <- cfg & asList
-        pure (xs <> [object ["domain" .= domain, "search_policy" .= "full_search"] | domain <- domains])
-    )
-    val
-
-fullSearchWithAll :: ServiceOverrides
-fullSearchWithAll =
-  def
-    { brigCfg = \val -> do
-        ownDomain <- asString =<< val %. "optSettings.setFederationDomain"
-        env <- ask
-        let remoteDomains = List.delete ownDomain $ [env.domain1, env.domain2] <> env.dynamicDomains
-        addFullSearchFor remoteDomains val
-    }
-
 withFederatingBackendsAllowDynamic :: HasCallStack => Int -> ((String, String, String) -> App a) -> App a
 withFederatingBackendsAllowDynamic n k = do
   let setFederationConfig =
         setField "optSettings.setFederationStrategy" "allowDynamic"
-          >=> removeField "optSettings.setFederationDomainConfigs"
           >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1)
   startDynamicBackends
     [ def {brigCfg = setFederationConfig},

integration/test/Test/Brig.hs

@@ -29,14 +29,8 @@ testSearchContactForExternalUsers = do
 testCrudFederationRemotes :: HasCallStack => App ()
 testCrudFederationRemotes = do
   otherDomain <- asString OtherDomain
-  let overrides =
-        def
-          { brigCfg =
-              setField
-                "optSettings.setFederationDomainConfigs"
-                [object ["domain" .= otherDomain, "search_policy" .= "full_search"]]
-          }
-  withModifiedBackend overrides $ \ownDomain -> do
+  withModifiedBackend def $ \ownDomain -> do
+    void $ Internal.createFedConn ownDomain (Internal.FedConn otherDomain "full_search")
     let parseFedConns :: HasCallStack => Response -> App [Value]
         parseFedConns resp =
           -- Pick out the list of federation domain configs
@@ -52,23 +46,13 @@ testCrudFederationRemotes = do
             res2 <- parseFedConns =<< Internal.readFedConns ownDomain
             sort res2 `shouldMatch` sort want
 
-        addFail :: HasCallStack => MakesValue fedConn => fedConn -> App ()
-        addFail fedConn = do
-          bindResponse (Internal.createFedConn' ownDomain fedConn) $ \res -> do
-            addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
-
         deleteOnce :: (Ord fedConn, ToJSON fedConn, MakesValue fedConn) => String -> [fedConn] -> App ()
         deleteOnce domain want = do
           bindResponse (Internal.deleteFedConn ownDomain domain) $ \res -> do
             addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 200
             res2 <- parseFedConns =<< Internal.readFedConns ownDomain
             sort res2 `shouldMatch` sort want
 
-        deleteFail :: HasCallStack => String -> App ()
-        deleteFail del = do
-          bindResponse (Internal.deleteFedConn' ownDomain del) $ \res -> do
-            addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533
-
         updateOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => String -> fedConn -> [fedConn2] -> App ()
         updateOnce domain fedConn want = do
           bindResponse (Internal.updateFedConn ownDomain domain fedConn) $ \res -> do
@@ -86,33 +70,28 @@ testCrudFederationRemotes = do
 
     let remote1, remote1', remote1'' :: Internal.FedConn
         remote1 = Internal.FedConn dom1 "no_search"
-        remote1' = remote1 {Internal.searchStrategy = "full_search"}
-        remote1'' = remote1 {Internal.domain = dom2}
-
-        cfgRemotesExpect :: Internal.FedConn
-        cfgRemotesExpect = Internal.FedConn (cs otherDomain) "full_search"
+        remote1' = Internal.FedConn dom1 "full_search"
+        remote1'' = Internal.FedConn dom2 "exact_handle_search"
 
     remote1J <- make remote1
     remote1J' <- make remote1'
+    remote1J'' <- make remote1''
 
     resetFedConns ownDomain
-    cfgRemotes <- parseFedConns =<< Internal.readFedConns ownDomain
-    cfgRemotes `shouldMatch` [cfgRemotesExpect]
-    -- entries present in the config file can be idempotently added if identical, but cannot be
-    -- updated, deleted or updated.
-    addOnce cfgRemotesExpect [cfgRemotesExpect]
-    addFail (cfgRemotesExpect {Internal.searchStrategy = "no_search"})
-    deleteFail (Internal.domain cfgRemotesExpect)
-    updateFail (Internal.domain cfgRemotesExpect) (cfgRemotesExpect {Internal.searchStrategy = "no_search"})
     -- create
-    addOnce remote1 $ (remote1J : cfgRemotes)
-    addOnce remote1 $ (remote1J : cfgRemotes) -- idempotency
+    addOnce remote1 [remote1J]
+    addOnce remote1 [remote1J] -- idempotency
+    addOnce remote1'' [remote1J, remote1J'']
     -- update
-    updateOnce (Internal.domain remote1) remote1' (remote1J' : cfgRemotes)
-    updateFail (Internal.domain remote1) remote1''
+    updateOnce (Internal.domain remote1) remote1' [remote1J', remote1J'']
+    updateOnce (Internal.domain remote1) remote1' [remote1J', remote1J''] -- idempotency
+    updateFail (Internal.domain remote1) remote1'' -- existing entries can't change domain
     -- delete
-    deleteOnce (Internal.domain remote1) cfgRemotes
-    deleteOnce (Internal.domain remote1) cfgRemotes -- idempotency
+    deleteOnce (Internal.domain remote1) [remote1J'']
+    deleteOnce (Internal.domain remote1) [remote1J''] -- idempotency
+    -- reset
+    resetFedConns ownDomain
+    deleteOnce (Internal.domain remote1) ([] :: [Internal.FedConn])
 
 testCrudOAuthClient :: HasCallStack => App ()
 testCrudOAuthClient = do
@@ -185,7 +164,6 @@ testRemoteUserSearch :: HasCallStack => App ()
 testRemoteUserSearch = do
   let overrides =
         setField "optSettings.setFederationStrategy" "allowDynamic"
-          >=> removeField "optSettings.setFederationDomainConfigs"
           >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1)
   startDynamicBackends [def {brigCfg = overrides}, def {brigCfg = overrides}] $ \dynDomains -> do
     domains@[d1, d2] <- pure dynDomains