diff --git a/Makefile b/Makefile index b2706f460a2..5f47141ff25 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ endif # # If you want to pass arguments to the test-suite call cabal-run-integration.sh directly. .PHONY: ci-fast -ci-fast: c db-migrate +ci-fast: c db-migrate cqlsh_clear_federation_remotes ifeq ("$(package)", "all") ./hack/bin/cabal-run-integration.sh all ./hack/bin/cabal-run-integration.sh integration @@ -270,6 +270,18 @@ cqlsh: @echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!" docker exec -it $(CASSANDRA_CONTAINER) /usr/bin/cqlsh +.PHONY: cqlsh_clear_federation_remotes +cqlsh_clear_federation_remotes: + $(eval CASSANDRA_CONTAINER := $(shell docker ps | grep '/cassandra:' | perl -ne '/^(\S+)\s/ && print $$1')) + @echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!" + for ts in brig_test{,2} brig_dyn1_test brig_dyn2_test brig_dyn3_test; do echo "truncate $ts.federation_remotes if exists" ; done + echo "truncate brig_test.federation_remotes; \ + truncate brig_test2.federation_remotes; \ + truncate brig_test_dyn_1.federation_remotes; \ + truncate brig_test_dyn_2.federation_remotes; \ + truncate brig_test_dyn_3.federation_remotes;" | \ + docker exec -i $(CASSANDRA_CONTAINER) /usr/bin/cqlsh + .PHONY: db-reset-package db-reset-package: @echo "Deprecated! Please use 'db-reset' instead" diff --git a/changelog.d/0-release-notes/WPB-3796 b/changelog.d/0-release-notes/WPB-3796 new file mode 100644 index 00000000000..fd8d7268e82 --- /dev/null +++ b/changelog.d/0-release-notes/WPB-3796 @@ -0,0 +1,15 @@ +New config value +`background-worker.backendNotificationPusher.remotesRefreshIntervalMs`. +If in doubt, set to 30000 ms. (This controls the delay between +backend-to-backend notification queue update pulls from rabbitMQ to +background-worker.) + +If you have enabled federation and therefore rabbitMQ before this +upgrade, you should remove all queues called +`backend-notifications.delete-federation` manually. + +Remote federation domains in the config file are no longer supported. +Before deploying this release, make sure you have completed the +migration steps from [the CHANGELOG of Chart Release 4.36.0 (search +for "remote connections can be configured via an internal REST +API")](https://github.com/wireapp/wire-server/releases/tag/v2023-08-11). diff --git a/changelog.d/3-bug-fixes/WPB-3796 b/changelog.d/3-bug-fixes/WPB-3796 new file mode 100644 index 00000000000..76392180c03 --- /dev/null +++ b/changelog.d/3-bug-fixes/WPB-3796 @@ -0,0 +1 @@ +[background-worker] Get federation remote domains from rabbitMQ, not from brig (this fixes a bug in federation policy `allowAll`) diff --git a/charts/background-worker/values.yaml b/charts/background-worker/values.yaml index fcae0115bfc..e85973290c1 100644 --- a/charts/background-worker/values.yaml +++ b/charts/background-worker/values.yaml @@ -26,6 +26,7 @@ config: backendNotificationPusher: pushBackoffMinWait: 10000 # in microseconds, so 10ms pushBackoffMaxWait: 300000000 # microseconds, so 300s + remotesRefreshInterval: 30000000 # microseconds, so 3s serviceAccount: # When setting this to 'false', either make sure that a service account named diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 2ed6eb6833a..a08d359f31c 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -253,12 +253,6 @@ data: {{- if .setFederationDomainConfigsUpdateFreq }} setFederationDomainConfigsUpdateFreq: {{ .setFederationDomainConfigsUpdateFreq }} {{- end }} - {{- if .setFederationDomainConfigs }} - # 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260. See - # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections - # for details. - setFederationDomainConfigs: {{ toYaml .setFederationDomainConfigs | nindent 8 }} - {{- end }} {{- if .setSearchSameTeamOnly }} setSearchSameTeamOnly: {{ .setSearchSameTeamOnly }} {{- end }} diff --git a/charts/brig/templates/tests/configmap.yaml b/charts/brig/templates/tests/configmap.yaml index 56667e55ed3..966b57bc48a 100644 --- a/charts/brig/templates/tests/configmap.yaml +++ b/charts/brig/templates/tests/configmap.yaml @@ -58,6 +58,8 @@ data: cert: /etc/wire/integration-secrets/provider-publiccert.pem botHost: https://brig-integration + originDomain: 612c6a18-4e3e-11ee-a205-97507f092421.example.com + backendTwo: brig: host: brig.{{ .Release.Namespace }}-fed2.svc.cluster.local @@ -87,3 +89,5 @@ data: federatorExternal: host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local port: 8081 + + originDomain: 6da818ea-4e26-11ee-9422-9b252c5bbf2b.example.com diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index f4ad967a975..dc30a869406 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -207,3 +207,7 @@ spec: secretKeyRef: name: brig key: rabbitmqPassword + - name: TEST_INCLUDE + value: "" + - name: TASTY_PATTERN + value: "(!/turn/&&!/user.auth.cookies.limit/)&&/search users on remote backend/" diff --git a/docs/src/developer/developer/how-to.md b/docs/src/developer/developer/how-to.md index 6c7cc0e1cb2..2d47e9857b0 100644 --- a/docs/src/developer/developer/how-to.md +++ b/docs/src/developer/developer/how-to.md @@ -108,9 +108,7 @@ make kube-integration-setup #### Deploy your local code to a kind cluster -This can be useful to get quicker feedback while working on multi-backend code or configuration (e.g. helm charts) than to wait an hour for CI. This allows you to test code without uploading it to github and waiting an hour for CI. - -FUTUREWORK: this process is in development (update this section after it's confirmed to work): +This can be useful to get quicker feedback while working on multi-backend code or configuration (e.g. helm charts), or integration tests. It saves you uploading everything to github and waiting an hour for CI. ##### Run tests in kind @@ -119,11 +117,33 @@ FUTUREWORK: this process is in development (update this section after it's confi *Note:* First time all the images need to be uploaded. When working on one service it can be selectively uploaded using `make kind-upload-image-` - (e.g. `make kind-upload-image-brig`). + (e.g. `make kind-upload-image-{brig,integration}`). 2. Install wire-server using `make kind-integration-setup`. 3. Run tests using `make kind-integration-test`. 4. Run end2end integration tests: `make kind-integration-e2e`. +If you want to run only a particular test suite, [edit this line](https://github.com/wireapp/wire-server/blob/2dfc268f233cc7b42a75243f2fd0071b81aef65b/hack/bin/integration-test.sh#L14). + +If you want to run only a particular test *case*, you can add env variables like `TASTY_PATTERN` or `TEST_INCLUDE` to [this block](https://github.com/wireapp/wire-server/blob/2dfc268f233cc7b42a75243f2fd0071b81aef65b/charts/integration/templates/integration-integration.yaml#L192-L209). + +To shut everything down (sometimes RAM / CPU use stays high indefinitely otherwise): + +1. `make kind-delete` + +---------------------------------------------------------------------- + +make kind-integration-setup +make kind-integration-test + + + +[needs kind] make ci-safe package=integration TEST_INCLUDE=Federation.testNotificationsForOfflineBackends + + +[fixed?] make ci-safe package=brig TASTY_PATTERN='(!/turn/&&!/user.auth.cookies.limit/)&&/search users on remote backend/' +[fixed?] make ci-safe package=brig TASTY_PATTERN='(!/turn/&&!/user.auth.cookies.limit/)&&/lookup user by qualified handle on remote backend/' + + #### Deploy your local code to a kubernetes cluster diff --git a/docs/src/understand/configure-federation.md b/docs/src/understand/configure-federation.md index 3477a10242a..98fa2401a62 100644 --- a/docs/src/understand/configure-federation.md +++ b/docs/src/understand/configure-federation.md @@ -506,9 +506,11 @@ search policy if you create an entry. #### If your instance has been federating before You only need to read this section if your instance has been -federating with other instances prior to -[PR#3260](https://github.com/wireapp/wire-server/pull/3260), and you -are upgrading to the release containing that PR. +federating with other instances prior to [Chart Release +4.36.0](https://github.com/wireapp/wire-server/releases/tag/v2023-08-11), +and you are upgrading to that PR. + +Ignore this section if your release is more recent than 4.38.0.. From now on the federation policy set in the federator config under `federationStrategy` is ignored. Instead, the federation strategy is diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh index 841a3172fd8..b3cc5bdee9b 100755 --- a/hack/bin/integration-test.sh +++ b/hack/bin/integration-test.sh @@ -11,7 +11,7 @@ UPLOAD_LOGS=${UPLOAD_LOGS:-0} echo "Running integration tests on wire-server with parallelism=${HELM_PARALLELISM} ..." CHART=wire-server -tests=(integration stern galley cargohold gundeck federator spar brig) +tests=(integration) # brig cleanup() { if (( CLEANUP_LOCAL_FILES > 0 )); then diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 4bd06d953fe..83b6302fa0e 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -77,21 +77,6 @@ brig: setMaxConvSize: 16 # See helmfile for the real value setFederationDomain: integration.example.com - setFederationDomainConfigs: - # 'setFederationDomainConfigs' is deprecated as of https://github.com/wireapp/wire-server/pull/3260. See - # https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections - # for details. - - domain: integration.example.com - search_policy: full_search - - domain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local - search_policy: full_search - # Remove these after fixing https://wearezeta.atlassian.net/browse/WPB-3796 - - domain: dyn-backend-1 - search_policy: full_search - - domain: dyn-backend-2 - search_policy: full_search - - domain: dyn-backend-3 - search_policy: full_search setFederationStrategy: allowAll setFederationDomainConfigsUpdateFreq: 10 set2FACodeGenerationDelaySecs: 5 @@ -321,6 +306,7 @@ background-worker: backendNotificationPusher: pushBackoffMinWait: 1000 # 1ms pushBackoffMaxWait: 500000 # 0.5s + remotesRefreshInterval: 500000 # 0.5s secrets: rabbitmq: username: {{ .Values.rabbitmqUsername }} diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index b40cd73d623..878cb016f50 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -127,14 +127,6 @@ releases: value: {{ .Values.federationDomain1 }} - name: cargohold.config.settings.federationDomain value: {{ .Values.federationDomain1 }} - - name: brig.config.optSettings.setFederationDomainConfigs[0].domain - value: {{ .Values.federationDomain2 }} - - name: brig.config.optSettings.setFederationDomainConfigs[2].domain - value: {{ .Values.dynBackendDomain1 }} - - name: brig.config.optSettings.setFederationDomainConfigs[3].domain - value: {{ .Values.dynBackendDomain2 }} - - name: brig.config.optSettings.setFederationDomainConfigs[4].domain - value: {{ .Values.dynBackendDomain3 }} needs: - 'databases-ephemeral' @@ -151,13 +143,5 @@ releases: value: {{ .Values.federationDomain2 }} - name: cargohold.config.settings.federationDomain value: {{ .Values.federationDomain2 }} - - name: brig.config.optSettings.setFederationDomainConfigs[0].domain - value: {{ .Values.federationDomain1 }} - - name: brig.config.optSettings.setFederationDomainConfigs[2].domain - value: {{ .Values.dynBackendDomain1 }} - - name: brig.config.optSettings.setFederationDomainConfigs[3].domain - value: {{ .Values.dynBackendDomain2 }} - - name: brig.config.optSettings.setFederationDomainConfigs[4].domain - value: {{ .Values.dynBackendDomain3 }} needs: - 'databases-ephemeral' diff --git a/integration/test/SetupHelpers.hs b/integration/test/SetupHelpers.hs index d5e324ea41c..854c2a5f1c8 100644 --- a/integration/test/SetupHelpers.hs +++ b/integration/test/SetupHelpers.hs @@ -9,7 +9,6 @@ import Data.Aeson hiding ((.=)) import Data.Aeson.Types qualified as Aeson import Data.Default import Data.Function -import Data.List qualified as List import Data.UUID.V1 (nextUUID) import Data.UUID.V4 (nextRandom) import GHC.Stack @@ -81,7 +80,7 @@ resetFedConns owndom = do rdoms :: [String] <- do rawlist <- resp.json %. "remotes" & asList (asString . (%. "domain")) `mapM` rawlist - Internal.deleteFedConn' owndom `mapM_` rdoms + Internal.deleteFedConn owndom `mapM_` rdoms randomId :: HasCallStack => App String randomId = liftIO (show <$> nextRandom) @@ -95,32 +94,10 @@ randomUserId domain = do uid <- randomId pure $ object ["id" .= uid, "domain" .= d] -addFullSearchFor :: [String] -> Value -> App Value -addFullSearchFor domains val = - modifyField - "optSettings.setFederationDomainConfigs" - ( \configs -> do - cfg <- assertJust "" configs - xs <- cfg & asList - pure (xs <> [object ["domain" .= domain, "search_policy" .= "full_search"] | domain <- domains]) - ) - val - -fullSearchWithAll :: ServiceOverrides -fullSearchWithAll = - def - { brigCfg = \val -> do - ownDomain <- asString =<< val %. "optSettings.setFederationDomain" - env <- ask - let remoteDomains = List.delete ownDomain $ [env.domain1, env.domain2] <> env.dynamicDomains - addFullSearchFor remoteDomains val - } - withFederatingBackendsAllowDynamic :: HasCallStack => Int -> ((String, String, String) -> App a) -> App a withFederatingBackendsAllowDynamic n k = do let setFederationConfig = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) startDynamicBackends [ def {brigCfg = setFederationConfig}, diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs index a229f8cfd6d..9df96a2c885 100644 --- a/integration/test/Test/Brig.hs +++ b/integration/test/Test/Brig.hs @@ -29,14 +29,8 @@ testSearchContactForExternalUsers = do testCrudFederationRemotes :: HasCallStack => App () testCrudFederationRemotes = do otherDomain <- asString OtherDomain - let overrides = - def - { brigCfg = - setField - "optSettings.setFederationDomainConfigs" - [object ["domain" .= otherDomain, "search_policy" .= "full_search"]] - } - withModifiedBackend overrides $ \ownDomain -> do + withModifiedBackend def $ \ownDomain -> do + void $ Internal.createFedConn ownDomain (Internal.FedConn otherDomain "full_search") let parseFedConns :: HasCallStack => Response -> App [Value] parseFedConns resp = -- Pick out the list of federation domain configs @@ -52,11 +46,6 @@ testCrudFederationRemotes = do res2 <- parseFedConns =<< Internal.readFedConns ownDomain sort res2 `shouldMatch` sort want - addFail :: HasCallStack => MakesValue fedConn => fedConn -> App () - addFail fedConn = do - bindResponse (Internal.createFedConn' ownDomain fedConn) $ \res -> do - addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533 - deleteOnce :: (Ord fedConn, ToJSON fedConn, MakesValue fedConn) => String -> [fedConn] -> App () deleteOnce domain want = do bindResponse (Internal.deleteFedConn ownDomain domain) $ \res -> do @@ -64,11 +53,6 @@ testCrudFederationRemotes = do res2 <- parseFedConns =<< Internal.readFedConns ownDomain sort res2 `shouldMatch` sort want - deleteFail :: HasCallStack => String -> App () - deleteFail del = do - bindResponse (Internal.deleteFedConn' ownDomain del) $ \res -> do - addFailureContext ("res = " <> show res) $ res.status `shouldMatchInt` 533 - updateOnce :: (MakesValue fedConn, Ord fedConn2, ToJSON fedConn2, MakesValue fedConn2, HasCallStack) => String -> fedConn -> [fedConn2] -> App () updateOnce domain fedConn want = do bindResponse (Internal.updateFedConn ownDomain domain fedConn) $ \res -> do @@ -86,33 +70,28 @@ testCrudFederationRemotes = do let remote1, remote1', remote1'' :: Internal.FedConn remote1 = Internal.FedConn dom1 "no_search" - remote1' = remote1 {Internal.searchStrategy = "full_search"} - remote1'' = remote1 {Internal.domain = dom2} - - cfgRemotesExpect :: Internal.FedConn - cfgRemotesExpect = Internal.FedConn (cs otherDomain) "full_search" + remote1' = Internal.FedConn dom1 "full_search" + remote1'' = Internal.FedConn dom2 "exact_handle_search" remote1J <- make remote1 remote1J' <- make remote1' + remote1J'' <- make remote1'' resetFedConns ownDomain - cfgRemotes <- parseFedConns =<< Internal.readFedConns ownDomain - cfgRemotes `shouldMatch` [cfgRemotesExpect] - -- entries present in the config file can be idempotently added if identical, but cannot be - -- updated, deleted or updated. - addOnce cfgRemotesExpect [cfgRemotesExpect] - addFail (cfgRemotesExpect {Internal.searchStrategy = "no_search"}) - deleteFail (Internal.domain cfgRemotesExpect) - updateFail (Internal.domain cfgRemotesExpect) (cfgRemotesExpect {Internal.searchStrategy = "no_search"}) -- create - addOnce remote1 $ (remote1J : cfgRemotes) - addOnce remote1 $ (remote1J : cfgRemotes) -- idempotency + addOnce remote1 [remote1J] + addOnce remote1 [remote1J] -- idempotency + addOnce remote1'' [remote1J, remote1J''] -- update - updateOnce (Internal.domain remote1) remote1' (remote1J' : cfgRemotes) - updateFail (Internal.domain remote1) remote1'' + updateOnce (Internal.domain remote1) remote1' [remote1J', remote1J''] + updateOnce (Internal.domain remote1) remote1' [remote1J', remote1J''] -- idempotency + updateFail (Internal.domain remote1) remote1'' -- existing entries can't change domain -- delete - deleteOnce (Internal.domain remote1) cfgRemotes - deleteOnce (Internal.domain remote1) cfgRemotes -- idempotency + deleteOnce (Internal.domain remote1) [remote1J''] + deleteOnce (Internal.domain remote1) [remote1J''] -- idempotency + -- reset + resetFedConns ownDomain + deleteOnce (Internal.domain remote1) ([] :: [Internal.FedConn]) testCrudOAuthClient :: HasCallStack => App () testCrudOAuthClient = do @@ -185,7 +164,6 @@ testRemoteUserSearch :: HasCallStack => App () testRemoteUserSearch = do let overrides = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) startDynamicBackends [def {brigCfg = overrides}, def {brigCfg = overrides}] $ \dynDomains -> do domains@[d1, d2] <- pure dynDomains diff --git a/integration/test/Test/Conversation.hs b/integration/test/Test/Conversation.hs index f956480dc85..c03041d5c5e 100644 --- a/integration/test/Test/Conversation.hs +++ b/integration/test/Test/Conversation.hs @@ -21,9 +21,7 @@ testDynamicBackendsFullyConnectedWhenAllowAll :: HasCallStack => App () testDynamicBackendsFullyConnectedWhenAllowAll = do let overrides = def {brigCfg = setField "optSettings.setFederationStrategy" "allowAll"} - <> fullSearchWithAll - startDynamicBackends [overrides, overrides, overrides] $ \dynDomains -> do - [domainA, domainB, domainC] <- pure dynDomains + startDynamicBackends [overrides, overrides, overrides] $ \[domainA, domainB, domainC] -> do uidA <- randomUser domainA def {team = True} uidB <- randomUser domainA def {team = True} uidC <- randomUser domainA def {team = True} @@ -61,7 +59,6 @@ testDynamicBackendsFullyConnectedWhenAllowDynamic :: HasCallStack => App () testDynamicBackendsFullyConnectedWhenAllowDynamic = do let overrides = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) startDynamicBackends [ def {brigCfg = overrides}, @@ -90,7 +87,6 @@ testDynamicBackendsNotFullyConnected = do def { brigCfg = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) } startDynamicBackends [overrides, overrides, overrides] $ @@ -139,7 +135,6 @@ testCreateConversationFullyConnected :: HasCallStack => App () testCreateConversationFullyConnected = do let setFederationConfig = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) startDynamicBackends [ def {brigCfg = setFederationConfig}, @@ -157,7 +152,6 @@ testCreateConversationNonFullyConnected :: HasCallStack => App () testCreateConversationNonFullyConnected = do let setFederationConfig = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) startDynamicBackends [ def {brigCfg = setFederationConfig}, @@ -180,7 +174,6 @@ testDefederationGroupConversation :: HasCallStack => App () testDefederationGroupConversation = do let setFederationConfig = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) startDynamicBackends [ def {brigCfg = setFederationConfig}, @@ -246,7 +239,6 @@ testDefederationOneOnOne :: HasCallStack => App () testDefederationOneOnOne = do let setFederationConfig = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) startDynamicBackends [ def {brigCfg = setFederationConfig}, @@ -345,7 +337,7 @@ testConvWithUnreachableRemoteUsers :: HasCallStack => App () testConvWithUnreachableRemoteUsers = do let overrides = def {brigCfg = setField "optSettings.setFederationStrategy" "allowAll"} - <> fullSearchWithAll + ([alice, alex, bob, charlie, dylan], domains) <- startDynamicBackends [overrides, overrides] $ \domains -> do own <- make OwnDomain & asString @@ -366,7 +358,6 @@ testAddReachableWithUnreachableRemoteUsers :: HasCallStack => App () testAddReachableWithUnreachableRemoteUsers = do let overrides = def {brigCfg = setField "optSettings.setFederationStrategy" "allowAll"} - <> fullSearchWithAll ([alex, bob], conv, domains) <- startDynamicBackends [overrides, overrides] $ \domains -> do own <- make OwnDomain & asString @@ -392,7 +383,6 @@ testAddUnreachable :: HasCallStack => App () testAddUnreachable = do let overrides = def {brigCfg = setField "optSettings.setFederationStrategy" "allowAll"} - <> fullSearchWithAll ([alex, charlie], [charlieDomain, dylanDomain], conv) <- startDynamicBackends [overrides, overrides] $ \domains -> do own <- make OwnDomain & asString @@ -416,7 +406,6 @@ testAddingUserNonFullyConnectedFederation = do def { brigCfg = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" } startDynamicBackends [overrides] $ \[dynBackend] -> do own <- asString OwnDomain diff --git a/integration/test/Test/Defederation.hs b/integration/test/Test/Defederation.hs index 513efe535f6..7ee9942fcb5 100644 --- a/integration/test/Test/Defederation.hs +++ b/integration/test/Test/Defederation.hs @@ -13,24 +13,22 @@ testDefederationRemoteNotifications :: HasCallStack => App () testDefederationRemoteNotifications = do let remoteDomain = "example.example.com" -- Setup federation between OtherDomain and the remote domain - bindResponse (createFedConn OtherDomain $ object ["domain" .= remoteDomain, "search_policy" .= "full_search"]) $ \resp -> - resp.status `shouldMatchInt` 200 + void $ createFedConn OtherDomain (FedConn remoteDomain "full_search") -- Setup a remote user we can get notifications for. user <- randomUser OtherDomain def withWebSocket user $ \ws -> do -- Defederate from a domain that doesn't exist. This won't do anything to the databases - -- But it will send out notifications that we can wait on. + -- and it won't send any bogus notifications. -- Begin the whole process at Brig, the same as an operator would. void $ deleteFedConn OwnDomain remoteDomain - void $ awaitNMatches 2 3 (\n -> nPayload n %. "type" `isEqual` "federation.connectionRemoved") ws + void $ awaitNMatches 0 3 (\_ -> pure True) ws testDefederationNonFullyConnectedGraph :: HasCallStack => App () testDefederationNonFullyConnectedGraph = do let setFederationConfig = setField "optSettings.setFederationStrategy" "allowDynamic" - >=> removeField "optSettings.setFederationDomainConfigs" >=> setField "optSettings.setFederationDomainConfigsUpdateFreq" (Aeson.Number 1) startDynamicBackends [ def {brigCfg = setFederationConfig}, diff --git a/integration/test/Test/Demo.hs b/integration/test/Test/Demo.hs index 69d3cf1b0c4..a61c8d1950f 100644 --- a/integration/test/Test/Demo.hs +++ b/integration/test/Test/Demo.hs @@ -1,3 +1,5 @@ +{-# OPTIONS_GHC -Wno-incomplete-uni-patterns #-} + -- | This module is meant to show how Testlib can be used module Test.Demo where @@ -173,8 +175,7 @@ testIndependentESIndices = do testDynamicBackendsFederation :: HasCallStack => App () testDynamicBackendsFederation = do - startDynamicBackends [def <> fullSearchWithAll, def <> fullSearchWithAll] $ \dynDomains -> do - [aDynDomain, anotherDynDomain] <- pure dynDomains + startDynamicBackends [def, def] $ \[aDynDomain, anotherDynDomain] -> do u1 <- randomUser aDynDomain def u2 <- randomUser anotherDynDomain def uid2 <- objId u2 diff --git a/integration/test/Test/Federation.hs b/integration/test/Test/Federation.hs index a16b3e9077f..f81e9ec8b86 100644 --- a/integration/test/Test/Federation.hs +++ b/integration/test/Test/Federation.hs @@ -93,14 +93,14 @@ testNotificationsForOfflineBackends = do isDelUserLeaveUpConvNotif = allPreds [isConvLeaveNotif, isNotifConv upBackendConv, isNotifForUser delUser] do - newMsgNotif <- awaitNotification otherUser otherClient noValue 1 isNewMessageNotif + newMsgNotif <- awaitNotification otherUser otherClient noValue 2 isNewMessageNotif newMsgNotif %. "payload.0.qualified_conversation" `shouldMatch` objQidObject upBackendConv newMsgNotif %. "payload.0.data.text" `shouldMatchBase64` "success message for other user" - void $ awaitNotification otherUser otherClient (Just newMsgNotif) 1 isOtherUser2LeaveUpConvNotif - void $ awaitNotification otherUser otherClient (Just newMsgNotif) 1 isDelUserLeaveUpConvNotif + void $ awaitNotification otherUser otherClient (Just newMsgNotif) 10 isOtherUser2LeaveUpConvNotif -- TODO: this is still not enough. something else wrong, then? but why only sometimes? + void $ awaitNotification otherUser otherClient (Just newMsgNotif) 10 isDelUserLeaveUpConvNotif - delUserDeletedNotif <- nPayload $ awaitNotification otherUser otherClient (Just newMsgNotif) 1 isDeleteUserNotif + delUserDeletedNotif <- nPayload $ awaitNotification otherUser otherClient (Just newMsgNotif) 2 isDeleteUserNotif objQid delUserDeletedNotif `shouldMatch` objQid delUser runCodensity (startDynamicBackend downBackend mempty) $ \_ -> do diff --git a/integration/test/Testlib/ModService.hs b/integration/test/Testlib/ModService.hs index b5e0b526004..f8f523e708d 100644 --- a/integration/test/Testlib/ModService.hs +++ b/integration/test/Testlib/ModService.hs @@ -172,7 +172,6 @@ startDynamicBackend resource beOverrides = do def { brigCfg = setField "optSettings.setFederationDomain" resource.berDomain - >=> setField "optSettings.setFederationDomainConfigs" ([] :: [Value]) >=> setField "federatorInternal.port" resource.berFederatorInternal >=> setField "federatorInternal.host" ("127.0.0.1" :: String) >=> setField "rabbitmq.vHost" resource.berVHost, diff --git a/integration/test/Testlib/RunServices.hs b/integration/test/Testlib/RunServices.hs index 5f2b5a3eb34..48a3c6d2103 100644 --- a/integration/test/Testlib/RunServices.hs +++ b/integration/test/Testlib/RunServices.hs @@ -4,7 +4,6 @@ module Testlib.RunServices where import Control.Concurrent import Control.Monad.Codensity (lowerCodensity) -import SetupHelpers import System.Directory import System.Environment (getArgs) import System.Exit (exitWith) @@ -63,9 +62,7 @@ main = do _modifyEnv <- traverseConcurrentlyCodensity ( \resource -> - -- We add the 'fullSerachWithAll' overrrides is a hack to get - -- around https://wearezeta.atlassian.net/browse/WPB-3796 - startDynamicBackend resource fullSearchWithAll + startDynamicBackend resource def ) [backendA, backendB] liftIO run diff --git a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs b/libs/wire-api-federation/src/Wire/API/BackgroundWorker.hs similarity index 90% rename from libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs rename to libs/wire-api-federation/src/Wire/API/BackgroundWorker.hs index 3560e2c5e44..93219b373b4 100644 --- a/libs/wire-api-federation/src/Wire/API/Federation/BackendNotifications.hs +++ b/libs/wire-api-federation/src/Wire/API/BackgroundWorker.hs @@ -1,7 +1,7 @@ {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE RecordWildCards #-} -module Wire.API.Federation.BackendNotifications where +module Wire.API.BackgroundWorker where import Control.Exception import Control.Monad.Except @@ -81,10 +81,10 @@ enqueue :: Q.Channel -> Domain -> Domain -> Q.DeliveryMode -> FedQueueClient c ( enqueue channel originDomain targetDomain deliveryMode (FedQueueClient action) = runReaderT action FedQueueEnv {..} -routingKey :: Text -> Text -routingKey t = "backend-notifications." <> t +backendNotificationQueueName :: Domain -> Text +backendNotificationQueueName (Domain t) = "backend-notifications." <> t --- Shared values for both brig and background worker so they are +-- | Shared values for both brig and background worker so they are -- kept in sync about what types they are expecting and where -- they are stored in Rabbit. type DefederationDomain = Domain @@ -92,6 +92,11 @@ type DefederationDomain = Domain defederationQueue :: Text defederationQueue = "delete-federation" +-- | Create a queue under routingKey (`backend-notifications`). If you want to do something +-- else use `ensureQueue`. +ensureBackendNotificationsQueue :: Q.Channel -> Domain -> IO () +ensureBackendNotificationsQueue chan = ensureQueue chan . backendNotificationQueueName + -- | If you ever change this function and modify -- queue parameters, know that it will start failing in the -- next release! So be prepared to write migrations. @@ -99,7 +104,7 @@ ensureQueue :: Q.Channel -> Text -> IO () ensureQueue chan queue = do let opts = Q.QueueOpts - { Q.queueName = routingKey queue, + { Q.queueName = queue, Q.queuePassive = False, Q.queueDurable = True, Q.queueExclusive = False, @@ -169,8 +174,8 @@ instance (KnownComponent c) => RunClient (FedQueueClient c) where -- Empty string means default exchange exchange = "" liftIO $ do - ensureQueue env.channel env.targetDomain._domainText - void $ Q.publishMsg env.channel exchange (routingKey env.targetDomain._domainText) msg + ensureBackendNotificationsQueue env.channel env.targetDomain + void $ Q.publishMsg env.channel exchange (backendNotificationQueueName env.targetDomain) msg pure $ Response { responseHttpVersion = http20, diff --git a/libs/wire-api-federation/src/Wire/API/Federation/API.hs b/libs/wire-api-federation/src/Wire/API/Federation/API.hs index f344a80ced2..c08aa45d0ed 100644 --- a/libs/wire-api-federation/src/Wire/API/Federation/API.hs +++ b/libs/wire-api-federation/src/Wire/API/Federation/API.hs @@ -39,10 +39,10 @@ import GHC.TypeLits import Imports import Servant.Client import Servant.Client.Core +import Wire.API.BackgroundWorker import Wire.API.Federation.API.Brig import Wire.API.Federation.API.Cargohold import Wire.API.Federation.API.Galley -import Wire.API.Federation.BackendNotifications import Wire.API.Federation.Client import Wire.API.MakesFederatedCall import Wire.API.Routes.Named diff --git a/libs/wire-api-federation/wire-api-federation.cabal b/libs/wire-api-federation/wire-api-federation.cabal index 53862a2f92f..41a29e9d2c5 100644 --- a/libs/wire-api-federation/wire-api-federation.cabal +++ b/libs/wire-api-federation/wire-api-federation.cabal @@ -16,12 +16,12 @@ build-type: Simple library -- cabal-fmt: expand src exposed-modules: + Wire.API.BackgroundWorker Wire.API.Federation.API Wire.API.Federation.API.Brig Wire.API.Federation.API.Cargohold Wire.API.Federation.API.Common Wire.API.Federation.API.Galley - Wire.API.Federation.BackendNotifications Wire.API.Federation.Client Wire.API.Federation.Component Wire.API.Federation.Domain diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal index 2dc7c897f45..6c1d712efae 100644 --- a/services/background-worker/background-worker.cabal +++ b/services/background-worker/background-worker.cabal @@ -31,7 +31,6 @@ library aeson , amqp , async - , base , bilge , bytestring-conversion , containers @@ -56,7 +55,6 @@ library , types-common , unliftio , wai-utilities - , wire-api , wire-api-federation default-extensions: diff --git a/services/background-worker/background-worker.integration.yaml b/services/background-worker/background-worker.integration.yaml index 9762cc70825..5fbdac7dfe1 100644 --- a/services/background-worker/background-worker.integration.yaml +++ b/services/background-worker/background-worker.integration.yaml @@ -25,3 +25,4 @@ rabbitmq: backendNotificationPusher: pushBackoffMinWait: 1000 pushBackoffMaxWait: 1000000 + remotesRefreshInterval: 500000 diff --git a/services/background-worker/default.nix b/services/background-worker/default.nix index de2217e45b6..945e4101bb2 100644 --- a/services/background-worker/default.nix +++ b/services/background-worker/default.nix @@ -56,7 +56,6 @@ mkDerivation { aeson amqp async - base bilge bytestring-conversion containers @@ -81,7 +80,6 @@ mkDerivation { types-common unliftio wai-utilities - wire-api wire-api-federation ]; executableHaskellDepends = [ HsOpenSSL imports types-common ]; diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs index f52f165dbbd..8b0ddc3a946 100644 --- a/services/background-worker/src/Wire/BackendNotificationPusher.hs +++ b/services/background-worker/src/Wire/BackendNotificationPusher.hs @@ -19,9 +19,8 @@ import Network.RabbitMqAdmin import Prometheus import System.Logger.Class qualified as Log import UnliftIO -import Wire.API.Federation.BackendNotifications +import Wire.API.BackgroundWorker import Wire.API.Federation.Client -import Wire.API.Routes.FederationDomainConfig import Wire.BackgroundWorker.Env import Wire.BackgroundWorker.Options import Wire.BackgroundWorker.Util @@ -32,8 +31,8 @@ startPushingNotifications :: Domain -> AppT IO Q.ConsumerTag startPushingNotifications runningFlag chan domain = do - lift $ ensureQueue chan domain._domainText - QL.consumeMsgs chan (routingKey domain._domainText) Q.Ack (void . pushNotification runningFlag domain) + lift $ ensureBackendNotificationsQueue chan domain + QL.consumeMsgs chan (backendNotificationQueueName domain) Q.Ack (void . pushNotification runningFlag domain) pushNotification :: RabbitMQEnvelope e => MVar () -> Domain -> (Q.Message, e) -> AppT IO (Async ()) pushNotification runningFlag targetDomain (msg, envelope) = do @@ -123,7 +122,7 @@ startPusher consumersRef chan = do throwM e -- If this thread is cancelled, catch the exception, kill the consumers, and carry on. - -- FUTUREWORK?: + -- -- If this throws an exception on the Chan / in the forever loop, the exception will -- bubble all the way up and kill the pod. Kubernetes should restart the pod automatically. flip @@ -132,29 +131,15 @@ startPusher consumersRef chan = do Handler $ cleanup @SomeAsyncException ] $ do - -- Get an initial set of domains from the sync thread - -- The Chan that we will be waiting on isn't initialised with a - -- value until the domain update loop runs the callback for the - -- first time. - initRemotes <- liftIO $ readIORef env.remoteDomains - -- Get an initial set of consumers for the domains pulled from the IORef - -- so that we aren't just sitting around not doing anything for a bit at - -- the start. - ensureConsumers consumersRef chan $ domain <$> initRemotes.remotes - -- Wait for updates to the domains, this is where the bulk of the action - -- is going to take place + consumers <- newIORef mempty forever $ do - -- Wait for a new set of domains. This is a blocking action - -- so we will only move past here when we get a new set of domains. - -- It is a bit nicer than having another timeout value, as Brig is - -- already providing one in the domain update message. - chanRemotes <- liftIO $ readChan env.remoteDomainsChan - -- Make new consumers for the new domains, clean up old ones from the consumer map. - ensureConsumers consumersRef chan $ domain <$> chanRemotes.remotes + remoteDomains <- getRemoteDomains + ensureConsumers consumers chan remoteDomains + threadDelay env.backendNotificationsConfig.remotesRefreshInterval ensureConsumers :: IORef (Map Domain (Q.ConsumerTag, MVar ())) -> Q.Channel -> [Domain] -> AppT IO () ensureConsumers consumers chan domains = do - keys' <- Set.fromList . Map.keys <$> readIORef consumers + keys' <- Map.keysSet <$> readIORef consumers let domains' = Set.fromList domains droppedDomains = Set.difference keys' domains' -- Loop over all of the new domains. We can check for existing consumers and add new ones. diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs index 117b135aa6b..368365a56f0 100644 --- a/services/background-worker/src/Wire/BackgroundWorker.hs +++ b/services/background-worker/src/Wire/BackgroundWorker.hs @@ -2,7 +2,6 @@ module Wire.BackgroundWorker where -import Control.Concurrent.Async (cancel) import Data.Domain import Data.Map.Strict qualified as Map import Data.Metrics.Servant qualified as Metrics @@ -22,14 +21,13 @@ import Wire.Defederation as Defederation -- FUTUREWORK: Start an http service with status and metrics endpoints run :: Opts -> IO () run opts = do - (env, syncThread) <- mkEnv opts + env <- mkEnv opts (defedChanRef, defedConsumerRef) <- runAppT env $ Defederation.startWorker opts.rabbitmq (notifChanRef, notifConsumersRef) <- runAppT env $ BackendNotificationPusher.startWorker opts.rabbitmq let -- cleanup will run in a new thread when the signal is caught, so we need to use IORefs and -- specific exception types to message threads to clean up l = logger env cleanup = do - cancel syncThread -- Cancel the consumers and wait for them to finish their processing step. -- Defederation thread Log.info (logger env) $ Log.msg (Log.val "Cancelling the defederation thread") diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs index 86a5b99ed57..2c9205c3762 100644 --- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs +++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs @@ -3,8 +3,6 @@ module Wire.BackgroundWorker.Env where -import Control.Concurrent.Async -import Control.Concurrent.Chan import Control.Monad.Base import Control.Monad.Catch import Control.Monad.Trans.Control @@ -24,8 +22,6 @@ import System.Logger qualified as Log import System.Logger.Class (Logger, MonadLogger (..)) import System.Logger.Extended qualified as Log import Util.Options -import Wire.API.FederationUpdate -import Wire.API.Routes.FederationDomainConfig import Wire.BackgroundWorker.Options type IsWorking = Bool @@ -45,10 +41,7 @@ data Env = Env federatorInternal :: Endpoint, httpManager :: Manager, galley :: Endpoint, - brig :: Endpoint, defederationTimeout :: ResponseTimeout, - remoteDomains :: IORef FederationDomainConfigs, - remoteDomainsChan :: Chan FederationDomainConfigs, backendNotificationMetrics :: BackendNotificationMetrics, -- This is needed so that the defederation worker can push -- connection-removed notifications into the notifications channels. @@ -71,12 +64,11 @@ mkBackendNotificationMetrics = <*> register (vector "targetDomain" $ counter $ Prometheus.Info "wire_backend_notifications_errors" "Number of errors that occurred while pushing notifications") <*> register (vector "targetDomain" $ gauge $ Prometheus.Info "wire_backend_notifications_stuck_queues" "Set to 1 when pushing notifications is stuck") -mkEnv :: Opts -> IO (Env, Async ()) +mkEnv :: Opts -> IO Env mkEnv opts = do http2Manager <- initHttp2Manager logger <- Log.mkLogger opts.logLevel Nothing opts.logFormat httpManager <- newManager defaultManagerSettings - remoteDomainsChan <- newChan let federatorInternal = opts.federatorInternal galley = opts.galley defederationTimeout = @@ -84,14 +76,7 @@ mkEnv opts = do responseTimeoutNone (\t -> responseTimeoutMicro $ 1000000 * t) -- seconds to microseconds opts.defederationTimeout - brig = opts.brig rabbitmqVHost = opts.rabbitmq.vHost - callback = - SyncFedDomainConfigsCallback - { fromFedUpdateCallback = \_old new -> do - writeChan remoteDomainsChan new - } - (remoteDomains, syncThread) <- syncFedDomainConfigs brig logger callback rabbitmqAdminClient <- mkRabbitMqAdminClientEnv opts.rabbitmq statuses <- newIORef $ @@ -103,7 +88,7 @@ mkEnv opts = do backendNotificationMetrics <- mkBackendNotificationMetrics notificationChannel <- mkRabbitMqChannelMVar logger $ demoteOpts opts.rabbitmq let backendNotificationsConfig = opts.backendNotificationPusher - pure (Env {..}, syncThread) + pure Env {..} initHttp2Manager :: IO Http2Manager initHttp2Manager = do diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs index 7cac93318db..e35c89f1e70 100644 --- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs +++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs @@ -13,7 +13,6 @@ data Opts = Opts federatorInternal :: !Endpoint, rabbitmq :: !RabbitMqAdminOpts, galley :: !Endpoint, - brig :: !Endpoint, -- | Seconds, Nothing for no timeout defederationTimeout :: Maybe Int, backendNotificationPusher :: BackendNotificationsConfig @@ -31,7 +30,11 @@ data BackendNotificationsConfig = BackendNotificationsConfig -- | Upper limit on amount of time (in microseconds) to wait before retrying -- any notification. This exists to ensure that exponential back-off doesn't -- cause wait times to be very big. - pushBackoffMaxWait :: Int + pushBackoffMaxWait :: Int, + -- | Number of microseconds between two calls to `getRemoteDomains`. (good + -- defaults are 3 seconds, or 30 seconds, but for testing we sometimes set + -- it below 1 second.) + remotesRefreshInterval :: Int } deriving (Show, Generic) diff --git a/services/background-worker/src/Wire/Defederation.hs b/services/background-worker/src/Wire/Defederation.hs index e8da0e9366b..67d141be841 100644 --- a/services/background-worker/src/Wire/Defederation.hs +++ b/services/background-worker/src/Wire/Defederation.hs @@ -7,8 +7,6 @@ import Control.Monad.Catch import Control.Retry import Data.Aeson qualified as A import Data.ByteString.Conversion -import Data.Domain -import Data.Text (unpack) import Data.Text.Encoding import Imports import Network.AMQP qualified as Q @@ -16,19 +14,16 @@ import Network.AMQP.Extended import Network.AMQP.Lifted qualified as QL import Network.HTTP.Client import Network.HTTP.Types -import Servant.Client (BaseUrl (..), ClientEnv, Scheme (Http), mkClientEnv) import System.Logger.Class qualified as Log -import Util.Options import Util.Options qualified as O -import Wire.API.Federation.BackendNotifications -import Wire.API.Routes.FederationDomainConfig qualified as Fed +import Wire.API.BackgroundWorker import Wire.BackgroundWorker.Env import Wire.BackgroundWorker.Util deleteFederationDomain :: MVar () -> Q.Channel -> AppT IO Q.ConsumerTag deleteFederationDomain runningFlag chan = do lift $ ensureQueue chan defederationQueue - QL.consumeMsgs chan (routingKey defederationQueue) Q.Ack $ deleteFederationDomainInner runningFlag + QL.consumeMsgs chan defederationQueue Q.Ack $ deleteFederationDomainInner runningFlag x3 :: RetryPolicy x3 = limitRetries 3 <> exponentialBackoff 100000 @@ -52,18 +47,6 @@ deleteFederationDomainInner' go (msg, envelope) = do Log.msg (Log.val "Failed to delete federation domain") . Log.field "error" err -mkBrigEnv :: AppT IO ClientEnv -mkBrigEnv = do - Endpoint brigHost brigPort <- asks brig - mkClientEnv - <$> asks httpManager - <*> pure (BaseUrl Http (unpack brigHost) (fromIntegral brigPort) "") - -getRemoteDomains :: AppT IO [Domain] -getRemoteDomains = do - ref <- asks remoteDomains - fmap Fed.domain . Fed.remotes <$> readIORef ref - callGalleyDelete :: ( MonadReader Env m, MonadMask m, diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs index f38680c1d43..82cc5ee47ad 100644 --- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs +++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs @@ -4,7 +4,6 @@ module Test.Wire.BackendNotificationPusherSpec where -import Control.Concurrent.Chan import Control.Exception import Control.Monad.Trans.Except import Data.Aeson qualified as Aeson @@ -37,12 +36,11 @@ import Test.QuickCheck import Test.Wire.Util import UnliftIO.Async import Util.Options +import Wire.API.BackgroundWorker import Wire.API.Federation.API import Wire.API.Federation.API.Brig import Wire.API.Federation.API.Common -import Wire.API.Federation.BackendNotifications import Wire.API.RawJson -import Wire.API.Routes.FederationDomainConfig import Wire.BackendNotificationPusher import Wire.BackgroundWorker.Env import Wire.BackgroundWorker.Options @@ -220,8 +218,6 @@ spec = do ] logger <- Logger.new Logger.defSettings httpManager <- newManager defaultManagerSettings - remoteDomains <- newIORef defFederationDomainConfigs - remoteDomainsChan <- newChan notificationChannel <- newEmptyMVar let federatorInternal = Endpoint "localhost" 8097 http2Manager = undefined @@ -231,8 +227,7 @@ spec = do rabbitmqVHost = "test-vhost" defederationTimeout = responseTimeoutNone galley = Endpoint "localhost" 8085 - brig = Endpoint "localhost" 8082 - backendNotificationsConfig = BackendNotificationsConfig 1000 500000 + backendNotificationsConfig = BackendNotificationsConfig 1000 500000 500 backendNotificationMetrics <- mkBackendNotificationMetrics domains <- runAppT Env {..} getRemoteDomains @@ -243,8 +238,6 @@ spec = do mockAdmin <- newMockRabbitMqAdmin True ["backend-notifications.foo.example"] logger <- Logger.new Logger.defSettings httpManager <- newManager defaultManagerSettings - remoteDomains <- newIORef defFederationDomainConfigs - remoteDomainsChan <- newChan notificationChannel <- newEmptyMVar let federatorInternal = Endpoint "localhost" 8097 http2Manager = undefined @@ -254,8 +247,7 @@ spec = do rabbitmqVHost = "test-vhost" defederationTimeout = responseTimeoutNone galley = Endpoint "localhost" 8085 - brig = Endpoint "localhost" 8082 - backendNotificationsConfig = BackendNotificationsConfig 1000 500000 + backendNotificationsConfig = BackendNotificationsConfig 1000 500000 500 backendNotificationMetrics <- mkBackendNotificationMetrics domainsThread <- async $ runAppT Env {..} getRemoteDomains diff --git a/services/background-worker/test/Test/Wire/DefederationSpec.hs b/services/background-worker/test/Test/Wire/DefederationSpec.hs index 8707414d442..343933d4eb5 100644 --- a/services/background-worker/test/Test/Wire/DefederationSpec.hs +++ b/services/background-worker/test/Test/Wire/DefederationSpec.hs @@ -8,8 +8,8 @@ import Network.AMQP qualified as Q import Test.HUnit.Lang import Test.Hspec import Test.Wire.Util +import Wire.API.BackgroundWorker import Wire.API.Federation.API.Common -import Wire.API.Federation.BackendNotifications import Wire.BackgroundWorker.Util import Wire.Defederation diff --git a/services/background-worker/test/Test/Wire/Util.hs b/services/background-worker/test/Test/Wire/Util.hs index d80121fdc69..eac2f85c6ea 100644 --- a/services/background-worker/test/Test/Wire/Util.hs +++ b/services/background-worker/test/Test/Wire/Util.hs @@ -2,12 +2,10 @@ module Test.Wire.Util where -import Control.Concurrent.Chan import Imports import Network.HTTP.Client import System.Logger.Class qualified as Logger import Util.Options (Endpoint (..)) -import Wire.API.Routes.FederationDomainConfig import Wire.BackgroundWorker.Env hiding (federatorInternal, galley) import Wire.BackgroundWorker.Env qualified as E import Wire.BackgroundWorker.Options @@ -20,17 +18,14 @@ testEnv = do statuses <- newIORef mempty backendNotificationMetrics <- mkBackendNotificationMetrics httpManager <- newManager defaultManagerSettings - remoteDomains <- newIORef defFederationDomainConfigs - remoteDomainsChan <- newChan notificationChannel <- newEmptyMVar let federatorInternal = Endpoint "localhost" 0 rabbitmqAdminClient = undefined rabbitmqVHost = undefined metrics = undefined galley = Endpoint "localhost" 8085 - brig = Endpoint "localhost" 8082 defederationTimeout = responseTimeoutNone - backendNotificationsConfig = BackendNotificationsConfig 1000 500000 + backendNotificationsConfig = BackendNotificationsConfig 1000 500000 500 pure Env {..} runTestAppT :: AppT IO a -> Int -> IO a diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml index 6114e56fa76..448d24b69aa 100644 --- a/services/brig/brig.integration.yaml +++ b/services/brig/brig.integration.yaml @@ -191,9 +191,6 @@ optSettings: setFeatureFlags: # see #RefConfigOptions in `/docs/reference` setFederationDomainConfigsUpdateFreq: 1 setFederationStrategy: allowAll - setFederationDomainConfigs: - - domain: example.com - search_policy: full_search set2FACodeGenerationDelaySecs: 5 setNonceTtlSecs: 5 setDpopMaxSkewSecs: 1 diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs index cd2393132ee..818c49bf5b8 100644 --- a/services/brig/src/Brig/API/Internal.hs +++ b/services/brig/src/Brig/API/Internal.hs @@ -83,11 +83,11 @@ import System.Logger qualified as Lg import System.Logger.Class qualified as Log import System.Random (randomRIO) import UnliftIO.Async +import Wire.API.BackgroundWorker import Wire.API.Connection import Wire.API.Error import Wire.API.Error.Brig qualified as E import Wire.API.Federation.API -import Wire.API.Federation.BackendNotifications import Wire.API.Federation.Error (FederationError (..)) import Wire.API.MLS.Credential import Wire.API.MLS.KeyPackage @@ -230,7 +230,6 @@ federationRemotesAPI = addFederationRemote :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) () addFederationRemote fedDomConf = do - assertNoDivergingDomainInConfigFiles fedDomConf result <- lift . wrapClient $ Data.addFederationRemote fedDomConf case result of Data.AddFederationRemoteSuccess -> pure () @@ -239,57 +238,13 @@ addFederationRemote fedDomConf = do "Maximum number of remote backends reached. If you need to create more connections, \ \please contact wire.com." --- | Compile config file list into a map indexed by domains. Use this to make sure the config --- file is consistent (ie., no two entries for the same domain). -remotesMapFromCfgFile :: AppT r (Map Domain FederationDomainConfig) -remotesMapFromCfgFile = do - cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings) - let dict = [(domain cnf, cnf) | cnf <- cfg] - merge c c' = - if c == c' - then c - else error $ "error in config file: conflicting parameters on domain: " <> show (c, c') - pure $ Map.fromListWith merge dict - --- | Return the config file list. Use this to make sure the config file is consistent (ie., --- no two entries for the same domain). Based on `remotesMapFromCfgFile`. -remotesListFromCfgFile :: AppT r [FederationDomainConfig] -remotesListFromCfgFile = Map.elems <$> remotesMapFromCfgFile - --- | If remote domain is registered in config file, the version that can be added to the --- database must be the same. -assertNoDivergingDomainInConfigFiles :: FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) () -assertNoDivergingDomainInConfigFiles fedComConf = do - cfg <- lift remotesMapFromCfgFile - let diverges = case Map.lookup (domain fedComConf) cfg of - Nothing -> False - Just fedComConf' -> fedComConf' /= fedComConf - when diverges $ do - throwError . fedError . FederationUnexpectedError $ - "keeping track of remote domains in the brig config file is deprecated, but as long as we \ - \do that, adding a domain with different settings than in the config file is nto allowed. want " - <> ( "Just " - <> cs (show fedComConf) - <> "or Nothing, " - ) - <> ( "got " - <> cs (show (Map.lookup (domain fedComConf) cfg)) - ) - getFederationRemotes :: ExceptT Brig.API.Error.Error (AppT r) FederationDomainConfigs getFederationRemotes = lift $ do - -- FUTUREWORK: we should solely rely on `db` in the future for remote domains; merging - -- remote domains from `cfg` is just for providing an easier, more robust migration path. - -- See - -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections, - -- http://docs.wire.com/developer/developer/federation-design-aspects.html#configuring-remote-connections-dev-perspective db <- wrapClient Data.getFederationRemotes - (ms :: Maybe FederationStrategy, mf :: [FederationDomainConfig], mu :: Maybe Int) <- do + (ms :: Maybe FederationStrategy, mu :: Maybe Int) <- do cfg <- ask - domcfgs <- remotesListFromCfgFile -- (it's not very elegant to prove the env twice here, but this code is transitory.) pure ( setFederationStrategy (cfg ^. settings), - domcfgs, setFederationDomainConfigsUpdateFreq (cfg ^. settings) ) @@ -303,14 +258,13 @@ getFederationRemotes = lift $ do defFederationDomainConfigs & maybe id (\v cfg -> cfg {strategy = v}) ms - & (\cfg -> cfg {remotes = nub $ db <> mf}) + & (\cfg -> cfg {remotes = db}) & maybe id (\v cfg -> cfg {updateInterval = min 1 v}) mu & pure updateFederationRemote :: Domain -> FederationDomainConfig -> ExceptT Brig.API.Error.Error (AppT r) () updateFederationRemote dom fedcfg = do assertDomainIsNotUpdated dom fedcfg - assertNoDomainsFromConfigFiles dom (lift . wrapClient . Data.updateFederationRemote $ fedcfg) >>= \case True -> pure () False -> @@ -323,15 +277,6 @@ assertDomainIsNotUpdated dom fedcfg = do throwError . fedError . FederationUnexpectedError . cs $ "federation domain of a given peer cannot be changed from " <> show (domain fedcfg) <> " to " <> show dom <> "." --- | FUTUREWORK: should go away in the future; see 'getFederationRemotes'. -assertNoDomainsFromConfigFiles :: Domain -> ExceptT Brig.API.Error.Error (AppT r) () -assertNoDomainsFromConfigFiles dom = do - cfg <- asks (fromMaybe [] . setFederationDomainConfigs . view settings) - when (dom `elem` (domain <$> cfg)) $ do - throwError . fedError . FederationUnexpectedError $ - "keeping track of remote domains in the brig config file is deprecated, but as long as we \ - \do that, removing or updating items listed in the config file is not allowed." - -- | Remove the entry from the database if present (or do nothing if not). This responds with -- 533 if the entry was also present in the config file, but only *after* it has removed the -- entry from cassandra. @@ -341,15 +286,13 @@ assertNoDomainsFromConfigFiles dom = do deleteFederationRemote :: Domain -> ExceptT Brig.API.Error.Error (AppT r) () deleteFederationRemote dom = do lift . wrapClient . Data.deleteFederationRemote $ dom - assertNoDomainsFromConfigFiles dom env <- ask ownDomain <- viewFederationDomain remoteDomains <- fmap domain . remotes <$> getFederationRemotes for_ (env ^. rabbitmqChannel) $ \chan -> liftIO . withMVar chan $ \chan' -> do - -- ensureQueue uses routingKey internally ensureQueue chan' defederationQueue void $ - Q.publishMsg chan' "" queue $ + Q.publishMsg chan' "" defederationQueue $ Q.newMsg { -- Check that this message type is compatible with what -- background worker is expecting @@ -362,7 +305,7 @@ deleteFederationRemote dom = do -- clean up their conversations and notify clients. -- Just to be safe! for_ (filter (/= dom) remoteDomains) $ \remoteDomain -> do - ensureQueue chan' $ domainText remoteDomain + ensureBackendNotificationsQueue chan' $ remoteDomain liftIO $ enqueue chan' ownDomain remoteDomain Q.Persistent . void @@ -371,11 +314,8 @@ deleteFederationRemote dom = do -- This will also drop all of the messages in the queue -- as we will no longer be able to communicate with this -- domain. - num <- Q.deleteQueue chan' . routingKey $ domainText dom + num <- Q.deleteQueue chan' $ backendNotificationQueueName dom Lg.info (env ^. applog) $ Log.msg @String "Dropped Notifications" . Log.field "domain" (domainText dom) . Log.field "count" (show num) - where - -- Ensure that this is kept in sync with background worker - queue = routingKey defederationQueue -- | Remove one-on-one conversations for the given remote domain. This is called from Galley as -- part of the defederation process, and should not be called during the initial domain removal diff --git a/services/brig/src/Brig/Federation/Client.hs b/services/brig/src/Brig/Federation/Client.hs index e824b4f53e9..693407ae8bc 100644 --- a/services/brig/src/Brig/Federation/Client.hs +++ b/services/brig/src/Brig/Federation/Client.hs @@ -39,9 +39,9 @@ import Data.Time.Units import Imports import Network.AMQP qualified as Q import System.Logger.Class qualified as Log +import Wire.API.BackgroundWorker import Wire.API.Federation.API import Wire.API.Federation.API.Brig as FederatedBrig -import Wire.API.Federation.BackendNotifications import Wire.API.Federation.Client import Wire.API.Federation.Error import Wire.API.User diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs index 6ca9938851b..d86e2298916 100644 --- a/services/brig/src/Brig/Options.hs +++ b/services/brig/src/Brig/Options.hs @@ -552,13 +552,6 @@ data Settings = Settings -- | See https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections -- default: AllowNone setFederationStrategy :: !(Maybe FederationStrategy), - -- | 'setFederationDomainConfigs' is introduced in - -- https://github.com/wireapp/wire-server/pull/3260 for the sole purpose of transitioning - -- to dynamic federation remote configuration. See - -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections - -- for details. - -- default: [] - setFederationDomainConfigs :: !(Maybe [FederationDomainConfig]), -- | In seconds. Default: 10 seconds. Values <1 are silently replaced by 1. See -- https://docs.wire.com/understand/federation/backend-communication.html#configuring-remote-connections setFederationDomainConfigsUpdateFreq :: !(Maybe Int), @@ -913,7 +906,6 @@ Lens.makeLensesFor ("setSqsThrottleMillis", "sqsThrottleMillis"), ("setSftStaticUrl", "sftStaticUrl"), ("setSftListAllServers", "sftListAllServers"), - ("setFederationDomainConfigs", "federationDomainConfigs"), ("setEnableDevelopmentVersions", "enableDevelopmentVersions"), ("setRestrictUserCreation", "restrictUserCreation"), ("setEnableMLS", "enableMLS"), diff --git a/services/brig/src/Brig/User/API/Handle.hs b/services/brig/src/Brig/User/API/Handle.hs index a554b61f0f6..908667f67d4 100644 --- a/services/brig/src/Brig/User/API/Handle.hs +++ b/services/brig/src/Brig/User/API/Handle.hs @@ -53,7 +53,7 @@ getHandleInfo self handle = do lself <- qualifyLocal self foldQualified lself - (getLocalHandleInfo lself . tUnqualified) + (getLocalHandleInfo lself . tUnqualified) -- does it call local or remote? how does local work? getRemoteHandleInfo handle diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs index 0cc5eaf9be1..57719a42722 100644 --- a/services/brig/test/integration/API/Federation.hs +++ b/services/brig/test/integration/API/Federation.hs @@ -24,7 +24,6 @@ import Bilge hiding (head) import Bilge.Assert import Brig.Options qualified as Opt import Control.Arrow (Arrow (first), (&&&)) -import Control.Lens ((?~)) import Data.Aeson import Data.Domain (Domain (Domain)) import Data.Handle (Handle (..)) @@ -49,7 +48,6 @@ import Wire.API.Federation.API.Brig qualified as FedBrig import Wire.API.Federation.API.Brig qualified as S import Wire.API.Federation.Component import Wire.API.Federation.Version -import Wire.API.Routes.FederationDomainConfig as FD import Wire.API.User import Wire.API.User.Client import Wire.API.User.Client.Prekey @@ -65,12 +63,12 @@ tests m opts brig cannon fedBrigClient = [ test m "POST /federation/search-users : Found" (testSearchSuccess opts brig), test m "POST /federation/search-users : Found (fulltext)" (testFulltextSearchSuccess opts brig), test m "POST /federation/search-users : Found (multiple users)" (testFulltextSearchMultipleUsers opts brig), - test m "POST /federation/search-users : NotFound" (testSearchNotFound opts), - test m "POST /federation/search-users : Empty Input - NotFound" (testSearchNotFoundEmpty opts), + test m "POST /federation/search-users : NotFound" (testSearchNotFound opts brig), + test m "POST /federation/search-users : Empty Input - NotFound" (testSearchNotFoundEmpty opts brig), test m "POST /federation/search-users : configured restrictions" (testSearchRestrictions opts brig), test m "POST /federation/get-user-by-handle : configured restrictions" (testGetUserByHandleRestrictions opts brig), test m "POST /federation/get-user-by-handle : Found" (testGetUserByHandleSuccess opts brig), - test m "POST /federation/get-user-by-handle : NotFound" (testGetUserByHandleNotFound opts), + test m "POST /federation/get-user-by-handle : NotFound" (testGetUserByHandleNotFound opts brig), test m "POST /federation/get-users-by-ids : 200 all found" (testGetUsersByIdsSuccess brig fedBrigClient), test m "POST /federation/get-users-by-ids : 200 partially found" (testGetUsersByIdsPartial brig fedBrigClient), test m "POST /federation/get-users-by-ids : 200 none found" (testGetUsersByIdsNoneFound fedBrigClient), @@ -83,10 +81,6 @@ tests m opts brig cannon fedBrigClient = test m "POST /federation/api-version : 200" (testAPIVersion brig fedBrigClient) ] -allowFullSearch :: Domain -> Opt.Opts -> Opt.Opts -allowFullSearch domain opts = - opts & Opt.optionSettings . Opt.federationDomainConfigs ?~ [FD.FederationDomainConfig domain FullSearch] - testSearchSuccess :: Opt.Opts -> Brig -> Http () testSearchSuccess opts brig = do (handle, user) <- createUserWithHandle brig @@ -95,7 +89,8 @@ testSearchSuccess opts brig = do let quid = userQualifiedId user let domain = Domain "example.com" - searchResponse <- withSettingsOverrides (allowFullSearch domain opts) $ do + searchResponse <- withSettingsOverrides opts $ do + allowFullSearch brig domain runWaiTestFedClient domain $ createWaiTestFedClient @"search-users" @'Brig $ SearchRequest (fromHandle handle) @@ -112,7 +107,8 @@ testFulltextSearchSuccess opts brig = do let quid = userQualifiedId user let domain = Domain "example.com" - searchResponse <- withSettingsOverrides (allowFullSearch domain opts) $ do + searchResponse <- withSettingsOverrides opts $ do + allowFullSearch brig domain runWaiTestFedClient domain $ createWaiTestFedClient @"search-users" @'Brig $ SearchRequest (fromName $ userDisplayName user) @@ -139,7 +135,8 @@ testFulltextSearchMultipleUsers opts brig = do let domain = Domain "example.com" - searchResponse <- withSettingsOverrides (allowFullSearch domain opts) $ do + searchResponse <- withSettingsOverrides opts $ do + allowFullSearch brig domain runWaiTestFedClient domain $ createWaiTestFedClient @"search-users" @'Brig $ SearchRequest (fromHandle handle) @@ -148,22 +145,24 @@ testFulltextSearchMultipleUsers opts brig = do let contacts = contactQualifiedId <$> S.contacts searchResponse assertEqual "should find both users" (sort [quid, userQualifiedId identityThief]) (sort contacts) -testSearchNotFound :: Opt.Opts -> Http () -testSearchNotFound opts = do +testSearchNotFound :: Opt.Opts -> Brig -> Http () +testSearchNotFound opts brig = do let domain = Domain "example.com" - searchResponse <- withSettingsOverrides (allowFullSearch domain opts) $ do + searchResponse <- withSettingsOverrides opts $ do + allowFullSearch brig domain runWaiTestFedClient domain $ createWaiTestFedClient @"search-users" @'Brig $ SearchRequest "this-handle-should-not-exist" liftIO $ assertEqual "should return empty array of users" [] (S.contacts searchResponse) -testSearchNotFoundEmpty :: Opt.Opts -> Http () -testSearchNotFoundEmpty opts = do +testSearchNotFoundEmpty :: Opt.Opts -> Brig -> Http () +testSearchNotFoundEmpty opts brig = do let domain = Domain "example.com" - searchResponse <- withSettingsOverrides (allowFullSearch domain opts) $ do + searchResponse <- withSettingsOverrides opts $ do + allowFullSearch brig domain runWaiTestFedClient domain $ createWaiTestFedClient @"search-users" @'Brig $ SearchRequest "this-handle-should-not-exist" @@ -181,14 +180,6 @@ testSearchRestrictions opts brig = do let quid = userQualifiedId user refreshIndex brig - let opts' = - opts - & Opt.optionSettings . Opt.federationDomainConfigs - ?~ [ FD.FederationDomainConfig domainNoSearch NoSearch, - FD.FederationDomainConfig domainExactHandle ExactHandleSearch, - FD.FederationDomainConfig domainFullSearch FullSearch - ] - let expectSearch :: HasCallStack => Domain -> Either Handle Name -> Maybe (Qualified UserId) -> FederatedUserSearchPolicy -> WaiTest.Session () expectSearch domain handleOrName mExpectedUser expectedSearchPolicy = do let squery = either fromHandle fromName handleOrName @@ -205,7 +196,11 @@ testSearchRestrictions opts brig = do assertEqual "Unexpected search result" (maybeToList mExpectedUser) (contactQualifiedId <$> S.contacts searchResponse) assertEqual "Unexpected search result" expectedSearchPolicy (S.searchPolicy searchResponse) - withSettingsOverrides opts' $ do + withSettingsOverrides opts $ do + setSearchPolicy brig domainNoSearch NoSearch + setSearchPolicy brig domainExactHandle ExactHandleSearch + setSearchPolicy brig domainFullSearch FullSearch + expectSearch domainNoSearch (Left handle) Nothing NoSearch expectSearch domainExactHandle (Left handle) (Just quid) ExactHandleSearch expectSearch domainExactHandle (Right (userDisplayName user)) Nothing ExactHandleSearch @@ -225,21 +220,17 @@ testGetUserByHandleRestrictions opts brig = do let quid = userQualifiedId user refreshIndex brig - let opts' = - opts - & Opt.optionSettings . Opt.federationDomainConfigs - ?~ [ FD.FederationDomainConfig domainNoSearch NoSearch, - FD.FederationDomainConfig domainExactHandle ExactHandleSearch, - FD.FederationDomainConfig domainFullSearch FullSearch - ] - let expectSearch domain expectedUser = do maybeUserProfile <- runWaiTestFedClient domain $ createWaiTestFedClient @"get-user-by-handle" @'Brig handle liftIO $ assertEqual "Unexpected search result" expectedUser (profileQualifiedId <$> maybeUserProfile) - withSettingsOverrides opts' $ do + withSettingsOverrides opts $ do + setSearchPolicy brig domainNoSearch NoSearch + setSearchPolicy brig domainExactHandle ExactHandleSearch + setSearchPolicy brig domainFullSearch FullSearch + expectSearch domainNoSearch Nothing expectSearch domainExactHandle (Just quid) expectSearch domainFullSearch (Just quid) @@ -251,7 +242,8 @@ testGetUserByHandleSuccess opts brig = do let quid = userQualifiedId user let domain = Domain "example.com" - maybeProfile <- withSettingsOverrides (allowFullSearch domain opts) $ do + maybeProfile <- withSettingsOverrides opts $ do + allowFullSearch brig domain runWaiTestFedClient domain $ createWaiTestFedClient @"get-user-by-handle" @'Brig $ handle @@ -263,12 +255,13 @@ testGetUserByHandleSuccess opts brig = do assertEqual "should return correct user Id" quid (profileQualifiedId profile) assertEqual "should not have email address" Nothing (profileEmail profile) -testGetUserByHandleNotFound :: Opt.Opts -> Http () -testGetUserByHandleNotFound opts = do +testGetUserByHandleNotFound :: Opt.Opts -> Brig -> Http () +testGetUserByHandleNotFound opts brig = do hdl <- randomHandle let domain = Domain "example.com" - maybeProfile <- withSettingsOverrides (allowFullSearch domain opts) $ do + maybeProfile <- withSettingsOverrides opts $ do + allowFullSearch brig domain runWaiTestFedClient domain $ createWaiTestFedClient @"get-user-by-handle" @'Brig $ Handle hdl diff --git a/services/brig/test/integration/Federation/End2end.hs b/services/brig/test/integration/Federation/End2end.hs index 25d95fd6bae..c19ecadc35a 100644 --- a/services/brig/test/integration/Federation/End2end.hs +++ b/services/brig/test/integration/Federation/End2end.hs @@ -22,7 +22,6 @@ import API.User.Util import Bilge import Bilge.Assert ((!!!), ( + Domain -> Manager -> Brig -> Galley -> @@ -87,12 +87,12 @@ spec :: CargoHold -> Cannon -> IO TestTree -spec _brigOpts mg brig galley cargohold cannon _federator brigTwo galleyTwo cargoholdTwo cannonTwo = +spec originDomain mg brig galley cargohold cannon _federator brigTwo galleyTwo cargoholdTwo cannonTwo = pure $ testGroup "federation-end2end-user" - [ test mg "lookup user by qualified handle on remote backend" $ testHandleLookup brig brigTwo, - test mg "search users on remote backend" $ testSearchUsers brig brigTwo, + [ test mg "lookup user by qualified handle on remote backend" $ testHandleLookup originDomain brig brigTwo, + test mg "search users on remote backend" $ testSearchUsers originDomain brig brigTwo, test mg "get users by ids on multiple backends" $ testGetUsersById brig brigTwo, test mg "claim client prekey" $ testClaimPrekeySuccess brig brigTwo, test mg "claim prekey bundle" $ testClaimPrekeyBundleSuccess brig brigTwo, @@ -117,24 +117,25 @@ spec _brigOpts mg brig galley cargohold cannon _federator brigTwo galleyTwo carg -- | brig | http2 |federator| http2 |federator| http | brig | -- | +-------->+ +------->+ +--------->+ | -- +------+ +-+-------+ +---------+ +------+ -testHandleLookup :: Brig -> Brig -> Http () -testHandleLookup brig brigTwo = do +testHandleLookup :: Domain -> Brig -> Brig -> Http () +testHandleLookup originDomain brig brigTwo = do -- Create a user on the "other side" using an internal brig endpoint from a -- second brig instance in backendTwo (in another namespace in kubernetes) (handle, userBrigTwo) <- createUserWithHandle brigTwo -- Get result from brig two for comparison - let domain = qDomain $ userQualifiedId userBrigTwo - resultViaBrigTwo <- getUserInfoFromHandle brigTwo domain handle + let backendTwoDomain = qDomain $ userQualifiedId userBrigTwo + mbResultViaBrigTwo <- getUserInfoFromHandle' brigTwo backendTwoDomain handle -- query the local-namespace brig for a user sitting on the other backend -- (which will exercise the network traffic via two federators to the remote brig) - resultViaBrigOne <- getUserInfoFromHandle brig domain handle + allowFullSearch brigTwo originDomain + mbResultViaBrigOne <- getUserInfoFromHandle' brig backendTwoDomain handle - liftIO $ assertEqual "remote handle lookup via federator should work in the happy case" (profileQualifiedId resultViaBrigOne) (userQualifiedId userBrigTwo) - liftIO $ assertEqual "querying brig1 or brig2 about the same user should give same result" resultViaBrigTwo resultViaBrigOne + liftIO $ assertEqual "querying brig1 or brig2 about the same user should give same result" mbResultViaBrigOne mbResultViaBrigTwo + liftIO $ assertEqual "remote handle lookup via federator should work in the happy case" (Just $ userQualifiedId userBrigTwo) (profileQualifiedId <$> mbResultViaBrigOne) -testSearchUsers :: Brig -> Brig -> Http () -testSearchUsers brig brigTwo = do +testSearchUsers :: Domain -> Brig -> Brig -> Http () +testSearchUsers originDomain brig brigTwo = do -- Create a user on the "other side" using an internal brig endpoint from a -- second brig instance in backendTwo (in another namespace in kubernetes) (handle, userBrigTwo) <- createUserWithHandle brigTwo @@ -142,13 +143,14 @@ testSearchUsers brig brigTwo = do searcher <- userId <$> randomUser brig let expectedUserId = userQualifiedId userBrigTwo searchTerm = fromHandle handle - domain = qDomain expectedUserId + domainTwo = qDomain expectedUserId liftIO $ putStrLn "search for user on brigTwo (directly)..." - assertCanFindWithDomain brigTwo searcher expectedUserId searchTerm domain + assertCanFindWithDomain brigTwo searcher expectedUserId searchTerm domainTwo -- exercises multi-backend network traffic liftIO $ putStrLn "search for user on brigOne via federators to remote brig..." - assertCanFindWithDomain brig searcher expectedUserId searchTerm domain + setSearchPolicy brigTwo originDomain FullSearch + assertCanFindWithDomain brig searcher expectedUserId searchTerm domainTwo testGetUsersById :: Brig -> Brig -> Http () testGetUsersById brig1 brig2 = do diff --git a/services/brig/test/integration/Main.hs b/services/brig/test/integration/Main.hs index dcdef16cd5a..f5f79cc599e 100644 --- a/services/brig/test/integration/Main.hs +++ b/services/brig/test/integration/Main.hs @@ -46,6 +46,7 @@ import Cassandra.Util (defInitCassandra) import Control.Lens import Data.Aeson import Data.ByteString.Char8 qualified as B8 +import Data.Domain import Data.Metrics.Test (pathsConsistencyCheck) import Data.Metrics.WaiRoute (treeToPaths) import Data.Text.Encoding (encodeUtf8) @@ -106,6 +107,7 @@ data Config = Config -- external provider provider :: Provider.Config, -- for federation + originDomain :: Domain, backendTwo :: BackendConf } deriving (Show, Generic) @@ -155,7 +157,7 @@ runTests iConf brigOpts otherArgs = do createIndex <- Index.Create.spec brigOpts browseTeam <- TeamUserSearch.tests brigOpts mg g b userPendingActivation <- UserPendingActivation.tests brigOpts mg db b g s - federationEnd2End <- Federation.End2end.spec brigOpts mg b g ch c f brigTwo galleyTwo ch2 cannonTwo + federationEnd2End <- Federation.End2end.spec iConf.originDomain mg b g ch c f brigTwo galleyTwo ch2 cannonTwo federationEndpoints <- API.Federation.tests mg brigOpts b c fedBrigClient internalApi <- API.Internal.tests brigOpts mg db b (brig iConf) gd g diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs index 6aafab8a991..550b4907c5b 100644 --- a/services/brig/test/integration/Util.hs +++ b/services/brig/test/integration/Util.hs @@ -108,6 +108,7 @@ import Wire.API.Conversation.Role (roleNameWireAdmin) import Wire.API.Federation.API import Wire.API.Federation.Domain import Wire.API.Internal.Notification +import Wire.API.Routes.FederationDomainConfig qualified as FD import Wire.API.Routes.MultiTablePaging import Wire.API.Team.Member hiding (userId) import Wire.API.User hiding (AccountStatus (..)) @@ -118,6 +119,7 @@ import Wire.API.User.Auth.LegalHold import Wire.API.User.Auth.Sso import Wire.API.User.Client import Wire.API.User.Client.Prekey +import Wire.API.User.Search import Wire.API.VersionInfo type Brig = Request -> Request @@ -647,6 +649,26 @@ getUserInfoFromHandle brig domain handle = do . expect2xx ) +getUserInfoFromHandle' :: + (MonadIO m, MonadCatch m, MonadHttp m, HasCallStack) => + Brig -> + Domain -> + Handle -> + m (Maybe UserProfile) +getUserInfoFromHandle' brig domain handle = do + u <- randomId + resp <- + get + ( apiVersion "v1" + . brig + . paths ["users", "by-handle", toByteString' (domainText domain), toByteString' handle] + . zUser u + ) + case HTTP.statusCode $ responseStatus resp of + 200 -> responseJsonError resp + 404 -> pure Nothing + bad -> error $ "unexpected: " <> show bad + addClient :: (MonadHttp m, HasCallStack) => Brig -> @@ -1338,3 +1360,12 @@ assertElem :: (HasCallStack, Eq a, Show a) => String -> a -> [a] -> Assertion assertElem msg x xs = unless (x `elem` xs) $ assertFailure (msg <> "\nExpected to find: \n" <> show x <> "\nin:\n" <> show xs) + +setSearchPolicy :: (MonadHttp m, MonadCatch m, MonadIO m) => Brig -> Domain -> FederatedUserSearchPolicy -> m () +setSearchPolicy brig domain policy = do + let req = brig . path "/i/federation/remotes" . Bilge.json (FD.FederationDomainConfig domain policy) + post req + !!! const 200 === statusCode + +allowFullSearch :: (MonadHttp m, MonadCatch m, MonadIO m) => Brig -> Domain -> m () +allowFullSearch brig domain = setSearchPolicy brig domain FullSearch diff --git a/services/federator/test/integration/Test/Federator/IngressSpec.hs b/services/federator/test/integration/Test/Federator/IngressSpec.hs index a93a4fa78bf..bb5bb8d852a 100644 --- a/services/federator/test/integration/Test/Federator/IngressSpec.hs +++ b/services/federator/test/integration/Test/Federator/IngressSpec.hs @@ -17,7 +17,7 @@ module Test.Federator.IngressSpec where -import Control.Lens (view) +import Control.Lens (to, view, (^.)) import Control.Monad.Catch (throwM) import Control.Monad.Codensity import Data.Aeson qualified as Aeson @@ -46,6 +46,7 @@ import Wire.API.Federation.Client import Wire.API.Federation.Component import Wire.API.Federation.Domain import Wire.API.User +import Wire.API.User.Search (FederatedUserSearchPolicy (..)) import Wire.Network.DNS.SRV -- | This module contains tests for the interface between federator and ingress. Ingress is @@ -56,6 +57,13 @@ spec env = do it "should be accessible using http2 and forward to the local brig" $ runTestFederator env $ do brig <- view teBrig <$> ask + let domain = + -- FUTUREWORK: we need to come up with a more + -- parallelisable way to test this. as of now, this + -- comes from the + env ^. teTstOpts . to originDomain + setSearchPolicyFor brig (Domain domain) FullSearch + user <- randomUser brig hdl <- randomHandle _ <- putHandle brig (userId user) hdl diff --git a/services/federator/test/integration/Test/Federator/InwardSpec.hs b/services/federator/test/integration/Test/Federator/InwardSpec.hs index ae267dd67e8..17cd5d46fa6 100644 --- a/services/federator/test/integration/Test/Federator/InwardSpec.hs +++ b/services/federator/test/integration/Test/Federator/InwardSpec.hs @@ -22,12 +22,13 @@ module Test.Federator.InwardSpec where import Bilge import Bilge.Assert -import Control.Lens (view) +import Control.Lens (to, view, (^.)) import Data.Aeson import Data.Aeson.Types qualified as Aeson import Data.ByteString qualified as BS import Data.ByteString.Conversion (toByteString') import Data.ByteString.Lazy qualified as LBS +import Data.Domain import Data.Handle import Data.LegalHold (UserLegalHoldStatus (UserLegalHoldNoConsent)) import Data.Text.Encoding @@ -42,6 +43,7 @@ import Util.Options (Endpoint (Endpoint)) import Wire.API.Federation.API.Cargohold import Wire.API.Federation.Domain import Wire.API.User +import Wire.API.User.Search (FederatedUserSearchPolicy (..)) -- FUTUREWORK(federation): move these tests to brig-integration (benefit: avoid duplicating all of the brig helper code) -- FUTUREWORK(fisx): better yet, reorganize integration tests (or at least the helpers) so @@ -70,6 +72,13 @@ spec env = it "should be able to call brig" $ runTestFederator env $ do brig <- view teBrig <$> ask + let domain = + -- FUTUREWORK: we need to come up with a more + -- parallelisable way to test this. as of now, this + -- comes from the + env ^. teTstOpts . to originDomain + setSearchPolicyFor brig (Domain domain) ExactHandleSearch + user <- randomUser brig hdl <- randomHandle _ <- putHandle brig (userId user) hdl @@ -77,7 +86,7 @@ spec env = let expectedProfile = (publicProfile user UserLegalHoldNoConsent) {profileHandle = Just (Handle hdl)} bdy <- responseJsonError - =<< inwardCall "/federation/brig/get-user-by-handle" (encode hdl) + =<< inwardCallWithOriginDomain (cs domain) "/federation/brig/get-user-by-handle" (encode hdl) Request @@ -347,3 +350,11 @@ assertNoError = runError >=> \case Left err -> embed @IO . assertFailure $ "Unexpected error: " <> show err Right x -> pure x + +setSearchPolicyFor :: BrigReq -> Domain -> FederatedUserSearchPolicy -> (Functor m, MonadHttp m) => m () +setSearchPolicyFor brig domain policy = + void $ + post $ + brig + . paths ["/i/federation/remotes"] + . Bilge.json (FederationDomainConfig domain policy) diff --git a/services/galley/src/Galley/Effects/BackendNotificationQueueAccess.hs b/services/galley/src/Galley/Effects/BackendNotificationQueueAccess.hs index c7ecdfcb771..69bd752882e 100644 --- a/services/galley/src/Galley/Effects/BackendNotificationQueueAccess.hs +++ b/services/galley/src/Galley/Effects/BackendNotificationQueueAccess.hs @@ -6,7 +6,7 @@ import Data.Qualified import Imports import Network.AMQP qualified as Q import Polysemy -import Wire.API.Federation.BackendNotifications +import Wire.API.BackgroundWorker import Wire.API.Federation.Component import Wire.API.Federation.Error diff --git a/services/galley/src/Galley/Intra/BackendNotificationQueue.hs b/services/galley/src/Galley/Intra/BackendNotificationQueue.hs index fb2e02605fc..60a43b97070 100644 --- a/services/galley/src/Galley/Intra/BackendNotificationQueue.hs +++ b/services/galley/src/Galley/Intra/BackendNotificationQueue.hs @@ -17,7 +17,7 @@ import Polysemy import Polysemy.Input import System.Logger.Class qualified as Log import UnliftIO.Timeout (timeout) -import Wire.API.Federation.BackendNotifications +import Wire.API.BackgroundWorker import Wire.API.Federation.Error interpretBackendNotificationQueueAccess ::