Document federation errors

CHANGELOG-draft.md

@@ -18,6 +18,8 @@ THIS FILE ACCUMULATES THE RELEASE NOTES FOR THE UPCOMING RELEASE.
 
 ## Documentation
 
+* Added documentation of federation errors (#1674)
+
 ## Internal changes
 
 * Rewrite the `POST /connections` endpoint to Servant (#1726)

libs/wire-api-federation/proto/router.proto

@@ -37,16 +37,16 @@ message OutwardError {
     DiscoveryFailed = 1;
     ConnectionRefused = 2;
     TLSFailure = 3;
-    InvalidCertificate = 4;
-    VersionMismatch = 5;
-    FederationDeniedByRemote = 6;
-    FederationDeniedLocally = 7;
-    RemoteFederatorError = 8;
+    VersionMismatch = 4;
+    FederationDeniedByRemote = 5;
+    FederationDeniedLocally = 6;
+    TooMuchConcurrency = 7;
+    GrpcError = 8;
     InvalidRequest = 9;
   }
 
   ErrorType type = 1;
-  ErrorPayload payload = 2;
+  string msg = 2;
 }
 
 message InwardError {
@@ -63,11 +63,6 @@ message InwardError {
   string msg = 2;
 }
 
-message ErrorPayload {
-  string label = 1;
-  string msg = 2;
-}
-
 // The envelope message which is sent from brig to the Outward service of a local federator
 message FederatedRequest {
   string domain = 1;

libs/wire-api-federation/src/Wire/API/Federation/Error.hs

@@ -38,7 +38,11 @@ federationErrorToWai FederationNotImplemented = federationNotImplemented
 federationErrorToWai FederationNotConfigured = federationNotConfigured
 federationErrorToWai (FederationCallFailure failure) = addErrorData $
   case fedFailError failure of
-    FederationClientRPCError msg -> federationRpcError msg
+    FederationClientRPCError msg ->
+      Wai.mkError
+        HTTP.status500
+        "client-rpc-error"
+        (LT.fromStrict msg)
     FederationClientInvalidMethod mth ->
       federationInvalidCall
         ("Unexpected method: " <> LT.fromStrict (T.decodeUtf8 mth))
@@ -117,13 +121,6 @@ federationNotConfigured =
     "federation-not-enabled"
     "no federator configured"
 
-federationRpcError :: Text -> Wai.Error
-federationRpcError msg =
-  Wai.mkError
-    HTTP.status500
-    "federation-rpc-error"
-    (LT.fromStrict msg)
-
 federationUnavailable :: Text -> Wai.Error
 federationUnavailable err =
   Wai.mkError
@@ -144,25 +141,25 @@ federationRemoteInwardError err = Wai.mkError status (LT.fromStrict label) (LT.f
       Proto.IOther -> (unexpectedFederationResponseStatus, "inward-other")
 
 federationRemoteError :: Proto.OutwardError -> Wai.Error
-federationRemoteError err = Wai.mkError status (LT.fromStrict label) (LT.fromStrict msg)
+federationRemoteError err = case Proto.outwardErrorType err of
+  Proto.RemoteNotFound -> mkErr HTTP.status422 "srv-record-not-found"
+  Proto.DiscoveryFailed -> mkErr HTTP.status500 "srv-lookup-dns-error"
+  Proto.ConnectionRefused ->
+    mkErr
+      (HTTP.Status 521 "Web Server Is Down")
+      "cannot-connect-to-remote-federator"
+  Proto.TLSFailure -> mkErr (HTTP.Status 525 "SSL Handshake Failure") "tls-failure"
+  Proto.VersionMismatch -> mkErr (HTTP.Status 531 "Version Mismatch") "version-mismatch"
+  Proto.FederationDeniedByRemote ->
+    mkErr
+      (HTTP.Status 532 "Federation Denied")
+      "federation-denied-remotely"
+  Proto.FederationDeniedLocally -> mkErr HTTP.status400 "federation-not-allowed"
+  Proto.TooMuchConcurrency -> mkErr unexpectedFederationResponseStatus "too-much-concurrency"
+  Proto.GrpcError -> mkErr unexpectedFederationResponseStatus "grpc-error"
+  Proto.InvalidRequest -> mkErr HTTP.status500 "invalid-request-to-federator"
   where
-    decodeError :: Maybe Proto.ErrorPayload -> (Text, Text)
-    decodeError Nothing = ("unknown-federation-error", "Unknown federation error")
-    decodeError (Just (Proto.ErrorPayload label' msg')) = (label', msg')
-
-    (label, msg) = decodeError (Proto.outwardErrorPayload err)
-
-    status = case Proto.outwardErrorType err of
-      Proto.RemoteNotFound -> HTTP.status422
-      Proto.DiscoveryFailed -> HTTP.status500
-      Proto.ConnectionRefused -> HTTP.Status 521 "Web Server Is Down"
-      Proto.TLSFailure -> HTTP.Status 525 "SSL Handshake Failure"
-      Proto.InvalidCertificate -> HTTP.Status 526 "Invalid SSL Certificate"
-      Proto.VersionMismatch -> HTTP.Status 531 "Version Mismatch"
-      Proto.FederationDeniedByRemote -> HTTP.Status 532 "Federation Denied"
-      Proto.FederationDeniedLocally -> HTTP.status400
-      Proto.RemoteFederatorError -> unexpectedFederationResponseStatus
-      Proto.InvalidRequest -> HTTP.status500
+    mkErr status label = Wai.mkError status label (LT.fromStrict (Proto.outwardErrorMessage err))
 
 federationInvalidCall :: LText -> Wai.Error
 federationInvalidCall = Wai.mkError HTTP.status500 "federation-invalid-call"

libs/wire-api-federation/src/Wire/API/Federation/GRPC/Types.hs

@@ -26,6 +26,7 @@ module Wire.API.Federation.GRPC.Types where
 import Data.Domain (Domain (..), domainText, mkDomain)
 import Data.Either.Validation
 import Data.List.NonEmpty (NonEmpty ((:|)))
+import qualified Data.Text as Text
 import Imports
 import Mu.Quasi.GRpc (grpc)
 import Mu.Schema
@@ -101,12 +102,12 @@ instance FromSchema Router "OutwardResponse" OutwardResponse where
 -- https://higherkindness.io/mu-haskell/schema/#mapping-haskell-types
 type OutwardErrorFieldMapping =
   '[ "outwardErrorType" ':-> "type",
-     "outwardErrorPayload" ':-> "payload"
+     "outwardErrorMessage" ':-> "msg"
    ]
 
 data OutwardError = OutwardError
   { outwardErrorType :: OutwardErrorType,
-    outwardErrorPayload :: Maybe ErrorPayload
+    outwardErrorMessage :: Text
   }
   deriving (Typeable, Show, Eq, Generic)
   deriving (Arbitrary) via (GenericUniform OutwardError)
@@ -119,22 +120,15 @@ data OutwardErrorType
   | DiscoveryFailed
   | ConnectionRefused
   | TLSFailure
-  | InvalidCertificate
   | VersionMismatch
   | FederationDeniedByRemote
   | FederationDeniedLocally
-  | RemoteFederatorError
+  | TooMuchConcurrency
+  | GrpcError
   | InvalidRequest
   deriving (Typeable, Show, Eq, Generic, ToSchema Router "OutwardError.ErrorType", FromSchema Router "OutwardError.ErrorType")
   deriving (Arbitrary) via (GenericUniform OutwardErrorType)
 
-data ErrorPayload = ErrorPayload
-  { label :: Text,
-    msg :: Text
-  }
-  deriving (Typeable, Show, Eq, Generic, ToSchema Router "ErrorPayload", FromSchema Router "ErrorPayload")
-  deriving (Arbitrary) via (GenericUniform ErrorPayload)
-
 -- See mu-haskell Custom Mapping documentation here:
 -- https://higherkindness.io/mu-haskell/schema/#mapping-haskell-types
 type InwardErrorFieldMapping =
@@ -190,6 +184,16 @@ data ValidatedFederatedRequest = ValidatedFederatedRequest
   }
   deriving (Typeable, Eq, Show)
 
+showFederatedRequestValidationError :: FederatedRequestValidationError -> Text
+showFederatedRequestValidationError (InvalidDomain msg) = "invalid domain: " <> Text.pack msg
+showFederatedRequestValidationError RequestMissing = "federation request is missing"
+
+showFederatedRequestValidationErrors :: NonEmpty FederatedRequestValidationError -> Text
+showFederatedRequestValidationErrors =
+  Text.intercalate "; "
+    . map showFederatedRequestValidationError
+    . toList
+
 validateFederatedRequest :: FederatedRequest -> Validation (NonEmpty FederatedRequestValidationError) ValidatedFederatedRequest
 validateFederatedRequest FederatedRequest {..} = do
   vDomain <- validateDomain

services/brig/src/Brig/API/Public.hs

@@ -163,6 +163,63 @@ Okta will ask you to provide two URLs when you set it up for talking to wireapp:
 #### centrify.com
 
 Centrify allows you to upload the metadata xml document that you get from the `/sso/metadata` end-point.  You can also enter the metadata url and have centrify retrieve the xml, but to guarantee integrity of the setup, the metadata should be copied from the team settings page and pasted into the centrify setup page without any URL indirections.
+
+## Federation errors
+
+Endpoints involving federated calls to other domains can return some extra failure responses, common to all endpoints. Instead of listing them as possible responses for each endpoint, we document them here.
+
+For errors that are more likely to be transient, we suggest clients to retry whatever request resulted in the error. Transient errors are indicated explicitly below.
+
+**Note**: when a failure occurs as a result of making a federated RPC to another backend, the error response contains the following extra fields:
+
+ - `domain`: the target backend of the RPC that failed;
+ - `path`: the path of the RPC that failed.
+
+### Domain errors
+
+Errors in this category result from trying to communicate with a backend that is considered non-existent or invalid. They can result from invalid user input or client issues, but they can also be a symptom of misconfiguration in one or multiple backends.
+
+ - **Remote backend not found** (status: 422, label: `srv-record-not-found`): This backend attempted to contact a backend which does not exist or is not properly configured. For the most part, clients can consider this error equivalent to a domain not existing, although it should be noted that certain mistakes in the DNS configuration on a remote backend can lead to the backend not being recognized, and hence to this error. It is therefore not advisable to take any destructive action upon encountering this error, such as deleting remote users from conversations.
+ - **Federation denied locally** (status: 400, label: `federation-not-allowed`): This backend attempted an RPC to a non-whitelisted backend. Similar considerations as for the previous error apply.
+
+### Local federation errors
+
+An error in this category likely indicates an issue with configuration of federation on the local backend. Possibly transient errors are indicated explicitly below.
+
+ - **Federation not enabled** (status: 400, label: `federation-not-enabled`): Federation has not been configured for this backend. This will happen if a federation-aware client tries to talk to a backend for which federation is disabled, or if federation was disabled on the backend after reaching a federation-specific state (e.g. conversations with remote users). There is no way to cleanly recover from these errors at this point.
+ - **Federation unavailable** (status: 500, label: `federation-not-available`): Federation is configured for this backend, but the local federator cannot be reached. This can be transient, so clients should retry the request.
+ - **Federation not implemented** (status: 403, label: `federation-not-implemented`): Federated behaviour for a certain endpoint is not yet implemented.
+ - **Federator discovery failed** (status: 500, label: `srv-lookup-dns-error`): A DNS error occurred during discovery of a remote backend. This can be transient, so clients should retry the request.
+ - **Too much concurrency** (status: 533, label: `too-much-concurrency`): Too many concurrent requests from this backend. This can be transient, so clients should retry the request.
+
+### Remote federation errors
+
+Errors in this category are returned in case of communication issues between the local backend and a remote one, or if the remote side encountered an error while processing an RPC. Some errors in this category might be caused by incorrect client behaviour or wrong user input. All of these errors can be transient, so clients should retry the request that caused them.
+
+ - **gRPC error** (status: 533, label: `grpc-error`): The current federator encountered an error when making an RPC to a remote one. Check the error message for more details.
+ - **Client RPC error** (status: 500, label: `client-rpc-error`): There was a non-specified error when making a request to another backend. Check the error message for more details.
+ - **Connection refused** (status: 521, label: `cannot-connect-to-remote-federator`): The local federator could not connect to a remote one.
+ - **Unknown remote error** (status: 500, label: `unknown-federation-error`): An RPC failed but no specific error was returned by the remote side. Check the error message for more details.
+
+### Backend compatibility errors
+
+An error in this category will be returned when this backend makes an invalid or unsupported RPC to another backend. This can indicate some incompatibility between backends or a backend bug. These errors are unlikely to be transient, so retrying requests is *not* advised.
+
+ - **Version mismatch** (status: 531): A remote backend is running an unsupported version of the federator.
+ - **Invalid method** / **Streaming not supported** (status: 500, label: `federation-invalid-call`): There was an error in the communication between a service on this backend and the local federator.
+ - **Invalid request** (status: 500, label: `invalid-request-to-federator`): The local federator made an invalid request to a remote one. Check the error message for more details.
+ - **Invalid content type** (status: 503, label: `federation-invalid-content-type-header`): An RPC to another backend returned an invalid content type.
+ - **Unsupported content type** (status: 503, label: `federation-unsupported-content-type`): An RPC to another backend returned an unsupported content type.
+ - **Invalid origin domain** (status: 533, label: `invalid-origin-domain`): The current backend attempted an RPC with an invalid origin domain field.
+ - **Forbidden endpoint** (status: 533, label: `forbidden-endpoint`): The current backend attempted an RPC to a forbidden or inaccessible remote endpoint.
+ - **Unknown federation error** (status: 503, label: `unknown-federation-error`): The target of an RPC returned an unexpected reponse. Check the error message for more details.
+
+### Authentication errors
+
+The errors in this category relate to authentication or authorization issues between backends. These errors are unlikely to be transient, so retrying requests is *not* advised.
+
+ - **TLS failure**: (status: 525): An error occurred during the TLS handshake between the local federator and a remote one. This is most likely due to an issue with the certificate on the remote end.
+ - **Federation denied remotely** (status: 532): The current backend made an unauthorized request to a remote one.
 |]
 
 servantSitemap :: ServerT ServantAPI Handler

services/brig/test/unit/Test/Brig/API/Error.hs

@@ -32,7 +32,7 @@ testFedError =
       testCase "when federation call fails due to requesting streaming" $
         assertFedErrorStatus (mkFailure FederationClientStreamingUnsupported) 500,
       testCase "when federation call fails due to discovery failure" $ do
-        let outwardErr = FederationClientOutwardError (Proto.OutwardError Proto.DiscoveryFailed Nothing)
+        let outwardErr = FederationClientOutwardError (Proto.OutwardError Proto.DiscoveryFailed "discovery failed")
         assertFedErrorStatus (mkFailure outwardErr) 500,
       testCase "when federation call fails due to decode failure" $
         assertFedErrorStatus (mkFailure (FederationClientServantError (Servant.DecodeFailure "some failure" emptyRes))) 533,
@@ -58,31 +58,23 @@ testOutwardError =
             assertOutwardErrorStatus Proto.ConnectionRefused 521,
           testCase "when TLS fails" $
             assertOutwardErrorStatus Proto.TLSFailure 525,
-          testCase "when remote certificate is invalid" $
-            assertOutwardErrorStatus Proto.InvalidCertificate 526,
           testCase "when remote returns version mismatch" $
             assertOutwardErrorStatus Proto.VersionMismatch 531,
           testCase "when remote denies federation" $
             assertOutwardErrorStatus Proto.FederationDeniedByRemote 532,
           testCase "when local federator denies federation" $
             assertOutwardErrorStatus Proto.FederationDeniedLocally 400,
-          testCase "when remote federator errors" $
-            assertOutwardErrorStatus Proto.RemoteFederatorError 533,
+          testCase "when there is too much concurrency" $
+            assertOutwardErrorStatus Proto.TooMuchConcurrency 533,
+          testCase "when gRPC fails" $
+            assertOutwardErrorStatus Proto.GrpcError 533,
           testCase "when federator returns invalid request" $
             assertOutwardErrorStatus Proto.InvalidRequest 500
         ],
-      testGroup "label & message" $
-        [ testCase "when error payload is specified" $ do
-            let outwardErr = Proto.OutwardError Proto.TLSFailure . Just $ Proto.ErrorPayload "an-interesting-label" "very interesting message"
-                waiErr = federationRemoteError outwardErr
-            assertEqual "label should be copied" (Wai.label waiErr) "an-interesting-label"
-            assertEqual "message should be copied" (Wai.message waiErr) "very interesting message",
-          testCase "when error payload is not specified" $ do
-            let outwardErr = Proto.OutwardError Proto.TLSFailure Nothing
-                waiErr = federationRemoteError outwardErr
-            assertEqual "label should be set as unknown" (Wai.label waiErr) "unknown-federation-error"
-            assertEqual "message should be set as unknown" (Wai.message waiErr) "Unknown federation error"
-        ]
+      testCase "error message" $ do
+        let outwardErr = Proto.OutwardError Proto.TLSFailure "something went wrong"
+            waiErr = federationRemoteError outwardErr
+        assertEqual "message should be copied" (Wai.message waiErr) "something went wrong"
     ]
 
 mkFailure :: FederationClientError -> FederationError
@@ -95,7 +87,7 @@ assertFedErrorStatus err sts = assertEqual ("http status should be " <> show sts
 
 assertOutwardErrorStatus :: HasCallStack => Proto.OutwardErrorType -> Int -> IO ()
 assertOutwardErrorStatus errType sts =
-  assertEqual ("http status should be " <> show sts) (HTTP.statusCode . Wai.code . federationRemoteError $ Proto.OutwardError errType Nothing) sts
+  assertEqual ("http status should be " <> show sts) (HTTP.statusCode . Wai.code . federationRemoteError $ Proto.OutwardError errType mempty) sts
 
 statusFor :: FederationError -> Int
 statusFor = HTTP.statusCode . errorStatus . fedError