diff --git a/changelog.d/5-internal/SQPIT-1544 b/changelog.d/5-internal/SQPIT-1544 new file mode 100644 index 00000000000..20449ff4397 --- /dev/null +++ b/changelog.d/5-internal/SQPIT-1544 @@ -0,0 +1,4 @@ +When sending a push message, stop deleting the push token and start recreating +ARN when ARN is reported as invalid on AWS, but push token still is present in +Cassandra. This allows on-demand migrations from one AWS account used for push +notifications to another one. diff --git a/services/gundeck/src/Gundeck/Aws.hs b/services/gundeck/src/Gundeck/Aws.hs index b34f3bf7492..7fea7d114ef 100644 --- a/services/gundeck/src/Gundeck/Aws.hs +++ b/services/gundeck/src/Gundeck/Aws.hs @@ -355,6 +355,7 @@ data PublishError = EndpointDisabled !EndpointArn | InvalidEndpoint !EndpointArn | PayloadTooLarge !EndpointArn + | UnauthorisedEndpoint !EndpointArn newtype Attributes = Attributes { setAttributes :: Endo (HashMap Text SNS.MessageAttributeValue) @@ -424,6 +425,9 @@ publish arn txt attrs = do && AWS.newErrorCode "InvalidParameter" == e ^. serviceError_code && isArnError (e ^. serviceError_message) -> pure (Left (InvalidEndpoint arn)) + | is "SNS" 403 x + && AWS.newErrorCode "AuthorizationError" == e ^. serviceError_code -> + pure (Left (UnauthorisedEndpoint arn)) Left x -> throwM (GeneralError x) where -- Thank you Amazon for not having granular error codes! diff --git a/services/gundeck/src/Gundeck/Push/Data.hs b/services/gundeck/src/Gundeck/Push/Data.hs index 6437e2466e7..bc30021ccc8 100644 --- a/services/gundeck/src/Gundeck/Push/Data.hs +++ b/services/gundeck/src/Gundeck/Push/Data.hs @@ -17,6 +17,7 @@ module Gundeck.Push.Data ( insert, + updateArn, delete, lookup, erase, @@ -29,7 +30,7 @@ import Data.ByteString.Conversion import Data.Id (ClientId, ConnId, UserId) import Gundeck.Instances () import Gundeck.Push.Native.Types -import Gundeck.Types +import Gundeck.Types hiding (token) import Imports hiding (lookup) import System.Logger.Class (MonadLogger, field, msg, val, (~~)) import qualified System.Logger.Class as Log @@ -47,6 +48,12 @@ insert u t a p e o c = retry x5 $ write q (params LocalQuorum (u, t, a, p, e, o, q :: PrepQuery W (UserId, Transport, AppName, Token, EndpointArn, ConnId, ClientId) () q = "insert into user_push (usr, transport, app, ptoken, arn, connection, client) values (?, ?, ?, ?, ?, ?, ?)" +updateArn :: MonadClient m => UserId -> Transport -> AppName -> Token -> EndpointArn -> m () +updateArn uid transport app token arn = retry x5 $ write q (params LocalQuorum (arn, uid, transport, app, token)) + where + q :: PrepQuery W (EndpointArn, UserId, Transport, AppName, Token) () + q = "update user_push set arn = ? where usr = ? and transport = ? and app = ? and ptoken = ?" + delete :: MonadClient m => UserId -> Transport -> AppName -> Token -> m () delete u t a p = retry x5 $ write q (params LocalQuorum (u, t, a, p)) where diff --git a/services/gundeck/src/Gundeck/Push/Native.hs b/services/gundeck/src/Gundeck/Push/Native.hs index c48fabd4690..9590eb044f6 100644 --- a/services/gundeck/src/Gundeck/Push/Native.hs +++ b/services/gundeck/src/Gundeck/Push/Native.hs @@ -43,7 +43,7 @@ import Gundeck.Push.Native.Types as Types import Gundeck.Types import Gundeck.Util import Imports -import System.Logger.Class (MonadLogger, field, msg, val, (~~)) +import System.Logger.Class (MonadLogger, field, msg, val, (.=), (~~)) import qualified System.Logger.Class as Log import UnliftIO (handleAny, mapConcurrently, pooledMapConcurrentlyN_) import Wire.API.Internal.Notification @@ -61,58 +61,103 @@ push m addrs = do Just chunkSize -> pooledMapConcurrentlyN_ chunkSize (push1 m) addrs push1 :: NativePush -> Address -> Gundeck () -push1 m a = do - e <- view awsEnv - r <- Aws.execute e $ publish m a - case r of - Success _ -> do - Log.debug $ - field "user" (toByteString (a ^. addrUser)) - ~~ field "notificationId" (toText (npNotificationid m)) - ~~ Log.msg (val "Native push success") - view monitor >>= counterIncr (path "push.native.success") - Failure EndpointDisabled _ -> onDisabled - Failure PayloadTooLarge _ -> onPayloadTooLarge - Failure EndpointInvalid _ -> onInvalidEndpoint - Failure (PushException ex) _ -> do - logError a "Native push failed" ex - view monitor >>= counterIncr (path "push.native.errors") +push1 = push1' 0 where - onDisabled = - handleAny (logError a "Failed to cleanup disabled endpoint") $ do - Log.info $ - field "user" (toByteString (a ^. addrUser)) - ~~ field "arn" (toText (a ^. addrEndpoint)) - ~~ field "cause" ("EndpointDisabled" :: Text) - ~~ msg (val "Removing disabled endpoint and token") - view monitor >>= counterIncr (path "push.native.disabled") - Data.delete (a ^. addrUser) (a ^. addrTransport) (a ^. addrApp) (a ^. addrToken) - onTokenRemoved - e <- view awsEnv - Aws.execute e (Aws.deleteEndpoint (a ^. addrEndpoint)) - onPayloadTooLarge = do - view monitor >>= counterIncr (path "push.native.too_large") - Log.warn $ - field "user" (toByteString (a ^. addrUser)) - ~~ field "arn" (toText (a ^. addrEndpoint)) - ~~ msg (val "Payload too large") - onInvalidEndpoint = - handleAny (logError a "Failed to cleanup orphaned push token") $ do - Log.warn $ - field "user" (toByteString (a ^. addrUser)) - ~~ field "arn" (toText (a ^. addrEndpoint)) - ~~ field "cause" ("InvalidEndpoint" :: Text) - ~~ msg (val "Invalid ARN. Deleting orphaned push token") - view monitor >>= counterIncr (path "push.native.invalid") - Data.delete (a ^. addrUser) (a ^. addrTransport) (a ^. addrApp) (a ^. addrToken) - onTokenRemoved - onTokenRemoved = do - i <- mkNotificationId - let c = a ^. addrClient - let r = singleton (target (a ^. addrUser) & targetClients .~ [c]) - let t = a ^. addrPushToken - let p = singletonPayload (PushRemove t) - Stream.add i r p =<< view (options . optSettings . setNotificationTTL) + push1' :: Int -> NativePush -> Address -> Gundeck () + push1' n m a = + if n > retryUnauthorisedThreshold + then onPersistentlyUnauthorisedEndpoint + else do + e <- view awsEnv + r <- Aws.execute e $ publish m a + case r of + Success _ -> onSuccess + Failure EndpointDisabled _ -> onDisabled + Failure PayloadTooLarge _ -> onPayloadTooLarge + Failure EndpointInvalid _ -> onInvalidEndpoint + Failure EndpointUnauthorised _ -> + if n < retryUnauthorisedThreshold + then onUnauthorisedEndpoint + else onPersistentlyUnauthorisedEndpoint + Failure (PushException ex) _ -> onPushException ex + where + onSuccess = do + Log.debug $ + field "user" (toByteString (a ^. addrUser)) + ~~ field "notificationId" (toText (npNotificationid m)) + ~~ Log.msg (val "Native push success") + view monitor >>= counterIncr (path "push.native.success") + onDisabled = + handleAny (logError a "Failed to cleanup disabled endpoint") $ do + Log.info $ + field "user" (toByteString (a ^. addrUser)) + ~~ field "arn" (toText (a ^. addrEndpoint)) + ~~ field "cause" ("EndpointDisabled" :: Text) + ~~ msg (val "Removing disabled endpoint and token") + view monitor >>= counterIncr (path "push.native.disabled") + Data.delete (a ^. addrUser) (a ^. addrTransport) (a ^. addrApp) (a ^. addrToken) + onTokenRemoved + e <- view awsEnv + Aws.execute e (Aws.deleteEndpoint (a ^. addrEndpoint)) + onPayloadTooLarge = do + view monitor >>= counterIncr (path "push.native.too_large") + Log.warn $ + field "user" (toByteString (a ^. addrUser)) + ~~ field "arn" (toText (a ^. addrEndpoint)) + ~~ msg (val "Payload too large") + onInvalidEndpoint = + handleAny (logError a "Failed to cleanup orphaned push token") $ do + Log.warn $ + field "user" (toByteString (a ^. addrUser)) + ~~ field "arn" (toText (a ^. addrEndpoint)) + ~~ field "cause" ("InvalidEndpoint" :: Text) + ~~ msg (val "Invalid ARN. Deleting orphaned push token") + view monitor >>= counterIncr (path "push.native.invalid") + Data.delete (a ^. addrUser) (a ^. addrTransport) (a ^. addrApp) (a ^. addrToken) + onTokenRemoved + retryUnauthorisedThreshold = 1 + onUnauthorisedEndpoint = do + -- try to recreate ARN (cf. Gundeck.Push.addToken.create) + let uid = a ^. addrUser + let t = a ^. addrPushToken + let trp = t ^. tokenTransport + let app = t ^. tokenApp + let tok = t ^. token + env <- view (options . optAws . awsArnEnv) + aws <- view awsEnv + ept <- Aws.execute aws (Aws.createEndpoint uid trp env app tok) + case ept of + Left (Aws.EndpointInUse arn) -> + Log.info $ "arn" .= toText arn ~~ msg (val "ARN in use") + Left (Aws.AppNotFound app') -> + Log.info $ msg ("Push token of unknown application: '" <> appNameText app' <> "'") + Left (Aws.InvalidToken _) -> + Log.info $ + "token" + .= tokenText tok + ~~ msg (val "Invalid push token.") + Left (Aws.TokenTooLong l) -> + Log.info $ msg ("Push token is too long: token length = " ++ show l) + Right arn -> do + Data.updateArn uid trp app tok arn + push1' (succ n) m (a & addrEndpoint .~ arn) -- try to send the push message with the new ARN + onPersistentlyUnauthorisedEndpoint = handleAny (logError a "Found orphaned push token") $ do + Log.warn $ + field "user" (toByteString (a ^. addrUser)) + ~~ field "arn" (toText (a ^. addrEndpoint)) + ~~ field "cause" ("UnauthorisedEndpoint" :: Text) + ~~ msg (val "Invalid ARN. Dropping push message.") + view monitor >>= counterIncr (path "push.native.unauthorized") + onPushException ex = do + logError a "Native push failed" ex + view monitor >>= counterIncr (path "push.native.errors") + onTokenRemoved = do + i <- mkNotificationId + let c = a ^. addrClient + let r = singleton (target (a ^. addrUser) & targetClients .~ [c]) + let t = a ^. addrPushToken + let p = singletonPayload (PushRemove t) + Stream.add i r p =<< view (options . optSettings . setNotificationTTL) publish :: NativePush -> Address -> Aws.Amazon Result publish m a = flip catches pushException $ do @@ -133,6 +178,7 @@ publish m a = flip catches pushException $ do toResult (Left (Aws.EndpointDisabled _)) = Failure EndpointDisabled a toResult (Left (Aws.PayloadTooLarge _)) = Failure PayloadTooLarge a toResult (Left (Aws.InvalidEndpoint _)) = Failure EndpointInvalid a + toResult (Left (Aws.UnauthorisedEndpoint _)) = Failure EndpointUnauthorised a toResult (Right ()) = Success a pushException = [ Handler (\(ex :: SomeAsyncException) -> throwM ex), diff --git a/services/gundeck/src/Gundeck/Push/Native/Types.hs b/services/gundeck/src/Gundeck/Push/Native/Types.hs index 07ac8a27af5..d191bfb0459 100644 --- a/services/gundeck/src/Gundeck/Push/Native/Types.hs +++ b/services/gundeck/src/Gundeck/Push/Native/Types.hs @@ -102,6 +102,7 @@ data Failure = PayloadTooLarge | EndpointInvalid | EndpointDisabled + | EndpointUnauthorised | PushException !SomeException deriving (Show)