From 218bae4be43ed5fc4bc674d005700806cff152e7 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 30 Jan 2023 12:58:16 +0000 Subject: [PATCH 1/6] fix conversation list parser in stern --- tools/stern/src/Stern/API.hs | 6 +++--- tools/stern/src/Stern/Intra.hs | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/stern/src/Stern/API.hs b/tools/stern/src/Stern/API.hs index 31b67199ac..aba2ca2918 100644 --- a/tools/stern/src/Stern/API.hs +++ b/tools/stern/src/Stern/API.hs @@ -402,15 +402,15 @@ getUserData uid = do convs <- Intra.getUserConversations uid clts <- Intra.getUserClients uid notfs <- Intra.getUserNotifications uid - consent <- Intra.getUserConsentValue uid - consentLog <- Intra.getUserConsentLog uid + consent <- (Intra.getUserConsentValue uid <&> Just) `catchE` \_ -> pure Nothing + consentLog <- (Intra.getUserConsentLog uid <&> Just) `catchE` \_ -> pure Nothing cookies <- Intra.getUserCookies uid properties <- Intra.getUserProperties uid -- Get all info from Marketo too let em = userEmail $ accountUser account marketo <- do let noEmail = MarketoResult $ KeyMap.singleton "results" emptyArray - maybe (pure noEmail) Intra.getMarketoResult em + maybe (pure noEmail) (\e -> Intra.getMarketoResult e `catchE` \_ -> pure noEmail) em pure . UserMetaInfo . KeyMap.fromList $ [ "account" .= account, "cookies" .= cookies, diff --git a/tools/stern/src/Stern/Intra.hs b/tools/stern/src/Stern/Intra.hs index 704892ba32..cc40ca1b90 100644 --- a/tools/stern/src/Stern/Intra.hs +++ b/tools/stern/src/Stern/Intra.hs @@ -100,6 +100,8 @@ import Wire.API.Internal.Notification import Wire.API.Properties import Wire.API.Routes.Internal.Brig.Connection import qualified Wire.API.Routes.Internal.Brig.EJPD as EJPD +import Wire.API.Routes.Version +import Wire.API.Routes.Versioned import Wire.API.Team import Wire.API.Team.Feature import qualified Wire.API.Team.Feature as Public @@ -744,12 +746,12 @@ getUserConversations uid = do b ( method GET . header "Z-User" (toByteString' uid) - . path "/conversations" + . path "/v2/conversations" . queryItem "size" (toByteString' batchSize) . maybe id (queryItem "start" . toByteString') start . expect2xx ) - parseResponse (mkError status502 "bad-upstream") r + unVersioned @'V2 <$> parseResponse (mkError status502 "bad-upstream") r batchSize = 100 :: Int getUserClients :: UserId -> Handler [Client] From 489e2b273850772082f197932cea6638e3736f67 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 30 Jan 2023 13:00:30 +0000 Subject: [PATCH 2/6] changelog --- changelog.d/5-internal/pr-3035 | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5-internal/pr-3035 diff --git a/changelog.d/5-internal/pr-3035 b/changelog.d/5-internal/pr-3035 new file mode 100644 index 0000000000..9930a21a2d --- /dev/null +++ b/changelog.d/5-internal/pr-3035 @@ -0,0 +1 @@ +Fixed broken stern endpoint `POST i/user/meta-info` From f98baa6f54cc1246cfc362453429989c69ca94e8 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 30 Jan 2023 14:28:53 +0000 Subject: [PATCH 3/6] add the error to the response --- tools/stern/src/Stern/API.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/stern/src/Stern/API.hs b/tools/stern/src/Stern/API.hs index aba2ca2918..ac43e8f52a 100644 --- a/tools/stern/src/Stern/API.hs +++ b/tools/stern/src/Stern/API.hs @@ -402,15 +402,15 @@ getUserData uid = do convs <- Intra.getUserConversations uid clts <- Intra.getUserClients uid notfs <- Intra.getUserNotifications uid - consent <- (Intra.getUserConsentValue uid <&> Just) `catchE` \_ -> pure Nothing - consentLog <- (Intra.getUserConsentLog uid <&> Just) `catchE` \_ -> pure Nothing + consent <- (Intra.getUserConsentValue uid <&> toJSON @ConsentValue) `catchE` (pure . String . cs . show) + consentLog <- (Intra.getUserConsentLog uid <&> toJSON @ConsentLog) `catchE` (pure . String . cs . show) cookies <- Intra.getUserCookies uid properties <- Intra.getUserProperties uid -- Get all info from Marketo too let em = userEmail $ accountUser account marketo <- do let noEmail = MarketoResult $ KeyMap.singleton "results" emptyArray - maybe (pure noEmail) (\e -> Intra.getMarketoResult e `catchE` \_ -> pure noEmail) em + maybe (pure $ toJSON noEmail) (\e -> (Intra.getMarketoResult e <&> toJSON) `catchE` (pure . String . cs . show)) em pure . UserMetaInfo . KeyMap.fromList $ [ "account" .= account, "cookies" .= cookies, From 8a4bd1bd10ad0eeba57ef27a2844ba237973c86a Mon Sep 17 00:00:00 2001 From: fisx Date: Mon, 30 Jan 2023 16:42:03 +0100 Subject: [PATCH 4/6] Make stern assert backend api version (#3036) --- changelog.d/5-internal/pr-3036 | 1 + libs/wire-api/src/Wire/API/Routes/Version.hs | 4 +++ tools/stern/src/Stern/API.hs | 1 + tools/stern/src/Stern/Intra.hs | 32 ++++++++++++++++++-- tools/stern/stern.cabal | 1 + 5 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 changelog.d/5-internal/pr-3036 diff --git a/changelog.d/5-internal/pr-3036 b/changelog.d/5-internal/pr-3036 new file mode 100644 index 0000000000..6d052dfcf0 --- /dev/null +++ b/changelog.d/5-internal/pr-3036 @@ -0,0 +1 @@ +Make stern fail on startup if supported backend api version needs bumping \ No newline at end of file diff --git a/libs/wire-api/src/Wire/API/Routes/Version.hs b/libs/wire-api/src/Wire/API/Routes/Version.hs index 68d46bf8ee..fa6b430394 100644 --- a/libs/wire-api/src/Wire/API/Routes/Version.hs +++ b/libs/wire-api/src/Wire/API/Routes/Version.hs @@ -43,6 +43,7 @@ import Control.Lens ((?~)) import Data.Aeson (FromJSON, ToJSON (..)) import qualified Data.Aeson as Aeson import Data.Bifunctor +import Data.ByteString.Conversion (ToByteString (builder)) import qualified Data.ByteString.Lazy as LBS import Data.Domain import Data.Schema @@ -83,6 +84,9 @@ instance ToHttpApiData Version where toHeader = LBS.toStrict . Aeson.encode toUrlPiece = Text.decodeUtf8 . toHeader +instance ToByteString Version where + builder = toEncodedUrlPiece + supportedVersions :: [Version] supportedVersions = [minBound .. maxBound] diff --git a/tools/stern/src/Stern/API.hs b/tools/stern/src/Stern/API.hs index ac43e8f52a..3852c1738f 100644 --- a/tools/stern/src/Stern/API.hs +++ b/tools/stern/src/Stern/API.hs @@ -74,6 +74,7 @@ default (ByteString) start :: Opts -> IO () start o = do e <- newEnv o + runAppT e $ Intra.assertBackendApiVersion s <- Server.newSettings (server e) Server.runSettingsWithShutdown s (servantApp e) Nothing where diff --git a/tools/stern/src/Stern/Intra.hs b/tools/stern/src/Stern/Intra.hs index cc40ca1b90..c27ef36829 100644 --- a/tools/stern/src/Stern/Intra.hs +++ b/tools/stern/src/Stern/Intra.hs @@ -22,7 +22,8 @@ -- with this program. If not, see . module Stern.Intra - ( putUser, + ( assertBackendApiVersion, + putUser, putUserStatus, getContacts, getUserConnections, @@ -61,10 +62,12 @@ module Stern.Intra ) where -import Bilge hiding (head, options, requestId) +import Bilge hiding (head, options, path, paths, requestId) +import qualified Bilge import Bilge.RPC import Brig.Types.Intra import Control.Error +import Control.Exception (ErrorCall (ErrorCall)) import Control.Lens (view, (^.)) import Control.Monad.Reader import Data.Aeson hiding (Error) @@ -94,6 +97,7 @@ import Stern.Types import System.Logger.Class hiding (Error, name, (.=)) import qualified System.Logger.Class as Log import UnliftIO.Exception hiding (Handler) +import UnliftIO.Retry (constantDelay, limitRetries, recoverAll) import Wire.API.Connection import Wire.API.Conversation import Wire.API.Internal.Notification @@ -114,6 +118,28 @@ import Wire.API.User.Search ------------------------------------------------------------------------------- +backendApiVersion :: Version +backendApiVersion = V2 + +-- | Make sure the backend supports `backendApiVersion`. Crash if it doesn't. (This is called +-- in `Stern.API` so problems make `./services/integration.sh` crash.) +assertBackendApiVersion :: App () +assertBackendApiVersion = recoverAll (constantDelay 1000000 <> limitRetries 5) $ \_retryStatus -> do + b <- view brig + vinfo :: VersionInfo <- + responseJsonError + =<< rpc' "brig" b (method GET . Bilge.path "/api-version" . contentJson . expect2xx) + unless (maximum (vinfoSupported vinfo) == backendApiVersion) $ do + throwIO . ErrorCall $ "newest supported backend api version must be " <> show backendApiVersion + +path :: ByteString -> Request -> Request +path = Bilge.path . ((toByteString' backendApiVersion <> "/") <>) + +paths :: [ByteString] -> Request -> Request +paths = Bilge.paths . (toByteString' backendApiVersion :) + +------------------------------------------------------------------------------- + putUser :: UserId -> UserUpdate -> Handler () putUser uid upd = do info $ userMsg uid . msg "Changing user state" @@ -746,7 +772,7 @@ getUserConversations uid = do b ( method GET . header "Z-User" (toByteString' uid) - . path "/v2/conversations" + . path "conversations" . queryItem "size" (toByteString' batchSize) . maybe id (queryItem "start" . toByteString') start . expect2xx diff --git a/tools/stern/stern.cabal b/tools/stern/stern.cabal index 2fc74138b4..d0924fbf3b 100644 --- a/tools/stern/stern.cabal +++ b/tools/stern/stern.cabal @@ -91,6 +91,7 @@ library , lens >=4.4 , metrics-wai >=0.3 , mtl >=2.1 + , retry , schema-profunctor , servant , servant-server From 1f18eedb8e745e5431881fdf1e91bf1eb946d81a Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 30 Jan 2023 16:13:50 +0000 Subject: [PATCH 5/6] gen nix --- tools/stern/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/stern/default.nix b/tools/stern/default.nix index 6eb54f2aa5..8b64ca1ecd 100644 --- a/tools/stern/default.nix +++ b/tools/stern/default.nix @@ -24,6 +24,7 @@ , lib , metrics-wai , mtl +, retry , schema-profunctor , servant , servant-server @@ -75,6 +76,7 @@ mkDerivation { lens metrics-wai mtl + retry schema-profunctor servant servant-server From 54ef38820488c68fd71723d333a2a90264d856f8 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Tue, 31 Jan 2023 07:24:49 +0000 Subject: [PATCH 6/6] hi ci