diff --git a/CHANGELOG.md b/CHANGELOG.md index 35e8d149785..3911236aac5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,10 +29,17 @@ ## API Changes +* Add `POST /conversations/list-ids` (#1686) +* Deprecate `GET /converstations/ids` (#1686) + ## Features ## Bug fixes and other updates +## Federation changes (alpha feature, do not use yet) + +* Add new API to list paginated qualified conversation ids (#1686) + ## Documentation * fix swagger: mark name in UserUpdate as optional (#1691) diff --git a/libs/cassandra-util/src/Cassandra.hs b/libs/cassandra-util/src/Cassandra.hs index 31b9c4ed856..2b166cc9b4c 100644 --- a/libs/cassandra-util/src/Cassandra.hs +++ b/libs/cassandra-util/src/Cassandra.hs @@ -30,6 +30,7 @@ import Cassandra.CQL as C Consistency (All, One, Quorum), Cql, Keyspace (Keyspace), + PagingState (..), QueryParams (QueryParams), QueryString (QueryString), R, @@ -58,6 +59,7 @@ import Cassandra.Exec as C ClientState, MonadClient, Page (..), + PageWithState (..), PrepQuery, Row, addPrepQuery, @@ -70,8 +72,11 @@ import Cassandra.Exec as C nextPage, paginate, paginateC, + paginateWithState, params, paramsP, + paramsPagingState, + pwsHasMore, query, query1, result, diff --git a/libs/cassandra-util/src/Cassandra/CQL.hs b/libs/cassandra-util/src/Cassandra/CQL.hs index 3680166d0da..058b6a5bd41 100644 --- a/libs/cassandra-util/src/Cassandra/CQL.hs +++ b/libs/cassandra-util/src/Cassandra/CQL.hs @@ -29,6 +29,7 @@ import Database.CQL.Protocol as C Consistency (All, One, Quorum), Cql, Keyspace (Keyspace), + PagingState (..), QueryParams (QueryParams), QueryString (QueryString), R, diff --git a/libs/cassandra-util/src/Cassandra/Exec.hs b/libs/cassandra-util/src/Cassandra/Exec.hs index bfe15af27f1..d39188bc039 100644 --- a/libs/cassandra-util/src/Cassandra/Exec.hs +++ b/libs/cassandra-util/src/Cassandra/Exec.hs @@ -26,6 +26,10 @@ module Cassandra.Exec x1, syncCassandra, paginateC, + PageWithState (..), + paginateWithState, + paramsPagingState, + pwsHasMore, module C, ) where @@ -34,10 +38,11 @@ import Cassandra.CQL (Consistency, R) import Control.Monad.Catch import Data.Conduit -- We only use these locally. -import Database.CQL.IO (RetrySettings, RunQ, defRetrySettings, eagerRetrySettings) +import Database.CQL.IO (ProtocolError (UnexpectedResponse), RetrySettings, RunQ, defRetrySettings, eagerRetrySettings, getResult, hrHost, hrResponse, runQ) -- Things we just import and re-export. import Database.CQL.IO as C (BatchM, Client, ClientState, MonadClient, Page (..), PrepQuery, Row, addPrepQuery, addQuery, adjustConsistency, adjustResponseTimeout, adjustSendTimeout, batch, emptyPage, init, liftClient, localState, paginate, prepared, query, query1, queryString, retry, runClient, schema, setConsistency, setSerialConsistency, setType, shutdown, trans, write) -import Database.CQL.Protocol (Error, QueryParams (QueryParams), Tuple) +import Database.CQL.Protocol (Error, QueryParams (QueryParams), Tuple, pagingState) +import qualified Database.CQL.Protocol as Protocol import Imports hiding (init) params :: Tuple a => Consistency -> a -> QueryParams a @@ -100,3 +105,29 @@ paginateC q p r = go =<< lift (retry r (paginate q p)) yield (result page) when (hasMore page) $ go =<< lift (retry r (liftClient (nextPage page))) + +data PageWithState a = PageWithState + { pwsResults :: [a], + pwsState :: Maybe Protocol.PagingState + } + deriving (Functor) + +-- | Like 'paginate' but exposes the paging state. This paging state can be +-- serialised and sent to consumers of the API. The state is not good for long +-- term storage as the bytestring format may change when the schema of a table +-- changes or when cassandra is upgraded. +paginateWithState :: (MonadClient m, Tuple a, Tuple b, RunQ q) => q R a b -> QueryParams a -> m (PageWithState b) +paginateWithState q p = do + let p' = p {Protocol.pageSize = Protocol.pageSize p <|> Just 10000} + r <- runQ q p' + getResult r >>= \case + Protocol.RowsResult m b -> + return $ PageWithState b (pagingState m) + _ -> throwM $ UnexpectedResponse (hrHost r) (hrResponse r) + +paramsPagingState :: Consistency -> a -> Int32 -> Maybe Protocol.PagingState -> QueryParams a +paramsPagingState c p n state = QueryParams c False p (Just n) state Nothing Nothing +{-# INLINE paramsPagingState #-} + +pwsHasMore :: PageWithState a -> Bool +pwsHasMore = isJust . pwsState diff --git a/libs/types-common/src/Data/Json/Util.hs b/libs/types-common/src/Data/Json/Util.hs index 21e1e9b7091..bed1399f7fb 100644 --- a/libs/types-common/src/Data/Json/Util.hs +++ b/libs/types-common/src/Data/Json/Util.hs @@ -37,6 +37,7 @@ module Data.Json.Util -- * Base64 Base64ByteString (..), fromBase64TextLenient, + fromBase64Text, toBase64Text, ) where @@ -200,5 +201,8 @@ instance Arbitrary Base64ByteString where fromBase64TextLenient :: Text -> ByteString fromBase64TextLenient = B64.decodeLenient . Text.encodeUtf8 +fromBase64Text :: Text -> Either String ByteString +fromBase64Text = B64.decode . Text.encodeUtf8 + toBase64Text :: ByteString -> Text toBase64Text = Text.decodeUtf8 . B64.encode diff --git a/libs/wire-api/src/Wire/API/Conversation.hs b/libs/wire-api/src/Wire/API/Conversation.hs index c4b8b627f7a..55e9c50fa72 100644 --- a/libs/wire-api/src/Wire/API/Conversation.hs +++ b/libs/wire-api/src/Wire/API/Conversation.hs @@ -1,4 +1,5 @@ {-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE StrictData #-} -- This file is part of the Wire Server implementation. @@ -18,14 +19,18 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . --- FUTUREWORK: --- There's still a lot of stuff we should factor out into separate modules. +-- FUTUREWORK: There's still a lot of stuff we should factor out into separate +-- modules. module Wire.API.Conversation ( -- * Conversation Conversation (..), ConversationCoverView (..), ConversationList (..), ListConversations (..), + GetPaginatedConversationIds (..), + ConversationPagingState (..), + ConversationPagingTable (..), + ConvIdsPage (..), -- * Conversation properties Access (..), @@ -73,13 +78,16 @@ import Control.Applicative import Control.Lens (at, (?~)) import Data.Aeson (FromJSON (..), ToJSON (..)) import qualified Data.Aeson as A +import qualified Data.Attoparsec.ByteString as AB +import qualified Data.ByteString as BS import Data.Id +import Data.Json.Util (fromBase64Text, toBase64Text) import Data.List.NonEmpty (NonEmpty) import Data.List1 import Data.Misc import Data.Proxy (Proxy (Proxy)) import Data.Qualified (Qualified (qUnqualified), deprecatedSchema) -import Data.Range (Range) +import Data.Range (Range, toRange) import Data.Schema import qualified Data.Set as Set import Data.String.Conversions (cs) @@ -225,6 +233,9 @@ instance ConversationListItem ConvId where instance ConversationListItem Conversation where convListItemName _ = "conversations" +instance ConversationListItem (Qualified ConvId) where + convListItemName _ = "qualified Conversation IDs" + instance (ConversationListItem a, S.ToSchema a) => S.ToSchema (ConversationList a) where declareNamedSchema _ = do listSchema <- S.declareSchemaRef (Proxy @[a]) @@ -252,6 +263,85 @@ instance FromJSON a => FromJSON (ConversationList a) where <$> o A..: "conversations" <*> o A..: "has_more" +data ConvIdsPage = ConvIdsPage + { pageConvIds :: [Qualified ConvId], + pageHasMore :: Bool, + pagePagingState :: ConversationPagingState + } + deriving (Show, Eq, Generic) + deriving (FromJSON, ToJSON, S.ToSchema) via Schema ConvIdsPage + +instance ToSchema ConvIdsPage where + schema = + object "ConvIdsPage" $ + ConvIdsPage + <$> pageConvIds .= field "qualified_conversations" (array schema) + <*> pageHasMore .= field "has_more" schema + <*> pagePagingState .= field "paging_state" schema + +data ConversationPagingState = ConversationPagingState + { cpsTable :: ConversationPagingTable, + cpsPagingState :: Maybe ByteString + } + deriving (Show, Eq) + deriving (FromJSON, ToJSON, S.ToSchema) via Schema ConversationPagingState + +instance ToSchema ConversationPagingState where + schema = + (toBase64Text . encodeConversationPagingState) + .= parsedText "ConversationPagingState" (parseConvesationPagingState <=< fromBase64Text) + +parseConvesationPagingState :: ByteString -> Either String ConversationPagingState +parseConvesationPagingState = AB.parseOnly conversationPagingStateParser + +conversationPagingStateParser :: AB.Parser ConversationPagingState +conversationPagingStateParser = do + cpsTable <- tableParser + cpsPagingState <- (AB.endOfInput $> Nothing) <|> (Just <$> AB.takeByteString <* AB.endOfInput) + pure ConversationPagingState {..} + where + tableParser :: AB.Parser ConversationPagingTable + tableParser = + (AB.word8 0 $> PagingLocals) + <|> (AB.word8 1 $> PagingRemotes) + +encodeConversationPagingState :: ConversationPagingState -> ByteString +encodeConversationPagingState ConversationPagingState {..} = + let table = encodeConversationPagingTable cpsTable + state = fromMaybe "" cpsPagingState + in BS.cons table state + +encodeConversationPagingTable :: ConversationPagingTable -> Word8 +encodeConversationPagingTable = \case + PagingLocals -> 0 + PagingRemotes -> 1 + +data ConversationPagingTable + = PagingLocals + | PagingRemotes + deriving (Show, Eq) + +data GetPaginatedConversationIds = GetPaginatedConversationIds + { gpciPagingState :: Maybe ConversationPagingState, + gpciSize :: Range 1 1000 Int32 + } + deriving stock (Eq, Show, Generic) + deriving (FromJSON, ToJSON, S.ToSchema) via Schema GetPaginatedConversationIds + +instance ToSchema GetPaginatedConversationIds where + schema = + let addPagingStateDoc = + description + ?~ "optional, when not specified first page of the conversation ids will be returned.\ + \Every returned page contains a paging_state, this should be supplied to retrieve the next page." + addSizeDoc = description ?~ "optional, must be <= 1000, defaults to 1000." + in objectWithDocModifier + "GetPaginatedConversationIds" + (description ?~ "A request to list some or all of a user's conversation ids, including remote ones") + $ GetPaginatedConversationIds + <$> gpciPagingState .= optFieldWithDocModifier "paging_state" Nothing addPagingStateDoc schema + <*> gpciSize .= (fieldWithDocModifier "size" addSizeDoc schema <|> pure (toRange (Proxy @1000))) + -- | Used on the POST /list-conversations endpoint -- FUTUREWORK: add to golden tests (how to generate them?) data ListConversations = ListConversations diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs index bb1be945aa7..e93b561e200 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs @@ -126,9 +126,9 @@ data Api routes = Api :> Capture "cnv" ConvId :> "roles" :> Get '[Servant.JSON] Public.ConversationRolesList, - getConversationIds :: + listConversationIdsUnqualified :: routes - :- Summary "Get all conversation IDs." + :- Summary "[deprecated] Get all local conversation IDs." -- FUTUREWORK: add bounds to swagger schema for Range :> ZUser :> "conversations" @@ -148,6 +148,15 @@ data Api routes = Api "size" (Range 1 1000 Int32) :> Get '[Servant.JSON] (Public.ConversationList ConvId), + listConversationIds :: + routes + :- Summary "Get all conversation IDs." + :> Description "To retrieve the next page, a client must pass the paging_state returned by the previous page." + :> ZUser + :> "conversations" + :> "list-ids" + :> ReqBody '[Servant.JSON] Public.GetPaginatedConversationIds + :> Post '[Servant.JSON] Public.ConvIdsPage, getConversations :: routes :- Summary "Get all *local* conversations." diff --git a/libs/wire-api/test/golden/testObject_ConvIdsPage_1.json b/libs/wire-api/test/golden/testObject_ConvIdsPage_1.json new file mode 100644 index 00000000000..e56981dcc2d --- /dev/null +++ b/libs/wire-api/test/golden/testObject_ConvIdsPage_1.json @@ -0,0 +1,5 @@ +{ + "has_more": false, + "paging_state": "AA==", + "qualified_conversations": [] +} \ No newline at end of file diff --git a/libs/wire-api/test/golden/testObject_ConvIdsPage_2.json b/libs/wire-api/test/golden/testObject_ConvIdsPage_2.json new file mode 100644 index 00000000000..54491bffe9d --- /dev/null +++ b/libs/wire-api/test/golden/testObject_ConvIdsPage_2.json @@ -0,0 +1,10 @@ +{ + "has_more": true, + "paging_state": "AA==", + "qualified_conversations": [ + { + "domain": "domain.example.com", + "id": "00000018-0000-0020-0000-000e00000002" + } + ] +} \ No newline at end of file diff --git a/libs/wire-api/test/golden/testObject_ConversationPagingState_1.json b/libs/wire-api/test/golden/testObject_ConversationPagingState_1.json new file mode 100644 index 00000000000..12ccb495bea --- /dev/null +++ b/libs/wire-api/test/golden/testObject_ConversationPagingState_1.json @@ -0,0 +1 @@ +"AA==" \ No newline at end of file diff --git a/libs/wire-api/test/golden/testObject_ConversationPagingState_2.json b/libs/wire-api/test/golden/testObject_ConversationPagingState_2.json new file mode 100644 index 00000000000..400bef583ca --- /dev/null +++ b/libs/wire-api/test/golden/testObject_ConversationPagingState_2.json @@ -0,0 +1 @@ +"AAABWGN9WA==" \ No newline at end of file diff --git a/libs/wire-api/test/golden/testObject_ConversationPagingState_3.json b/libs/wire-api/test/golden/testObject_ConversationPagingState_3.json new file mode 100644 index 00000000000..3ca82884d86 --- /dev/null +++ b/libs/wire-api/test/golden/testObject_ConversationPagingState_3.json @@ -0,0 +1 @@ +"AQ==" \ No newline at end of file diff --git a/libs/wire-api/test/golden/testObject_ConversationPagingState_4.json b/libs/wire-api/test/golden/testObject_ConversationPagingState_4.json new file mode 100644 index 00000000000..f04aaae71e3 --- /dev/null +++ b/libs/wire-api/test/golden/testObject_ConversationPagingState_4.json @@ -0,0 +1 @@ +"AVgMFw==" \ No newline at end of file diff --git a/libs/wire-api/test/golden/testObject_GetPaginatedConversationIds_1.json b/libs/wire-api/test/golden/testObject_GetPaginatedConversationIds_1.json new file mode 100644 index 00000000000..57dfcf1da08 --- /dev/null +++ b/libs/wire-api/test/golden/testObject_GetPaginatedConversationIds_1.json @@ -0,0 +1,3 @@ +{ + "size": 50 +} \ No newline at end of file diff --git a/libs/wire-api/test/golden/testObject_GetPaginatedConversationIds_2.json b/libs/wire-api/test/golden/testObject_GetPaginatedConversationIds_2.json new file mode 100644 index 00000000000..5de0d4bea98 --- /dev/null +++ b/libs/wire-api/test/golden/testObject_GetPaginatedConversationIds_2.json @@ -0,0 +1,4 @@ +{ + "size": 1000, + "paging_state": "AA==" +} \ No newline at end of file diff --git a/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual.hs b/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual.hs index 8ba523d2d02..bd7fd0d8b03 100644 --- a/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual.hs +++ b/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual.hs @@ -22,8 +22,11 @@ import Test.Tasty import Test.Tasty.HUnit import Test.Wire.API.Golden.Manual.ClientCapability import Test.Wire.API.Golden.Manual.ClientCapabilityList +import Test.Wire.API.Golden.Manual.ConvIdsPage import Test.Wire.API.Golden.Manual.ConversationCoverView +import Test.Wire.API.Golden.Manual.ConversationPagingState import Test.Wire.API.Golden.Manual.FeatureConfigEvent +import Test.Wire.API.Golden.Manual.GetPaginatedConversationIds import Test.Wire.API.Golden.Manual.QualifiedUserClientPrekeyMap import Test.Wire.API.Golden.Manual.UserClientPrekeyMap import Test.Wire.API.Golden.Runner @@ -54,6 +57,23 @@ tests = (testObject_ConversationCoverView_2, "testObject_ConversationCoverView_2.json"), (testObject_ConversationCoverView_3, "testObject_ConversationCoverView_3.json") ], + testCase "GetPaginatedConversationIds" $ + testObjects + [ (testObject_GetPaginatedConversationIds_1, "testObject_GetPaginatedConversationIds_1.json"), + (testObject_GetPaginatedConversationIds_2, "testObject_GetPaginatedConversationIds_2.json") + ], + testCase "ConversationPagingState" $ + testObjects + [ (testObject_ConversationPagingState_1, "testObject_ConversationPagingState_1.json"), + (testObject_ConversationPagingState_2, "testObject_ConversationPagingState_2.json"), + (testObject_ConversationPagingState_3, "testObject_ConversationPagingState_3.json"), + (testObject_ConversationPagingState_4, "testObject_ConversationPagingState_4.json") + ], + testCase "ConvIdsPage" $ + testObjects + [ (testObject_ConvIdsPage_1, "testObject_ConvIdsPage_1.json"), + (testObject_ConvIdsPage_2, "testObject_ConvIdsPage_2.json") + ], testCase "ClientCapability" $ testObjects [(testObject_ClientCapability_1, "testObject_ClientCapability_1.json")], diff --git a/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/ConvIdsPage.hs b/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/ConvIdsPage.hs new file mode 100644 index 00000000000..e81ba688147 --- /dev/null +++ b/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/ConvIdsPage.hs @@ -0,0 +1,15 @@ +module Test.Wire.API.Golden.Manual.ConvIdsPage where + +import Data.Domain (Domain (Domain)) +import Data.Id (Id (Id)) +import Data.Qualified +import qualified Data.UUID as UUID +import Imports +import Test.Wire.API.Golden.Manual.ConversationPagingState (testObject_ConversationPagingState_1) +import Wire.API.Conversation + +testObject_ConvIdsPage_1 :: ConvIdsPage +testObject_ConvIdsPage_1 = ConvIdsPage [] False testObject_ConversationPagingState_1 + +testObject_ConvIdsPage_2 :: ConvIdsPage +testObject_ConvIdsPage_2 = ConvIdsPage [Qualified (Id (fromJust (UUID.fromString "00000018-0000-0020-0000-000e00000002"))) (Domain "domain.example.com")] True testObject_ConversationPagingState_1 diff --git a/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/ConversationPagingState.hs b/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/ConversationPagingState.hs new file mode 100644 index 00000000000..bcf75ee423d --- /dev/null +++ b/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/ConversationPagingState.hs @@ -0,0 +1,17 @@ +module Test.Wire.API.Golden.Manual.ConversationPagingState where + +import qualified Data.ByteString as BS +import Imports +import Wire.API.Conversation + +testObject_ConversationPagingState_1 :: ConversationPagingState +testObject_ConversationPagingState_1 = ConversationPagingState PagingLocals Nothing + +testObject_ConversationPagingState_2 :: ConversationPagingState +testObject_ConversationPagingState_2 = ConversationPagingState PagingLocals (Just (BS.pack [0, 1, 88, 99, 125, 88])) + +testObject_ConversationPagingState_3 :: ConversationPagingState +testObject_ConversationPagingState_3 = ConversationPagingState PagingRemotes Nothing + +testObject_ConversationPagingState_4 :: ConversationPagingState +testObject_ConversationPagingState_4 = ConversationPagingState PagingRemotes (Just (BS.pack [88, 12, 23])) diff --git a/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/GetPaginatedConversationIds.hs b/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/GetPaginatedConversationIds.hs new file mode 100644 index 00000000000..5848a8b0fb2 --- /dev/null +++ b/libs/wire-api/test/unit/Test/Wire/API/Golden/Manual/GetPaginatedConversationIds.hs @@ -0,0 +1,12 @@ +module Test.Wire.API.Golden.Manual.GetPaginatedConversationIds where + +import Data.Proxy +import Data.Range +import Imports +import Wire.API.Conversation + +testObject_GetPaginatedConversationIds_1 :: GetPaginatedConversationIds +testObject_GetPaginatedConversationIds_1 = GetPaginatedConversationIds Nothing (toRange (Proxy @50)) + +testObject_GetPaginatedConversationIds_2 :: GetPaginatedConversationIds +testObject_GetPaginatedConversationIds_2 = GetPaginatedConversationIds (Just $ ConversationPagingState PagingLocals Nothing) (toRange (Proxy @1000)) diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal index f09feee5562..76a60c29555 100644 --- a/libs/wire-api/wire-api.cabal +++ b/libs/wire-api/wire-api.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: c6b3b26309d37316c1e5f29b10e0a9fba3ab066898d161965fd20fa531237a89 +-- hash: 9152de17654638a8439538e5d42de1b136055ce828561a44f3ca3e3b3d28fdba name: wire-api version: 0.1.0 @@ -399,7 +399,10 @@ test-suite wire-api-tests Test.Wire.API.Golden.Manual.ClientCapability Test.Wire.API.Golden.Manual.ClientCapabilityList Test.Wire.API.Golden.Manual.ConversationCoverView + Test.Wire.API.Golden.Manual.ConversationPagingState + Test.Wire.API.Golden.Manual.ConvIdsPage Test.Wire.API.Golden.Manual.FeatureConfigEvent + Test.Wire.API.Golden.Manual.GetPaginatedConversationIds Test.Wire.API.Golden.Manual.ListConversations Test.Wire.API.Golden.Manual.QualifiedUserClientPrekeyMap Test.Wire.API.Golden.Manual.UserClientPrekeyMap diff --git a/services/galley/src/Galley/API/Public.hs b/services/galley/src/Galley/API/Public.hs index b2b5ffb1474..b256288b0ac 100644 --- a/services/galley/src/Galley/API/Public.hs +++ b/services/galley/src/Galley/API/Public.hs @@ -80,7 +80,8 @@ servantSitemap = { GalleyAPI.getUnqualifiedConversation = Query.getUnqualifiedConversation, GalleyAPI.getConversation = Query.getConversation, GalleyAPI.getConversationRoles = Query.getConversationRoles, - GalleyAPI.getConversationIds = Query.getConversationIds, + GalleyAPI.listConversationIdsUnqualified = Query.conversationIdsPageFromUnqualified, + GalleyAPI.listConversationIds = Query.conversationIdsPageFrom, GalleyAPI.getConversations = Query.getConversations, GalleyAPI.getConversationByReusableCode = Query.getConversationByReusableCode, GalleyAPI.listConversations = Query.listConversations, diff --git a/services/galley/src/Galley/API/Query.hs b/services/galley/src/Galley/API/Query.hs index eafebb67281..d401ed47306 100644 --- a/services/galley/src/Galley/API/Query.hs +++ b/services/galley/src/Galley/API/Query.hs @@ -14,13 +14,15 @@ -- -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . +{-# LANGUAGE RecordWildCards #-} module Galley.API.Query ( getBotConversationH, getUnqualifiedConversation, getConversation, getConversationRoles, - getConversationIds, + conversationIdsPageFromUnqualified, + conversationIdsPageFrom, getConversations, listConversations, iterateConversations, @@ -31,7 +33,9 @@ module Galley.API.Query ) where +import qualified Cassandra as C import Control.Monad.Catch (throwM) +import qualified Data.ByteString.Lazy as LBS import Data.Code import Data.CommaSeparatedList import Data.Domain (Domain) @@ -118,8 +122,8 @@ getConversationRoles zusr cnv = do -- be merged with the team roles (if they exist) pure $ Public.ConversationRolesList wireConvRoles -getConversationIds :: UserId -> Maybe ConvId -> Maybe (Range 1 1000 Int32) -> Galley (Public.ConversationList ConvId) -getConversationIds zusr start msize = do +conversationIdsPageFromUnqualified :: UserId -> Maybe ConvId -> Maybe (Range 1 1000 Int32) -> Galley (Public.ConversationList ConvId) +conversationIdsPageFromUnqualified zusr start msize = do let size = fromMaybe (toRange (Proxy @1000)) msize ids <- Data.conversationIdsFrom zusr start size pure $ @@ -127,6 +131,50 @@ getConversationIds zusr start msize = do (Data.resultSetResult ids) (Data.resultSetType ids == Data.ResultSetTruncated) +-- | Lists conversation ids for the logged in user in a paginated way. +-- +-- Pagination requires an order, in this case the order is defined as: +-- +-- - First all the local conversations are listed ordered by their id +-- +-- - After local conversations, remote conversations are listed ordered +-- - lexicographically by their domain and then by their id. +conversationIdsPageFrom :: UserId -> Public.GetPaginatedConversationIds -> Galley Public.ConvIdsPage +conversationIdsPageFrom zusr Public.GetPaginatedConversationIds {..} = do + localDomain <- viewFederationDomain + case gpciPagingState of + Just (Public.ConversationPagingState Public.PagingRemotes stateBS) -> remotesOnly (mkState <$> stateBS) (fromRange gpciSize) + _ -> localsAndRemotes localDomain (fmap mkState . Public.cpsPagingState =<< gpciPagingState) gpciSize + where + mkState :: ByteString -> C.PagingState + mkState = C.PagingState . LBS.fromStrict + + localsAndRemotes :: Domain -> Maybe C.PagingState -> Range 1 1000 Int32 -> Galley Public.ConvIdsPage + localsAndRemotes localDomain pagingState size = do + localPage <- pageToConvIdPage Public.PagingLocals . fmap (`Qualified` localDomain) <$> Data.localConversationIdsPageFrom zusr pagingState size + let remainingSize = fromRange size - fromIntegral (length (Public.pageConvIds localPage)) + if Public.pageHasMore localPage || remainingSize <= 0 + then pure localPage {Public.pageHasMore = True} -- We haven't check the remotes yet, so has_more must always be True here. + else do + remotePage <- remotesOnly Nothing remainingSize + pure $ remotePage {Public.pageConvIds = Public.pageConvIds localPage <> Public.pageConvIds remotePage} + + remotesOnly :: Maybe C.PagingState -> Int32 -> Galley Public.ConvIdsPage + remotesOnly pagingState size = + pageToConvIdPage Public.PagingRemotes <$> Data.remoteConversationIdsPageFrom zusr pagingState size + + pageToConvIdPage :: Public.ConversationPagingTable -> Data.PageWithState (Qualified ConvId) -> Public.ConvIdsPage + pageToConvIdPage table page@Data.PageWithState {..} = + Public.ConvIdsPage + { pageConvIds = pwsResults, + pageHasMore = C.pwsHasMore page, + pagePagingState = + Public.ConversationPagingState + { cpsTable = table, + cpsPagingState = LBS.toStrict . C.unPagingState <$> pwsState + } + } + getConversations :: UserId -> Maybe (Range 1 32 (CommaSeparatedList ConvId)) -> Maybe ConvId -> Maybe (Range 1 500 Int32) -> Galley (Public.ConversationList Public.Conversation) getConversations user mids mstart msize = do ConversationList cs more <- getConversationsInternal user mids mstart msize diff --git a/services/galley/src/Galley/Data.hs b/services/galley/src/Galley/Data.hs index 84175711055..2e14434e5d1 100644 --- a/services/galley/src/Galley/Data.hs +++ b/services/galley/src/Galley/Data.hs @@ -20,6 +20,7 @@ module Galley.Data ( ResultSet, ResultSetType (..), + PageWithState (..), mkResultSet, resultSetType, resultSetResult, @@ -59,6 +60,7 @@ module Galley.Data acceptConnect, conversation, conversationIdsFrom, + localConversationIdsPageFrom, conversationIdRowsForPagination, conversationIdsOf, conversationMeta, @@ -75,6 +77,7 @@ module Galley.Data updateConversationMessageTimer, deleteConversation, lookupReceiptMode, + remoteConversationIdsPageFrom, -- * Conversation Members addMember, @@ -538,8 +541,9 @@ conversationMeta conv = where toConvMeta (t, c, a, r, n, i, _, mt, rm) = ConversationMeta conv t c (defAccess t a) (maybeRole t r) n i mt rm +-- | Deprecated, use 'localConversationIdsPageFrom' conversationIdsFrom :: - (MonadClient m, Log.MonadLogger m, MonadThrow m) => + (MonadClient m) => UserId -> Maybe ConvId -> Range 1 1000 Int32 -> @@ -551,6 +555,19 @@ conversationIdsFrom usr start (fromRange -> max) = where strip p = p {result = take (fromIntegral max) (result p)} +localConversationIdsPageFrom :: + (MonadClient m) => + UserId -> + Maybe PagingState -> + Range 1 1000 Int32 -> + m (PageWithState ConvId) +localConversationIdsPageFrom usr pagingState (fromRange -> max) = + fmap runIdentity <$> paginateWithState Cql.selectUserConvs (paramsPagingState Quorum (Identity usr) max pagingState) + +remoteConversationIdsPageFrom :: (MonadClient m) => UserId -> Maybe PagingState -> Int32 -> m (PageWithState (Qualified ConvId)) +remoteConversationIdsPageFrom usr pagingState max = + uncurry (flip Qualified) <$$> paginateWithState Cql.selectUserRemoteConvs (paramsPagingState Quorum (Identity usr) max pagingState) + conversationIdRowsForPagination :: MonadClient m => UserId -> Maybe ConvId -> Range 1 1000 Int32 -> m (Page ConvId) conversationIdRowsForPagination usr start (fromRange -> max) = runIdentity diff --git a/services/galley/src/Galley/Data/Queries.hs b/services/galley/src/Galley/Data/Queries.hs index 6af659366bc..bd8cdd6277e 100644 --- a/services/galley/src/Galley/Data/Queries.hs +++ b/services/galley/src/Galley/Data/Queries.hs @@ -305,7 +305,7 @@ insertUserRemoteConv :: PrepQuery W (UserId, Domain, ConvId) () insertUserRemoteConv = "insert into user_remote_conv (user, conv_remote_domain, conv_remote_id) values (?, ?, ?)" selectUserRemoteConvs :: PrepQuery R (Identity UserId) (Domain, ConvId) -selectUserRemoteConvs = "select conv_remote_domain, conv_remote_id from user_remote_conv where user = ? order by conv_remote_domain" +selectUserRemoteConvs = "select conv_remote_domain, conv_remote_id from user_remote_conv where user = ?" selectRemoteConvMembership :: PrepQuery R (UserId, Domain, ConvId) (Identity UserId) selectRemoteConvMembership = "select user from user_remote_conv where user = ? and conv_remote_domain = ? and conv_remote_id = ?" diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs index d950cf252ca..28b6471eb95 100644 --- a/services/galley/test/integration/API.hs +++ b/services/galley/test/integration/API.hs @@ -50,6 +50,7 @@ import Data.List.NonEmpty (NonEmpty (..)) import Data.List1 import qualified Data.List1 as List1 import qualified Data.Map.Strict as Map +import Data.Proxy (Proxy (..)) import Data.Qualified import Data.Range import qualified Data.Set as Set @@ -116,7 +117,10 @@ tests s = test s "list-conversations by ids" listConvsOk2, test s "fail to get >500 conversations" getConvsFailMaxSize, test s "get conversation ids" getConvIdsOk, + test s "get conversation ids v2" listConvIdsOk, test s "paginate through conversation ids" paginateConvIds, + test s "paginate through /conversations/list-ids" paginateConvListIds, + test s "paginate through /conversations/list-ids - page ending at locals and remote domain" paginateConvListIdsPageEndingAtLocalsAndDomain, test s "fail to get >1000 conversation ids" getConvIdsFailMaxSize, test s "page through conversations" getConvsPagingOk, test s "page through list-conversations (local conversations only)" listConvsPagingOk, @@ -667,12 +671,12 @@ postMessageQualifiedLocalOwningBackendRedundantAndDeletedClients = do let expectedRedundant = QualifiedUserClients . Map.fromList $ [ ( owningDomain, - Map.fromList $ + Map.fromList [ (nonMemberUnqualified, Set.singleton nonMemberOwningDomainClient) ] ), ( remoteDomain, - Map.fromList $ + Map.fromList [ (nonMemberRemoteUnqualified, Set.singleton nonMemberRemoteClient) ] ) @@ -1220,17 +1224,27 @@ paginateConvIds = do [alice, bob, eve] <- randomUsers 3 connectUsers alice (singleton bob) connectUsers alice (singleton eve) - replicateM_ 256 $ + replicateM_ 253 $ postConv alice [bob, eve] (Just "gossip") [] Nothing Nothing !!! const 201 === statusCode - foldM_ (getChunk 16 alice) Nothing [15 .. 0 :: Int] + -- 1 self conv, 2 convs with bob and eve, 253 gossips = 256 convs + foldM_ (getChunk 16 alice) Nothing [15, 14 .. 0 :: Int] where getChunk size alice start n = do resp <- getConvIds alice start (Just size) 0 + -- This is because of the way this test is setup, we always get 16 + -- convs, even on the last one + assertEqual + ("Number of convs should match the requested size, " <> show n <> " more gets to go") + (fromIntegral size) + (length (convList c)) + + if n > 0 + then assertEqual "hasMore should be True" True (convHasMore c) + else assertEqual ("hasMore should be False, " <> show n <> " more chunks to go") False (convHasMore c) + return (Just (Right (last (convList c)))) getConvIdsFailMaxSize :: TestM () @@ -1239,6 +1253,148 @@ getConvIdsFailMaxSize = do getConvIds usr Nothing (Just 1001) !!! const 400 === statusCode +listConvIdsOk :: TestM () +listConvIdsOk = do + [alice, bob] <- randomUsers 2 + connectUsers alice (singleton bob) + void $ postO2OConv alice bob (Just "gossip") + let paginationOpts = GetPaginatedConversationIds Nothing (toRange (Proxy @5)) + listConvIds alice paginationOpts !!! do + const 200 === statusCode + const (Right 2) === fmap length . decodeQualifiedConvIdList + listConvIds bob paginationOpts !!! do + const 200 === statusCode + const (Right 2) === fmap length . decodeQualifiedConvIdList + +paginateConvListIds :: TestM () +paginateConvListIds = do + [alice, bob, eve] <- randomUsers 3 + connectUsers alice (list1 bob [eve]) + localDomain <- viewFederationDomain + let qAlice = Qualified alice localDomain + now <- liftIO getCurrentTime + fedGalleyClient <- view tsFedGalleyClient + + replicateM_ 197 $ + postConv alice [bob, eve] (Just "gossip") [] Nothing Nothing + !!! const 201 === statusCode + + remoteChad <- randomId + let chadDomain = Domain "chad.example.com" + qChad = Qualified remoteChad chadDomain + replicateM_ 25 $ do + conv <- randomId + let cmu = + FederatedGalley.ConversationMemberUpdate + { FederatedGalley.cmuTime = now, + FederatedGalley.cmuOrigUserId = qChad, + FederatedGalley.cmuConvId = Qualified conv chadDomain, + FederatedGalley.cmuAlreadyPresentUsers = [], + FederatedGalley.cmuUsersAdd = [(qAlice, roleNameWireMember)], + FederatedGalley.cmuUsersRemove = [] + } + FederatedGalley.updateConversationMemberships fedGalleyClient cmu + + remoteDee <- randomId + let deeDomain = Domain "dee.example.com" + qDee = Qualified remoteDee deeDomain + replicateM_ 31 $ do + conv <- randomId + let cmu = + FederatedGalley.ConversationMemberUpdate + { FederatedGalley.cmuTime = now, + FederatedGalley.cmuOrigUserId = qDee, + FederatedGalley.cmuConvId = Qualified conv deeDomain, + FederatedGalley.cmuAlreadyPresentUsers = [], + FederatedGalley.cmuUsersAdd = [(qAlice, roleNameWireMember)], + FederatedGalley.cmuUsersRemove = [] + } + FederatedGalley.updateConversationMemberships fedGalleyClient cmu + + -- 1 self conv + 2 convs with bob and eve + 197 local convs + 25 convs on + -- chad.example.com + 31 on dee.example = 256 convs. Getting them 16 at a time + -- should get all them in 16 times. + foldM_ (getChunkedConvs 16 0 alice) Nothing [16, 15 .. 0 :: Int] + +-- This test ensures to setup conversations so that a page would end exactly +-- when local convs are exhausted and then exactly when another remote domain's +-- convs are exhausted. As the local convs and remote convs are stored in two +-- different tables, this is an important edge case to test. +paginateConvListIdsPageEndingAtLocalsAndDomain :: TestM () +paginateConvListIdsPageEndingAtLocalsAndDomain = do + [alice, bob, eve] <- randomUsers 3 + connectUsers alice (list1 bob [eve]) + localDomain <- viewFederationDomain + let qAlice = Qualified alice localDomain + now <- liftIO getCurrentTime + fedGalleyClient <- view tsFedGalleyClient + + -- With page size 16, 29 group convs + 2 one-to-one convs + 1 self conv, we + -- get 32 convs. The 2nd page should end here. + replicateM_ 29 $ + postConv alice [bob, eve] (Just "gossip") [] Nothing Nothing + !!! const 201 === statusCode + + -- We should be able to page through current state in 2 pages exactly + foldM_ (getChunkedConvs 16 0 alice) Nothing [2, 1, 0 :: Int] + + remoteChad <- randomId + let chadDomain = Domain "chad.example.com" + qChad = Qualified remoteChad chadDomain + -- The 3rd page will end with this domain + replicateM_ 16 $ do + conv <- randomId + let cmu = + FederatedGalley.ConversationMemberUpdate + { FederatedGalley.cmuTime = now, + FederatedGalley.cmuOrigUserId = qChad, + FederatedGalley.cmuConvId = Qualified conv chadDomain, + FederatedGalley.cmuAlreadyPresentUsers = [], + FederatedGalley.cmuUsersAdd = [(qAlice, roleNameWireMember)], + FederatedGalley.cmuUsersRemove = [] + } + FederatedGalley.updateConversationMemberships fedGalleyClient cmu + + remoteDee <- randomId + let deeDomain = Domain "dee.example.com" + qDee = Qualified remoteDee deeDomain + -- The 4th and last page will end with this domain + replicateM_ 16 $ do + conv <- randomId + let cmu = + FederatedGalley.ConversationMemberUpdate + { FederatedGalley.cmuTime = now, + FederatedGalley.cmuOrigUserId = qDee, + FederatedGalley.cmuConvId = Qualified conv deeDomain, + FederatedGalley.cmuAlreadyPresentUsers = [], + FederatedGalley.cmuUsersAdd = [(qAlice, roleNameWireMember)], + FederatedGalley.cmuUsersRemove = [] + } + FederatedGalley.updateConversationMemberships fedGalleyClient cmu + + foldM_ (getChunkedConvs 16 0 alice) Nothing [4, 3, 2, 1, 0 :: Int] + +-- | Gets chunked conversation ids given size of each chunk, size of the last +-- chunk, requesting user and @n@ which represents how many chunks are remaining +-- to go, when this is 0, it is assumed that this chunk is last and the response +-- must set @has_more@ to 'False' and the number of conv ids returned should +-- match @lastSize@. +getChunkedConvs :: HasCallStack => Int32 -> Int -> UserId -> Maybe ConversationPagingState -> Int -> TestM (Maybe ConversationPagingState) +getChunkedConvs size lastSize alice pagingState n = do + let paginationOpts = GetPaginatedConversationIds pagingState (unsafeRange size) + resp <- listConvIds alice paginationOpts 0 + then assertEqual ("Number of convs should match the requested size, " <> show n <> " more chunks to go") (fromIntegral size) (length (pageConvIds c)) + else assertEqual "Number of convs should match the last size, no more chunks to go" lastSize (length (pageConvIds c)) + + if n > 0 + then assertEqual ("hasMore should be True, " <> show n <> " more chunk(s) to go") True (pageHasMore c) + else assertEqual "hasMore should be False, no more chunks to go" False (pageHasMore c) + + return . Just $ pagePagingState c + getConvsPagingOk :: TestM () getConvsPagingOk = do [ally, bill, carl] <- randomUsers 3 @@ -1312,7 +1468,7 @@ postConvFailNumMembers :: TestM () postConvFailNumMembers = do n <- fromIntegral <$> view tsMaxConvSize alice <- randomUser - bob : others <- replicateM n (randomUser) + bob : others <- replicateM n randomUser connectUsers alice (list1 bob others) postConv alice (bob : others) Nothing [] Nothing Nothing !!! do const 400 === statusCode @@ -1504,7 +1660,7 @@ postRepeatConnectConvCancel = do let cnv = responseJsonUnsafeWithMsg "conversation" rsp1 liftIO $ do ConnectConv @=? cnvType cnv - (Just "A") @=? cnvName cnv + Just "A" @=? cnvName cnv [] @=? cmOthers (cnvMembers cnv) privateAccess @=? cnvAccess cnv -- Alice blocks / cancels @@ -1514,7 +1670,7 @@ postRepeatConnectConvCancel = do let cnv2 = responseJsonUnsafeWithMsg "conversation" rsp2 liftIO $ do ConnectConv @=? cnvType cnv2 - (Just "A2") @=? cnvName cnv2 + Just "A2" @=? cnvName cnv2 [] @=? cmOthers (cnvMembers cnv2) privateAccess @=? cnvAccess cnv2 -- Alice blocks / cancels again @@ -1524,7 +1680,7 @@ postRepeatConnectConvCancel = do let cnv3 = responseJsonUnsafeWithMsg "conversation" rsp3 liftIO $ do ConnectConv @=? cnvType cnv3 - (Just "B") @=? cnvName cnv3 + Just "B" @=? cnvName cnv3 privateAccess @=? cnvAccess cnv3 -- Bob accepting is a no-op, since he is already a member let convId = qUnqualified . cnvQualifiedId $ cnv @@ -1532,14 +1688,14 @@ postRepeatConnectConvCancel = do cnvX <- responseJsonUnsafeWithMsg "conversation" <$> getConv bob convId liftIO $ do ConnectConv @=? cnvType cnvX - (Just "B") @=? cnvName cnvX + Just "B" @=? cnvName cnvX privateAccess @=? cnvAccess cnvX -- Alice accepts, finally turning it into a 1-1 putConvAccept alice convId !!! const 200 === statusCode cnv4 <- responseJsonUnsafeWithMsg "conversation" <$> getConv alice convId liftIO $ do One2OneConv @=? cnvType cnv4 - (Just "B") @=? cnvName cnv4 + Just "B" @=? cnvName cnv4 privateAccess @=? cnvAccess cnv4 where cancel u c = do @@ -1620,7 +1776,7 @@ leaveConnectConversation = do alice <- randomUser bob <- randomUser bdy <- postConnectConv alice bob "alice" "ni" Nothing responseJsonUnsafe bdy) + let c = maybe (error "invalid connect conversation") (qUnqualified . cnvQualifiedId) (responseJsonUnsafe bdy) deleteMember alice alice c !!! const 403 === statusCode -- FUTUREWORK: Add more tests for scenarios of federation. @@ -1806,7 +1962,7 @@ testAddRemoteMemberFederationDisabled = do -- federator endpoint not configured is equivalent to federation being disabled -- This is the case on staging/production in May 2021. let federatorNotConfigured :: Opts = opts & optFederator .~ Nothing - withSettingsOverrides federatorNotConfigured $ do + withSettingsOverrides federatorNotConfigured $ postQualifiedMembers' g alice (remoteBob :| []) convId !!! do const 400 === statusCode const (Just "federation-not-enabled") === fmap label . responseJsonUnsafe @@ -1815,7 +1971,7 @@ testAddRemoteMemberFederationDisabled = do -- misconfiguration of federator. That should give a 500. -- Port 1 should always be wrong hopefully. let federatorUnavailable :: Opts = opts & optFederator ?~ Endpoint "127.0.0.1" 1 - withSettingsOverrides federatorUnavailable $ do + withSettingsOverrides federatorUnavailable $ postQualifiedMembers' g alice (remoteBob :| []) convId !!! do const 500 === statusCode const (Just "federation-not-available") === fmap label . responseJsonUnsafe @@ -2002,12 +2158,12 @@ putMemberOk update = do Member { memId = bob, memService = Nothing, - memOtrMuted = fromMaybe False (mupOtrMute update), + memOtrMuted = Just True == mupOtrMute update, memOtrMutedStatus = mupOtrMuteStatus update, memOtrMutedRef = mupOtrMuteRef update, - memOtrArchived = fromMaybe False (mupOtrArchive update), + memOtrArchived = Just True == mupOtrArchive update, memOtrArchivedRef = mupOtrArchiveRef update, - memHidden = fromMaybe False (mupHidden update), + memHidden = Just True == mupHidden update, memHiddenRef = mupHiddenRef update, memConvRoleName = fromMaybe roleNameWireAdmin (mupConvRoleName update) } diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs index f52b2b2fdbd..fd7a8cebb88 100644 --- a/services/galley/test/integration/API/Util.hs +++ b/services/galley/test/integration/API/Util.hs @@ -834,6 +834,15 @@ getConvIds u r s = do . zType "access" . convRange r s +listConvIds :: UserId -> Public.GetPaginatedConversationIds -> TestM ResponseLBS +listConvIds u paginationOpts = do + g <- view tsGalley + post $ + g + . path "/conversations/list-ids" + . zUser u + . json paginationOpts + postQualifiedMembers :: UserId -> NonEmpty (Qualified UserId) -> ConvId -> TestM ResponseLBS postQualifiedMembers zusr invitees conv = do g <- view tsGalley @@ -1286,6 +1295,9 @@ decodeConvList = convList . responseJsonUnsafeWithMsg "conversations" decodeConvIdList :: Response (Maybe Lazy.ByteString) -> [ConvId] decodeConvIdList = convList . responseJsonUnsafeWithMsg "conversation-ids" +decodeQualifiedConvIdList :: Response (Maybe Lazy.ByteString) -> Either String [Qualified ConvId] +decodeQualifiedConvIdList = fmap pageConvIds . responseJsonEither + zUser :: UserId -> Request -> Request zUser = header "Z-User" . toByteString'