Sqservices 1548 be csv export from tm created on date missing in some instances

changelog.d/0-release-notes/pr-2543

@@ -0,0 +1 @@
+Deploy spar before galley

changelog.d/2-features/pr-2543

@@ -0,0 +1 @@
+The team member CSV export now fills `created_on` for SCIM users

libs/brig-types/src/Brig/Types/Intra.hs

@@ -33,7 +33,7 @@ where
 import Data.Aeson
 import qualified Data.Aeson.KeyMap as KeyMap
 import Data.Code as Code
-import Data.Id (TeamId, UserId)
+import Data.Id (TeamId)
 import Data.Misc (PlainTextPassword (..))
 import qualified Data.Text as Text
 import Imports
@@ -144,26 +144,6 @@ instance ToJSON NewUserScimInvitation where
         "email" .= email
       ]
 
--------------------------------------------------------------------------------
--- UserList
-
--- | Set of user ids, can be used for different purposes (e.g., used on the internal
--- APIs for listing user's clients)
-data UserSet = UserSet
-  { usUsrs :: !(Set UserId)
-  }
-  deriving (Eq, Show, Generic)
-
-instance FromJSON UserSet where
-  parseJSON = withObject "user-set" $ \o ->
-    UserSet <$> o .: "users"
-
-instance ToJSON UserSet where
-  toJSON ac =
-    object
-      [ "users" .= usUsrs ac
-      ]
-
 -------------------------------------------------------------------------------
 -- ReAuthUser
 

libs/wire-api/src/Wire/API/Routes/Public/Spar.hs

@@ -34,6 +34,7 @@ import Web.Scim.Class.User as Scim.User
 import Wire.API.Error
 import Wire.API.Error.Brig
 import Wire.API.Routes.Public
+import Wire.API.User (ScimUserInfos, UserSet)
 import Wire.API.User.IdentityProvider
 import Wire.API.User.Saml
 import Wire.API.User.Scim
@@ -124,6 +125,7 @@ type APIINTERNAL =
   "status" :> Get '[JSON] NoContent
     :<|> "teams" :> Capture "team" TeamId :> DeleteNoContent
     :<|> "sso" :> "settings" :> ReqBody '[JSON] SsoSettings :> Put '[JSON] NoContent
+    :<|> "scim" :> "userinfos" :> ReqBody '[JSON] UserSet :> Post '[JSON] ScimUserInfos
 
 sparSPIssuer :: SAML.HasConfig m => Maybe TeamId -> m SAML.Issuer
 sparSPIssuer Nothing =

libs/wire-api/src/Wire/API/User.hs

@@ -23,6 +23,9 @@ module Wire.API.User
   ( UserIdList (..),
     QualifiedUserIdList (..),
     LimitedQualifiedUserIdList (..),
+    ScimUserInfo (..),
+    ScimUserInfos (..),
+    UserSet (..),
     -- Profiles
     UserProfile (..),
     SelfProfile (..),
@@ -892,6 +895,53 @@ instance ToSchema BindingNewTeamUser where
         <$> bnuTeam .= bindingNewTeamObjectSchema
         <*> bnuCurrency .= maybe_ (optField "currency" genericToSchema)
 
+--------------------------------------------------------------------------------
+-- SCIM User Info
+
+data ScimUserInfo = ScimUserInfo
+  { suiUserId :: UserId,
+    suiCreatedOn :: Maybe UTCTimeMillis
+  }
+  deriving stock (Eq, Show, Generic)
+  deriving (Arbitrary) via (GenericUniform ScimUserInfo)
+  deriving (ToJSON, FromJSON, S.ToSchema) via (Schema ScimUserInfo)
+
+instance ToSchema ScimUserInfo where
+  schema =
+    object "ScimUserInfo" $
+      ScimUserInfo
+        <$> suiUserId .= field "id" schema
+        <*> suiCreatedOn .= maybe_ (optField "created_on" schema)
+
+newtype ScimUserInfos = ScimUserInfos {scimUserInfos :: [ScimUserInfo]}
+  deriving stock (Eq, Show, Generic)
+  deriving (Arbitrary) via (GenericUniform ScimUserInfos)
+  deriving (ToJSON, FromJSON, S.ToSchema) via (Schema ScimUserInfos)
+
+instance ToSchema ScimUserInfos where
+  schema =
+    object "ScimUserInfos" $
+      ScimUserInfos
+        <$> scimUserInfos .= field "scim_user_infos" (array schema)
+
+-------------------------------------------------------------------------------
+-- UserSet
+
+-- | Set of user ids, can be used for different purposes (e.g., used on the internal
+-- APIs for listing user's clients)
+newtype UserSet = UserSet
+  { usUsrs :: Set UserId
+  }
+  deriving stock (Eq, Show, Generic)
+  deriving newtype (Arbitrary)
+  deriving (ToJSON, FromJSON, S.ToSchema) via (Schema UserSet)
+
+instance ToSchema UserSet where
+  schema =
+    object "UserSet" $
+      UserSet
+        <$> usUsrs .= field "users" (set schema)
+
 --------------------------------------------------------------------------------
 -- Profile Updates
 

libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs

@@ -225,6 +225,7 @@ tests =
       testRoundTrip @(User.LimitedQualifiedUserIdList 20),
       testRoundTrip @User.UserProfile,
       testRoundTrip @User.User,
+      testRoundTrip @User.UserSet,
       testRoundTrip @User.SelfProfile,
       testRoundTrip @User.InvitationCode,
       testRoundTrip @User.BindingNewTeamUser,

services/galley/src/Galley/API/Teams.hs

@@ -146,7 +146,7 @@ import Wire.API.Team.Permission (Perm (..), Permissions (..), SPerm (..), copy,
 import Wire.API.Team.Role
 import Wire.API.Team.SearchVisibility
 import qualified Wire.API.Team.SearchVisibility as Public
-import Wire.API.User (User, UserIdList, UserSSOId (UserScimExternalId), userSCIMExternalId, userSSOId)
+import Wire.API.User (ScimUserInfo (..), User, UserIdList, UserSSOId (UserScimExternalId), userSCIMExternalId, userSSOId)
 import qualified Wire.API.User as U
 import Wire.API.User.Identity (UserSSOId (UserSSOId))
 import Wire.API.User.RichInfo (RichInfo)
@@ -505,7 +505,7 @@ outputToStreamingBody action = withWeavingToFinal @IO $ \state weave _inspect ->
     void . weave . (<$ state) $ runOutputSem writeChunk action
 
 getTeamMembersCSV ::
-  (Members '[BrigAccess, ErrorS 'AccessDenied, TeamMemberStore InternalPaging, TeamStore, Final IO] r) =>
+  (Members '[BrigAccess, ErrorS 'AccessDenied, TeamMemberStore InternalPaging, TeamStore, Final IO, SparAccess] r) =>
   Local UserId ->
   TeamId ->
   Sem r StreamingBody
@@ -522,16 +522,18 @@ getTeamMembersCSV lusr tid = do
     output headerLine
     E.withChunks (\mps -> E.listTeamMembers @InternalPaging tid mps maxBound) $
       \members -> do
-        inviters <- lookupInviterHandle members
-        users <-
-          lookupUser <$> E.lookupActivatedUsers (fmap (view userId) members)
-        richInfos <-
-          lookupRichInfo <$> E.getRichInfoMultiUser (fmap (view userId) members)
-        numUserClients <- lookupClients <$> E.lookupClients (fmap (view userId) members)
+        let uids = fmap (view userId) members
+        teamExportUser <-
+          mkTeamExportUser
+            <$> (lookupUser <$> E.lookupActivatedUsers uids)
+            <*> lookupInviterHandle members
+            <*> (lookupRichInfo <$> E.getRichInfoMultiUser uids)
+            <*> (lookupClients <$> E.lookupClients uids)
+            <*> (lookupScimUserInfo <$> Spar.lookupScimUserInfos uids)
         output @LByteString
           ( encodeDefaultOrderedByNameWith
               defaultEncodeOptions
-              (mapMaybe (teamExportUser users inviters richInfos numUserClients) members)
+              (mapMaybe teamExportUser members)
           )
   where
     headerLine :: LByteString
@@ -546,14 +548,15 @@ getTeamMembersCSV lusr tid = do
           encQuoting = QuoteAll
         }
 
-    teamExportUser ::
+    mkTeamExportUser ::
       (UserId -> Maybe User) ->
       (UserId -> Maybe Handle.Handle) ->
       (UserId -> Maybe RichInfo) ->
       (UserId -> Int) ->
+      (UserId -> Maybe ScimUserInfo) ->
       TeamMember ->
       Maybe TeamExportUser
-    teamExportUser users inviters richInfos numClients member = do
+    mkTeamExportUser users inviters richInfos numClients scimUserInfo member = do
       let uid = member ^. userId
       user <- users uid
       pure $
@@ -562,7 +565,7 @@ getTeamMembersCSV lusr tid = do
             tExportHandle = U.userHandle user,
             tExportEmail = U.userIdentity user >>= U.emailIdentity,
             tExportRole = permissionsRole . view permissions $ member,
-            tExportCreatedOn = fmap snd . view invitation $ member,
+            tExportCreatedOn = maybe (scimUserInfo uid >>= suiCreatedOn) (Just . snd) (view invitation member),
             tExportInvitedBy = inviters . fst =<< member ^. invitation,
             tExportIdpIssuer = userToIdPIssuer user,
             tExportManagedBy = U.userManagedBy user,
@@ -593,6 +596,9 @@ getTeamMembersCSV lusr tid = do
       Just _ -> Nothing
       Nothing -> Nothing
 
+    lookupScimUserInfo :: [ScimUserInfo] -> (UserId -> Maybe ScimUserInfo)
+    lookupScimUserInfo infos = (`M.lookup` M.fromList (infos <&> (\sui -> (suiUserId sui, sui))))
+
     lookupUser :: [U.User] -> (UserId -> Maybe U.User)
     lookupUser users = (`M.lookup` M.fromList (users <&> \user -> (U.userId user, user)))
 

services/galley/src/Galley/Effects/SparAccess.hs

@@ -21,8 +21,10 @@ module Galley.Effects.SparAccess where
 
 import Data.Id
 import Polysemy
+import Wire.API.User (ScimUserInfo)
 
 data SparAccess m a where
   DeleteTeam :: TeamId -> SparAccess m ()
+  LookupScimUserInfos :: [UserId] -> SparAccess m [ScimUserInfo]
 
 makeSem ''SparAccess

services/galley/src/Galley/Intra/Effects.hs

@@ -91,6 +91,7 @@ interpretSparAccess ::
   Sem r a
 interpretSparAccess = interpret $ \case
   DeleteTeam tid -> embedApp $ deleteTeam tid
+  LookupScimUserInfos uids -> embedApp $ lookupScimUserInfos uids
 
 interpretBotAccess ::
   Members '[Embed IO, Input Env] r =>

services/galley/src/Galley/Intra/Spar.hs

@@ -17,16 +17,19 @@
 
 module Galley.Intra.Spar
   ( deleteTeam,
+    lookupScimUserInfos,
   )
 where
 
 import Bilge
 import Data.ByteString.Conversion
 import Data.Id
+import qualified Data.Set as Set
 import Galley.Intra.Util
 import Galley.Monad
 import Imports
 import Network.HTTP.Types.Method
+import Wire.API.User (ScimUserInfo, UserSet (..), scimUserInfos)
 
 -- | Notify Spar that a team is being deleted.
 deleteTeam :: TeamId -> App ()
@@ -35,3 +38,13 @@ deleteTeam tid = do
     method DELETE
       . paths ["i", "teams", toByteString' tid]
       . expect2xx
+
+-- | Get the SCIM user info for a user.
+lookupScimUserInfos :: [UserId] -> App [ScimUserInfo]
+lookupScimUserInfos uids = do
+  response <-
+    call Spar $
+      method POST
+        . paths ["i", "scim", "userinfos"]
+        . json (UserSet $ Set.fromList uids)
+  pure $ maybe mempty scimUserInfos $ responseJsonMaybe response

services/galley/test/integration/API/Teams.hs

@@ -292,6 +292,7 @@ testListTeamMembersDefaultLimit = do
 
 -- | for ad-hoc load-testing, set @numMembers@ to, say, 10k and see what
 -- happens.  but please don't give that number to our ci!  :)
+-- for additional tests of the CSV download particularly with SCIM users, please refer to 'Test.Spar.Scim.UserSpec'
 testListTeamMembersCsv :: HasCallStack => Int -> TestM ()
 testListTeamMembersCsv numMembers = do
   let teamSize = numMembers + 1

services/galley/test/integration/API/Util.hs

@@ -29,7 +29,7 @@ import Bilge.TestSession
 -- import Galley.Types.Teams hiding (Event, EventType (..), self)
 
 import Brig.Types.Connection
-import Brig.Types.Intra (UserAccount (..), UserSet (..))
+import Brig.Types.Intra (UserAccount (..))
 import Control.Concurrent.Async
 import Control.Exception (throw)
 import Control.Lens hiding (from, to, (#), (.=))

services/spar/src/Spar/API.hs

@@ -53,6 +53,7 @@ import qualified Data.ByteString as SBS
 import Data.ByteString.Builder (toLazyByteString)
 import Data.Id
 import Data.Proxy
+import qualified Data.Set as Set
 import Data.String.Conversions
 import Data.Time
 import Galley.Types.Teams (HiddenPerm (CreateUpdateDeleteIdp, ReadIdp))
@@ -92,6 +93,7 @@ import Spar.Sem.ScimExternalIdStore (ScimExternalIdStore)
 import Spar.Sem.ScimTokenStore (ScimTokenStore)
 import qualified Spar.Sem.ScimTokenStore as ScimTokenStore
 import Spar.Sem.ScimUserTimesStore (ScimUserTimesStore)
+import qualified Spar.Sem.ScimUserTimesStore as ScimUserTimesStore
 import Spar.Sem.VerdictFormatStore (VerdictFormatStore)
 import qualified Spar.Sem.VerdictFormatStore as VerdictFormatStore
 import System.Logger (Msg)
@@ -205,14 +207,16 @@ apiINTERNAL ::
        DefaultSsoCode,
        IdPConfigStore,
        Error SparError,
-       SAMLUserStore
+       SAMLUserStore,
+       ScimUserTimesStore
      ]
     r =>
   ServerT APIINTERNAL (Sem r)
 apiINTERNAL =
   internalStatus
     :<|> internalDeleteTeam
     :<|> internalPutSsoSettings
+    :<|> internalGetScimUserInfo
 
 appName :: ST
 appName = "spar"
@@ -766,3 +770,9 @@ internalPutSsoSettings SsoSettings {defaultSsoCode = Just code} =
   IdPConfigStore.getConfig code
     *> DefaultSsoCode.store code
     $> NoContent
+
+internalGetScimUserInfo :: Members '[ScimUserTimesStore] r => UserSet -> Sem r ScimUserInfos
+internalGetScimUserInfo (UserSet uids) = do
+  results <- ScimUserTimesStore.readMulti (Set.toList uids)
+  let scimUserInfos = results <&> (\(uid, t, _) -> ScimUserInfo uid (Just t))
+  pure $ ScimUserInfos scimUserInfos

services/spar/src/Spar/Sem/ScimUserTimesStore.hs

@@ -21,6 +21,7 @@ module Spar.Sem.ScimUserTimesStore
   ( ScimUserTimesStore (..),
     write,
     read,
+    readMulti,
     delete,
   )
 where
@@ -35,6 +36,7 @@ import Web.Scim.Schema.Meta (WithMeta)
 data ScimUserTimesStore m a where
   Write :: WithMeta (WithId UserId t) -> ScimUserTimesStore m ()
   Read :: UserId -> ScimUserTimesStore m (Maybe (UTCTimeMillis, UTCTimeMillis))
+  ReadMulti :: [UserId] -> ScimUserTimesStore m [(UserId, UTCTimeMillis, UTCTimeMillis)]
   Delete :: UserId -> ScimUserTimesStore m ()
 
 makeSem ''ScimUserTimesStore

services/spar/src/Spar/Sem/ScimUserTimesStore/Cassandra.hs

@@ -35,6 +35,7 @@ scimUserTimesStoreToCassandra =
     embed @m . \case
       Write wm -> writeScimUserTimes wm
       Read uid -> readScimUserTimes uid
+      ReadMulti uids -> readScimUserTimesMulti uids
       Delete uid -> deleteScimUserTimes uid
 
 ----------------------------------------------------------------------
@@ -64,6 +65,13 @@ readScimUserTimes uid = do
     sel :: PrepQuery R (Identity UserId) (UTCTimeMillis, UTCTimeMillis)
     sel = "SELECT created_at, last_updated_at FROM scim_user_times WHERE uid = ?"
 
+readScimUserTimesMulti :: (HasCallStack, MonadClient m) => [UserId] -> m [(UserId, UTCTimeMillis, UTCTimeMillis)]
+readScimUserTimesMulti uid = do
+  retry x1 . query sel $ params LocalQuorum (Identity uid)
+  where
+    sel :: PrepQuery R (Identity [UserId]) (UserId, UTCTimeMillis, UTCTimeMillis)
+    sel = "SELECT uid, created_at, last_updated_at FROM scim_user_times WHERE uid IN ?"
+
 -- | Delete a SCIM user's access times by id.
 -- You'll also want to ensure they are deleted in Brig and in the SAML Users table.
 deleteScimUserTimes ::

services/spar/src/Spar/Sem/ScimUserTimesStore/Mem.hs

@@ -39,4 +39,5 @@ scimUserTimesStoreToMem = (runState mempty .) $
   reinterpret $ \case
     Write (WithMeta meta (WithId uid _)) -> modify $ M.insert uid (toUTCTimeMillis $ created meta, toUTCTimeMillis $ lastModified meta)
     Read uid -> gets $ M.lookup uid
+    ReadMulti uids -> gets $ map (\(u, (a, b)) -> (u, a, b)) . filter ((`elem` uids) . fst) . M.toList
     Delete uid -> modify $ M.delete uid