diff --git a/.envrc b/.envrc index e01698201b..a8b29ed74b 100644 --- a/.envrc +++ b/.envrc @@ -23,7 +23,6 @@ if [[ ! -d "$env_dir" || ! -f "$layout_dir/nix-rebuild" || "$store_paths" != $(< else nix build -f nix wireServer.devEnv -Lv --out-link ./.env fi - echo "$store_paths" > "$layout_dir/nix-rebuild" fi diff --git a/changelog.d/5-internal/WPB-5042 b/changelog.d/5-internal/WPB-5042 new file mode 100644 index 0000000000..bc192a27c7 --- /dev/null +++ b/changelog.d/5-internal/WPB-5042 @@ -0,0 +1 @@ +upgrade nixpkgs to upgrade haskell-language-server diff --git a/integration/default.nix b/integration/default.nix index 979f965d39..15ef55849c 100644 --- a/integration/default.nix +++ b/integration/default.nix @@ -20,6 +20,7 @@ , cryptonite , data-default , data-timeout +, deriving-aeson , directory , errors , exceptions @@ -92,6 +93,7 @@ mkDerivation { cryptonite data-default data-timeout + deriving-aeson directory errors exceptions diff --git a/integration/integration.cabal b/integration/integration.cabal index 43de978287..0cd573e316 100644 --- a/integration/integration.cabal +++ b/integration/integration.cabal @@ -27,6 +27,7 @@ common common-all NoImplicitPrelude AllowAmbiguousTypes BangPatterns + BlockArguments ConstraintKinds DataKinds DefaultSignatures @@ -156,6 +157,7 @@ library , cryptonite , data-default , data-timeout + , deriving-aeson , directory , errors , exceptions diff --git a/integration/test/MLS/Util.hs b/integration/test/MLS/Util.hs index 8e3f850ad0..17ffca1b70 100644 --- a/integration/test/MLS/Util.hs +++ b/integration/test/MLS/Util.hs @@ -76,7 +76,7 @@ randomFileName = do bd <- getBaseDir (bd ) . UUID.toString <$> liftIO UUIDV4.nextRandom -mlscli :: HasCallStack => ClientIdentity -> [String] -> Maybe ByteString -> App ByteString +mlscli :: (HasCallStack) => ClientIdentity -> [String] -> Maybe ByteString -> App ByteString mlscli cid args mbstdin = do groupOut <- randomFileName let substOut = argSubst "" groupOut @@ -136,7 +136,7 @@ instance MakesValue CredentialType where make BasicCredentialType = make "basic" make X509CredentialType = make "x509" -instance HasTests x => HasTests (CredentialType -> x) where +instance (HasTests x) => HasTests (CredentialType -> x) where mkTests m n s f x = mkTests m (n <> "[ctype=basic]") s f (x BasicCredentialType) <> mkTests m (n <> "[ctype=x509]") s f (x X509CredentialType) @@ -147,7 +147,7 @@ data InitMLSClient = InitMLSClient instance Default InitMLSClient where def = InitMLSClient {credType = BasicCredentialType} -initMLSClient :: HasCallStack => InitMLSClient -> ClientIdentity -> App () +initMLSClient :: (HasCallStack) => InitMLSClient -> ClientIdentity -> App () initMLSClient opts cid = do bd <- getBaseDir mls <- getMLSState @@ -175,7 +175,7 @@ createMLSClient opts u = do pure cid -- | create and upload to backend -uploadNewKeyPackage :: HasCallStack => ClientIdentity -> App String +uploadNewKeyPackage :: (HasCallStack) => ClientIdentity -> App String uploadNewKeyPackage cid = do mls <- getMLSState (kp, ref) <- generateKeyPackage cid mls.ciphersuite @@ -186,7 +186,7 @@ uploadNewKeyPackage cid = do pure ref -generateKeyPackage :: HasCallStack => ClientIdentity -> Ciphersuite -> App (ByteString, String) +generateKeyPackage :: (HasCallStack) => ClientIdentity -> Ciphersuite -> App (ByteString, String) generateKeyPackage cid suite = do kp <- mlscli cid ["key-package", "create", "--ciphersuite", suite.code] Nothing ref <- B8.unpack . Base64.encode <$> mlscli cid ["key-package", "ref", "-"] (Just kp) @@ -195,7 +195,7 @@ generateKeyPackage cid suite = do pure (kp, ref) -- | Create conversation and corresponding group. -createNewGroup :: HasCallStack => ClientIdentity -> App (String, Value) +createNewGroup :: (HasCallStack) => ClientIdentity -> App (String, Value) createNewGroup cid = do conv <- postConversation cid defMLS >>= getJSON 201 groupId <- conv %. "group_id" & asString @@ -204,7 +204,7 @@ createNewGroup cid = do pure (groupId, convId) -- | Retrieve self conversation and create the corresponding group. -createSelfGroup :: HasCallStack => ClientIdentity -> App (String, Value) +createSelfGroup :: (HasCallStack) => ClientIdentity -> App (String, Value) createSelfGroup cid = do conv <- getSelfConversation cid >>= getJSON 200 conv %. "epoch" `shouldMatchInt` 0 @@ -213,7 +213,7 @@ createSelfGroup cid = do createGroup cid conv pure (groupId, convId) -createGroup :: MakesValue conv => ClientIdentity -> conv -> App () +createGroup :: (MakesValue conv) => ClientIdentity -> conv -> App () createGroup cid conv = do mls <- getMLSState case mls.groupId of @@ -221,14 +221,14 @@ createGroup cid conv = do Nothing -> pure () resetGroup cid conv -createSubConv :: HasCallStack => ClientIdentity -> String -> App () +createSubConv :: (HasCallStack) => ClientIdentity -> String -> App () createSubConv cid subId = do mls <- getMLSState sub <- getSubConversation cid mls.convId subId >>= getJSON 200 resetGroup cid sub void $ createPendingProposalCommit cid >>= sendAndConsumeCommitBundle -resetGroup :: MakesValue conv => ClientIdentity -> conv -> App () +resetGroup :: (MakesValue conv) => ClientIdentity -> conv -> App () resetGroup cid conv = do convId <- objSubConvObject conv groupId <- conv %. "group_id" & asString @@ -261,7 +261,7 @@ resetClientGroup cid gid = do ] Nothing -keyPackageFile :: HasCallStack => ClientIdentity -> String -> App FilePath +keyPackageFile :: (HasCallStack) => ClientIdentity -> String -> App FilePath keyPackageFile cid ref = do let ref' = map urlSafe ref bd <- getBaseDir @@ -271,7 +271,7 @@ keyPackageFile cid ref = do urlSafe '/' = '_' urlSafe c = c -unbundleKeyPackages :: HasCallStack => Value -> App [(ClientIdentity, ByteString)] +unbundleKeyPackages :: (HasCallStack) => Value -> App [(ClientIdentity, ByteString)] unbundleKeyPackages bundle = do let entryIdentity be = do d <- be %. "domain" & asString @@ -290,7 +290,7 @@ unbundleKeyPackages bundle = do -- Note that this alters the state of the group immediately. If we want to test -- a scenario where the commit is rejected by the backend, we can restore the -- group to the previous state by using an older version of the group file. -createAddCommit :: HasCallStack => ClientIdentity -> [Value] -> App MessagePackage +createAddCommit :: (HasCallStack) => ClientIdentity -> [Value] -> App MessagePackage createAddCommit cid users = do mls <- getMLSState kps <- fmap concat . for users $ \user -> do @@ -310,7 +310,7 @@ withTempKeyPackageFile bs = do k fp createAddCommitWithKeyPackages :: - HasCallStack => + (HasCallStack) => ClientIdentity -> [(ClientIdentity, ByteString)] -> App MessagePackage @@ -352,7 +352,7 @@ createAddCommitWithKeyPackages cid clientsAndKeyPackages = do groupInfo = Just gi } -createRemoveCommit :: HasCallStack => ClientIdentity -> [ClientIdentity] -> App MessagePackage +createRemoveCommit :: (HasCallStack) => ClientIdentity -> [ClientIdentity] -> App MessagePackage createRemoveCommit cid targets = do bd <- getBaseDir welcomeFile <- liftIO $ emptyTempFile bd "welcome" @@ -393,14 +393,14 @@ createRemoveCommit cid targets = do groupInfo = Just gi } -createAddProposals :: HasCallStack => ClientIdentity -> [Value] -> App [MessagePackage] +createAddProposals :: (HasCallStack) => ClientIdentity -> [Value] -> App [MessagePackage] createAddProposals cid users = do mls <- getMLSState bundles <- for users $ (claimKeyPackages mls.ciphersuite cid >=> getJSON 200) kps <- concat <$> traverse unbundleKeyPackages bundles traverse (createAddProposalWithKeyPackage cid) kps -createReInitProposal :: HasCallStack => ClientIdentity -> App MessagePackage +createReInitProposal :: (HasCallStack) => ClientIdentity -> App MessagePackage createReInitProposal cid = do prop <- mlscli @@ -433,7 +433,7 @@ createAddProposalWithKeyPackage cid (_, kp) = do groupInfo = Nothing } -createPendingProposalCommit :: HasCallStack => ClientIdentity -> App MessagePackage +createPendingProposalCommit :: (HasCallStack) => ClientIdentity -> App MessagePackage createPendingProposalCommit cid = do bd <- getBaseDir welcomeFile <- liftIO $ emptyTempFile bd "welcome" @@ -464,7 +464,7 @@ createPendingProposalCommit cid = do } createExternalCommit :: - HasCallStack => + (HasCallStack) => ClientIdentity -> Maybe ByteString -> App MessagePackage @@ -509,7 +509,7 @@ data MLSNotificationTag = MLSNotificationMessageTag | MLSNotificationWelcomeTag -- | Extract a conversation ID (including an optional subconversation) from an -- event object. -eventSubConv :: HasCallStack => MakesValue event => event -> App Value +eventSubConv :: (HasCallStack) => (MakesValue event) => event -> App Value eventSubConv event = do sub <- lookupField event "subconv" conv <- event %. "qualified_conversation" @@ -519,7 +519,7 @@ eventSubConv event = do "subconv_id" .= sub ] -consumingMessages :: HasCallStack => MessagePackage -> Codensity App () +consumingMessages :: (HasCallStack) => MessagePackage -> Codensity App () consumingMessages mp = Codensity $ \k -> do mls <- getMLSState -- clients that should receive the message itself @@ -555,16 +555,14 @@ consumingMessages mp = Codensity $ \k -> do -- at this point we know that every new user has been added to the -- conversation for_ (zip clients wss) $ \((cid, t), ws) -> case t of - MLSNotificationMessageTag -> void $ consumeMessage cid (Just mp) ws + MLSNotificationMessageTag -> void $ consumeMessageNoExternal cid (Just mp) ws MLSNotificationWelcomeTag -> consumeWelcome cid mp ws pure r --- | Get a single MLS message from a websocket and consume it. Return a JSON --- representation of the message. -consumeMessage :: HasCallStack => ClientIdentity -> Maybe MessagePackage -> WebSocket -> App Value -consumeMessage cid mmp ws = do +consumeMessageWithPredicate :: (HasCallStack) => (Value -> App Bool) -> ClientIdentity -> Maybe MessagePackage -> WebSocket -> App Value +consumeMessageWithPredicate p cid mmp ws = do mls <- getMLSState - notif <- awaitMatch isNewMLSMessageNotif ws + notif <- awaitMatch p ws event <- notif %. "payload.0" for_ mmp $ \mp -> do @@ -573,30 +571,55 @@ consumeMessage cid mmp ws = do shouldMatch (event %. "data") (B8.unpack (Base64.encode mp.message)) msgData <- event %. "data" & asByteString - void $ - mlscli - cid - [ "consume", - "--group", - "", - "--group-out", - "", - "-" - ] - (Just msgData) + _ <- mlsCliConsume cid msgData showMessage cid msgData +-- | Get a single MLS message from a websocket and consume it. Return a JSON +-- representation of the message. +consumeMessage :: (HasCallStack) => ClientIdentity -> Maybe MessagePackage -> WebSocket -> App Value +consumeMessage = consumeMessageWithPredicate isNewMLSMessageNotif + +-- | like 'consumeMessage' but but will not consume a message where the sender is the backend +consumeMessageNoExternal :: (HasCallStack) => ClientIdentity -> Maybe MessagePackage -> WebSocket -> App Value +consumeMessageNoExternal cid = consumeMessageWithPredicate isNewMLSMessageNotifButNoProposal cid + where + -- the backend (correctly) reacts to a commit removing someone from a parent conversation with a + -- remove proposal, however, we don't want to consume this here + isNewMLSMessageNotifButNoProposal :: Value -> App Bool + isNewMLSMessageNotifButNoProposal n = do + isNotif <- isNewMLSMessageNotif n + if isNotif + then do + msg <- n %. "payload.0.data" & asByteString >>= showMessage cid + sender <- msg `lookupField` "message.content.sender" `catch` \(_ :: AssertionFailure) -> pure Nothing + let backendSender = object ["External" .= Number 0] + pure $ sender /= Just backendSender + else pure False + +mlsCliConsume :: ClientIdentity -> ByteString -> App ByteString +mlsCliConsume cid msgData = + mlscli + cid + [ "consume", + "--group", + "", + "--group-out", + "", + "-" + ] + (Just msgData) + -- | Send an MLS message, wait for clients to receive it, then consume it on -- the client side. If the message is a commit, the -- 'sendAndConsumeCommitBundle' function should be used instead. -sendAndConsumeMessage :: HasCallStack => MessagePackage -> App Value +sendAndConsumeMessage :: (HasCallStack) => MessagePackage -> App Value sendAndConsumeMessage mp = lowerCodensity $ do consumingMessages mp lift $ postMLSMessage mp.sender mp.message >>= getJSON 201 -- | Send an MLS commit bundle, wait for clients to receive it, consume it, and -- update the test state accordingly. -sendAndConsumeCommitBundle :: HasCallStack => MessagePackage -> App Value +sendAndConsumeCommitBundle :: (HasCallStack) => MessagePackage -> App Value sendAndConsumeCommitBundle mp = do lowerCodensity $ do consumingMessages mp @@ -620,7 +643,7 @@ sendAndConsumeCommitBundle mp = do pure r -consumeWelcome :: HasCallStack => ClientIdentity -> MessagePackage -> WebSocket -> App () +consumeWelcome :: (HasCallStack) => ClientIdentity -> MessagePackage -> WebSocket -> App () consumeWelcome cid mp ws = do mls <- getMLSState notif <- awaitMatch isWelcomeNotif ws @@ -663,7 +686,7 @@ mkBundle mp = mp.message <> foldMap mkGroupInfoMessage mp.groupInfo <> fold mp.w mkGroupInfoMessage :: ByteString -> ByteString mkGroupInfoMessage gi = BS.pack [0x00, 0x01, 0x00, 0x04] <> gi -spawn :: HasCallStack => CreateProcess -> Maybe ByteString -> App ByteString +spawn :: (HasCallStack) => CreateProcess -> Maybe ByteString -> App ByteString spawn cp minput = do (mout, ex) <- liftIO $ withCreateProcess @@ -680,22 +703,22 @@ spawn cp minput = do (Just out, ExitSuccess) -> pure out _ -> assertFailure "Failed spawning process" -getClientGroupState :: HasCallStack => ClientIdentity -> App ClientGroupState +getClientGroupState :: (HasCallStack) => ClientIdentity -> App ClientGroupState getClientGroupState cid = do mls <- getMLSState pure $ Map.findWithDefault emptyClientGroupState cid mls.clientGroupState -setClientGroupState :: HasCallStack => ClientIdentity -> ClientGroupState -> App () +setClientGroupState :: (HasCallStack) => ClientIdentity -> ClientGroupState -> App () setClientGroupState cid g = modifyMLSState $ \s -> s {clientGroupState = Map.insert cid g (clientGroupState s)} -showMessage :: HasCallStack => ClientIdentity -> ByteString -> App Value +showMessage :: (HasCallStack) => ClientIdentity -> ByteString -> App Value showMessage cid msg = do bs <- mlscli cid ["show", "message", "-"] (Just msg) assertOne (Aeson.decode (BS.fromStrict bs)) -readGroupState :: HasCallStack => ByteString -> App [(ClientIdentity, Word32)] +readGroupState :: (HasCallStack) => ByteString -> App [(ClientIdentity, Word32)] readGroupState gs = do v :: Value <- assertJust "Could not decode group state" (Aeson.decode (BS.fromStrict gs)) lnodes <- v %. "group" %. "public_group" %. "treesync" %. "tree" %. "leaf_nodes" & asList @@ -716,7 +739,7 @@ readGroupState gs = do pure Nothing createApplicationMessage :: - HasCallStack => + (HasCallStack) => ClientIdentity -> String -> App MessagePackage @@ -739,7 +762,7 @@ setMLSCiphersuite :: Ciphersuite -> App () setMLSCiphersuite suite = modifyMLSState $ \mls -> mls {ciphersuite = suite} leaveCurrentConv :: - HasCallStack => + (HasCallStack) => ClientIdentity -> App () leaveCurrentConv cid = do @@ -755,7 +778,7 @@ leaveCurrentConv cid = do { members = Set.difference mls.members (Set.singleton cid) } -getCurrentConv :: HasCallStack => ClientIdentity -> App Value +getCurrentConv :: (HasCallStack) => ClientIdentity -> App Value getCurrentConv cid = do mls <- getMLSState (conv, mSubId) <- objSubConv mls.convId diff --git a/integration/test/Test/MLS/SubConversation.hs b/integration/test/Test/MLS/SubConversation.hs index c66af61118..4f5767f4af 100644 --- a/integration/test/Test/MLS/SubConversation.hs +++ b/integration/test/Test/MLS/SubConversation.hs @@ -1,6 +1,9 @@ module Test.MLS.SubConversation where import API.Galley +import Control.Monad.Trans (lift) +import Control.Monad.Trans.Maybe (MaybeT (runMaybeT)) +import Data.Set qualified as Set import MLS.Util import Notifications import SetupHelpers @@ -170,3 +173,73 @@ testLeaveSubConv variant = do conv <- getCurrentConv (head others) mems <- conv %. "members" & asList length mems `shouldMatchInt` 2 + +testCreatorRemovesUserFromParent :: App () +testCreatorRemovesUserFromParent = do + [alice, bob, charlie] <- createAndConnectUsers [OwnDomain, OwnDomain, OtherDomain] + [alice1, bob1, bob2, charlie1, charlie2] <- traverse (createMLSClient def) [alice, bob, bob, charlie, charlie] + traverse_ uploadNewKeyPackage [bob1, bob2, charlie1, charlie2] + (_, qcnv) <- createNewGroup alice1 + + _ <- createAddCommit alice1 [bob, charlie] >>= sendAndConsumeCommitBundle + + -- save the state of the parent group + parentState <- getMLSState + -- switch to the subgroup + let subConvName = "conference" + createSubConv alice1 subConvName + + for_ [bob1, bob2, charlie1, charlie2] \c -> + createExternalCommit c Nothing >>= sendAndConsumeCommitBundle + -- save the state of the subgroup and switch to the parent context + childState <- getMLSState <* setMLSState parentState + withWebSockets [alice1, charlie1, charlie2] \wss -> do + removeCommitEvents <- createRemoveCommit alice1 [bob1, bob2] >>= sendAndConsumeCommitBundle + modifyMLSState $ \s -> s {members = s.members Set.\\ Set.fromList [bob1, bob2]} + + removeCommitEvents %. "events.0.type" `shouldMatch` "conversation.member-leave" + removeCommitEvents %. "events.0.data.reason" `shouldMatch` "removed" + removeCommitEvents %. "events.0.from" `shouldMatch` alice1.user + + for_ wss \ws -> do + n <- awaitMatch isConvLeaveNotif ws + n %. "payload.0.data.reason" `shouldMatch` "removed" + n %. "payload.0.from" `shouldMatch` alice1.user + + setMLSState childState + let idxBob1 :: Int = 1 + idxBob2 :: Int = 2 + for_ ((,) <$> [idxBob1, idxBob2] <*> [alice1, charlie1, charlie2] `zip` wss) \(idx, (consumer, ws)) -> do + msg <- + awaitMatch + do + \n -> + isJust <$> runMaybeT do + msg <- lift $ n %. "payload.0.data" & asByteString >>= showMessage alice1 + guard =<< lift do + isNewMLSMessageNotif n + + prop <- + maybe mzero pure =<< lift do + lookupField msg "message.content.body.Proposal" + + lift do + (== idx) <$> (prop %. "Remove.removed" & asInt) + ws + msg %. "payload.0.data" + & asByteString + >>= mlsCliConsume consumer + + -- remove bob from the child state + modifyMLSState $ \s -> s {members = s.members Set.\\ Set.fromList [bob1, bob2]} + + _ <- createPendingProposalCommit alice1 >>= sendAndConsumeCommitBundle + + getSubConversation bob qcnv subConvName >>= flip withResponse \resp -> + assertBool "access to the conversation for bob should be denied" (resp.status == 403) + + for_ [charlie, alice] \m -> do + resp <- getSubConversation m qcnv subConvName + assertBool "alice and charlie should have access to the conversation" (resp.status == 200) + mems <- resp.jsonBody %. "members" & asList + mems `shouldMatchSet` ((renameField "id" "user_id" <=< make) `traverse` [alice1, charlie1, charlie2]) diff --git a/integration/test/Testlib/JSON.hs b/integration/test/Testlib/JSON.hs index debb96fbac..a0b8778e99 100644 --- a/integration/test/Testlib/JSON.hs +++ b/integration/test/Testlib/JSON.hs @@ -175,6 +175,13 @@ assertField :: (HasCallStack, MakesValue a) => a -> String -> Maybe Value -> App assertField x k Nothing = assertFailureWithJSON x $ "Field \"" <> k <> "\" is missing from object:" assertField _ _ (Just x) = pure x +-- rename a field if it exists, else return the old object +renameField :: String -> String -> Value -> App Value +renameField old new obj = + fromMaybe obj <$> runMaybeT do + o :: Value <- maybe mzero pure =<< lift (lookupField obj old) + lift (removeField old obj >>= setField new o) + -- | Look up (nested) field of a JSON object -- -- If the field key has no dots then returns Nothing if the key is missing from the diff --git a/integration/test/Testlib/Types.hs b/integration/test/Testlib/Types.hs index c88ac445f6..b4f0711753 100644 --- a/integration/test/Testlib/Types.hs +++ b/integration/test/Testlib/Types.hs @@ -1,6 +1,7 @@ {- NOTE: Don't import any other Testlib modules here. Use this module to break dependency cycles. -} + module Testlib.Types where import Control.Concurrent (QSemN) @@ -193,7 +194,7 @@ data Response = Response headers :: [HTTP.Header], request :: HTTP.Request } - deriving (Show) + deriving stock (Show) instance HasField "json" Response (App Aeson.Value) where getField response = maybe (assertFailure "Response has no json body") pure response.jsonBody @@ -203,7 +204,7 @@ data ClientIdentity = ClientIdentity user :: String, client :: String } - deriving (Show, Eq, Ord) + deriving stock (Show, Eq, Ord, Generic) newtype Ciphersuite = Ciphersuite {code :: String} deriving (Eq, Ord, Show) @@ -265,7 +266,7 @@ instance Exception AssertionFailure where displayException (AssertionFailure _ _ msg) = msg newtype App a = App {unApp :: ReaderT Env IO a} - deriving + deriving newtype ( Functor, Applicative, Monad, diff --git a/libs/extended/src/System/Logger/Extended.hs b/libs/extended/src/System/Logger/Extended.hs index bfad629408..e360da4e85 100644 --- a/libs/extended/src/System/Logger/Extended.hs +++ b/libs/extended/src/System/Logger/Extended.hs @@ -127,7 +127,7 @@ structuredJSONRenderer _sep _dateFmt _lvlThreshold logElems = in case parseLevel buildMsg of Nothing -> o {msgs = builderToText b : msgs o} Just lvl -> o {lvl = Just lvl} - Field k v -> o {fields = Map.insertWith (<>) (Key.fromText $ builderToText k) (map builderToText [v]) (fields o)} + Field k v -> o {fields = Map.insertWith (<>) (Key.fromText $ builderToText k) ([builderToText v]) (fields o)} ) (StructuredJSONOutput Nothing [] mempty) diff --git a/libs/gundeck-types/src/Gundeck/Types/Push/V2.hs b/libs/gundeck-types/src/Gundeck/Types/Push/V2.hs index 9b26ab7154..ab9ac4de26 100644 --- a/libs/gundeck-types/src/Gundeck/Types/Push/V2.hs +++ b/libs/gundeck-types/src/Gundeck/Types/Push/V2.hs @@ -312,9 +312,9 @@ instance ToJSON Push where # "origin" .= _pushOrigin p # "connections" .= ifNot Set.null (_pushConnections p) # "origin_connection" .= _pushOriginConnection p - # "transient" .= ifNot (== False) (_pushTransient p) - # "native_include_origin" .= ifNot (== True) (_pushNativeIncludeOrigin p) - # "native_encrypt" .= ifNot (== True) (_pushNativeEncrypt p) + # "transient" .= ifNot not (_pushTransient p) + # "native_include_origin" .= ifNot id (_pushNativeIncludeOrigin p) + # "native_encrypt" .= ifNot id (_pushNativeEncrypt p) # "native_aps" .= _pushNativeAps p # "native_priority" .= ifNot (== HighPriority) (_pushNativePriority p) # "payload" .= _pushPayload p diff --git a/libs/http2-manager/http2-manager.cabal b/libs/http2-manager/http2-manager.cabal index f967858813..4aa3b54347 100644 --- a/libs/http2-manager/http2-manager.cabal +++ b/libs/http2-manager/http2-manager.cabal @@ -48,6 +48,10 @@ library default-language: Haskell2010 +flag test-trailing-dot + description: "Whether or not include the trainling dot test" + default: True + test-suite http2-manager-tests type: exitcode-stdio-1.0 main-is: Main.hs @@ -57,6 +61,9 @@ test-suite http2-manager-tests -Wredundant-constraints -Wunused-packages -threaded -with-rtsopts=-N + if flag(test-trailing-dot) + cpp-options: -DENABLE-TRAILING-DOT-TEST + -- cabal-fmt: expand test other-modules: Main diff --git a/libs/http2-manager/test/Test/HTTP2/Client/ManagerSpec.hs b/libs/http2-manager/test/Test/HTTP2/Client/ManagerSpec.hs index 331110d3e2..f839619b9b 100644 --- a/libs/http2-manager/test/Test/HTTP2/Client/ManagerSpec.hs +++ b/libs/http2-manager/test/Test/HTTP2/Client/ManagerSpec.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE CPP #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiWayIf #-} {-# LANGUAGE NumericUnderscores #-} @@ -71,10 +72,12 @@ spec = do withTestServer (Just ctx) $ \TestServer {..} -> echoTest mgr True serverPort `shouldThrow` (\(_ :: SSL.SomeSSLException) -> True) +#if DENABLE-TRAILING-DOT-TEST it "should allow accepting a certificate without a trailing dot" $ do mgr <- setSSLRemoveTrailingDot True <$> mkTestManager withTestServer (Just localhostCtx) $ \TestServer {..} -> echoTest' "localhost." "/echo" "some body" mgr True serverPort +#endif specTemplate :: Maybe SSL.SSLContext -> Spec specTemplate mCtx = do diff --git a/libs/libzauth/libzauth-c/Cargo.nix b/libs/libzauth/libzauth-c/Cargo.nix index 13374265f8..ac5c917bd2 100644 --- a/libs/libzauth/libzauth-c/Cargo.nix +++ b/libs/libzauth/libzauth-c/Cargo.nix @@ -1,10 +1,9 @@ - # This file was @generated by crate2nix 0.11.0 with the command: # "generate" # See https://github.com/kolloch/crate2nix for more info. { nixpkgs ? -, pkgs ? import nixpkgs { config = {}; } +, pkgs ? import nixpkgs { config = { }; } , lib ? pkgs.lib , stdenv ? pkgs.stdenv , buildRustCrateForPkgs ? pkgs: pkgs.buildRustCrate @@ -15,14 +14,13 @@ # If true, throw errors instead of issueing deprecation warnings. , strictDeprecation ? false # Used for conditional compilation based on CPU feature detection. -, targetFeatures ? [] +, targetFeatures ? [ ] # Whether to perform release builds: longer compile times, faster binaries. , release ? true # Additional crate2nix configuration if it exists. -, crateConfig - ? if builtins.pathExists ./crate-config.nix - then pkgs.callPackage ./crate-config.nix {} - else {} +, crateConfig ? if builtins.pathExists ./crate-config.nix + then pkgs.callPackage ./crate-config.nix { } + else { } }: rec { @@ -61,10 +59,10 @@ rec { # A derivation that joins the outputs of all workspace members together. allWorkspaceMembers = pkgs.symlinkJoin { - name = "all-workspace-members"; - paths = - let members = builtins.attrValues workspaceMembers; - in builtins.map (m: m.build) members; + name = "all-workspace-members"; + paths = + let members = builtins.attrValues workspaceMembers; + in builtins.map (m: m.build) members; }; # @@ -251,7 +249,7 @@ rec { crateName = "cc"; version = "1.0.83"; edition = "2018"; - crateBin = []; + crateBin = [ ]; sha256 = "1l643zidlb5iy1dskc5ggqs4wqa29a02f44piczqc8zcnsq4y5zi"; authors = [ "Alex Crichton " @@ -312,8 +310,7 @@ rec { target = { target, features }: ((("wasm32" == target."arch") || ("wasm64" == target."arch")) && ("unknown" == target."os")); } ]; - features = { - }; + features = { }; }; "const-oid" = rec { crateName = "const-oid"; @@ -1811,13 +1808,13 @@ rec { { name = "cc"; packageId = "cc"; - target = {target, features}: (!("msvc" == target."env")); + target = { target, features }: (!("msvc" == target."env")); } { name = "libc"; packageId = "libc"; usesDefaultFeatures = false; - target = {target, features}: ("msvc" == target."env"); + target = { target, features }: ("msvc" == target."env"); } { name = "pkg-config"; @@ -1828,8 +1825,7 @@ rec { packageId = "walkdir"; } ]; - features = { - }; + features = { }; }; "log" = rec { crateName = "log"; @@ -3194,8 +3190,7 @@ rec { packageId = "syn"; } ]; - features = { - }; + features = { }; resolvedDefaultFeatures = [ "default" ]; }; "serde_json" = rec { @@ -3850,8 +3845,9 @@ rec { edition = "2015"; # We can't filter paths with references in Nix 2.4 # See https://github.com/NixOS/nix/issues/5410 - src = if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion )) - then lib.cleanSourceWith { filter = sourceFilter; src = ../libzauth; } + src = + if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion)) + then lib.cleanSourceWith { filter = sourceFilter; src = ../libzauth; } else ../libzauth; authors = [ "Wire Swiss GmbH " @@ -3906,10 +3902,12 @@ rec { edition = "2015"; # We can't filter paths with references in Nix 2.4 # See https://github.com/NixOS/nix/issues/5410 - src = if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion )) - then lib.cleanSourceWith { filter = sourceFilter; src = ./.; } + src = + if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion)) + then lib.cleanSourceWith { filter = sourceFilter; src = ./.; } else ./.; - libName = "zauth";type = [ "cdylib" ]; + libName = "zauth"; + type = [ "cdylib" ]; authors = [ "Wire Swiss GmbH " ]; @@ -3949,676 +3947,676 @@ rec { }; # -# crate2nix/default.nix (excerpt start) -# + # crate2nix/default.nix (excerpt start) + # - /* Target (platform) data for conditional dependencies. - This corresponds roughly to what buildRustCrate is setting. - */ - makeDefaultTarget = platform: { - unix = platform.isUnix; - windows = platform.isWindows; - fuchsia = true; - test = false; + /* Target (platform) data for conditional dependencies. + This corresponds roughly to what buildRustCrate is setting. + */ + makeDefaultTarget = platform: { + unix = platform.isUnix; + windows = platform.isWindows; + fuchsia = true; + test = false; - /* We are choosing an arbitrary rust version to grab `lib` from, + /* We are choosing an arbitrary rust version to grab `lib` from, which is unfortunate, but `lib` has been version-agnostic the whole time so this is good enough for now. - */ - os = pkgs.rust.lib.toTargetOs platform; - arch = pkgs.rust.lib.toTargetArch platform; - family = pkgs.rust.lib.toTargetFamily platform; - env = "gnu"; - endian = - if platform.parsed.cpu.significantByte.name == "littleEndian" - then "little" else "big"; - pointer_width = toString platform.parsed.cpu.bits; - vendor = platform.parsed.vendor.name; - debug_assertions = false; - }; + */ + os = pkgs.rust.lib.toTargetOs platform; + arch = pkgs.rust.lib.toTargetArch platform; + family = pkgs.rust.lib.toTargetFamily platform; + env = "gnu"; + endian = + if platform.parsed.cpu.significantByte.name == "littleEndian" + then "little" else "big"; + pointer_width = toString platform.parsed.cpu.bits; + vendor = platform.parsed.vendor.name; + debug_assertions = false; + }; - /* Filters common temp files and build files. */ - # TODO(pkolloch): Substitute with gitignore filter - sourceFilter = name: type: - let - baseName = builtins.baseNameOf (builtins.toString name); - in - ! ( - # Filter out git - baseName == ".gitignore" - || (type == "directory" && baseName == ".git") + /* Filters common temp files and build files. */ + # TODO(pkolloch): Substitute with gitignore filter + sourceFilter = name: type: + let + baseName = builtins.baseNameOf (builtins.toString name); + in + ! ( + # Filter out git + baseName == ".gitignore" + || (type == "directory" && baseName == ".git") - # Filter out build results - || ( - type == "directory" && ( - baseName == "target" - || baseName == "_site" - || baseName == ".sass-cache" - || baseName == ".jekyll-metadata" - || baseName == "build-artifacts" + # Filter out build results + || ( + type == "directory" && ( + baseName == "target" + || baseName == "_site" + || baseName == ".sass-cache" + || baseName == ".jekyll-metadata" + || baseName == "build-artifacts" + ) ) - ) - # Filter out nix-build result symlinks - || ( - type == "symlink" && lib.hasPrefix "result" baseName - ) - - # Filter out IDE config - || ( - type == "directory" && ( - baseName == ".idea" || baseName == ".vscode" + # Filter out nix-build result symlinks + || ( + type == "symlink" && lib.hasPrefix "result" baseName ) - ) || lib.hasSuffix ".iml" baseName - # Filter out nix build files - || baseName == "Cargo.nix" + # Filter out IDE config + || ( + type == "directory" && ( + baseName == ".idea" || baseName == ".vscode" + ) + ) || lib.hasSuffix ".iml" baseName - # Filter out editor backup / swap files. - || lib.hasSuffix "~" baseName - || builtins.match "^\\.sw[a-z]$$" baseName != null - || builtins.match "^\\..*\\.sw[a-z]$$" baseName != null - || lib.hasSuffix ".tmp" baseName - || lib.hasSuffix ".bak" baseName - || baseName == "tests.nix" - ); + # Filter out nix build files + || baseName == "Cargo.nix" - /* Returns a crate which depends on successful test execution - of crate given as the second argument. + # Filter out editor backup / swap files. + || lib.hasSuffix "~" baseName + || builtins.match "^\\.sw[a-z]$$" baseName != null + || builtins.match "^\\..*\\.sw[a-z]$$" baseName != null + || lib.hasSuffix ".tmp" baseName + || lib.hasSuffix ".bak" baseName + || baseName == "tests.nix" + ); - testCrateFlags: list of flags to pass to the test exectuable - testInputs: list of packages that should be available during test execution - */ - crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: - assert builtins.typeOf testCrateFlags == "list"; - assert builtins.typeOf testInputs == "list"; - assert builtins.typeOf testPreRun == "string"; - assert builtins.typeOf testPostRun == "string"; - let - # override the `crate` so that it will build and execute tests instead of - # building the actual lib and bin targets We just have to pass `--test` - # to rustc and it will do the right thing. We execute the tests and copy - # their log and the test executables to $out for later inspection. - test = - let - drv = testCrate.override - ( - _: { - buildTests = true; - release = false; - } - ); - # If the user hasn't set any pre/post commands, we don't want to - # insert empty lines. This means that any existing users of crate2nix - # don't get a spurious rebuild unless they set these explicitly. - testCommand = pkgs.lib.concatStringsSep "\n" - (pkgs.lib.filter (s: s != "") [ - testPreRun - "$f $testCrateFlags 2>&1 | tee -a $out" - testPostRun - ]); - in - pkgs.runCommand "run-tests-${testCrate.name}" - { - inherit testCrateFlags; - buildInputs = testInputs; - } '' - set -e + /* Returns a crate which depends on successful test execution + of crate given as the second argument. - export RUST_BACKTRACE=1 + testCrateFlags: list of flags to pass to the test exectuable + testInputs: list of packages that should be available during test execution + */ + crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: + assert builtins.typeOf testCrateFlags == "list"; + assert builtins.typeOf testInputs == "list"; + assert builtins.typeOf testPreRun == "string"; + assert builtins.typeOf testPostRun == "string"; + let + # override the `crate` so that it will build and execute tests instead of + # building the actual lib and bin targets We just have to pass `--test` + # to rustc and it will do the right thing. We execute the tests and copy + # their log and the test executables to $out for later inspection. + test = + let + drv = testCrate.override + ( + _: { + buildTests = true; + release = false; + } + ); + # If the user hasn't set any pre/post commands, we don't want to + # insert empty lines. This means that any existing users of crate2nix + # don't get a spurious rebuild unless they set these explicitly. + testCommand = pkgs.lib.concatStringsSep "\n" + (pkgs.lib.filter (s: s != "") [ + testPreRun + "$f $testCrateFlags 2>&1 | tee -a $out" + testPostRun + ]); + in + pkgs.runCommand "run-tests-${testCrate.name}" + { + inherit testCrateFlags; + buildInputs = testInputs; + } '' + set -e - # recreate a file hierarchy as when running tests with cargo + export RUST_BACKTRACE=1 - # the source for test data - ${pkgs.buildPackages.xorg.lndir}/bin/lndir ${crate.src} + # recreate a file hierarchy as when running tests with cargo - # build outputs - testRoot=target/debug - mkdir -p $testRoot + # the source for test data + ${pkgs.buildPackages.xorg.lndir}/bin/lndir ${crate.src} - # executables of the crate - # we copy to prevent std::env::current_exe() to resolve to a store location - for i in ${crate}/bin/*; do - cp "$i" "$testRoot" - done - chmod +w -R . + # build outputs + testRoot=target/debug + mkdir -p $testRoot - # test harness executables are suffixed with a hash, like cargo does - # this allows to prevent name collision with the main - # executables of the crate - hash=$(basename $out) - for file in ${drv}/tests/*; do - f=$testRoot/$(basename $file)-$hash - cp $file $f - ${testCommand} - done - ''; - in - pkgs.runCommand "${crate.name}-linked" - { - inherit (crate) outputs crateName; - passthru = (crate.passthru or { }) // { - inherit test; - }; - } - (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' - echo tested by ${test} - '' + '' - ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} - ''); + # executables of the crate + # we copy to prevent std::env::current_exe() to resolve to a store location + for i in ${crate}/bin/*; do + cp "$i" "$testRoot" + done + chmod +w -R . - /* A restricted overridable version of builtRustCratesWithFeatures. */ - buildRustCrateWithFeatures = - { packageId - , features ? rootFeatures - , crateOverrides ? defaultCrateOverrides - , buildRustCrateForPkgsFunc ? null - , runTests ? false - , testCrateFlags ? [ ] - , testInputs ? [ ] - # Any command to run immediatelly before a test is executed. - , testPreRun ? "" - # Any command run immediatelly after a test is executed. - , testPostRun ? "" - }: - lib.makeOverridable - ( - { features - , crateOverrides - , runTests - , testCrateFlags - , testInputs - , testPreRun - , testPostRun - }: - let - buildRustCrateForPkgsFuncOverriden = - if buildRustCrateForPkgsFunc != null - then buildRustCrateForPkgsFunc - else - ( - if crateOverrides == pkgs.defaultCrateOverrides - then buildRustCrateForPkgs - else - pkgs: (buildRustCrateForPkgs pkgs).override { - defaultCrateOverrides = crateOverrides; - } - ); - builtRustCrates = builtRustCratesWithFeatures { - inherit packageId features; - buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; - runTests = false; - }; - builtTestRustCrates = builtRustCratesWithFeatures { - inherit packageId features; - buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; - runTests = true; + # test harness executables are suffixed with a hash, like cargo does + # this allows to prevent name collision with the main + # executables of the crate + hash=$(basename $out) + for file in ${drv}/tests/*; do + f=$testRoot/$(basename $file)-$hash + cp $file $f + ${testCommand} + done + ''; + in + pkgs.runCommand "${crate.name}-linked" + { + inherit (crate) outputs crateName; + passthru = (crate.passthru or { }) // { + inherit test; }; - drv = builtRustCrates.crates.${packageId}; - testDrv = builtTestRustCrates.crates.${packageId}; - derivation = - if runTests then - crateWithTest - { - crate = drv; - testCrate = testDrv; - inherit testCrateFlags testInputs testPreRun testPostRun; - } - else drv; - in - derivation - ) - { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + } + (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + echo tested by ${test} + '' + '' + ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} + ''); - /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc - for the corresponding crate. - */ - builtRustCratesWithFeatures = - { packageId - , features - , crateConfigs ? crates - , buildRustCrateForPkgsFunc - , runTests - , makeTarget ? makeDefaultTarget - } @ args: - assert (builtins.isAttrs crateConfigs); - assert (builtins.isString packageId); - assert (builtins.isList features); - assert (builtins.isAttrs (makeTarget stdenv.hostPlatform)); - assert (builtins.isBool runTests); - let - rootPackageId = packageId; - mergedFeatures = mergePackageFeatures - ( - args // { - inherit rootPackageId; - target = makeTarget stdenv.hostPlatform // { test = runTests; }; - } - ); - # Memoize built packages so that reappearing packages are only built once. - builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; - mkBuiltByPackageIdByPkgs = pkgs: + /* A restricted overridable version of builtRustCratesWithFeatures. */ + buildRustCrateWithFeatures = + { packageId + , features ? rootFeatures + , crateOverrides ? defaultCrateOverrides + , buildRustCrateForPkgsFunc ? null + , runTests ? false + , testCrateFlags ? [ ] + , testInputs ? [ ] + # Any command to run immediatelly before a test is executed. + , testPreRun ? "" + # Any command run immediatelly after a test is executed. + , testPostRun ? "" + }: + lib.makeOverridable + ( + { features + , crateOverrides + , runTests + , testCrateFlags + , testInputs + , testPreRun + , testPostRun + }: let - self = { - crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; - target = makeTarget pkgs.stdenv.hostPlatform; - build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; + buildRustCrateForPkgsFuncOverriden = + if buildRustCrateForPkgsFunc != null + then buildRustCrateForPkgsFunc + else + ( + if crateOverrides == pkgs.defaultCrateOverrides + then buildRustCrateForPkgs + else + pkgs: (buildRustCrateForPkgs pkgs).override { + defaultCrateOverrides = crateOverrides; + } + ); + builtRustCrates = builtRustCratesWithFeatures { + inherit packageId features; + buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; + runTests = false; }; + builtTestRustCrates = builtRustCratesWithFeatures { + inherit packageId features; + buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; + runTests = true; + }; + drv = builtRustCrates.crates.${packageId}; + testDrv = builtTestRustCrates.crates.${packageId}; + derivation = + if runTests then + crateWithTest + { + crate = drv; + testCrate = testDrv; + inherit testCrateFlags testInputs testPreRun testPostRun; + } + else drv; in - self; - buildByPackageIdForPkgsImpl = self: pkgs: packageId: - let - features = mergedFeatures."${packageId}" or [ ]; - crateConfig' = crateConfigs."${packageId}"; - crateConfig = - builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; - devDependencies = - lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig'.devDependencies or [ ]); - dependencies = - dependencyDerivations { - inherit features; - inherit (self) target; - buildByPackageId = depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false - then self.build.crates.${depPackageId} - else self.crates.${depPackageId}; - dependencies = - (crateConfig.dependencies or [ ]) - ++ devDependencies; - }; - buildDependencies = - dependencyDerivations { - inherit features; - inherit (self.build) target; - buildByPackageId = depPackageId: - self.build.crates.${depPackageId}; - dependencies = crateConfig.buildDependencies or [ ]; + derivation + ) + { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + + /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc + for the corresponding crate. + */ + builtRustCratesWithFeatures = + { packageId + , features + , crateConfigs ? crates + , buildRustCrateForPkgsFunc + , runTests + , makeTarget ? makeDefaultTarget + } @ args: + assert (builtins.isAttrs crateConfigs); + assert (builtins.isString packageId); + assert (builtins.isList features); + assert (builtins.isAttrs (makeTarget stdenv.hostPlatform)); + assert (builtins.isBool runTests); + let + rootPackageId = packageId; + mergedFeatures = mergePackageFeatures + ( + args // { + inherit rootPackageId; + target = makeTarget stdenv.hostPlatform // { test = runTests; }; + } + ); + # Memoize built packages so that reappearing packages are only built once. + builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; + mkBuiltByPackageIdByPkgs = pkgs: + let + self = { + crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; + target = makeTarget pkgs.stdenv.hostPlatform; + build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; - dependenciesWithRenames = - let - buildDeps = filterEnabledDependencies { + in + self; + buildByPackageIdForPkgsImpl = self: pkgs: packageId: + let + features = mergedFeatures."${packageId}" or [ ]; + crateConfig' = crateConfigs."${packageId}"; + crateConfig = + builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; + devDependencies = + lib.optionals + (runTests && packageId == rootPackageId) + (crateConfig'.devDependencies or [ ]); + dependencies = + dependencyDerivations { inherit features; inherit (self) target; - dependencies = crateConfig.dependencies or [ ] ++ devDependencies; + buildByPackageId = depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false + then self.build.crates.${depPackageId} + else self.crates.${depPackageId}; + dependencies = + (crateConfig.dependencies or [ ]) + ++ devDependencies; }; - hostDeps = filterEnabledDependencies { + buildDependencies = + dependencyDerivations { inherit features; inherit (self.build) target; + buildByPackageId = depPackageId: + self.build.crates.${depPackageId}; dependencies = crateConfig.buildDependencies or [ ]; }; - in - lib.filter (d: d ? "rename") (hostDeps ++ buildDeps); - # Crate renames have the form: - # - # { - # crate_name = [ - # { version = "1.2.3"; rename = "crate_name01"; } - # ]; - # # ... - # } - crateRenames = - let - grouped = - lib.groupBy - (dependency: dependency.name) - dependenciesWithRenames; - versionAndRename = dep: - let - package = crateConfigs."${dep.packageId}"; - in - { inherit (dep) rename; version = package.version; }; - in - lib.mapAttrs (name: choices: builtins.map versionAndRename choices) grouped; - in - buildRustCrateForPkgsFunc pkgs - ( - crateConfig // { - # https://github.com/NixOS/nixpkgs/issues/218712 - dontStrip = stdenv.hostPlatform.isDarwin; - src = crateConfig.src or ( - pkgs.fetchurl rec { - name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; - # https://www.pietroalbini.org/blog/downloading-crates-io/ - # Not rate-limited, CDN URL. - url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; - sha256 = - assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); - crateConfig.sha256; - } - ); - extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; - inherit features dependencies buildDependencies crateRenames release; - } - ); - in - builtByPackageIdByPkgs; - - /* Returns the actual derivations for the given dependencies. */ - dependencyDerivations = - { buildByPackageId - , features - , dependencies - , target - }: - assert (builtins.isList features); - assert (builtins.isList dependencies); - assert (builtins.isAttrs target); - let - enabledDependencies = filterEnabledDependencies { - inherit dependencies features target; - }; - depDerivation = dependency: buildByPackageId dependency.packageId; - in - map depDerivation enabledDependencies; - - /* Returns a sanitized version of val with all values substituted that cannot - be serialized as JSON. - */ - sanitizeForJson = val: - if builtins.isAttrs val - then lib.mapAttrs (n: v: sanitizeForJson v) val - else if builtins.isList val - then builtins.map sanitizeForJson val - else if builtins.isFunction val - then "function" - else val; - - /* Returns various tools to debug a crate. */ - debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: - assert (builtins.isString packageId); - let - debug = rec { - # The built tree as passed to buildRustCrate. - buildTree = buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: lib.id; - inherit packageId; - }; - sanitizedBuildTree = sanitizeForJson buildTree; - dependencyTree = sanitizeForJson - ( - buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: crate: { - "01_crateName" = crate.crateName or false; - "02_features" = crate.features or [ ]; - "03_dependencies" = crate.dependencies or [ ]; - }; - inherit packageId; - } - ); - mergedPackageFeatures = mergePackageFeatures { - features = rootFeatures; - inherit packageId target; - }; - diffedDefaultPackageFeatures = diffDefaultPackageFeatures { - inherit packageId target; - }; - }; - in - { internal = debug; }; - - /* Returns differences between cargo default features and crate2nix default - features. + dependenciesWithRenames = + let + buildDeps = filterEnabledDependencies { + inherit features; + inherit (self) target; + dependencies = crateConfig.dependencies or [ ] ++ devDependencies; + }; + hostDeps = filterEnabledDependencies { + inherit features; + inherit (self.build) target; + dependencies = crateConfig.buildDependencies or [ ]; + }; + in + lib.filter (d: d ? "rename") (hostDeps ++ buildDeps); + # Crate renames have the form: + # + # { + # crate_name = [ + # { version = "1.2.3"; rename = "crate_name01"; } + # ]; + # # ... + # } + crateRenames = + let + grouped = + lib.groupBy + (dependency: dependency.name) + dependenciesWithRenames; + versionAndRename = dep: + let + package = crateConfigs."${dep.packageId}"; + in + { inherit (dep) rename; version = package.version; }; + in + lib.mapAttrs (name: choices: builtins.map versionAndRename choices) grouped; + in + buildRustCrateForPkgsFunc pkgs + ( + crateConfig // { + # https://github.com/NixOS/nixpkgs/issues/218712 + dontStrip = stdenv.hostPlatform.isDarwin; + src = crateConfig.src or ( + pkgs.fetchurl rec { + name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; + # https://www.pietroalbini.org/blog/downloading-crates-io/ + # Not rate-limited, CDN URL. + url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; + sha256 = + assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); + crateConfig.sha256; + } + ); + extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; + inherit features dependencies buildDependencies crateRenames release; + } + ); + in + builtByPackageIdByPkgs; - This is useful for verifying the feature resolution in crate2nix. - */ - diffDefaultPackageFeatures = - { crateConfigs ? crates - , packageId - , target - }: - assert (builtins.isAttrs crateConfigs); - let - prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); - mergedFeatures = - prefixValues - "crate2nix" - (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); - configs = prefixValues "cargo" crateConfigs; - combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; - onlyInCargo = - builtins.attrNames - (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); - onlyInCrate2Nix = - builtins.attrNames - (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); - differentFeatures = lib.filterAttrs - ( - n: v: - (v ? "crate2nix") - && (v ? "cargo") - && (v.crate2nix.features or [ ]) != (v."cargo".resolved_default_features or [ ]) - ) - combined; - in - builtins.toJSON { - inherit onlyInCargo onlyInCrate2Nix differentFeatures; - }; + /* Returns the actual derivations for the given dependencies. */ + dependencyDerivations = + { buildByPackageId + , features + , dependencies + , target + }: + assert (builtins.isList features); + assert (builtins.isList dependencies); + assert (builtins.isAttrs target); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies features target; + }; + depDerivation = dependency: buildByPackageId dependency.packageId; + in + map depDerivation enabledDependencies; - /* Returns an attrset mapping packageId to the list of enabled features. + /* Returns a sanitized version of val with all values substituted that cannot + be serialized as JSON. + */ + sanitizeForJson = val: + if builtins.isAttrs val + then lib.mapAttrs (n: v: sanitizeForJson v) val + else if builtins.isList val + then builtins.map sanitizeForJson val + else if builtins.isFunction val + then "function" + else val; - If multiple paths to a dependency enable different features, the - corresponding feature sets are merged. Features in rust are additive. - */ - mergePackageFeatures = - { crateConfigs ? crates - , packageId - , rootPackageId ? packageId - , features ? rootFeatures - , dependencyPath ? [ crates.${packageId}.crateName ] - , featuresByPackageId ? { } - , target - # Adds devDependencies to the crate with rootPackageId. - , runTests ? false - , ... - } @ args: - assert (builtins.isAttrs crateConfigs); + /* Returns various tools to debug a crate. */ + debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: assert (builtins.isString packageId); - assert (builtins.isString rootPackageId); - assert (builtins.isList features); - assert (builtins.isList dependencyPath); - assert (builtins.isAttrs featuresByPackageId); - assert (builtins.isAttrs target); - assert (builtins.isBool runTests); let - crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); - expandedFeatures = expandFeatures (crateConfig.features or { }) features; - enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; - depWithResolvedFeatures = dependency: - let - packageId = dependency.packageId; - features = dependencyFeatures enabledFeatures dependency; - in - { inherit packageId features; }; - resolveDependencies = cache: path: dependencies: - assert (builtins.isAttrs cache); - assert (builtins.isList dependencies); - let - enabledDependencies = filterEnabledDependencies { - inherit dependencies target; - features = enabledFeatures; - }; - directDependencies = map depWithResolvedFeatures enabledDependencies; - foldOverCache = op: lib.foldl op cache directDependencies; - in - foldOverCache - ( - cache: { packageId, features }: - let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); - in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures - then cache - else - mergePackageFeatures { - features = combinedFeatures; - featuresByPackageId = cache; - inherit crateConfigs packageId target runTests rootPackageId; - } - ); - cacheWithSelf = - let - cacheFeatures = featuresByPackageId.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); - in - featuresByPackageId // { - "${packageId}" = combinedFeatures; + debug = rec { + # The built tree as passed to buildRustCrate. + buildTree = buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: lib.id; + inherit packageId; }; - cacheWithDependencies = - resolveDependencies cacheWithSelf "dep" + sanitizedBuildTree = sanitizeForJson buildTree; + dependencyTree = sanitizeForJson ( - crateConfig.dependencies or [ ] - ++ lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig.devDependencies or [ ]) + buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: crate: { + "01_crateName" = crate.crateName or false; + "02_features" = crate.features or [ ]; + "03_dependencies" = crate.dependencies or [ ]; + }; + inherit packageId; + } ); - cacheWithAll = - resolveDependencies - cacheWithDependencies "build" - (crateConfig.buildDependencies or [ ]); + mergedPackageFeatures = mergePackageFeatures { + features = rootFeatures; + inherit packageId target; + }; + diffedDefaultPackageFeatures = diffDefaultPackageFeatures { + inherit packageId target; + }; + }; in - cacheWithAll; + { internal = debug; }; - /* Returns the enabled dependencies given the enabled features. */ - filterEnabledDependencies = { dependencies, features, target }: - assert (builtins.isList dependencies); - assert (builtins.isList features); - assert (builtins.isAttrs target); + /* Returns differences between cargo default features and crate2nix default + features. - lib.filter - ( - dep: + This is useful for verifying the feature resolution in crate2nix. + */ + diffDefaultPackageFeatures = + { crateConfigs ? crates + , packageId + , target + }: + assert (builtins.isAttrs crateConfigs); let - targetFunc = dep.target or (features: true); + prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); + mergedFeatures = + prefixValues + "crate2nix" + (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); + configs = prefixValues "cargo" crateConfigs; + combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; + onlyInCargo = + builtins.attrNames + (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); + onlyInCrate2Nix = + builtins.attrNames + (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); + differentFeatures = lib.filterAttrs + ( + n: v: + (v ? "crate2nix") + && (v ? "cargo") + && (v.crate2nix.features or [ ]) != (v."cargo".resolved_default_features or [ ]) + ) + combined; in - targetFunc { inherit features target; } - && ( - !(dep.optional or false) - || builtins.any (doesFeatureEnableDependency dep) features - ) - ) - dependencies; + builtins.toJSON { + inherit onlyInCargo onlyInCrate2Nix differentFeatures; + }; + + /* Returns an attrset mapping packageId to the list of enabled features. - /* Returns whether the given feature should enable the given dependency. */ - doesFeatureEnableDependency = dependency: feature: - let - name = dependency.rename or dependency.name; - prefix = "${name}/"; - len = builtins.stringLength prefix; - startsWithPrefix = builtins.substring 0 len feature == prefix; - in - feature == name || feature == "dep:" + name || startsWithPrefix; + If multiple paths to a dependency enable different features, the + corresponding feature sets are merged. Features in rust are additive. + */ + mergePackageFeatures = + { crateConfigs ? crates + , packageId + , rootPackageId ? packageId + , features ? rootFeatures + , dependencyPath ? [ crates.${packageId}.crateName ] + , featuresByPackageId ? { } + , target + # Adds devDependencies to the crate with rootPackageId. + , runTests ? false + , ... + } @ args: + assert (builtins.isAttrs crateConfigs); + assert (builtins.isString packageId); + assert (builtins.isString rootPackageId); + assert (builtins.isList features); + assert (builtins.isList dependencyPath); + assert (builtins.isAttrs featuresByPackageId); + assert (builtins.isAttrs target); + assert (builtins.isBool runTests); + let + crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); + expandedFeatures = expandFeatures (crateConfig.features or { }) features; + enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; + depWithResolvedFeatures = dependency: + let + packageId = dependency.packageId; + features = dependencyFeatures enabledFeatures dependency; + in + { inherit packageId features; }; + resolveDependencies = cache: path: dependencies: + assert (builtins.isAttrs cache); + assert (builtins.isList dependencies); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies target; + features = enabledFeatures; + }; + directDependencies = map depWithResolvedFeatures enabledDependencies; + foldOverCache = op: lib.foldl op cache directDependencies; + in + foldOverCache + ( + cache: { packageId, features }: + let + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); + in + if cache ? ${packageId} && cache.${packageId} == combinedFeatures + then cache + else + mergePackageFeatures { + features = combinedFeatures; + featuresByPackageId = cache; + inherit crateConfigs packageId target runTests rootPackageId; + } + ); + cacheWithSelf = + let + cacheFeatures = featuresByPackageId.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); + in + featuresByPackageId // { + "${packageId}" = combinedFeatures; + }; + cacheWithDependencies = + resolveDependencies cacheWithSelf "dep" + ( + crateConfig.dependencies or [ ] + ++ lib.optionals + (runTests && packageId == rootPackageId) + (crateConfig.devDependencies or [ ]) + ); + cacheWithAll = + resolveDependencies + cacheWithDependencies "build" + (crateConfig.buildDependencies or [ ]); + in + cacheWithAll; - /* Returns the expanded features for the given inputFeatures by applying the - rules in featureMap. + /* Returns the enabled dependencies given the enabled features. */ + filterEnabledDependencies = { dependencies, features, target }: + assert (builtins.isList dependencies); + assert (builtins.isList features); + assert (builtins.isAttrs target); - featureMap is an attribute set which maps feature names to lists of further - feature names to enable in case this feature is selected. - */ - expandFeatures = featureMap: inputFeatures: - assert (builtins.isAttrs featureMap); - assert (builtins.isList inputFeatures); - let - expandFeaturesNoCycle = oldSeen: inputFeatures: - if inputFeatures != [ ] - then + lib.filter + ( + dep: let - # The feature we're currently expanding. - feature = builtins.head inputFeatures; - # All the features we've seen/expanded so far, including the one - # we're currently processing. - seen = oldSeen // { ${feature} = 1; }; - # Expand the feature but be careful to not re-introduce a feature - # that we've already seen: this can easily cause a cycle, see issue - # #209. - enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + targetFunc = dep.target or (features: true); in - [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) - # No more features left, nothing to expand to. - else [ ]; - outFeatures = expandFeaturesNoCycle { } inputFeatures; - in - sortedUnique outFeatures; + targetFunc { inherit features target; } + && ( + !(dep.optional or false) + || builtins.any (doesFeatureEnableDependency dep) features + ) + ) + dependencies; - /* This function adds optional dependencies as features if they are enabled - indirectly by dependency features. This function mimics Cargo's behavior - described in a note at: - https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features - */ - enableFeatures = dependencies: features: - assert (builtins.isList features); - assert (builtins.isList dependencies); - let - additionalFeatures = lib.concatMap - ( - dependency: - assert (builtins.isAttrs dependency); + /* Returns whether the given feature should enable the given dependency. */ + doesFeatureEnableDependency = dependency: feature: + let + name = dependency.rename or dependency.name; + prefix = "${name}/"; + len = builtins.stringLength prefix; + startsWithPrefix = builtins.substring 0 len feature == prefix; + in + feature == name || feature == "dep:" + name || startsWithPrefix; + + /* Returns the expanded features for the given inputFeatures by applying the + rules in featureMap. + + featureMap is an attribute set which maps feature names to lists of further + feature names to enable in case this feature is selected. + */ + expandFeatures = featureMap: inputFeatures: + assert (builtins.isAttrs featureMap); + assert (builtins.isList inputFeatures); + let + expandFeaturesNoCycle = oldSeen: inputFeatures: + if inputFeatures != [ ] + then let - enabled = builtins.any (doesFeatureEnableDependency dependency) features; + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { ${feature} = 1; }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); in - if (dependency.optional or false) && enabled - then [ (dependency.rename or dependency.name) ] - else [ ] - ) - dependencies; - in - sortedUnique (features ++ additionalFeatures); + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; + in + sortedUnique outFeatures; - /* - Returns the actual features for the given dependency. + /* This function adds optional dependencies as features if they are enabled + indirectly by dependency features. This function mimics Cargo's behavior + described in a note at: + https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features + */ + enableFeatures = dependencies: features: + assert (builtins.isList features); + assert (builtins.isList dependencies); + let + additionalFeatures = lib.concatMap + ( + dependency: + assert (builtins.isAttrs dependency); + let + enabled = builtins.any (doesFeatureEnableDependency dependency) features; + in + if (dependency.optional or false) && enabled + then [ (dependency.rename or dependency.name) ] + else [ ] + ) + dependencies; + in + sortedUnique (features ++ additionalFeatures); - features: The features of the crate that refers this dependency. - */ - dependencyFeatures = features: dependency: - assert (builtins.isList features); - assert (builtins.isAttrs dependency); - let - defaultOrNil = - if dependency.usesDefaultFeatures or true - then [ "default" ] - else [ ]; - explicitFeatures = dependency.features or [ ]; - additionalDependencyFeatures = - let - name = dependency.rename or dependency.name; - stripPrefixMatch = prefix: s: - if lib.hasPrefix prefix s - then lib.removePrefix prefix s - else null; - extractFeature = feature: lib.findFirst - (f: f != null) - null - (map (prefix: stripPrefixMatch prefix feature) [ - (name + "/") - (name + "?/") - ]); - dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); - in - dependencyFeatures; - in - defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; + /* + Returns the actual features for the given dependency. - /* Sorts and removes duplicates from a list of strings. */ - sortedUnique = features: - assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; + features: The features of the crate that refers this dependency. + */ + dependencyFeatures = features: dependency: + assert (builtins.isList features); + assert (builtins.isAttrs dependency); + let + defaultOrNil = + if dependency.usesDefaultFeatures or true + then [ "default" ] + else [ ]; + explicitFeatures = dependency.features or [ ]; + additionalDependencyFeatures = + let + name = dependency.rename or dependency.name; + stripPrefixMatch = prefix: s: + if lib.hasPrefix prefix s + then lib.removePrefix prefix s + else null; + extractFeature = feature: lib.findFirst + (f: f != null) + null + (map (prefix: stripPrefixMatch prefix feature) [ + (name + "/") + (name + "?/") + ]); + dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); + in + dependencyFeatures; + in + defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - deprecationWarning = message: value: - if strictDeprecation - then builtins.throw "strictDeprecation enabled, aborting: ${message}" - else builtins.trace message value; + /* Sorts and removes duplicates from a list of strings. */ + sortedUnique = features: + assert (builtins.isList features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; - # - # crate2nix/default.nix (excerpt end) - # + deprecationWarning = message: value: + if strictDeprecation + then builtins.throw "strictDeprecation enabled, aborting: ${message}" + else builtins.trace message value; + + # + # crate2nix/default.nix (excerpt end) + # }; } diff --git a/libs/wire-api/src/Wire/API/User/Client/Prekey.hs b/libs/wire-api/src/Wire/API/User/Client/Prekey.hs index f58eaa000e..ff861a3078 100644 --- a/libs/wire-api/src/Wire/API/User/Client/Prekey.hs +++ b/libs/wire-api/src/Wire/API/User/Client/Prekey.hs @@ -32,11 +32,16 @@ module Wire.API.User.Client.Prekey ) where +import Crypto.Hash (SHA256, hash) import Data.Aeson (FromJSON (..), ToJSON (..)) -import Data.Hashable (hash) +import Data.ByteArray (convert) +import Data.ByteString qualified as BS +import Data.ByteString.Conversion (toByteString') import Data.Id import Data.OpenApi qualified as S import Data.Schema +import Data.Text.Ascii (encodeBase16) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) import Imports import Wire.Arbitrary (Arbitrary (arbitrary), GenericUniform (..)) @@ -63,7 +68,16 @@ instance ToSchema Prekey where <*> prekeyKey .= field "key" schema clientIdFromPrekey :: Prekey -> ClientId -clientIdFromPrekey = newClientId . fromIntegral . hash . prekeyKey +clientIdFromPrekey = + ClientId + . decodeUtf8 + . toByteString' + . encodeBase16 + . BS.take 8 + . convert + . hash @ByteString @SHA256 + . encodeUtf8 + . prekeyKey -------------------------------------------------------------------------------- -- LastPrekey diff --git a/nix/manual-overrides.nix b/nix/manual-overrides.nix index 9002298620..8648f5c5e9 100644 --- a/nix/manual-overrides.nix +++ b/nix/manual-overrides.nix @@ -2,9 +2,9 @@ # FUTUREWORK: Figure out a way to detect if some of these packages are not # actually marked broken, so we can cleanup this file on every nixpkgs bump. hself: hsuper: { - aeson = (hlib.doJailbreak hsuper.aeson_2_1_2_1); binary-parsers = hlib.markUnbroken (hlib.doJailbreak hsuper.binary-parsers); bytestring-arbitrary = hlib.markUnbroken (hlib.doJailbreak hsuper.bytestring-arbitrary); + bytestring-conversion = hlib.markUnbroken (hsuper.bytestring-conversion); openapi3 = hlib.markUnbroken (hlib.dontCheck hsuper.openapi3); cql = hlib.appendPatch (hlib.markUnbroken hsuper.cql) (fetchpatch { url = "https://gitlab.com/twittner/cql/-/merge_requests/11.patch"; @@ -13,23 +13,35 @@ hself: hsuper: { hashtables = hsuper.hashtables_1_3; invertible = hlib.markUnbroken hsuper.invertible; lens-datetime = hlib.markUnbroken (hlib.doJailbreak hsuper.lens-datetime); + monoidal-containers = hlib.doJailbreak hsuper.monoidal-containers; network-arbitrary = hlib.markUnbroken (hlib.doJailbreak hsuper.network-arbitrary); one-liner = hlib.doJailbreak hsuper.one-liner; + linear-generics = hsuper.linear-generics_0_2_2; polysemy = hlib.doJailbreak hsuper.polysemy; polysemy-check = hlib.markUnbroken (hlib.doJailbreak hsuper.polysemy-check); polysemy-plugin = hlib.doJailbreak hsuper.polysemy-plugin; quickcheck-state-machine = hlib.dontCheck hsuper.quickcheck-state-machine; + servant = hlib.doJailbreak hsuper.servant; + servant-client = hlib.doJailbreak hsuper.servant-client; + servant-client-core = hlib.doJailbreak hsuper.servant-client-core; servant-foreign = hlib.doJailbreak hsuper.servant-foreign; servant-multipart = hlib.doJailbreak hsuper.servant-multipart; servant-swagger-ui = hlib.doJailbreak hsuper.servant-swagger-ui; servant-swagger-ui-core = hlib.doJailbreak hsuper.servant-swagger-ui-core; + singletons-th = hlib.doJailbreak hsuper.singletons-th; + singletons-base = hlib.dontCheck (hlib.doJailbreak hsuper.singletons-base); sodium-crypto-sign = hlib.addPkgconfigDepend hsuper.sodium-crypto-sign libsodium.dev; text-icu-translit = hlib.markUnbroken (hlib.dontCheck hsuper.text-icu-translit); text-short = hlib.dontCheck hsuper.text-short; + template = hlib.markUnbroken hsuper.template; type-errors = hlib.dontCheck hsuper.type-errors; + th-abstraction = hsuper.th-abstraction_0_5_0_0; + th-desugar = hlib.doJailbreak hsuper.th-desugar; wai-middleware-prometheus = hlib.doJailbreak hsuper.wai-middleware-prometheus; wai-predicates = hlib.markUnbroken hsuper.wai-predicates; + http2-manager = hlib.enableCabalFlag hsuper.http2-manager "-f-test-trailing-dot"; + # PR with fix: https://github.com/freckle/hspec-junit-formatter/pull/23 hspec-junit-formatter = hlib.markUnbroken (hlib.dontCheck hsuper.hspec-junit-formatter); diff --git a/nix/overlay.nix b/nix/overlay.nix index 35acc2f23c..20aa6eb799 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -69,10 +69,19 @@ self: super: { name = "zauth"; src = ../services/nginz/third_party/nginx-zauth-module; inputs = [ self.pkg-config self.zauth.lib ]; + meta = { + license = [ self.lib.licenses.agpl3Only ]; + }; }; }; - nginz = super.nginx.override { + nginz = (super.nginx.overrideAttrs rec { + version = "1.22.1"; + src = super.fetchurl { + url = "https://nginx.org/download/nginx-${version}.tar.gz"; + hash = "sha256-nrszOp6CuVKs0+K0rrHU/2QG9ySRurbNn+afDepzfzE="; + }; + }).override { modules = [ self.nginxModules.vts self.nginxModules.moreheaders diff --git a/nix/pkgs/cryptobox/Cargo.nix b/nix/pkgs/cryptobox/Cargo.nix index 3ba01131fb..0f803cf1b3 100644 --- a/nix/pkgs/cryptobox/Cargo.nix +++ b/nix/pkgs/cryptobox/Cargo.nix @@ -1,15 +1,14 @@ - # This file was @generated by crate2nix 0.10.0 with the command: # "generate" # See https://github.com/kolloch/crate2nix for more info. { nixpkgs ? -, pkgs ? import nixpkgs { config = {}; } +, pkgs ? import nixpkgs { config = { }; } , lib ? pkgs.lib , stdenv ? pkgs.stdenv , buildRustCrateForPkgs ? if buildRustCrate != null - then lib.warn "crate2nix: Passing `buildRustCrate` as argument to Cargo.nix is deprecated. If you don't customize `buildRustCrate`, replace `callPackage ./Cargo.nix {}` by `import ./Cargo.nix { inherit pkgs; }`, and if you need to customize `buildRustCrate`, use `buildRustCrateForPkgs` instead." (_: buildRustCrate) - else pkgs: pkgs.buildRustCrate + then lib.warn "crate2nix: Passing `buildRustCrate` as argument to Cargo.nix is deprecated. If you don't customize `buildRustCrate`, replace `callPackage ./Cargo.nix {}` by `import ./Cargo.nix { inherit pkgs; }`, and if you need to customize `buildRustCrate`, use `buildRustCrateForPkgs` instead." (_: buildRustCrate) + else pkgs: pkgs.buildRustCrate # Deprecated , buildRustCrate ? null # This is used as the `crateOverrides` argument for `buildRustCrate`. @@ -19,14 +18,13 @@ # If true, throw errors instead of issueing deprecation warnings. , strictDeprecation ? false # Used for conditional compilation based on CPU feature detection. -, targetFeatures ? [] +, targetFeatures ? [ ] # Whether to perform release builds: longer compile times, faster binaries. , release ? true # Additional crate2nix configuration if it exists. -, crateConfig - ? if builtins.pathExists ./crate-config.nix - then pkgs.callPackage ./crate-config.nix {} - else {} +, crateConfig ? if builtins.pathExists ./crate-config.nix + then pkgs.callPackage ./crate-config.nix { } + else { } }: rec { @@ -65,10 +63,10 @@ rec { # A derivation that joins the outputs of all workspace members together. allWorkspaceMembers = pkgs.symlinkJoin { - name = "all-workspace-members"; - paths = - let members = builtins.attrValues workspaceMembers; - in builtins.map (m: m.build) members; + name = "all-workspace-members"; + paths = + let members = builtins.attrValues workspaceMembers; + in builtins.map (m: m.build) members; }; # @@ -162,7 +160,8 @@ rec { url = "https://github.com/wireapp/cryptobox-c"; rev = "4067ad96b125942545dbdec8c1a89f1e1b65d013"; sha256 = "1i9dlhw0xk1viglyhail9fb36v1awrypps8jmhrkz8k1bhx98ci3"; - };type = [ "cdylib" ]; + }; + type = [ "cdylib" ]; authors = [ "Wire Swiss GmbH " ]; @@ -317,643 +316,643 @@ rec { }; # -# crate2nix/default.nix (excerpt start) -# - - /* Target (platform) data for conditional dependencies. - This corresponds roughly to what buildRustCrate is setting. - */ - defaultTarget = { - unix = true; - windows = false; - fuchsia = true; - test = false; - - # This doesn't appear to be officially documented anywhere yet. - # See https://github.com/rust-lang-nursery/rust-forge/issues/101. - os = - if stdenv.hostPlatform.isDarwin - then "macos" - else stdenv.hostPlatform.parsed.kernel.name; - arch = stdenv.hostPlatform.parsed.cpu.name; - family = "unix"; - env = "gnu"; - endian = - if stdenv.hostPlatform.parsed.cpu.significantByte.name == "littleEndian" - then "little" else "big"; - pointer_width = toString stdenv.hostPlatform.parsed.cpu.bits; - vendor = stdenv.hostPlatform.parsed.vendor.name; - debug_assertions = false; - }; + # crate2nix/default.nix (excerpt start) + # - /* Filters common temp files and build files. */ - # TODO(pkolloch): Substitute with gitignore filter - sourceFilter = name: type: - let - baseName = builtins.baseNameOf (builtins.toString name); - in - ! ( - # Filter out git - baseName == ".gitignore" - || (type == "directory" && baseName == ".git") - - # Filter out build results - || ( - type == "directory" && ( - baseName == "target" - || baseName == "_site" - || baseName == ".sass-cache" - || baseName == ".jekyll-metadata" - || baseName == "build-artifacts" - ) - ) + /* Target (platform) data for conditional dependencies. + This corresponds roughly to what buildRustCrate is setting. + */ + defaultTarget = { + unix = true; + windows = false; + fuchsia = true; + test = false; + + # This doesn't appear to be officially documented anywhere yet. + # See https://github.com/rust-lang-nursery/rust-forge/issues/101. + os = + if stdenv.hostPlatform.isDarwin + then "macos" + else stdenv.hostPlatform.parsed.kernel.name; + arch = stdenv.hostPlatform.parsed.cpu.name; + family = "unix"; + env = "gnu"; + endian = + if stdenv.hostPlatform.parsed.cpu.significantByte.name == "littleEndian" + then "little" else "big"; + pointer_width = toString stdenv.hostPlatform.parsed.cpu.bits; + vendor = stdenv.hostPlatform.parsed.vendor.name; + debug_assertions = false; + }; - # Filter out nix-build result symlinks - || ( - type == "symlink" && lib.hasPrefix "result" baseName - ) + /* Filters common temp files and build files. */ + # TODO(pkolloch): Substitute with gitignore filter + sourceFilter = name: type: + let + baseName = builtins.baseNameOf (builtins.toString name); + in + ! ( + # Filter out git + baseName == ".gitignore" + || (type == "directory" && baseName == ".git") + + # Filter out build results + || ( + type == "directory" && ( + baseName == "target" + || baseName == "_site" + || baseName == ".sass-cache" + || baseName == ".jekyll-metadata" + || baseName == "build-artifacts" + ) + ) - # Filter out IDE config - || ( - type == "directory" && ( - baseName == ".idea" || baseName == ".vscode" + # Filter out nix-build result symlinks + || ( + type == "symlink" && lib.hasPrefix "result" baseName ) - ) || lib.hasSuffix ".iml" baseName - - # Filter out nix build files - || baseName == "Cargo.nix" - - # Filter out editor backup / swap files. - || lib.hasSuffix "~" baseName - || builtins.match "^\\.sw[a-z]$$" baseName != null - || builtins.match "^\\..*\\.sw[a-z]$$" baseName != null - || lib.hasSuffix ".tmp" baseName - || lib.hasSuffix ".bak" baseName - || baseName == "tests.nix" - ); - - /* Returns a crate which depends on successful test execution - of crate given as the second argument. - - testCrateFlags: list of flags to pass to the test exectuable - testInputs: list of packages that should be available during test execution - */ - crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: - assert builtins.typeOf testCrateFlags == "list"; - assert builtins.typeOf testInputs == "list"; - assert builtins.typeOf testPreRun == "string"; - assert builtins.typeOf testPostRun == "string"; - let - # override the `crate` so that it will build and execute tests instead of - # building the actual lib and bin targets We just have to pass `--test` - # to rustc and it will do the right thing. We execute the tests and copy - # their log and the test executables to $out for later inspection. - test = + + # Filter out IDE config + || ( + type == "directory" && ( + baseName == ".idea" || baseName == ".vscode" + ) + ) || lib.hasSuffix ".iml" baseName + + # Filter out nix build files + || baseName == "Cargo.nix" + + # Filter out editor backup / swap files. + || lib.hasSuffix "~" baseName + || builtins.match "^\\.sw[a-z]$$" baseName != null + || builtins.match "^\\..*\\.sw[a-z]$$" baseName != null + || lib.hasSuffix ".tmp" baseName + || lib.hasSuffix ".bak" baseName + || baseName == "tests.nix" + ); + + /* Returns a crate which depends on successful test execution + of crate given as the second argument. + + testCrateFlags: list of flags to pass to the test exectuable + testInputs: list of packages that should be available during test execution + */ + crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: + assert builtins.typeOf testCrateFlags == "list"; + assert builtins.typeOf testInputs == "list"; + assert builtins.typeOf testPreRun == "string"; + assert builtins.typeOf testPostRun == "string"; + let + # override the `crate` so that it will build and execute tests instead of + # building the actual lib and bin targets We just have to pass `--test` + # to rustc and it will do the right thing. We execute the tests and copy + # their log and the test executables to $out for later inspection. + test = + let + drv = testCrate.override + ( + _: { + buildTests = true; + } + ); + # If the user hasn't set any pre/post commands, we don't want to + # insert empty lines. This means that any existing users of crate2nix + # don't get a spurious rebuild unless they set these explicitly. + testCommand = pkgs.lib.concatStringsSep "\n" + (pkgs.lib.filter (s: s != "") [ + testPreRun + "$f $testCrateFlags 2>&1 | tee -a $out" + testPostRun + ]); + in + pkgs.runCommand "run-tests-${testCrate.name}" + { + inherit testCrateFlags; + buildInputs = testInputs; + } '' + set -ex + + export RUST_BACKTRACE=1 + + # recreate a file hierarchy as when running tests with cargo + + # the source for test data + ${pkgs.xorg.lndir}/bin/lndir ${crate.src} + + # build outputs + testRoot=target/debug + mkdir -p $testRoot + + # executables of the crate + # we copy to prevent std::env::current_exe() to resolve to a store location + for i in ${crate}/bin/*; do + cp "$i" "$testRoot" + done + chmod +w -R . + + # test harness executables are suffixed with a hash, like cargo does + # this allows to prevent name collision with the main + # executables of the crate + hash=$(basename $out) + for file in ${drv}/tests/*; do + f=$testRoot/$(basename $file)-$hash + cp $file $f + ${testCommand} + done + ''; + in + pkgs.runCommand "${crate.name}-linked" + { + inherit (crate) outputs crateName; + passthru = (crate.passthru or { }) // { + inherit test; + }; + } '' + echo tested by ${test} + ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} + ''; + + /* A restricted overridable version of builtRustCratesWithFeatures. */ + buildRustCrateWithFeatures = + { packageId + , features ? rootFeatures + , crateOverrides ? defaultCrateOverrides + , buildRustCrateForPkgsFunc ? null + , runTests ? false + , testCrateFlags ? [ ] + , testInputs ? [ ] + # Any command to run immediatelly before a test is executed. + , testPreRun ? "" + # Any command run immediatelly after a test is executed. + , testPostRun ? "" + }: + lib.makeOverridable + ( + { features + , crateOverrides + , runTests + , testCrateFlags + , testInputs + , testPreRun + , testPostRun + }: + let + buildRustCrateForPkgsFuncOverriden = + if buildRustCrateForPkgsFunc != null + then buildRustCrateForPkgsFunc + else + ( + if crateOverrides == pkgs.defaultCrateOverrides + then buildRustCrateForPkgs + else + pkgs: (buildRustCrateForPkgs pkgs).override { + defaultCrateOverrides = crateOverrides; + } + ); + builtRustCrates = builtRustCratesWithFeatures { + inherit packageId features; + buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; + runTests = false; + }; + builtTestRustCrates = builtRustCratesWithFeatures { + inherit packageId features; + buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; + runTests = true; + }; + drv = builtRustCrates.crates.${packageId}; + testDrv = builtTestRustCrates.crates.${packageId}; + derivation = + if runTests then + crateWithTest + { + crate = drv; + testCrate = testDrv; + inherit testCrateFlags testInputs testPreRun testPostRun; + } + else drv; + in + derivation + ) + { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + + /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc + for the corresponding crate. + */ + builtRustCratesWithFeatures = + { packageId + , features + , crateConfigs ? crates + , buildRustCrateForPkgsFunc + , runTests + , target ? defaultTarget + } @ args: + assert (builtins.isAttrs crateConfigs); + assert (builtins.isString packageId); + assert (builtins.isList features); + assert (builtins.isAttrs target); + assert (builtins.isBool runTests); let - drv = testCrate.override + rootPackageId = packageId; + mergedFeatures = mergePackageFeatures ( - _: { - buildTests = true; + args // { + inherit rootPackageId; + target = target // { test = runTests; }; } ); - # If the user hasn't set any pre/post commands, we don't want to - # insert empty lines. This means that any existing users of crate2nix - # don't get a spurious rebuild unless they set these explicitly. - testCommand = pkgs.lib.concatStringsSep "\n" - (pkgs.lib.filter (s: s != "") [ - testPreRun - "$f $testCrateFlags 2>&1 | tee -a $out" - testPostRun - ]); - in - pkgs.runCommand "run-tests-${testCrate.name}" - { - inherit testCrateFlags; - buildInputs = testInputs; - } '' - set -ex - - export RUST_BACKTRACE=1 - - # recreate a file hierarchy as when running tests with cargo - - # the source for test data - ${pkgs.xorg.lndir}/bin/lndir ${crate.src} - - # build outputs - testRoot=target/debug - mkdir -p $testRoot - - # executables of the crate - # we copy to prevent std::env::current_exe() to resolve to a store location - for i in ${crate}/bin/*; do - cp "$i" "$testRoot" - done - chmod +w -R . - - # test harness executables are suffixed with a hash, like cargo does - # this allows to prevent name collision with the main - # executables of the crate - hash=$(basename $out) - for file in ${drv}/tests/*; do - f=$testRoot/$(basename $file)-$hash - cp $file $f - ${testCommand} - done - ''; - in - pkgs.runCommand "${crate.name}-linked" - { - inherit (crate) outputs crateName; - passthru = (crate.passthru or { }) // { - inherit test; - }; - } '' - echo tested by ${test} - ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} - ''; - - /* A restricted overridable version of builtRustCratesWithFeatures. */ - buildRustCrateWithFeatures = - { packageId - , features ? rootFeatures - , crateOverrides ? defaultCrateOverrides - , buildRustCrateForPkgsFunc ? null - , runTests ? false - , testCrateFlags ? [ ] - , testInputs ? [ ] - # Any command to run immediatelly before a test is executed. - , testPreRun ? "" - # Any command run immediatelly after a test is executed. - , testPostRun ? "" - }: - lib.makeOverridable - ( - { features - , crateOverrides - , runTests - , testCrateFlags - , testInputs - , testPreRun - , testPostRun - }: - let - buildRustCrateForPkgsFuncOverriden = - if buildRustCrateForPkgsFunc != null - then buildRustCrateForPkgsFunc - else + # Memoize built packages so that reappearing packages are only built once. + builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; + mkBuiltByPackageIdByPkgs = pkgs: + let + self = { + crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; + build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; + }; + in + self; + buildByPackageIdForPkgsImpl = self: pkgs: packageId: + let + features = mergedFeatures."${packageId}" or [ ]; + crateConfig' = crateConfigs."${packageId}"; + crateConfig = + builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; + devDependencies = + lib.optionals + (runTests && packageId == rootPackageId) + (crateConfig'.devDependencies or [ ]); + dependencies = + dependencyDerivations { + inherit features target; + buildByPackageId = depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false + then self.build.crates.${depPackageId} + else self.crates.${depPackageId}; + dependencies = + (crateConfig.dependencies or [ ]) + ++ devDependencies; + }; + buildDependencies = + dependencyDerivations { + inherit features target; + buildByPackageId = depPackageId: + self.build.crates.${depPackageId}; + dependencies = crateConfig.buildDependencies or [ ]; + }; + filterEnabledDependenciesForThis = dependencies: filterEnabledDependencies { + inherit dependencies features target; + }; + dependenciesWithRenames = + lib.filter (d: d ? "rename") + ( + filterEnabledDependenciesForThis + ( + (crateConfig.buildDependencies or [ ]) + ++ (crateConfig.dependencies or [ ]) + ++ devDependencies + ) + ); + # Crate renames have the form: + # + # { + # crate_name = [ + # { version = "1.2.3"; rename = "crate_name01"; } + # ]; + # # ... + # } + crateRenames = + let + grouped = + lib.groupBy + (dependency: dependency.name) + dependenciesWithRenames; + versionAndRename = dep: + let + package = crateConfigs."${dep.packageId}"; + in + { inherit (dep) rename; version = package.version; }; + in + lib.mapAttrs (name: choices: builtins.map versionAndRename choices) grouped; + in + buildRustCrateForPkgsFunc pkgs ( - if crateOverrides == pkgs.defaultCrateOverrides - then buildRustCrateForPkgs - else - pkgs: (buildRustCrateForPkgs pkgs).override { - defaultCrateOverrides = crateOverrides; - } + crateConfig // { + src = crateConfig.src or ( + pkgs.fetchurl rec { + name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; + # https://www.pietroalbini.org/blog/downloading-crates-io/ + # Not rate-limited, CDN URL. + url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; + sha256 = + assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); + crateConfig.sha256; + } + ); + extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; + inherit features dependencies buildDependencies crateRenames release; + } ); - builtRustCrates = builtRustCratesWithFeatures { - inherit packageId features; - buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; - runTests = false; - }; - builtTestRustCrates = builtRustCratesWithFeatures { - inherit packageId features; - buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; - runTests = true; + in + builtByPackageIdByPkgs; + + /* Returns the actual derivations for the given dependencies. */ + dependencyDerivations = + { buildByPackageId + , features + , dependencies + , target + }: + assert (builtins.isList features); + assert (builtins.isList dependencies); + assert (builtins.isAttrs target); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies features target; }; - drv = builtRustCrates.crates.${packageId}; - testDrv = builtTestRustCrates.crates.${packageId}; - derivation = - if runTests then - crateWithTest - { - crate = drv; - testCrate = testDrv; - inherit testCrateFlags testInputs testPreRun testPostRun; - } - else drv; + depDerivation = dependency: buildByPackageId dependency.packageId; in - derivation - ) - { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; - - /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc - for the corresponding crate. - */ - builtRustCratesWithFeatures = - { packageId - , features - , crateConfigs ? crates - , buildRustCrateForPkgsFunc - , runTests - , target ? defaultTarget - } @ args: - assert (builtins.isAttrs crateConfigs); + map depDerivation enabledDependencies; + + /* Returns a sanitized version of val with all values substituted that cannot + be serialized as JSON. + */ + sanitizeForJson = val: + if builtins.isAttrs val + then lib.mapAttrs (n: v: sanitizeForJson v) val + else if builtins.isList val + then builtins.map sanitizeForJson val + else if builtins.isFunction val + then "function" + else val; + + /* Returns various tools to debug a crate. */ + debugCrate = { packageId, target ? defaultTarget }: assert (builtins.isString packageId); - assert (builtins.isList features); - assert (builtins.isAttrs target); - assert (builtins.isBool runTests); let - rootPackageId = packageId; - mergedFeatures = mergePackageFeatures - ( - args // { - inherit rootPackageId; - target = target // { test = runTests; }; - } - ); - # Memoize built packages so that reappearing packages are only built once. - builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; - mkBuiltByPackageIdByPkgs = pkgs: - let - self = { - crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; - build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; - }; - in - self; - buildByPackageIdForPkgsImpl = self: pkgs: packageId: - let - features = mergedFeatures."${packageId}" or [ ]; - crateConfig' = crateConfigs."${packageId}"; - crateConfig = - builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; - devDependencies = - lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig'.devDependencies or [ ]); - dependencies = - dependencyDerivations { - inherit features target; - buildByPackageId = depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false - then self.build.crates.${depPackageId} - else self.crates.${depPackageId}; - dependencies = - (crateConfig.dependencies or [ ]) - ++ devDependencies; - }; - buildDependencies = - dependencyDerivations { - inherit features target; - buildByPackageId = depPackageId: - self.build.crates.${depPackageId}; - dependencies = crateConfig.buildDependencies or [ ]; - }; - filterEnabledDependenciesForThis = dependencies: filterEnabledDependencies { - inherit dependencies features target; - }; - dependenciesWithRenames = - lib.filter (d: d ? "rename") - ( - filterEnabledDependenciesForThis - ( - (crateConfig.buildDependencies or [ ]) - ++ (crateConfig.dependencies or [ ]) - ++ devDependencies - ) - ); - # Crate renames have the form: - # - # { - # crate_name = [ - # { version = "1.2.3"; rename = "crate_name01"; } - # ]; - # # ... - # } - crateRenames = - let - grouped = - lib.groupBy - (dependency: dependency.name) - dependenciesWithRenames; - versionAndRename = dep: - let - package = crateConfigs."${dep.packageId}"; - in - { inherit (dep) rename; version = package.version; }; - in - lib.mapAttrs (name: choices: builtins.map versionAndRename choices) grouped; - in - buildRustCrateForPkgsFunc pkgs + debug = rec { + # The built tree as passed to buildRustCrate. + buildTree = buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: lib.id; + inherit packageId; + }; + sanitizedBuildTree = sanitizeForJson buildTree; + dependencyTree = sanitizeForJson ( - crateConfig // { - src = crateConfig.src or ( - pkgs.fetchurl rec { - name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; - # https://www.pietroalbini.org/blog/downloading-crates-io/ - # Not rate-limited, CDN URL. - url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; - sha256 = - assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); - crateConfig.sha256; - } - ); - extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; - inherit features dependencies buildDependencies crateRenames release; + buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: crate: { + "01_crateName" = crate.crateName or false; + "02_features" = crate.features or [ ]; + "03_dependencies" = crate.dependencies or [ ]; + }; + inherit packageId; } ); + mergedPackageFeatures = mergePackageFeatures { + features = rootFeatures; + inherit packageId target; + }; + diffedDefaultPackageFeatures = diffDefaultPackageFeatures { + inherit packageId target; + }; + }; in - builtByPackageIdByPkgs; - - /* Returns the actual derivations for the given dependencies. */ - dependencyDerivations = - { buildByPackageId - , features - , dependencies - , target - }: - assert (builtins.isList features); + { internal = debug; }; + + /* Returns differences between cargo default features and crate2nix default + features. + + This is useful for verifying the feature resolution in crate2nix. + */ + diffDefaultPackageFeatures = + { crateConfigs ? crates + , packageId + , target + }: + assert (builtins.isAttrs crateConfigs); + let + prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); + mergedFeatures = + prefixValues + "crate2nix" + (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); + configs = prefixValues "cargo" crateConfigs; + combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; + onlyInCargo = + builtins.attrNames + (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); + onlyInCrate2Nix = + builtins.attrNames + (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); + differentFeatures = lib.filterAttrs + ( + n: v: + (v ? "crate2nix") + && (v ? "cargo") + && (v.crate2nix.features or [ ]) != (v."cargo".resolved_default_features or [ ]) + ) + combined; + in + builtins.toJSON { + inherit onlyInCargo onlyInCrate2Nix differentFeatures; + }; + + /* Returns an attrset mapping packageId to the list of enabled features. + + If multiple paths to a dependency enable different features, the + corresponding feature sets are merged. Features in rust are additive. + */ + mergePackageFeatures = + { crateConfigs ? crates + , packageId + , rootPackageId ? packageId + , features ? rootFeatures + , dependencyPath ? [ crates.${packageId}.crateName ] + , featuresByPackageId ? { } + , target + # Adds devDependencies to the crate with rootPackageId. + , runTests ? false + , ... + } @ args: + assert (builtins.isAttrs crateConfigs); + assert (builtins.isString packageId); + assert (builtins.isString rootPackageId); + assert (builtins.isList features); + assert (builtins.isList dependencyPath); + assert (builtins.isAttrs featuresByPackageId); + assert (builtins.isAttrs target); + assert (builtins.isBool runTests); + let + crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); + expandedFeatures = expandFeatures (crateConfig.features or { }) features; + enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; + depWithResolvedFeatures = dependency: + let + packageId = dependency.packageId; + features = dependencyFeatures enabledFeatures dependency; + in + { inherit packageId features; }; + resolveDependencies = cache: path: dependencies: + assert (builtins.isAttrs cache); + assert (builtins.isList dependencies); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies target; + features = enabledFeatures; + }; + directDependencies = map depWithResolvedFeatures enabledDependencies; + foldOverCache = op: lib.foldl op cache directDependencies; + in + foldOverCache + ( + cache: { packageId, features }: + let + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); + in + if cache ? ${packageId} && cache.${packageId} == combinedFeatures + then cache + else + mergePackageFeatures { + features = combinedFeatures; + featuresByPackageId = cache; + inherit crateConfigs packageId target runTests rootPackageId; + } + ); + cacheWithSelf = + let + cacheFeatures = featuresByPackageId.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); + in + featuresByPackageId // { + "${packageId}" = combinedFeatures; + }; + cacheWithDependencies = + resolveDependencies cacheWithSelf "dep" + ( + crateConfig.dependencies or [ ] + ++ lib.optionals + (runTests && packageId == rootPackageId) + (crateConfig.devDependencies or [ ]) + ); + cacheWithAll = + resolveDependencies + cacheWithDependencies "build" + (crateConfig.buildDependencies or [ ]); + in + cacheWithAll; + + /* Returns the enabled dependencies given the enabled features. */ + filterEnabledDependencies = { dependencies, features, target }: assert (builtins.isList dependencies); + assert (builtins.isList features); assert (builtins.isAttrs target); + + lib.filter + ( + dep: + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } + && ( + !(dep.optional or false) + || builtins.any (doesFeatureEnableDependency dep) features + ) + ) + dependencies; + + /* Returns whether the given feature should enable the given dependency. */ + doesFeatureEnableDependency = { name, rename ? null, ... }: feature: let - enabledDependencies = filterEnabledDependencies { - inherit dependencies features target; - }; - depDerivation = dependency: buildByPackageId dependency.packageId; + prefix = "${name}/"; + len = builtins.stringLength prefix; + startsWithPrefix = builtins.substring 0 len feature == prefix; in - map depDerivation enabledDependencies; - - /* Returns a sanitized version of val with all values substituted that cannot - be serialized as JSON. - */ - sanitizeForJson = val: - if builtins.isAttrs val - then lib.mapAttrs (n: v: sanitizeForJson v) val - else if builtins.isList val - then builtins.map sanitizeForJson val - else if builtins.isFunction val - then "function" - else val; - - /* Returns various tools to debug a crate. */ - debugCrate = { packageId, target ? defaultTarget }: - assert (builtins.isString packageId); - let - debug = rec { - # The built tree as passed to buildRustCrate. - buildTree = buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: lib.id; - inherit packageId; - }; - sanitizedBuildTree = sanitizeForJson buildTree; - dependencyTree = sanitizeForJson - ( - buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: crate: { - "01_crateName" = crate.crateName or false; - "02_features" = crate.features or [ ]; - "03_dependencies" = crate.dependencies or [ ]; - }; - inherit packageId; - } - ); - mergedPackageFeatures = mergePackageFeatures { - features = rootFeatures; - inherit packageId target; - }; - diffedDefaultPackageFeatures = diffDefaultPackageFeatures { - inherit packageId target; - }; - }; - in - { internal = debug; }; - - /* Returns differences between cargo default features and crate2nix default - features. - - This is useful for verifying the feature resolution in crate2nix. - */ - diffDefaultPackageFeatures = - { crateConfigs ? crates - , packageId - , target - }: - assert (builtins.isAttrs crateConfigs); + (rename == null && feature == name) + || (rename != null && rename == feature) + || startsWithPrefix; + + /* Returns the expanded features for the given inputFeatures by applying the + rules in featureMap. + + featureMap is an attribute set which maps feature names to lists of further + feature names to enable in case this feature is selected. + */ + expandFeatures = featureMap: inputFeatures: + assert (builtins.isAttrs featureMap); + assert (builtins.isList inputFeatures); + let + expandFeature = feature: + assert (builtins.isString feature); + [ feature ] ++ (expandFeatures featureMap (featureMap."${feature}" or [ ])); + outFeatures = lib.concatMap expandFeature inputFeatures; + in + sortedUnique outFeatures; + + /* This function adds optional dependencies as features if they are enabled + indirectly by dependency features. This function mimics Cargo's behavior + described in a note at: + https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features + */ + enableFeatures = dependencies: features: + assert (builtins.isList features); + assert (builtins.isList dependencies); let - prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); - mergedFeatures = - prefixValues - "crate2nix" - (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); - configs = prefixValues "cargo" crateConfigs; - combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; - onlyInCargo = - builtins.attrNames - (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); - onlyInCrate2Nix = - builtins.attrNames - (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); - differentFeatures = lib.filterAttrs + additionalFeatures = lib.concatMap ( - n: v: - (v ? "crate2nix") - && (v ? "cargo") - && (v.crate2nix.features or [ ]) != (v."cargo".resolved_default_features or [ ]) + dependency: + assert (builtins.isAttrs dependency); + let + enabled = builtins.any (doesFeatureEnableDependency dependency) features; + in + if (dependency.optional or false) && enabled then [ dependency.name ] else [ ] ) - combined; + dependencies; in - builtins.toJSON { - inherit onlyInCargo onlyInCrate2Nix differentFeatures; - }; + sortedUnique (features ++ additionalFeatures); - /* Returns an attrset mapping packageId to the list of enabled features. - - If multiple paths to a dependency enable different features, the - corresponding feature sets are merged. Features in rust are additive. - */ - mergePackageFeatures = - { crateConfigs ? crates - , packageId - , rootPackageId ? packageId - , features ? rootFeatures - , dependencyPath ? [ crates.${packageId}.crateName ] - , featuresByPackageId ? { } - , target - # Adds devDependencies to the crate with rootPackageId. - , runTests ? false - , ... - } @ args: - assert (builtins.isAttrs crateConfigs); - assert (builtins.isString packageId); - assert (builtins.isString rootPackageId); + /* + Returns the actual features for the given dependency. + + features: The features of the crate that refers this dependency. + */ + dependencyFeatures = features: dependency: assert (builtins.isList features); - assert (builtins.isList dependencyPath); - assert (builtins.isAttrs featuresByPackageId); - assert (builtins.isAttrs target); - assert (builtins.isBool runTests); + assert (builtins.isAttrs dependency); let - crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); - expandedFeatures = expandFeatures (crateConfig.features or { }) features; - enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; - depWithResolvedFeatures = dependency: + defaultOrNil = + if dependency.usesDefaultFeatures or true + then [ "default" ] + else [ ]; + explicitFeatures = dependency.features or [ ]; + additionalDependencyFeatures = let - packageId = dependency.packageId; - features = dependencyFeatures enabledFeatures dependency; - in - { inherit packageId features; }; - resolveDependencies = cache: path: dependencies: - assert (builtins.isAttrs cache); - assert (builtins.isList dependencies); - let - enabledDependencies = filterEnabledDependencies { - inherit dependencies target; - features = enabledFeatures; - }; - directDependencies = map depWithResolvedFeatures enabledDependencies; - foldOverCache = op: lib.foldl op cache directDependencies; + dependencyPrefix = (dependency.rename or dependency.name) + "/"; + dependencyFeatures = + builtins.filter (f: lib.hasPrefix dependencyPrefix f) features; in - foldOverCache - ( - cache: { packageId, features }: - let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); - in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures - then cache - else - mergePackageFeatures { - features = combinedFeatures; - featuresByPackageId = cache; - inherit crateConfigs packageId target runTests rootPackageId; - } - ); - cacheWithSelf = - let - cacheFeatures = featuresByPackageId.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); - in - featuresByPackageId // { - "${packageId}" = combinedFeatures; - }; - cacheWithDependencies = - resolveDependencies cacheWithSelf "dep" - ( - crateConfig.dependencies or [ ] - ++ lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig.devDependencies or [ ]) - ); - cacheWithAll = - resolveDependencies - cacheWithDependencies "build" - (crateConfig.buildDependencies or [ ]); + builtins.map (lib.removePrefix dependencyPrefix) dependencyFeatures; in - cacheWithAll; + defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - /* Returns the enabled dependencies given the enabled features. */ - filterEnabledDependencies = { dependencies, features, target }: - assert (builtins.isList dependencies); - assert (builtins.isList features); - assert (builtins.isAttrs target); + /* Sorts and removes duplicates from a list of strings. */ + sortedUnique = features: + assert (builtins.isList features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; - lib.filter - ( - dep: - let - targetFunc = dep.target or (features: true); - in - targetFunc { inherit features target; } - && ( - !(dep.optional or false) - || builtins.any (doesFeatureEnableDependency dep) features - ) - ) - dependencies; - - /* Returns whether the given feature should enable the given dependency. */ - doesFeatureEnableDependency = { name, rename ? null, ... }: feature: - let - prefix = "${name}/"; - len = builtins.stringLength prefix; - startsWithPrefix = builtins.substring 0 len feature == prefix; - in - (rename == null && feature == name) - || (rename != null && rename == feature) - || startsWithPrefix; - - /* Returns the expanded features for the given inputFeatures by applying the - rules in featureMap. - - featureMap is an attribute set which maps feature names to lists of further - feature names to enable in case this feature is selected. - */ - expandFeatures = featureMap: inputFeatures: - assert (builtins.isAttrs featureMap); - assert (builtins.isList inputFeatures); - let - expandFeature = feature: - assert (builtins.isString feature); - [ feature ] ++ (expandFeatures featureMap (featureMap."${feature}" or [ ])); - outFeatures = lib.concatMap expandFeature inputFeatures; - in - sortedUnique outFeatures; - - /* This function adds optional dependencies as features if they are enabled - indirectly by dependency features. This function mimics Cargo's behavior - described in a note at: - https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features - */ - enableFeatures = dependencies: features: - assert (builtins.isList features); - assert (builtins.isList dependencies); - let - additionalFeatures = lib.concatMap - ( - dependency: - assert (builtins.isAttrs dependency); - let - enabled = builtins.any (doesFeatureEnableDependency dependency) features; - in - if (dependency.optional or false) && enabled then [ dependency.name ] else [ ] - ) - dependencies; - in - sortedUnique (features ++ additionalFeatures); - - /* - Returns the actual features for the given dependency. - - features: The features of the crate that refers this dependency. - */ - dependencyFeatures = features: dependency: - assert (builtins.isList features); - assert (builtins.isAttrs dependency); - let - defaultOrNil = - if dependency.usesDefaultFeatures or true - then [ "default" ] - else [ ]; - explicitFeatures = dependency.features or [ ]; - additionalDependencyFeatures = - let - dependencyPrefix = (dependency.rename or dependency.name) + "/"; - dependencyFeatures = - builtins.filter (f: lib.hasPrefix dependencyPrefix f) features; - in - builtins.map (lib.removePrefix dependencyPrefix) dependencyFeatures; - in - defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - - /* Sorts and removes duplicates from a list of strings. */ - sortedUnique = features: - assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; - - deprecationWarning = message: value: - if strictDeprecation - then builtins.throw "strictDeprecation enabled, aborting: ${message}" - else builtins.trace message value; + deprecationWarning = message: value: + if strictDeprecation + then builtins.throw "strictDeprecation enabled, aborting: ${message}" + else builtins.trace message value; - # - # crate2nix/default.nix (excerpt end) - # + # + # crate2nix/default.nix (excerpt end) + # }; } diff --git a/nix/pkgs/cryptobox/crate2nix-sources.nix b/nix/pkgs/cryptobox/crate2nix-sources.nix index 0af7754ab0..3e1364ba61 100644 --- a/nix/pkgs/cryptobox/crate2nix-sources.nix +++ b/nix/pkgs/cryptobox/crate2nix-sources.nix @@ -1,4 +1,3 @@ - # Support functions to create a nix generated workspace for out-of-tree sources. # # You do not need to check this in since it will be regenerated every time it is @@ -10,45 +9,48 @@ # See https://github.com/kolloch/crate2nix for more info. { nixpkgs ? -, pkgs ? import nixpkgs {} +, pkgs ? import nixpkgs { } , lib ? pkgs.lib -# The path to crate2nix.json. + # The path to crate2nix.json. , crate2nixJson ? ./crate2nix.json }: -let config = builtins.fromJSON (builtins.readFile crate2nixJson); - sources = config.sources or (builtins.throw "no sources in ${crate2nixJson}"); +let + config = builtins.fromJSON (builtins.readFile crate2nixJson); + sources = config.sources or (builtins.throw "no sources in ${crate2nixJson}"); in rec { - /* An attrset mapping a source name to its source (as a derivation). */ - fetchedSourcesByName = lib.mapAttrs internal.sourceFromConfig sources; + /* An attrset mapping a source name to its source (as a derivation). */ + fetchedSourcesByName = lib.mapAttrs internal.sourceFromConfig sources; - /* A derivation building a directory symlinking all workspace member sources + /* A derivation building a directory symlinking all workspace member sources by their name. */ - fetchedSources = - let sources = lib.mapAttrsToList (name: path: { inherit name path; }) fetchedSourcesByName; - in - pkgs.linkFarm "crate2nix-sources" sources; - - internal = rec { - sourceFromConfig = name: { type, ... } @ source: - assert builtins.isString name; - assert builtins.isString type; - - if type == "Git" - then pkgs.fetchgit { - url = source.url; - rev = source.rev; - sha256 = source.sha256; - } - else if type == "CratesIo" - then downloadFromCratesIo source - else if type == "Nix" - then resolveNix source - else builtins.throw "Unexpected source type '${type}' for source: ${builtins.toJSON source}"; - - /* Resolves a source configuration of type "Nix". + fetchedSources = + let sources = lib.mapAttrsToList (name: path: { inherit name path; }) fetchedSourcesByName; + in + pkgs.linkFarm "crate2nix-sources" sources; + + internal = rec { + sourceFromConfig = name: { type, ... } @ source: + assert builtins.isString name; + assert builtins.isString type; + + if type == "Git" + then + pkgs.fetchgit + { + url = source.url; + rev = source.rev; + sha256 = source.sha256; + } + else if type == "CratesIo" + then downloadFromCratesIo source + else if type == "Nix" + then resolveNix source + else builtins.throw "Unexpected source type '${type}' for source: ${builtins.toJSON source}"; + + /* Resolves a source configuration of type "Nix". It can either have @@ -67,43 +69,47 @@ rec { } ``` */ - resolveNix = { type, ... } @ source: - assert type == "Nix"; - - let attrs = - if source ? package - then pkgs.callPackage (./. + "/${source.package}") {} - else if source ? "import" - then import (./. + ''/${source."import"}'') - else builtins.throw "Neither import nor package in nix source."; - attrPath = lib.splitString "." source.attr; - sourceDerivation = - if source ? attr - then lib.attrByPath - attrPath - (builtins.throw - '' - Did not find attribute '${source.attr or ""}' - in '${source.package or source.import or "missing file"}'. - '') - attrs - else attrs; - in - sourceDerivation; - - downloadFromCratesIo = { type, name, version, sha256 }: - assert type == "CratesIo"; - - let archive = pkgs.fetchurl { - name = "${name}-${version}.tar.gz"; - url = "https://crates.io/api/v1/crates/${name}/${version}/download"; - inherit sha256; - }; - in pkgs.runCommand (lib.removeSuffix ".tar.gz" name) {} - '' - mkdir -p $out - tar -xzf ${archive} --strip-components=1 -C $out - ''; - }; + resolveNix = { type, ... } @ source: + assert type == "Nix"; + + let + attrs = + if source ? package + then pkgs.callPackage (./. + "/${source.package}") { } + else if source ? "import" + then import (./. + ''/${source."import"}'') + else builtins.throw "Neither import nor package in nix source."; + attrPath = lib.splitString "." source.attr; + sourceDerivation = + if source ? attr + then + lib.attrByPath + attrPath + (builtins.throw + '' + Did not find attribute '${source.attr or ""}' + in '${source.package or source.import or "missing file"}'. + '') + attrs + else attrs; + in + sourceDerivation; + + downloadFromCratesIo = { type, name, version, sha256 }: + assert type == "CratesIo"; + + let + archive = pkgs.fetchurl { + name = "${name}-${version}.tar.gz"; + url = "https://crates.io/api/v1/crates/${name}/${version}/download"; + inherit sha256; + }; + in + pkgs.runCommand (lib.removeSuffix ".tar.gz" name) { } + '' + mkdir -p $out + tar -xzf ${archive} --strip-components=1 -C $out + ''; + }; } diff --git a/nix/pkgs/mls-test-cli/default.nix b/nix/pkgs/mls-test-cli/default.nix index 7f1330ab7d..308904450d 100644 --- a/nix/pkgs/mls-test-cli/default.nix +++ b/nix/pkgs/mls-test-cli/default.nix @@ -4,22 +4,17 @@ # TODO: migrate to crate2nix once # https://github.com/nix-community/crate2nix/issues/310 is fixed -let - version = "0.7.0"; +rustPlatform.buildRustPackage rec { src = fetchFromGitHub { owner = "wireapp"; repo = "mls-test-cli"; rev = "baaa5c78411a5bf6d697803276b991523c111631"; sha256 = "sha256-M6bWB5hWl+WSblcH6L+AyGD+7ef9TvRs8wKYq7lJyS8="; }; - cargoLockFile = builtins.toFile "cargo.lock" (builtins.readFile "${src}/Cargo.lock"); -in -rustPlatform.buildRustPackage rec { - name = "mls-test-cli-${version}"; - inherit version src; - + pname = "mls-test-cli"; + version = "0.9.0"; cargoLock = { - lockFile = cargoLockFile; + lockFile = "${src}/Cargo.lock"; outputHashes = { "hpke-0.10.0" = "sha256-T1+BFwX6allljNZ/8T3mrWhOejnUU27BiWQetqU+0fY="; "openmls-1.0.0" = "sha256-tAIm8+IgubNnU2M2A5cxHY5caiEQmisw73I9/cqfvUc="; @@ -27,9 +22,5 @@ rustPlatform.buildRustPackage rec { "tls_codec-0.3.0" = "sha256-IO6tenXKkC14EoUDp/+DtFNOVzDfOlLu8K1EJI7sOzs="; }; }; - - postPatch = '' - cp ${cargoLockFile} Cargo.lock - ''; doCheck = false; } diff --git a/nix/sources.json b/nix/sources.json index e1adedd230..3f76c8b401 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -5,10 +5,10 @@ "homepage": "https://github.com/NixOS/nixpkgs", "owner": "NixOS", "repo": "nixpkgs", - "rev": "402cc3633cc60dfc50378197305c984518b30773", - "sha256": "0yvlprdkqg1kizg83j7nivlc58zk7llrbf82jqvgjimrwhsva1m9", + "rev": "01441e14af5e29c9d27ace398e6dd0b293e25a54", + "sha256": "0yvkamjbk3aj4lvhm6vdgdk4b2j0xdv3gx9n4p7wfky52j2529dy", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/402cc3633cc60dfc50378197305c984518b30773.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/01441e14af5e29c9d27ace398e6dd0b293e25a54.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-cargo": { @@ -17,10 +17,10 @@ "homepage": "https://github.com/NixOS/nixpkgs", "owner": "NixOS", "repo": "nixpkgs", - "rev": "efd23a1c9ae8c574e2ca923c2b2dc336797f4cc4", - "sha256": "0pb1dgdgfsnsngw2ci807wln2jnlsha4zkm1y14x497qbw4izir3", + "rev": "01441e14af5e29c9d27ace398e6dd0b293e25a54", + "sha256": "0yvkamjbk3aj4lvhm6vdgdk4b2j0xdv3gx9n4p7wfky52j2529dy", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/efd23a1c9ae8c574e2ca923c2b2dc336797f4cc4.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/01441e14af5e29c9d27ace398e6dd0b293e25a54.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/nix/wire-server.nix b/nix/wire-server.nix index 6a643b1131..f22c260365 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -304,6 +304,7 @@ let gnutar gzip openssl + nix-output-monitor which ]; @@ -388,6 +389,17 @@ let }; }; + # FIXME: when upgrading the ghc version, we + # should have to upgrade ormolu to support + # the new parser and get rid of these (then unnecessary) + # overrides + inherit (pkgs.haskell.packages.ghc92.override { + overrides = hfinal: hprev: { + ormolu = hfinal.ormolu_0_5_0_1; + ghc-lib-parser = hprev.ghc-lib-parser_9_2_8_20230729; + }; + }) ormolu; + # Tools common between CI and developers commonTools = [ pkgs.cabal2nix @@ -404,7 +416,7 @@ let pkgs.kubelogin-oidc pkgs.nixpkgs-fmt pkgs.openssl - pkgs.ormolu + ormolu pkgs.shellcheck pkgs.treefmt pkgs.gawk @@ -439,6 +451,14 @@ let }; ghcWithPackages = shell.nativeBuildInputs ++ shell.buildInputs; + inherit (pkgs.haskellPackages.override { + overrides = _hfinal: hprev: { + base-compat = hprev.base-compat_0_13_0; + base-compat-batteries = hprev.base-compat-batteries_0_13_0; + cabal-plan = hlib.markUnbroken (hlib.doJailbreak hprev.cabal-plan); + }; + }) cabal-plan; + profileEnv = pkgs.writeTextFile { name = "profile-env"; destination = "/.profile"; @@ -495,8 +515,8 @@ in pkgs.rabbitmqadmin pkgs.cabal-install - pkgs.haskellPackages.cabal-plan pkgs.nix-prefetch-git + cabal-plan profileEnv ] ++ ghcWithPackages @@ -510,4 +530,14 @@ in inherit brig-templates; haskellPackages = hPkgs localModsEnableAll; haskellPackagesUnoptimizedNoDocs = hPkgs localModsOnlyTests; -} // attrsets.genAttrs (wireServerPackages) (e: hPkgs.${e}) + + allLocalPackages = pkgs.symlinkJoin { + name = "all-local-packages"; + paths = map (e: (hPkgs localModsEnableAll).${e}) wireServerPackages; + }; + + allImages = pkgs.symlinkJoin { + name = "all-images"; + paths = builtins.attrValues (images localModsEnableAll); + }; +} // attrsets.genAttrs wireServerPackages (e: (hPkgs localModsEnableAll).${e}) diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index 9351b4f65b..4ffa019461 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -28,6 +28,7 @@ common common-all NoImplicitPrelude AllowAmbiguousTypes BangPatterns + BlockArguments ConstraintKinds DataKinds DefaultSignatures diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index f084f528a8..cbcb1161cb 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -232,7 +232,8 @@ newEnv o = do clock <- mkAutoUpdate defaultUpdateSettings {updateAction = getCurrentTime} w <- FS.startManagerConf $ - FS.defaultConfig {FS.confDebounce = FS.Debounce 0.5, FS.confPollInterval = 10000000} + -- FIXME: should this be changed from polling to OS based? + FS.defaultConfig {FS.confWatchMode = FS.WatchModePoll 10000000} g <- geoSetup lgr w $ Opt.geoDb o let turnOpts = Opt.turn o turnSecret <- Text.encodeUtf8 . Text.strip <$> Text.readFile (Opt.secret turnOpts) @@ -335,6 +336,9 @@ startWatching w p = void . FS.watchDir w (Path.dropFileName p) predicate predicate (FS.Modified f _ _) = Path.equalFilePath f p predicate FS.Removed {} = False predicate FS.Unknown {} = False + predicate FS.ModifiedAttributes {} = False + predicate FS.WatchedDirectoryRemoved {} = False + predicate FS.CloseWrite {} = False replaceGeoDb :: Logger -> IORef GeoIp.GeoDB -> FS.Event -> IO () replaceGeoDb g ref e = do @@ -449,13 +453,13 @@ initCredentials secretFile = do dat <- loadSecret secretFile pure $ either (\e -> error $ "Could not load secrets from " ++ show secretFile ++ ": " ++ e) id dat -userTemplates :: MonadReader Env m => Maybe Locale -> m (Locale, UserTemplates) +userTemplates :: (MonadReader Env m) => Maybe Locale -> m (Locale, UserTemplates) userTemplates l = forLocale l <$> view usrTemplates -providerTemplates :: MonadReader Env m => Maybe Locale -> m (Locale, ProviderTemplates) +providerTemplates :: (MonadReader Env m) => Maybe Locale -> m (Locale, ProviderTemplates) providerTemplates l = forLocale l <$> view provTemplates -teamTemplates :: MonadReader Env m => Maybe Locale -> m (Locale, TeamTemplates) +teamTemplates :: (MonadReader Env m) => Maybe Locale -> m (Locale, TeamTemplates) teamTemplates l = forLocale l <$> view tmTemplates closeEnv :: Env -> IO () @@ -469,7 +473,7 @@ closeEnv e = do -- App Monad newtype AppT r a = AppT - { unAppT :: Member (Final IO) r => ReaderT Env (Sem r) a + { unAppT :: (Member (Final IO) r) => ReaderT Env (Sem r) a } deriving ( Semigroup, @@ -477,7 +481,7 @@ newtype AppT r a = AppT ) via (Ap (AppT r) a) -lowerAppT :: Member (Final IO) r => Env -> AppT r a -> Sem r a +lowerAppT :: (Member (Final IO) r) => Env -> AppT r a -> Sem r a lowerAppT env (AppT r) = runReaderT r env temporaryGetEnv :: AppT r Env @@ -499,10 +503,10 @@ instance MonadIO (AppT r) where instance MonadThrow (AppT r) where throwM = liftIO . throwM -instance Member (Final IO) r => MonadThrow (Sem r) where +instance (Member (Final IO) r) => MonadThrow (Sem r) where throwM = embedFinal . throwM @IO -instance Member (Final IO) r => MonadCatch (Sem r) where +instance (Member (Final IO) r) => MonadCatch (Sem r) where catch m handler = withStrategicToFinal @IO $ do m' <- runS m st <- getInitialStateS @@ -521,7 +525,7 @@ instance MonadReader Env (AppT r) where liftSem :: Sem r a -> AppT r a liftSem sem = AppT $ lift sem -instance MonadIO m => MonadLogger (ReaderT Env m) where +instance (MonadIO m) => MonadLogger (ReaderT Env m) where log l m = do g <- view applog r <- view requestId @@ -613,14 +617,14 @@ wrapHttpClient = wrapHttp wrapHttpClientE :: ExceptT e HttpClientIO a -> ExceptT e (AppT r) a wrapHttpClientE = mapExceptT wrapHttpClient -instance MonadIO m => MonadIndexIO (ReaderT Env m) where +instance (MonadIO m) => MonadIndexIO (ReaderT Env m) where liftIndexIO m = view indexEnv >>= \e -> runIndexIO e m instance MonadIndexIO (AppT r) where liftIndexIO m = do AppT $ mapReaderT (embedToFinal @IO) $ liftIndexIO m -instance MonadIndexIO (AppT r) => MonadIndexIO (ExceptT err (AppT r)) where +instance (MonadIndexIO (AppT r)) => MonadIndexIO (ExceptT err (AppT r)) where liftIndexIO m = view indexEnv >>= \e -> runIndexIO e m instance HasRequestId (AppT r) where @@ -639,8 +643,8 @@ locationOf ip = -------------------------------------------------------------------------------- -- Federation -viewFederationDomain :: MonadReader Env m => m Domain +viewFederationDomain :: (MonadReader Env m) => m Domain viewFederationDomain = view (settings . Opt.federationDomain) -qualifyLocal :: MonadReader Env m => a -> m (Local a) +qualifyLocal :: (MonadReader Env m) => a -> m (Local a) qualifyLocal a = toLocalUnsafe <$> viewFederationDomain <*> pure a diff --git a/services/brig/src/Brig/Calling.hs b/services/brig/src/Brig/Calling.hs index 405c8faf72..1af030f701 100644 --- a/services/brig/src/Brig/Calling.hs +++ b/services/brig/src/Brig/Calling.hs @@ -106,7 +106,7 @@ type MaximumSFTServers = 100 -- servers in this list. Limit this list to a smaller subset in case many -- SFT servers are advertised in a given environment. getRandomElements :: - MonadRandom f => + (MonadRandom f) => Range 1 MaximumSFTServers Int -> NonEmpty a -> f (NonEmpty a) @@ -141,12 +141,12 @@ data Discovery a | Discovered a deriving (Show, Eq) -instance Semigroup a => Semigroup (Discovery a) where +instance (Semigroup a) => Semigroup (Discovery a) where NotDiscoveredYet <> other = other other <> NotDiscoveredYet = other Discovered x <> Discovered y = Discovered (x <> y) -instance Semigroup a => Monoid (Discovery a) where +instance (Semigroup a) => Monoid (Discovery a) where mempty = NotDiscoveredYet discoveryToMaybe :: Discovery a -> Maybe a @@ -154,7 +154,7 @@ discoveryToMaybe = \case NotDiscoveredYet -> Nothing Discovered x -> Just x -discoverSRVRecords :: Members [DNSLookup, TinyLog] r => DNS.Domain -> Sem r (Maybe (NonEmpty SrvEntry)) +discoverSRVRecords :: (Members [DNSLookup, TinyLog] r) => DNS.Domain -> Sem r (Maybe (NonEmpty SrvEntry)) discoverSRVRecords domain = lookupSRV domain >>= \case SrvAvailable es -> pure $ Just es @@ -176,7 +176,7 @@ discoverSRVRecords domain = . Log.field "domain" domain pure Nothing -srvDiscoveryLoop :: Members [DNSLookup, TinyLog, Delay] r => DNS.Domain -> Int -> (NonEmpty SrvEntry -> Sem r ()) -> Sem r () +srvDiscoveryLoop :: (Members [DNSLookup, TinyLog, Delay] r) => DNS.Domain -> Int -> (NonEmpty SrvEntry -> Sem r ()) -> Sem r () srvDiscoveryLoop domain discoveryInterval saveAction = forever $ do servers <- discoverSRVRecords domain forM_ servers saveAction @@ -185,7 +185,7 @@ srvDiscoveryLoop domain discoveryInterval saveAction = forever $ do mkSFTDomain :: SFTOptions -> DNS.Domain mkSFTDomain SFTOptions {..} = DNS.normalize $ maybe defSftServiceName ("_" <>) sftSRVServiceName <> "._tcp." <> sftBaseDomain -sftDiscoveryLoop :: Members [DNSLookup, TinyLog, Delay, Embed IO] r => SFTEnv -> Sem r () +sftDiscoveryLoop :: (Members [DNSLookup, TinyLog, Delay, Embed IO] r) => SFTEnv -> Sem r () sftDiscoveryLoop SFTEnv {..} = srvDiscoveryLoop sftDomain sftDiscoveryInterval $ atomicWriteIORef sftServers . Discovered . SFTServers @@ -244,7 +244,7 @@ mkTurnEnv serversSource _turnTokenTTL _turnConfigTTL _turnSecret _turnSHA512 = d _turnPrng <- createSystemRandom pure $ TurnEnv {..} -turnServersV1 :: MonadIO m => TurnServers -> m (Discovery (NonEmpty TurnURI)) +turnServersV1 :: (MonadIO m) => TurnServers -> m (Discovery (NonEmpty TurnURI)) turnServersV1 = readIORef . \case TurnServersFromFiles _opts v1Ref _v2Ref -> @@ -252,7 +252,7 @@ turnServersV1 = TurnServersFromDNS _opts v1UdpRef _v2UdpRef _tcpRef _tlsRef -> v1UdpRef -turnServersV2 :: MonadIO m => TurnServers -> m (Discovery (NonEmpty TurnURI)) +turnServersV2 :: (MonadIO m) => TurnServers -> m (Discovery (NonEmpty TurnURI)) turnServersV2 = \case TurnServersFromFiles _opts _v1Udref v2Ref -> readIORef v2Ref @@ -303,7 +303,7 @@ startDNSBasedTurnDiscovery logger opts deprecatedUdpRef udpRef tcpRef tlsRef = d atomicWriteIORef tlsRef . Discovered . fmap (turnURIFromSRV SchemeTurns (Just TransportTCP)) pure [udpLoop, tcpLoop, tlsLoop] where - withNonZeroWeightRecords :: Monad m => (NonEmpty SrvEntry -> m ()) -> NonEmpty SrvEntry -> m () + withNonZeroWeightRecords :: (Monad m) => (NonEmpty SrvEntry -> m ()) -> NonEmpty SrvEntry -> m () withNonZeroWeightRecords action records = case NonEmpty.filter (\e -> srvWeight e /= 0) records of [] -> pure () @@ -369,6 +369,9 @@ startWatching w p = void . FS.watchDir w (Path.dropFileName p) predicate predicate (FS.Modified f _ _) = Path.equalFilePath f p predicate FS.Removed {} = False predicate FS.Unknown {} = False + predicate FS.ModifiedAttributes {} = False + predicate FS.WatchedDirectoryRemoved {} = False + predicate FS.CloseWrite {} = False readTurnList :: FilePath -> IO (Maybe (NonEmpty TurnURI)) readTurnList = Text.readFile >=> pure . fn . mapMaybe (fromByteString . Text.encodeUtf8) . Text.lines diff --git a/services/brig/src/Brig/SMTP.hs b/services/brig/src/Brig/SMTP.hs index bfad48af6b..75694ee3c1 100644 --- a/services/brig/src/Brig/SMTP.hs +++ b/services/brig/src/Brig/SMTP.hs @@ -30,6 +30,7 @@ module Brig.SMTP ) where +import Control.Concurrent.Async (wait, withAsyncWithUnmask) import Control.Exception qualified as CE (throw) import Control.Lens import Control.Monad.Catch @@ -124,7 +125,7 @@ initSMTP' timeoutDuration lg host port credentials connType = do flush lg error $ "Failed to close test connection with SMTP server: " ++ show e ) - SMTP <$> createPool create destroy 1 5 5 + SMTP <$> newPool (setNumStripes (Just 1) (defaultPoolConfig create destroy 5 5)) where ensureTimeout :: IO a -> IO a ensureTimeout = ensureSMTPConnectionTimeout timeoutDuration @@ -158,10 +159,18 @@ initSMTP' timeoutDuration lg host port credentials connType = do ("Creating pooled SMTP connection to " ++ unpack host) establishConnection + -- NOTE: because `Data.Pool` masks the async exceptions for the resource deallocation function, + -- the timeout function called in `ensureTimeOut` will be masked as well and cannot + -- ever terminate, so `destroy` itself never terminates destroy :: SMTP.SMTPConnection -> IO () destroy c = - logExceptionOrResult lg ("Closing pooled SMTP connection to " ++ unpack host) $ - (ensureTimeout . SMTP.gracefullyCloseSMTP) c + withAsyncWithUnmask + do + \unmask -> do + logExceptionOrResult lg ("Closing pooled SMTP connection to " ++ unpack host) $ + unmask do + ensureTimeout $ SMTP.gracefullyCloseSMTP c + do wait logExceptionOrResult :: (MonadIO m, MonadCatch m) => Logger -> String -> m a -> m a logExceptionOrResult lg actionString action = do @@ -169,24 +178,22 @@ logExceptionOrResult lg actionString action = do catches action [ Handler - ( \(e :: SMTPPoolException) -> do - let resultLog = case e of - SMTPUnauthorized -> - ("Failed to establish connection, check your credentials." :: String) - SMTPConnectionTimeout -> ("Connection timeout." :: String) - doLog Logger.Warn resultLog - CE.throw e - ), + \(e :: SMTPPoolException) -> do + let resultLog = case e of + SMTPUnauthorized -> + ("Failed to establish connection, check your credentials." :: String) + SMTPConnectionTimeout -> ("Connection timeout." :: String) + doLog Logger.Warn resultLog + CE.throw e, Handler - ( \(e :: SomeException) -> do - doLog Logger.Warn ("Caught exception : " ++ show e) - CE.throw e - ) + \(e :: SomeException) -> do + doLog Logger.Warn ("Caught exception : " ++ show e) + CE.throw e ] doLog Logger.Debug ("Succeeded." :: String) pure res where - doLog :: MonadIO m => Logger.Level -> String -> m () + doLog :: (MonadIO m) => Logger.Level -> String -> m () doLog lvl result = let msg' = msg ("SMTP connection result" :: String) in Logger.log lg lvl (msg' . field "action" actionString . field "result" result) @@ -214,14 +221,14 @@ ensureSMTPConnectionTimeout timeoutDuration action = -- a timeout happens and on every other network failure. -- -- `defaultTimeoutDuration` is used as timeout duration for all actions. -sendMail :: MonadIO m => Logger -> SMTP -> Mail -> m () +sendMail :: (MonadIO m) => Logger -> SMTP -> Mail -> m () sendMail = sendMail' defaultTimeoutDuration -- | `sendMail` with configurable timeout duration -- -- This is mostly useful for testing. (We don't want to waste the amount of -- `defaultTimeoutDuration` in tests with waiting.) -sendMail' :: (MonadIO m, TimeUnit t) => t -> Logger -> SMTP -> Mail -> m () +sendMail' :: forall t m. (MonadIO m, TimeUnit t) => t -> Logger -> SMTP -> Mail -> m () sendMail' timeoutDuration lg s m = liftIO $ withResource (s ^. pool) sendMail'' where sendMail'' :: SMTP.SMTPConnection -> IO () diff --git a/services/brig/test/integration/SMTP.hs b/services/brig/test/integration/SMTP.hs index abbda817f4..6acc128823 100644 --- a/services/brig/test/integration/SMTP.hs +++ b/services/brig/test/integration/SMTP.hs @@ -14,6 +14,7 @@ import Data.Streaming.Network (bindRandomPortTCP) import Data.Text (unpack) import Data.Text.Lazy (fromStrict) import Data.Time.Units +import Debug.Trace (traceIO) import Imports import Network.Mail.Mime import Network.Mail.Postie qualified as Postie @@ -77,11 +78,13 @@ testSendMailNoReceiver lg = do $ \(port, sock) -> withMailServer sock (mailStoringApp receivedMailRef) $ do + traceIO "before initSMTP" conPool <- initSMTP lg "localhost" (Just port) Nothing Plain + traceIO "finished initSMTP" caughtException <- handle @SomeException (const (pure True)) - (sendMail lg conPool (emptyMail (Address Nothing "foo@example.com")) >> pure False) + (sendMail' @Second 1 lg conPool (emptyMail (Address Nothing "foo@example.com")) >> pure False) caughtException @? "Expected exception due to missing mail receiver." testSendMailTransactionFailed :: Logger.Logger -> Bilge.Http () diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs index 4eaebb02b7..fa4eb40b49 100644 --- a/services/galley/src/Galley/API/Action.hs +++ b/services/galley/src/Galley/API/Action.hs @@ -552,7 +552,7 @@ performConversationJoin qusr lconv (ConversationJoin invited role) = do ensureAccessRole (convAccessRoles conv) userMembershipMap ensureConnectedToLocalsOrSameTeam lusr newUsers checkLocals lusr Nothing newUsers = do - ensureAccessRole (convAccessRoles conv) (zip newUsers $ repeat Nothing) + ensureAccessRole (convAccessRoles conv) (map (,Nothing) newUsers) ensureConnectedToLocalsOrSameTeam lusr newUsers checkRemotes :: diff --git a/services/galley/src/Galley/API/Create.hs b/services/galley/src/Galley/API/Create.hs index f785269e6b..23c6681db4 100644 --- a/services/galley/src/Galley/API/Create.hs +++ b/services/galley/src/Galley/API/Create.hs @@ -520,7 +520,7 @@ createConnectConversation lusr conn j = do NewConversation { -- We add only one member, second one gets added later, -- when the other user accepts the connection request. - ncUsers = ulFromLocals (map (toUserRole . tUnqualified) [lusr]), + ncUsers = ulFromLocals ([(toUserRole . tUnqualified) lusr]), ncProtocol = BaseProtocolProteusTag, ncMetadata = meta } diff --git a/services/galley/src/Galley/API/Teams.hs b/services/galley/src/Galley/API/Teams.hs index 410c015285..a5aa8ecf9b 100644 --- a/services/galley/src/Galley/API/Teams.hs +++ b/services/galley/src/Galley/API/Teams.hs @@ -465,7 +465,7 @@ uncheckedDeleteTeam lusr zcon tid = do let e = Conv.Event qconvId Nothing (tUntagged lusr) now Conv.EdConvDelete -- This event always contains all the required recipients let p = newPushLocal ListComplete (tUnqualified lusr) (ConvEvent e) (map recipient mm) - let ee' = bots `zip` repeat e + let ee' = map (,e) bots let pp' = maybe pp (\x -> (x & pushConn .~ zcon) : pp) p pure (pp', ee' ++ ee) @@ -1044,7 +1044,7 @@ uncheckedDeleteTeamMember lusr zcon tid remove mems = do let y = Conv.Event qconvId Nothing (tUntagged lusr) now edata for_ (newPushLocal (mems ^. teamMemberListType) (tUnqualified lusr) (ConvEvent y) (recipient <$> x)) $ \p -> E.push1 $ p & pushConn .~ zcon - E.deliverAsync (bots `zip` repeat y) + E.deliverAsync (map (,y) bots) getTeamConversations :: ( Member (ErrorS 'NotATeamMember) r, diff --git a/services/galley/src/Galley/API/Update.hs b/services/galley/src/Galley/API/Update.hs index bbb54cd7b3..9b287381a0 100644 --- a/services/galley/src/Galley/API/Update.hs +++ b/services/galley/src/Galley/API/Update.hs @@ -1616,7 +1616,7 @@ addBot lusr zcon b = do ) for_ (newPushLocal ListComplete (tUnqualified lusr) (ConvEvent e) (recipient <$> users)) $ \p -> E.push1 $ p & pushConn ?~ zcon - E.deliverAsync ((bm : bots) `zip` repeat e) + E.deliverAsync (map (,e) (bm : bots)) pure e where regularConvChecks c = do @@ -1690,7 +1690,7 @@ rmBot lusr zcon b = do E.push1 $ p & pushConn .~ zcon E.deleteMembers (Data.convId c) (UserList [botUserId (b ^. rmBotId)] []) E.deleteClients (botUserId (b ^. rmBotId)) - E.deliverAsync (bots `zip` repeat e) + E.deliverAsync (map (,e) bots) pure $ Updated e ------------------------------------------------------------------------------- diff --git a/services/galley/src/Galley/API/Util.hs b/services/galley/src/Galley/API/Util.hs index a97672cc94..f3c0c1cfe7 100644 --- a/services/galley/src/Galley/API/Util.hs +++ b/services/galley/src/Galley/API/Util.hs @@ -642,7 +642,7 @@ pushConversationEvent :: pushConversationEvent conn e lusers bots = do for_ (newConversationEventPush e (fmap toList lusers)) $ \p -> push1 $ p & set pushConn conn - deliverAsync (toList bots `zip` repeat e) + deliverAsync (map (,e) (toList bots)) verifyReusableCode :: ( Member CodeStore r, diff --git a/services/galley/test/integration/API/Federation.hs b/services/galley/test/integration/API/Federation.hs index 6c71f87448..47424e80af 100644 --- a/services/galley/test/integration/API/Federation.hs +++ b/services/galley/test/integration/API/Federation.hs @@ -279,7 +279,7 @@ addUnconnectedUsersOnly = do -- Bob is connected to Alice -- Bob is not connected to Charlie connectWithRemoteUser alice qBob - let requestMembers = Set.fromList (map asOtherMember [qAlice]) + let requestMembers = Set.fromList ([asOtherMember qAlice]) now <- liftIO getCurrentTime fedGalleyClient <- view tsFedGalleyClient @@ -623,7 +623,7 @@ notifyDeletedConversation = do qconv bob (Just "gossip") - (Set.fromList (map mkMember [qalice])) + (Set.fromList ([mkMember qalice])) fedGalleyClient <- view tsFedGalleyClient diff --git a/services/galley/test/integration/API/MLS.hs b/services/galley/test/integration/API/MLS.hs index 2342735c50..eff357a58a 100644 --- a/services/galley/test/integration/API/MLS.hs +++ b/services/galley/test/integration/API/MLS.hs @@ -195,8 +195,7 @@ tests s = test s "last to leave a subconversation" testLastLeaverSubConv, test s "leave a subconversation as a non-member" testLeaveSubConvNonMember, test s "remove user from parent conversation" testRemoveUserParent, - test s "remove creator from parent conversation" testRemoveCreatorParent, - test s "creator removes user from parent conversation" testCreatorRemovesUserFromParent + test s "remove creator from parent conversation" testRemoveCreatorParent ], testGroup "Local Sender/Remote Subconversation" @@ -2355,100 +2354,3 @@ testRemoveCreatorParent = do "2. subconv membership mismatch after removal" (sort [charlie1, charlie2, bob1, bob2]) (sort $ pscMembers sub1) - -testCreatorRemovesUserFromParent :: TestM () -testCreatorRemovesUserFromParent = do - [alice, bob, charlie] <- createAndConnectUsers [Nothing, Nothing, Nothing] - - runMLSTest $ - do - [alice1, bob1, bob2, charlie1, charlie2] <- - traverse - createMLSClient - [alice, bob, bob, charlie, charlie] - traverse_ uploadNewKeyPackage [bob1, bob2, charlie1, charlie2] - (_, qcnv) <- setupMLSGroup alice1 - void $ createAddCommit alice1 [bob, charlie] >>= sendAndConsumeCommitBundle - - stateParent <- State.get - - let subId = SubConvId "conference" - qcs <- createSubConv qcnv alice1 subId - liftTest $ - getSubConv (qUnqualified alice) qcnv subId - !!! do const 200 === statusCode - - for_ [bob1, bob2, charlie1, charlie2] $ \c -> do - void $ createExternalCommit c Nothing qcs >>= sendAndConsumeCommitBundle - - stateSub <- State.get - State.put stateParent - - mlsBracket [alice1, charlie1, charlie2] $ \wss -> do - events <- createRemoveCommit alice1 [bob1, bob2] >>= sendAndConsumeCommitBundle - State.modify $ \s -> s {mlsMembers = Set.difference (mlsMembers s) (Set.fromList [bob1, bob2])} - - liftIO $ assertOne events >>= assertLeaveEvent qcnv alice [bob] - - WS.assertMatchN_ (5 # Second) wss $ \n -> do - wsAssertMemberLeave qcnv alice [bob] EdReasonRemoved n - - State.put stateSub - -- Get client state for alice and fetch bob client identities - [(_, idxBob1), (_, idxBob2)] <- getClientsFromGroupState alice1 bob - - -- handle bob1 removal - msgs <- WS.assertMatchN (5 # Second) wss $ \n -> do - -- it was an alice proposal for the parent, - -- but it's a backend proposal for the sub - wsAssertBackendRemoveProposal bob qcs idxBob1 n - - traverse_ (uncurry consumeMessage1) (zip [alice1, charlie1, charlie2] msgs) - - -- handle bob2 removal - msgs2 <- WS.assertMatchN (5 # Second) wss $ \n -> do - -- it was an alice proposal for the parent, - -- but it's a backend proposal for the sub - wsAssertBackendRemoveProposal bob qcs idxBob2 n - - traverse_ (uncurry consumeMessage1) (zip [alice1, charlie1, charlie2] msgs2) - - -- Remove bob from our state as well - State.modify $ \mls -> - mls - { mlsMembers = Set.difference (mlsMembers mls) (Set.fromList [bob1, bob2]) - } - -- alice commits the proposal and sends over for the backend to also process it - void $ - createPendingProposalCommit alice1 - >>= sendAndConsumeCommitBundle - - liftTest $ do - getSubConv (qUnqualified bob) qcnv (SubConvId "conference") - !!! const 403 === statusCode - - -- charlie sees updated memberlist - sub1 :: PublicSubConversation <- - responseJsonError - =<< getSubConv (qUnqualified charlie) qcnv (SubConvId "conference") - (show . length . pscMembers $ sub1) - ) - (sort [alice1, charlie1, charlie2]) - (sort $ pscMembers sub1) - - -- alice also sees updated memberlist - sub2 :: PublicSubConversation <- - responseJsonError - =<< getSubConv (qUnqualified alice) qcnv (SubConvId "conference") - (show . length . pscMembers $ sub2) - ) - (sort [alice1, charlie1, charlie2]) - (sort $ pscMembers sub2) diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs index 4b4d4eb28a..ba24e3da46 100644 --- a/services/galley/test/integration/API/Util.hs +++ b/services/galley/test/integration/API/Util.hs @@ -2419,7 +2419,7 @@ retryWhileN n f m = -- | Changing this will break tests; all prekeys and client Id must match the same -- fingerprint someClientId :: ClientId -someClientId = ClientId "550d8c614fd20299" +someClientId = ClientId "cc6e640e296e8bba" -- | Changing these will break tests; all prekeys and client Id must match the same -- fingerprint @@ -2939,7 +2939,7 @@ spawn cp minput = do in snd <$> concurrently writeInput readOutput case (mout, ex) of (Just out, ExitSuccess) -> pure out - _ -> assertFailure "Failed spawning process" + _ -> assertFailure "Process didn't finish successfully" decodeMLSError :: ParseMLS a => ByteString -> IO a decodeMLSError s = case decodeMLS' s of @@ -2986,7 +2986,7 @@ wsAssertBackendRemoveProposal fromUser cnvOrSubCnv idx n = do where getMLSMessageData :: Conv.EventData -> ByteString getMLSMessageData (EdMLSMessage bs) = bs - getMLSMessageData d = error ("Excepected EdMLSMessage, but got " <> show d) + getMLSMessageData d = error ("Expected EdMLSMessage, but got " <> show d) wsAssertAddProposal :: HasCallStack => diff --git a/services/gundeck/test/unit/DelayQueue.hs b/services/gundeck/test/unit/DelayQueue.hs index a073bf2bda..c9557dcea6 100644 --- a/services/gundeck/test/unit/DelayQueue.hs +++ b/services/gundeck/test/unit/DelayQueue.hs @@ -50,7 +50,7 @@ enqueueUniqueProp (Positive n) = ioProperty $ do q <- DelayQueue.new (Clock (pure 1)) (Delay 1) (Limit (n + 1)) r <- forM [1 .. n] $ \(i :: Int) -> DelayQueue.enqueue q (1 :: Int) i l <- DelayQueue.length q - pure $ all (== True) r && l == 1 + pure $ and r && l == 1 enqueueCancelProp :: Int -> Int -> Property enqueueCancelProp k v = ioProperty $ do diff --git a/tools/test-stats/Main.hs b/tools/test-stats/Main.hs index 03a4773efa..7c97134cbb 100644 --- a/tools/test-stats/Main.hs +++ b/tools/test-stats/Main.hs @@ -163,7 +163,7 @@ pushToPostgresql opts (reports, failedRuns, successfulRuns) = do [(testCase, suiteRunId, report.failure, report.success)] void $ executeMany conn "INSERT INTO test_case_failures (test_case_run_id, failure_log) VALUES (?,?)" $ - zip (repeat testCaseRunId) report.failureDesc + map (testCaseRunId,) report.failureDesc void $ MonoidalMap.traverseWithKey saveTestCaseRun reports extractId :: HasCallStack => [Only Int] -> IO Int