MLS upgrade

Makefile

@@ -225,11 +225,7 @@ git-add-cassandra-schema: db-migrate git-add-cassandra-schema-impl
 
 .PHONY: git-add-cassandra-schema-impl
 git-add-cassandra-schema-impl:
-	$(eval CASSANDRA_CONTAINER := $(shell docker ps | grep '/cassandra:' | perl -ne '/^(\S+)\s/ && print $$1'))
-	( echo '-- automatically generated with `make git-add-cassandra-schema`'; \
-      docker exec -i $(CASSANDRA_CONTAINER) /usr/bin/cqlsh -e "DESCRIBE schema;" ) \
-    | sed "s/CREATE TABLE galley_test.member_client/-- NOTE: this table is unused. It was replaced by mls_group_member_client\nCREATE TABLE galley_test.member_client/g" \
-      > ./cassandra-schema.cql
+	./hack/bin/cassandra_dump_schema > ./cassandra-schema.cql
 	git add ./cassandra-schema.cql
 
 .PHONY: cqlsh

cassandra-schema.cql

@@ -444,6 +444,8 @@ CREATE TABLE galley_test.mls_group_member_client (
     user uuid,
     client text,
     key_package_ref blob,
+    leaf_node_index int,
+    removal_pending boolean,
     PRIMARY KEY (group_id, user_domain, user, client)
 ) WITH CLUSTERING ORDER BY (user_domain ASC, user ASC, client ASC)
     AND bloom_filter_fp_chance = 0.01

changelog.d/1-api-changes/mls-upgrade

@@ -0,0 +1,7 @@
+Switch to MLS draft 20. The following endpoints are affected by the change:
+
+ - All endpoints with `message/mls` content type now expect and return draft-20 MLS structures.
+ - `POST /conversations` does not require `creator_client` anymore.
+ - `POST /mls/commit-bundles` now expects a "stream" of MLS messages, i.e. a sequence of TLS-serialised messages, one after the other, in any order. Its protobuf interface has been removed.
+ - `POST /mls/welcome` has been removed. Welcome messages can now only be sent as part of a commit bundle.
+ - `POST /mls/message` does not accept commit messages anymore. All commit messages must be sent as part of a commit bundle.

changelog.d/5-internal/key-package-mapping

@@ -0,0 +1 @@
+Brig does not perform key package ref mapping anymore. Claimed key packages are simply removed from the `mls_key_packages` table. The `mls_key_package_refs` table is now unused, and will be removed in the future.

hack/bin/cassandra_dump_schema

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import subprocess
+from subprocess import PIPE
+from itertools import zip_longest
+import re
+
+def run_cqlsh(container, expr):
+     p = subprocess.run(["docker", "exec", "-i", container, '/usr/bin/cqlsh', '-e', expr], stdout=PIPE, check=True).stdout.decode('utf8').strip()
+     return p
+
+def transpose(a):
+    return [x for col in zip_longest(*a, fillvalue='') for x in col]
+
+def main():
+    container = subprocess.run(["docker", "ps", "--filter=name=cassandra", "--format={{.ID}}"], stdout=PIPE, check=True).stdout.decode('utf8').rstrip()
+    s = run_cqlsh(container, 'DESCRIBE keyspaces;')
+
+    ks = []
+    for line in s.splitlines():
+        ks.append(re.split('\s+', line))
+
+    keyspaces = transpose(ks)
+    print("-- automatically generated with `make git-add-cassandra-schema`\n")
+    for keyspace in keyspaces:
+        if keyspace.endswith('_test'):
+            s = run_cqlsh(container, f'DESCRIBE keyspace {keyspace}')
+            print(s.replace('CREATE TABLE galley_test.member_client','-- NOTE: this table is unused. It was replaced by mls_group_member_client\nCREATE TABLE galley_test.member_client'))
+            print()
+
+if __name__ == '__main__':
+    main()

hack/python/wire/mlscli.py

@@ -189,7 +189,7 @@ def add_member(state, kpfiles):
         "<group-in>",
         "--welcome-out",
         welcome_file,
-        "--group-state-out",
+        "--group-info-out",
         pgs_file,
         "--group-out",
         "<group-out>",

libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs

@@ -27,7 +27,7 @@ import Test.QuickCheck (Arbitrary)
 import Wire.API.Federation.API.Common
 import Wire.API.Federation.Endpoint
 import Wire.API.Federation.Version
-import Wire.API.MLS.Credential
+import Wire.API.MLS.CipherSuite
 import Wire.API.MLS.KeyPackage
 import Wire.API.User (UserProfile)
 import Wire.API.User.Client

libs/wire-api-federation/src/Wire/API/Federation/API/Galley.hs

@@ -445,8 +445,11 @@ data ConversationUpdateResponse
     via (CustomEncoded ConversationUpdateResponse)
 
 -- | A wrapper around a raw welcome message
-newtype MLSWelcomeRequest = MLSWelcomeRequest
-  { unMLSWelcomeRequest :: Base64ByteString
+data MLSWelcomeRequest = MLSWelcomeRequest
+  { -- | A serialised welcome message.
+    welcomeMessage :: Base64ByteString,
+    -- | Recipients local to the target backend.
+    recipients :: [(UserId, ClientId)]
   }
   deriving stock (Eq, Generic, Show)
   deriving (Arbitrary) via (GenericUniform MLSWelcomeRequest)

libs/wire-api/default.nix

@@ -247,6 +247,7 @@ mkDerivation {
     process
     proto-lens
     QuickCheck
+    random
     saml2-web-sso
     schema-profunctor
     servant

libs/wire-api/src/Wire/API/Error/Galley.hs

@@ -72,7 +72,7 @@ data GalleyError
     MLSNotEnabled
   | MLSNonEmptyMemberList
   | MLSDuplicatePublicKey
-  | MLSKeyPackageRefNotFound
+  | MLSInvalidLeafNodeIndex
   | MLSUnsupportedMessage
   | MLSProposalNotFound
   | MLSUnsupportedProposal
@@ -85,7 +85,6 @@ data GalleyError
   | MLSClientSenderUserMismatch
   | MLSWelcomeMismatch
   | MLSMissingGroupInfo
-  | MLSMissingSenderClient
   | MLSUnexpectedSenderClient
   | MLSSubConvUnsupportedConvType
   | MLSSubConvClientNotInParent
@@ -201,7 +200,7 @@ type instance MapError 'MLSNonEmptyMemberList = 'StaticError 400 "non-empty-memb
 
 type instance MapError 'MLSDuplicatePublicKey = 'StaticError 400 "mls-duplicate-public-key" "MLS public key for the given signature scheme already exists"
 
-type instance MapError 'MLSKeyPackageRefNotFound = 'StaticError 404 "mls-key-package-ref-not-found" "A referenced key package could not be mapped to a known client"
+type instance MapError 'MLSInvalidLeafNodeIndex = 'StaticError 400 "mls-invalid-leaf-node-index" "A referenced leaf node index points to a blank or non-existing node"
 
 type instance MapError 'MLSUnsupportedMessage = 'StaticError 422 "mls-unsupported-message" "Attempted to send a message with an unsupported combination of content type and wire format"
 
@@ -227,8 +226,6 @@ type instance MapError 'MLSWelcomeMismatch = 'StaticError 400 "mls-welcome-misma
 
 type instance MapError 'MLSMissingGroupInfo = 'StaticError 404 "mls-missing-group-info" "The conversation has no group information"
 
-type instance MapError 'MLSMissingSenderClient = 'StaticError 403 "mls-missing-sender-client" "The client has to refresh their access token and provide their client ID"
-
 type instance MapError 'MLSSubConvUnsupportedConvType = 'StaticError 403 "mls-subconv-unsupported-convtype" "MLS subconversations are only supported for regular conversations"
 
 type instance MapError 'MLSSubConvClientNotInParent = 'StaticError 403 "mls-subconv-join-parent-missing" "MLS client cannot join the subconversation because it is not member of the parent conversation"

libs/wire-api/src/Wire/API/MLS/AuthenticatedContent.hs

@@ -0,0 +1,111 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Wire.API.MLS.AuthenticatedContent
+  ( AuthenticatedContent (..),
+    TaggedSender (..),
+    authContentRef,
+    publicMessageRef,
+    mkSignedPublicMessage,
+  )
+where
+
+import Crypto.PubKey.Ed25519
+import Imports
+import Wire.API.MLS.CipherSuite
+import Wire.API.MLS.Context
+import Wire.API.MLS.Epoch
+import Wire.API.MLS.Group
+import Wire.API.MLS.LeafNode
+import Wire.API.MLS.Message
+import Wire.API.MLS.Proposal
+import Wire.API.MLS.ProtocolVersion
+import Wire.API.MLS.Serialisation
+
+-- | Needed to compute proposal refs.
+-- https://messaginglayersecurity.rocks/mls-protocol/draft-ietf-mls-protocol-20/draft-ietf-mls-protocol.html#section-6-7
+data AuthenticatedContent = AuthenticatedContent
+  { wireFormat :: WireFormatTag,
+    content :: RawMLS FramedContent,
+    authData :: RawMLS FramedContentAuthData
+  }
+  deriving (Eq, Show)
+
+instance SerialiseMLS AuthenticatedContent where
+  serialiseMLS ac = do
+    serialiseMLS ac.wireFormat
+    serialiseMLS ac.content
+    serialiseMLS ac.authData
+
+msgAuthContent :: PublicMessage -> AuthenticatedContent
+msgAuthContent msg =
+  AuthenticatedContent
+    { wireFormat = WireFormatPublicTag,
+      content = msg.content,
+      authData = msg.authData
+    }
+
+-- | Compute the proposal ref given a ciphersuite and the raw proposal data.
+authContentRef :: CipherSuiteTag -> AuthenticatedContent -> ProposalRef
+authContentRef cs = ProposalRef . csHash cs proposalContext . mkRawMLS
+
+publicMessageRef :: CipherSuiteTag -> PublicMessage -> ProposalRef
+publicMessageRef cs = authContentRef cs . msgAuthContent
+
+-- | Sender, plus with a membership tag in the case of a member sender.
+data TaggedSender
+  = TaggedSenderMember LeafIndex ByteString
+  | TaggedSenderExternal Word32
+  | TaggedSenderNewMemberProposal
+  | TaggedSenderNewMemberCommit
+
+taggedSenderToSender :: TaggedSender -> Sender
+taggedSenderToSender (TaggedSenderMember i _) = SenderMember i
+taggedSenderToSender (TaggedSenderExternal n) = SenderExternal n
+taggedSenderToSender TaggedSenderNewMemberProposal = SenderNewMemberProposal
+taggedSenderToSender TaggedSenderNewMemberCommit = SenderNewMemberCommit
+
+taggedSenderMembershipTag :: TaggedSender -> Maybe ByteString
+taggedSenderMembershipTag (TaggedSenderMember _ t) = Just t
+taggedSenderMembershipTag _ = Nothing
+
+-- | Craft a message with the backend itself as a sender. Return the message and its ref.
+mkSignedPublicMessage ::
+  SecretKey -> PublicKey -> GroupId -> Epoch -> TaggedSender -> FramedContentData -> PublicMessage
+mkSignedPublicMessage priv pub gid epoch sender payload =
+  let framedContent =
+        mkRawMLS
+          FramedContent
+            { groupId = gid,
+              epoch = epoch,
+              sender = taggedSenderToSender sender,
+              content = payload,
+              authenticatedData = mempty
+            }
+      tbs =
+        FramedContentTBS
+          { protocolVersion = defaultProtocolVersion,
+            wireFormat = WireFormatPublicTag,
+            content = framedContent,
+            groupContext = Nothing
+          }
+      sig = signWithLabel "FramedContentTBS" priv pub (mkRawMLS tbs)
+   in PublicMessage
+        { content = framedContent,
+          authData = mkRawMLS (FramedContentAuthData sig Nothing),
+          membershipTag = taggedSenderMembershipTag sender
+        }

libs/wire-api/src/Wire/API/MLS/Capabilities.hs

@@ -0,0 +1,55 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Wire.API.MLS.Capabilities where
+
+import Imports
+import Test.QuickCheck
+import Wire.API.MLS.CipherSuite
+import Wire.API.MLS.Credential
+import Wire.API.MLS.ProposalTag
+import Wire.API.MLS.ProtocolVersion
+import Wire.API.MLS.Serialisation
+import Wire.Arbitrary
+
+-- | https://messaginglayersecurity.rocks/mls-protocol/draft-ietf-mls-protocol-20/draft-ietf-mls-protocol.html#section-7.2-2
+data Capabilities = Capabilities
+  { versions :: [ProtocolVersion],
+    ciphersuites :: [CipherSuite],
+    extensions :: [Word16],
+    proposals :: [ProposalTag],
+    credentials :: [CredentialTag]
+  }
+  deriving (Show, Eq, Generic)
+  deriving (Arbitrary) via (GenericUniform Capabilities)
+
+instance ParseMLS Capabilities where
+  parseMLS =
+    Capabilities
+      <$> parseMLSVector @VarInt parseMLS
+      <*> parseMLSVector @VarInt parseMLS
+      <*> parseMLSVector @VarInt parseMLS
+      <*> parseMLSVector @VarInt parseMLS
+      <*> parseMLSVector @VarInt parseMLS
+
+instance SerialiseMLS Capabilities where
+  serialiseMLS caps = do
+    serialiseMLSVector @VarInt serialiseMLS caps.versions
+    serialiseMLSVector @VarInt serialiseMLS caps.ciphersuites
+    serialiseMLSVector @VarInt serialiseMLS caps.extensions
+    serialiseMLSVector @VarInt serialiseMLS caps.proposals
+    serialiseMLSVector @VarInt serialiseMLS caps.credentials