From 3dab320889383565658cd5e333591175a1c84ddb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 22 Feb 2017 07:42:44 +0000 Subject: [PATCH 01/11] Key management stuff, from e2e branch Extract the key management stuff from the draft e2e branch, in an attempt to make it easier to land --- .../definitions/device_keys.yaml | 67 +++++ api/client-server/keys.yaml | 282 ++++++++++++++++++ event-schemas/examples/m.new_device | 7 + event-schemas/schema/m.new_device | 32 ++ .../modules/end_to_end_encryption.rst | 134 +++++++++ specification/modules/send_to_device.rst | 5 +- templating/matrix_templates/units.py | 14 +- 7 files changed, 533 insertions(+), 8 deletions(-) create mode 100644 api/client-server/definitions/device_keys.yaml create mode 100644 api/client-server/keys.yaml create mode 100644 event-schemas/examples/m.new_device create mode 100644 event-schemas/schema/m.new_device diff --git a/api/client-server/definitions/device_keys.yaml b/api/client-server/definitions/device_keys.yaml new file mode 100644 index 0000000000..57af9507e0 --- /dev/null +++ b/api/client-server/definitions/device_keys.yaml @@ -0,0 +1,67 @@ +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: DeviceKeys +description: Device identity keys +properties: + user_id: + type: string + description: |- + The ID of the user the device belongs to. Must match the user ID used + when logging in. + example: "@alice:example.com" + device_id: + type: string + description: |- + The ID of the device these keys belong to. Must match the device ID used + when logging in. + example: "JLAFKJWSCS" + algorithms: + type: array + items: + type: string + description: |- + The encryption algorithms supported by this device. + example: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"] + keys: + type: object + description: |- + Base64-encoded public identity keys. The names of the properties + should be in the format ``:``. + additionalProperties: + type: string + example: + "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI" + "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" + signatures: + type: object + description: |- + Base64-encoded signatures for the device key object. A map from user ID, + to a map from ``:`` to the signature. + + The signature is calculated using the process described at `Signing + JSON`_. + additionalProperties: + type: object + additionalProperties: + type: string + example: + "@alice:example.com": + "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" +required: + - user_id + - device_id + - algorithms + - keys + - signatures diff --git a/api/client-server/keys.yaml b/api/client-server/keys.yaml new file mode 100644 index 0000000000..48a8e34ece --- /dev/null +++ b/api/client-server/keys.yaml @@ -0,0 +1,282 @@ +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server Client Config API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +securityDefinitions: + $ref: definitions/security.yaml +paths: + "/keys/upload": + post: + summary: Upload end-to-end encryption keys + description: |- + Publishes end-to-end encryption keys for the device. + security: + - accessToken: [] + parameters: + - in: body + name: keys + description: |- + The keys to be published + schema: + type: object + properties: + device_keys: + description: |- + Identity keys for the device. May be absent if no new + identity keys are required. + allOf: + - $ref: definitions/device_keys.yaml + one_time_keys: + type: object + description: |- + One-time public keys for "pre-key" messages. The names of + the properties should be in the format + ``:``. The format of the key is determined + by the key algorithm. + + May be absent if no new one-time keys are required. + additionalProperties: + type: + - string + - object + example: + "curve25519:AAAAAQ": "/qyvZvwjiTxGdGU0RCguDCLeR+nmsb3FfNG3/Ve4vU8" + signed_curve25519:AAAAHg: + key: "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs" + signatures: + "@alice:example.com": + ed25519:JLAFKJWSCS: "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" + signed_curve25519:AAAAHQ: + key: "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw" + signatures: + "@alice:example.com": + ed25519:JLAFKJWSCS: "IQeCEPb9HFk217cU9kw9EOiusC6kMIkoIRnbnfOh5Oc63S1ghgyjShBGpu34blQomoalCyXWyhaaT3MrLZYQAA" + responses: + 200: + description: + The provided keys were sucessfully uploaded. + schema: + type: object + properties: + one_time_key_counts: + type: object + additionalProperties: + type: integer + description: |- + For each key algorighm, the number of unclaimed one-time keys + of that type currently held on the server for this device. + example: + curve25519: 10 + signed_curve25519: 20 + required: + - one_time_key_counts + + tags: + - End-to-end encryption + "/keys/query": + post: + summary: Download device identity keys + description: |- + Returns the current devices and identity keys for the given users. + security: + - accessToken: [] + parameters: + - in: body + name: query + description: |- + Query defining the keys to be downloaded + schema: + type: object + properties: + timeout: + type: integer + description: |- + The time (in milliseconds) to wait when downloading keys from + remote servers. 10 seconds is the recommended default. + example: 10000 + device_keys: + type: object + description: |- + The keys to be downloaded. A map from user ID, to a list of + device IDs, or to an empty list to indicate all devices for the + corresponding user. + additionalProperties: + type: array + items: + type: string + description: "device ID" + example: + "@alice:example.com": [] + required: + - device_keys + + responses: + 200: + description: + The device information + schema: + type: object + properties: + failures: + type: object + description: |- + If any remote homeservers could not be reached, they are + recorded here. The names of the properties are the names of + the unreachable servers. + + If the homeserver could be reached, but the user or device + was unknown, no failure is recorded. Instead, the corresponding + user or device is missing from the ``device_keys`` result. + additionalProperties: + type: object + example: {} + device_keys: + type: object + description: |- + Information on the queried devices. A map from user ID, to a + map from device ID to device information. For each device, + the information returned will be the same as uploaded via + ``/keys/upload``, with the addition of an ``unsigned`` + property. + additionalProperties: + type: object + additionalProperties: + allOf: + - $ref: definitions/device_keys.yaml + properties: + unsigned: + title: UnsignedDeviceInfo + type: object + description: |- + Additional data added to the device key information + by intermediate servers, and not covered by the + signatures. + properties: + device_display_name: + type: string + description: + The display name which the user set on the device. + example: + "@alice:example.com": + JLAFKJWSCS: { + "user_id": "@alice:example.com", + "device_id": "JLAFKJWSCS", + "algorithms": [ + "m.olm.curve25519-aes-sha256", + "m.megolm.v1.aes-sha" + ], + "keys": { + "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", + "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" + }, + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" + } + }, + "unsigned": { + "device_display_name": "Alice's mobile phone" + } + } + + tags: + - End-to-end encryption + "/keys/claim": + post: + summary: Claim one-time encryption keys + description: |- + Claims one-time keys for use in pre-key messages. + security: + - accessToken: [] + parameters: + - in: body + name: query + description: |- + Query defining the keys to be claimed + schema: + type: object + properties: + timeout: + type: integer + description: |- + The time (in milliseconds) to wait when downloading keys from + remote servers. 10 seconds is the recommended default. + example: 10000 + one_time_keys: + type: object + description: |- + The keys to be claimed. A map from user ID, to a map from + device ID to algorithm name. + additionalProperties: + type: object + additionalProperties: + type: string + description: algorithm + example: "signed_curve25519" + example: + "@alice:example.com": { "JLAFKJWSCS": "curve25519" } + required: + - one_time_keys + responses: + 200: + description: + The claimed keys + schema: + type: object + properties: + failures: + type: object + description: |- + If any remote homeservers could not be reached, they are + recorded here. The names of the properties are the names of + the unreachable servers. + + If the homeserver could be reached, but the user or device + was unknown, no failure is recorded. Instead, the corresponding + user or device is missing from the ``one_time_keys`` result. + additionalProperties: + type: object + example: {} + one_time_keys: + type: object + description: |- + One-time keys for the queried devices. A map from user ID, to a + map from ``:`` to the key object. + additionalProperties: + type: object + additionalProperties: + type: + - string + - object + example: + "@alice:example.com": + JLAFKJWSCS: + signed_curve25519:AAAAHg: + key: "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs" + signatures: + "@alice:example.com": + ed25519:JLAFKJWSCS: "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" + tags: + - End-to-end encryption diff --git a/event-schemas/examples/m.new_device b/event-schemas/examples/m.new_device new file mode 100644 index 0000000000..680a8cd9a5 --- /dev/null +++ b/event-schemas/examples/m.new_device @@ -0,0 +1,7 @@ +{ + "content": { + "device_id": "RJYKSTBOIE", + "rooms": ["!UCnwUWwIKhcpaPTHtR:sw1v.org"] + }, + "type": "m.new_device" +} diff --git a/event-schemas/schema/m.new_device b/event-schemas/schema/m.new_device new file mode 100644 index 0000000000..22d7854a45 --- /dev/null +++ b/event-schemas/schema/m.new_device @@ -0,0 +1,32 @@ +--- +allOf: + - $ref: core-event-schema/event.yaml + +description: |- + This event type is used to announce that a user has logged in with a new + device. It is sent as a `to-device`_ event to all + users which share an encrypted room with the user, to request that encryption + keys be shared with the new device. +properties: + content: + properties: + device_id: + type: string + description: |- + ID of the new device + rooms: + type: list + items: + type: string + description: |- + A list of rooms that the new device is requesting encryption keys + for. + required: + - device_id + - rooms + type: object + type: + enum: + - m.new_device + type: string +type: object diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 48416da43a..9be6d8648b 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -20,3 +20,137 @@ End-to-End Encryption End to end encryption is being worked on and will be coming soon. You can read about what's underway at http://matrix.org/speculator/spec/drafts%2Fe2e/client_server/unstable.html#end-to-end-encryption. + +Key Distribution +---------------- +Encryption and Authentication in Matrix is based around public-key +cryptography. The Matrix protocol provides a basic mechanism for exchange of +public keys, though an out-of-band channel is required to exchange fingerprints +between users to build a web of trust. + +Overview +~~~~~~~~ + +.. code:: + + 1) Bob publishes the public keys and supported algorithms for his + device. This may include long-term identity keys, and/or one-time + keys. + + +----------+ +--------------+ + | Bob's HS | | Bob's Device | + +----------+ +--------------+ + | | + |<=============| + /keys/upload + + 2) Alice requests Bob's public identity keys and supported algorithms. + + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/query + + 3) Alice selects an algorithm and claims any one-time keys needed. + + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/claim + + +Key Algorithms +~~~~~~~~~~~~~~ + +The name ``ed25519`` corresponds to the `Ed25519`_ signature algorithm. The key +is a Base64-encoded 32-byte Ed25519 public key. + +The name ``curve25519`` corresponds to the `Curve25519`_ ECDH algorithm. The +key is a Base64-encoded 32-byte Curve25519 public key. + +The name ``signed_curve25519`` also corresponds to the Curve25519 algorithm, +but keys using this algorithm are objects with the properties ``key`` (giving +the Base64-encoded 32-byte Curve25519 public key), and ``signatures`` (giving a +signature for the key object, as described in `Signing JSON`_). + +Device keys +~~~~~~~~~~~ + +Each device should have one Ed25519 signing key. This key should be generated +on the device from a cryptographically secure source, and the private part of +the key should never be exported from the device. This key is used as the +fingerprint for a device by other clients. + +A device will generally need to generate a number of additional keys. Details +of these will vary depending on the messaging algorithm in use. + +Algorithms generally require device identity keys as well as signing keys. Some +algorithms also require one-time keys to improve their secrecy and deniability. +These keys are used once during session establishment, and are then thrown +away. + +For Olm version 1 (see |m.olm.v1.curve25519-aes-sha2|_), each device requires a single +Curve25519 identity key, and a number of Curve25519 one-time keys. + +Uploading Keys +~~~~~~~~~~~~~~ + +A device uploads the public parts of identity keys to their homeserver as a +signed JSON object, using the |/keys/upload|_ API. +The JSON object must include the public part of the device's Ed25519 key, and +must be signed by that key, as described in `Signing JSON`_. + +One-time keys are also uploaded to the homeserver. + +Devices must store the private part of each key they upload. They can +discard the private part of a one-time key when they receive a message using +that key. However it's possible that a one-time key given out by a homeserver +will never be used, so the device that generates the key will never know that +it can discard the key. Therefore a device could end up trying to store too +many private keys. A device that is trying to store too many private keys may +discard keys starting with the oldest. + +Downloading Keys +~~~~~~~~~~~~~~~~ + +Keys are downloaded as a collection of signed JSON objects, using the +|/keys/query|_ API. + +Claiming One-Time Keys +~~~~~~~~~~~~~~~~~~~~~~ + +A client wanting to set up a session with another device can claim a one-time +key for that device. This is done by making a request to the |/keys/claim|_ +API. + +A homeserver should rate-limit the number of one-time keys that a given user or +remote server can claim. A homeserver should discard the public part of a one +time key once it has given that key to another user. + +Key management API +------------------ + +{{keys_cs_http_api}} + + +.. References + +.. _ed25519: http://ed25519.cr.yp.to/ +.. _curve25519: https://cr.yp.to/ecdh.html + +.. _`Signing JSON`: ../appendices.html#signing-json + +.. |m.olm.v1.curve25519-aes-sha2| replace:: ``m.olm.v1.curve25519-aes-sha2`` + +.. |/keys/upload| replace:: ``/keys/upload`` +.. _/keys/upload: #post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-upload + +.. |/keys/query| replace:: ``/keys/query`` +.. _/keys/query: #post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-query + +.. |/keys/claim| replace:: ``/keys/claim`` +.. _/keys/claim: #post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-claim diff --git a/specification/modules/send_to_device.rst b/specification/modules/send_to_device.rst index dd85a9929f..54a27ad531 100644 --- a/specification/modules/send_to_device.rst +++ b/specification/modules/send_to_device.rst @@ -12,11 +12,12 @@ .. See the License for the specific language governing permissions and .. limitations under the License. +.. _module:to_device: +.. _`to-device`: + Send-to-Device messaging ======================== -.. _module:to_device: - This module provides a means by which clients can exchange signalling messages without them being stored permanently as part of a shared communication history. A message is delivered exactly once to each client device. diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 051b38d7c4..93668ece92 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -123,11 +123,6 @@ def get_json_schema_object_fields(obj, enforce_title=False): logger.debug("Processing object with title '%s'", obj_title) - if enforce_title and not obj_title: - # Force a default titile of "NO_TITLE" to make it obvious in the - # specification output which parts of the schema are missing a title - obj_title = 'NO_TITLE' - additionalProps = obj.get("additionalProperties") props = obj.get("properties") if additionalProps and not props: @@ -151,14 +146,21 @@ def get_json_schema_object_fields(obj, enforce_title=False): props[pretty_key] = props[key_name] del props[key_name] + + # Sometimes you just want to specify that a thing is an object without # doing all the keys. if not props: return { - "type": obj_title, + "type": obj_title if obj_title else 'object', "tables": [], } + if enforce_title and not obj_title: + # Force a default titile of "NO_TITLE" to make it obvious in the + # specification output which parts of the schema are missing a title + obj_title = 'NO_TITLE' + required_keys = set(obj.get("required", [])) first_table_rows = [] From 2df19073f75345feae68cb75096411eadb3dde6f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 24 Mar 2017 08:38:41 +0000 Subject: [PATCH 02/11] Remove broken link to olm olm isn't specified yet, so we can't link to it. --- specification/modules/end_to_end_encryption.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 9be6d8648b..1129c6349a 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -93,8 +93,8 @@ algorithms also require one-time keys to improve their secrecy and deniability. These keys are used once during session establishment, and are then thrown away. -For Olm version 1 (see |m.olm.v1.curve25519-aes-sha2|_), each device requires a single -Curve25519 identity key, and a number of Curve25519 one-time keys. +For Olm version 1, each device requires a single Curve25519 identity key, and a +number of signed Curve25519 one-time keys. Uploading Keys ~~~~~~~~~~~~~~ From 957287138e5262a42ec9e9517b32f978e7991d51 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 22 Mar 2017 07:40:26 +0000 Subject: [PATCH 03/11] Swagger docs for /keys/changes --- .../definitions/device_keys.yaml | 9 +-- api/client-server/keys.yaml | 60 ++++++++++++++++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/api/client-server/definitions/device_keys.yaml b/api/client-server/definitions/device_keys.yaml index 57af9507e0..888c93a44d 100644 --- a/api/client-server/definitions/device_keys.yaml +++ b/api/client-server/definitions/device_keys.yaml @@ -37,8 +37,9 @@ properties: keys: type: object description: |- - Base64-encoded public identity keys. The names of the properties - should be in the format ``:``. + Public identity keys. The names of the properties should be in the + format ``:``. The keys themselves should be + encoded as specified by the key algorithm. additionalProperties: type: string example: @@ -47,8 +48,8 @@ properties: signatures: type: object description: |- - Base64-encoded signatures for the device key object. A map from user ID, - to a map from ``:`` to the signature. + Signatures for the device key object. A map from user ID, to a map from + ``:`` to the signature. The signature is calculated using the process described at `Signing JSON`_. diff --git a/api/client-server/keys.yaml b/api/client-server/keys.yaml index 48a8e34ece..f3766d9ca6 100644 --- a/api/client-server/keys.yaml +++ b/api/client-server/keys.yaml @@ -29,7 +29,7 @@ securityDefinitions: paths: "/keys/upload": post: - summary: Upload end-to-end encryption keys + summary: Upload end-to-end encryption keys. description: |- Publishes end-to-end encryption keys for the device. security: @@ -97,7 +97,7 @@ paths: - End-to-end encryption "/keys/query": post: - summary: Download device identity keys + summary: Download device identity keys. description: |- Returns the current devices and identity keys for the given users. security: @@ -205,7 +205,7 @@ paths: - End-to-end encryption "/keys/claim": post: - summary: Claim one-time encryption keys + summary: Claim one-time encryption keys. description: |- Claims one-time keys for use in pre-key messages. security: @@ -280,3 +280,57 @@ paths: ed25519:JLAFKJWSCS: "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" tags: - End-to-end encryption + "/keys/changes": + get: + summary: Query users with recent device key updates. + description: |- + Gets a list of users who have updated their device identity keys since a + previous sync token. + + The server should include in the results any users who: + + * currently share a room with the calling user (ie, both users have + membership state ``join``); *and* + * added new device identity keys or removed an existing device with + identity keys, between ``from`` and ``to``. + security: + - accessToken: [] + parameters: + - in: query + name: from + type: string + description: |- + The sync token corresponding to the start point of the list. Users + who have not uploaded new device identity keys since this point, + nor deleted existing devices with identity keys since then, will + be excluded from the results. + required: true + x-example: "s72594_4483_1934" + - in: query + name: to + type: string + description: |- + The sync token corresponding to the point in time to list changes + up to. Typically this will be the last known sync token. This may + be used by the server as a hint to check its caches are up to + date. + required: true + x-example: "s75689_5632_2435" + responses: + 200: + description: + The list of users who updated their devices. + schema: + type: object + title: KeyChangesResults + properties: + changes: + type: array + items: + type: string + description: |- + The Matrix User IDs of all users who updated their device + identity keys. + example: ["@alice:example.com", "@bob:example.org"] + tags: + - End-to-end encryption From 4bb47d704ab282a2a2ee25623dc80d4623be8f80 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 24 Mar 2017 08:36:19 +0000 Subject: [PATCH 04/11] Spec device_lists extension to /sync --- api/client-server/sync.yaml | 6 +++ .../modules/end_to_end_encryption.rst | 43 +++++++++++++++++++ specification/modules/send_to_device.rst | 5 ++- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/api/client-server/sync.yaml b/api/client-server/sync.yaml index 66f7d935bf..8da06f400f 100644 --- a/api/client-server/sync.yaml +++ b/api/client-server/sync.yaml @@ -242,6 +242,12 @@ paths: description: |- Information on the send-to-device messages for the client device, as defined in |send_to_device_sync|_. + device_lists: + title: DeviceLists + type: object + description: |- + Information on end-to-end device updates, as specified in + |device_lists_sync|_. examples: application/json: |- { diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 1129c6349a..4d27eb2865 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -137,6 +137,49 @@ Key management API {{keys_cs_http_api}} +.. anchor for link from /sync api spec +.. |device_lists_sync| replace:: End-to-end encryption +.. _device_lists_sync: + +Extensions to /sync +~~~~~~~~~~~~~~~~~~~ + +This module adds the following properties to the |/sync|_ response: + +.. todo: generate this from a swagger definition? + +.. device_lists: { changed: ["@user:server", ... ]}, + +============ =========== ===================================================== +Parameter Type Description +============ =========== ===================================================== +device_lists DeviceLists Optional. Information on e2e device updates. +============ =========== ===================================================== + +``DeviceLists`` + +========= ========= ============================================= +Parameter Type Description +========= ========= ============================================= +changed [string] List of users who have updated their device identity keys + since the previous sync response. +========= ========= ============================================= + + +Example response: + +.. code:: json + + { + "next_batch": "s72595_4483_1934", + "rooms": {"leave": {}, "join": {}, "invite": {}}, + "device_lists": { + "changed": [ + "@alice:example.com", + ], + }, + } + .. References .. _ed25519: http://ed25519.cr.yp.to/ diff --git a/specification/modules/send_to_device.rst b/specification/modules/send_to_device.rst index 54a27ad531..232becae90 100644 --- a/specification/modules/send_to_device.rst +++ b/specification/modules/send_to_device.rst @@ -36,10 +36,13 @@ have the same event type. The device ID in the request body can be set to ``*`` to request that the message be sent to all known devices. If there are send-to-device messages waiting for a client, they will be -returned by |/sync|_, as detailed in `Extensions to /sync`_. Clients should +returned by |/sync|_, as detailed in |Extensions|_. Clients should inspect the ``type`` of each returned event, and ignore any they do not understand. +.. |Extensions| replace:: Extensions to /sync +.. _Extensions: `send_to_device_sync`_ + Server behaviour ---------------- Servers should store pending messages for local users until they are From 58162b7f421c1336ba1adaefb44b598381c01012 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 24 Mar 2017 08:37:39 +0000 Subject: [PATCH 05/11] Key management wording tweaks --- specification/client_server_api.rst | 2 + .../modules/end_to_end_encryption.rst | 45 ++++++++++++++----- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index bbb6982a71..16ce0a7d11 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -1436,3 +1436,5 @@ have to wait in milliseconds before they can try again. .. |/user//account_data/| replace:: ``/user//account_data/`` .. _/user//account_data/: #put-matrix-client-%CLIENT_MAJOR_VERSION%-user-userid-account-data-type + +.. _`Unpadded Base64`: ../appendices.html#unpadded-base64 diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 4d27eb2865..4b86f55cad 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -63,19 +63,39 @@ Overview /keys/claim -Key Algorithms +Key algorithms ~~~~~~~~~~~~~~ The name ``ed25519`` corresponds to the `Ed25519`_ signature algorithm. The key -is a Base64-encoded 32-byte Ed25519 public key. +is a 32-byte Ed25519 public key, encoded using `unpadded Base64`_. Example: + +.. code:: json + + "SogYyrkTldLz0BXP+GYWs0qaYacUI0RleEqNT8J3riQ" The name ``curve25519`` corresponds to the `Curve25519`_ ECDH algorithm. The -key is a Base64-encoded 32-byte Curve25519 public key. +key is a 32-byte Curve25519 public key, encoded using `unpadded +Base64`_. Example: + +.. code:: json + + "JGLn/yafz74HB2AbPLYJWIVGnKAtqECOBf11yyXac2Y" The name ``signed_curve25519`` also corresponds to the Curve25519 algorithm, but keys using this algorithm are objects with the properties ``key`` (giving the Base64-encoded 32-byte Curve25519 public key), and ``signatures`` (giving a -signature for the key object, as described in `Signing JSON`_). +signature for the key object, as described in `Signing JSON`_). Example: + +.. code:: json + + { + "key":"06UzBknVHFMwgi7AVloY7ylC+xhOhEX4PkNge14Grl8", + "signatures": { + "@user:example.com": { + "ed25519:EGURVBUNJP": "YbJva03ihSj5mPk+CHMJKUKlCXCPFXjXOK6VqBnN9nA2evksQcTGn6hwQfrgRHIDDXO2le49x7jnWJHMJrJoBQ" + } + } + } Device keys ~~~~~~~~~~~ @@ -96,7 +116,7 @@ away. For Olm version 1, each device requires a single Curve25519 identity key, and a number of signed Curve25519 one-time keys. -Uploading Keys +Uploading keys ~~~~~~~~~~~~~~ A device uploads the public parts of identity keys to their homeserver as a @@ -104,7 +124,8 @@ signed JSON object, using the |/keys/upload|_ API. The JSON object must include the public part of the device's Ed25519 key, and must be signed by that key, as described in `Signing JSON`_. -One-time keys are also uploaded to the homeserver. +One-time keys are also uploaded to the homeserver using the |/keys/upload|_ +API. Devices must store the private part of each key they upload. They can discard the private part of a one-time key when they receive a message using @@ -114,13 +135,13 @@ it can discard the key. Therefore a device could end up trying to store too many private keys. A device that is trying to store too many private keys may discard keys starting with the oldest. -Downloading Keys -~~~~~~~~~~~~~~~~ +Downloading Identity Keys +~~~~~~~~~~~~~~~~~~~~~~~~~ -Keys are downloaded as a collection of signed JSON objects, using the +Identity keys are downloaded as a collection of signed JSON objects, using the |/keys/query|_ API. -Claiming One-Time Keys +Claiming one-time keys ~~~~~~~~~~~~~~~~~~~~~~ A client wanting to set up a session with another device can claim a one-time @@ -131,8 +152,8 @@ A homeserver should rate-limit the number of one-time keys that a given user or remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. -Key management API ------------------- +Protocol definitions +-------------------- {{keys_cs_http_api}} From 883f5a933e17d534b6935d7e96f597962d61dc0c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 12 Apr 2017 21:21:13 +0100 Subject: [PATCH 06/11] Document how a client is suppsed to do device-list tracking --- api/client-server/keys.yaml | 18 ++-- .../modules/end_to_end_encryption.rst | 94 +++++++++++++++++-- 2 files changed, 96 insertions(+), 16 deletions(-) diff --git a/api/client-server/keys.yaml b/api/client-server/keys.yaml index f3766d9ca6..a0ca6c9caa 100644 --- a/api/client-server/keys.yaml +++ b/api/client-server/keys.yaml @@ -300,20 +300,21 @@ paths: name: from type: string description: |- - The sync token corresponding to the start point of the list. Users - who have not uploaded new device identity keys since this point, - nor deleted existing devices with identity keys since then, will - be excluded from the results. + The desired start point of the list. Should be the ``next_batch`` field + from a response to an earlier call to |/sync|. Users who have not + uploaded new device identity keys since this point, nor deleted + existing devices with identity keys since then, will be excluded + from the results. required: true x-example: "s72594_4483_1934" - in: query name: to type: string description: |- - The sync token corresponding to the point in time to list changes - up to. Typically this will be the last known sync token. This may - be used by the server as a hint to check its caches are up to - date. + The desired end point of the list. Should be the ``next_batch`` + field from a recent call to |/sync| - typically the most recent + such call. This may be used by the server as a hint to check its + caches are up to date. required: true x-example: "s75689_5632_2435" responses: @@ -322,7 +323,6 @@ paths: The list of users who updated their devices. schema: type: object - title: KeyChangesResults properties: changes: type: array diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 4b86f55cad..f616e02707 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -135,11 +135,81 @@ it can discard the key. Therefore a device could end up trying to store too many private keys. A device that is trying to store too many private keys may discard keys starting with the oldest. -Downloading Identity Keys -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Identity keys are downloaded as a collection of signed JSON objects, using the -|/keys/query|_ API. +Tracking the device list for a user +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before Alice can send an encrypted message to Bob, she needs a list of each of +his devices and the associated identity keys, so that she can establish an +encryption session with each device. This list can be obtained by calling +|/keys/query|_, passing Bob's user ID in the ``device_keys`` parameter. + +From time to time, Bob may add new devices, and Alice will need to know this so +that she can include his new devices for later encrypted messages. A naive +solution to this would be to call |/keys/query|_ before sending each message - +however, the number of users and devices may be large and this would be +inefficient. + +It is therefore expected that each client will maintain a list of devices for a +number of users (in practice, typically each user with whom we share an +encrypted room). Furthermore, it is likely that this list will need to be +persisted between invocations of the client application (to preserve `device +verification`_ data and to alert Alice if Bob suddently gets a new +device). + +Alice's client can maintain a list of Bob's devices via the following +process: + +#. It first sets a flag to record that it is now tracking Bob's device list, + and a separate flag to indicate that its list of Bob's devices is + outdated. Both flags should be in storage which persists over client + restarts. + +#. It then makes a request to |/keys/query|_, passing Bob's user ID in the + ``device_keys`` parameter. When the request completes, it stores the + resulting list of devices in persistent storage, and clears the 'outdated' + flag. + +#. During its normal processing of responses to |/sync|_, Alice's client + inspects the |device_lists|_ field. If it is tracking the device lists of + any of the listed users, then it marks the device lists for those users + outdated, and initiates another request to |/keys/query|_ for them. + +#. Periodically, Alice's client stores the ``next_batch`` field of the result + from |/sync|_ in persistent storage. If Alice later restarts her client, it + can obtain a list of the users who have updated their device list while it + was offline by calling |/keys/changes|_, passing the recorded ``next_batch`` + field as the ``from`` parameter. If the client is tracking the device list + of any of the users listed in the response, it marks them as outdated. It + combines this list with those already flagged as outdated, and initiates a + |/keys/query|_ requests for all of them. + +.. Warning:: + + Bob may update one of his devices while Alice has a request to + ``/keys/query`` in flight. Alice's client may therefore see Bob's user ID in + the ``device_lists`` field of the ``/sync`` response while the first request + is in flight, and initiate a second request to ``/keys/query``. This may + lead to either of two related problems. + + The first problem is that, when the first request completes, the client will + clear the 'outdated' flag for Bob's devices. If the second request fails, or + the client is shut down before it completes, this could lead to Alice using + an outdated list of Bob's devices. + + The second possibility is that, under certain conditions, the second request + may complete *before* the first one. When the first request completes, the + client could overwrite the later results from the second request with those + from the first request. + + Clients MUST guard against these situations. For example, a client could + ensure that only one request to ``/keys/query`` is in flight at a time for + each user, by queuing additional requests until the first completes. + Alternatively, the client could make a new request immediately, but ensure + that the first request's results are ignored (possibly by cancelling the + request). + +.. |device_lists| replace:: ``device_lists`` +.. _`device_lists`: `device_lists_sync`_ Claiming one-time keys ~~~~~~~~~~~~~~~~~~~~~~ @@ -152,6 +222,7 @@ A homeserver should rate-limit the number of one-time keys that a given user or remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. + Protocol definitions -------------------- @@ -165,7 +236,12 @@ Protocol definitions Extensions to /sync ~~~~~~~~~~~~~~~~~~~ -This module adds the following properties to the |/sync|_ response: +This module adds an optional ``device_lists`` property to the |/sync|_ +response, as specified below. The server need only populate this property for +an incremental ``/sync`` (ie, one where the ``since`` parameter was +specified). The client is expected to use |/keys/query|_ or |/keys/changes|_ +for the equivalent functionality after an initial sync, as documented in +`Tracking the device list for a user`_. .. todo: generate this from a swagger definition? @@ -174,7 +250,8 @@ This module adds the following properties to the |/sync|_ response: ============ =========== ===================================================== Parameter Type Description ============ =========== ===================================================== -device_lists DeviceLists Optional. Information on e2e device updates. +device_lists DeviceLists Optional. Information on e2e device updates. Note: + only present on an incremental sync. ============ =========== ===================================================== ``DeviceLists`` @@ -218,3 +295,6 @@ Example response: .. |/keys/claim| replace:: ``/keys/claim`` .. _/keys/claim: #post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-claim + +.. |/keys/changes| replace:: ``/keys/changes`` +.. _/keys/changes: #get-matrix-client-%CLIENT_MAJOR_VERSION%-keys-changes From fcaa1195e20f7fc5255f686d4de6e622287f44d6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 19 Apr 2017 22:35:24 +0100 Subject: [PATCH 07/11] e2e spec: Reword in-progress warning --- specification/modules/end_to_end_encryption.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index f616e02707..639e8ff03d 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -17,9 +17,15 @@ End-to-End Encryption .. _module:e2e: -End to end encryption is being worked on and will be coming soon. +Matrix optionally supports end-to-end encryption, allowing rooms to be created +whose conversation contents is not decryptable or interceptable on any of the +participating homeservers. -You can read about what's underway at http://matrix.org/speculator/spec/drafts%2Fe2e/client_server/unstable.html#end-to-end-encryption. +.. WARNING:: + + End to end encryption is being worked on and will be coming soon. This + section is incomplete. You can read more about what's underway at + http://matrix.org/speculator/spec/drafts%2Fe2e/client_server/unstable.html#end-to-end-encryption. Key Distribution ---------------- From d36fad1e70a836719acb99f39095ee720e2d8064 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 19 Apr 2017 22:50:23 +0100 Subject: [PATCH 08/11] update changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 952a72e2ba..9bc027cd4d 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -46,6 +46,8 @@ - Allow guest accounts to use a number of endpoints which are required for end-to-end encryption. (`#751 `_). + - Add key distribution APIs, for use with end-to-end encryption. + (`#894 `_). - Spec clarifications: From 838ea84c0b0c748aac24d25f5f42d4981fb8aadf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 19 Apr 2017 23:01:19 +0100 Subject: [PATCH 09/11] Remove m.new_device event spec We're not going to use m.new_device. --- event-schemas/examples/m.new_device | 7 ------- event-schemas/schema/m.new_device | 32 ----------------------------- 2 files changed, 39 deletions(-) delete mode 100644 event-schemas/examples/m.new_device delete mode 100644 event-schemas/schema/m.new_device diff --git a/event-schemas/examples/m.new_device b/event-schemas/examples/m.new_device deleted file mode 100644 index 680a8cd9a5..0000000000 --- a/event-schemas/examples/m.new_device +++ /dev/null @@ -1,7 +0,0 @@ -{ - "content": { - "device_id": "RJYKSTBOIE", - "rooms": ["!UCnwUWwIKhcpaPTHtR:sw1v.org"] - }, - "type": "m.new_device" -} diff --git a/event-schemas/schema/m.new_device b/event-schemas/schema/m.new_device deleted file mode 100644 index 22d7854a45..0000000000 --- a/event-schemas/schema/m.new_device +++ /dev/null @@ -1,32 +0,0 @@ ---- -allOf: - - $ref: core-event-schema/event.yaml - -description: |- - This event type is used to announce that a user has logged in with a new - device. It is sent as a `to-device`_ event to all - users which share an encrypted room with the user, to request that encryption - keys be shared with the new device. -properties: - content: - properties: - device_id: - type: string - description: |- - ID of the new device - rooms: - type: list - items: - type: string - description: |- - A list of rooms that the new device is requesting encryption keys - for. - required: - - device_id - - rooms - type: object - type: - enum: - - m.new_device - type: string -type: object From ea6c933850d52364910f170e724d40446f8dd8e2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 19 Apr 2017 23:04:09 +0100 Subject: [PATCH 10/11] Remove broken 'device verification' link --- specification/modules/end_to_end_encryption.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 639e8ff03d..1f778bc2c7 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -158,8 +158,8 @@ inefficient. It is therefore expected that each client will maintain a list of devices for a number of users (in practice, typically each user with whom we share an encrypted room). Furthermore, it is likely that this list will need to be -persisted between invocations of the client application (to preserve `device -verification`_ data and to alert Alice if Bob suddently gets a new +persisted between invocations of the client application (to preserve device +verification data and to alert Alice if Bob suddently gets a new device). Alice's client can maintain a list of Bob's devices via the following From 48ae66b2dea1dfc3a7c820f9cc8178ddd073c4ba Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 20 Apr 2017 16:50:29 +0100 Subject: [PATCH 11/11] Fix typo --- api/client-server/keys.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/keys.yaml b/api/client-server/keys.yaml index a0ca6c9caa..e77d53254c 100644 --- a/api/client-server/keys.yaml +++ b/api/client-server/keys.yaml @@ -85,7 +85,7 @@ paths: additionalProperties: type: integer description: |- - For each key algorighm, the number of unclaimed one-time keys + For each key algorithm, the number of unclaimed one-time keys of that type currently held on the server for this device. example: curve25519: 10