From 36e035c79e5efccb76a429194efdc19fcc450d0b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Jul 2015 19:31:11 +0100 Subject: [PATCH 001/989] Add some specification for end-to-end --- specification/41_end_to_end_encryption.rst | 215 +++++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 023881527bc..a09c8fc60ce 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -13,3 +13,218 @@ participating homeservers. End-to-end crypto is still being designed and prototyped - notes on the design may be found at https://lwn.net/Articles/634144/ + +Overview +======== + +.. code:: + + 1) Bob publishes the public keys and supported algorithms for his device. + + +----------+ +--------------+ + | Bob's HS | | Bob's Device | + +----------+ +--------------+ + | | + |<=============| + /keys/upload + + 2) Alice requests Bob's public key and supported algorithms. + + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/query + + 3) Alice selects an algorithm takes any one time keys needed. + + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/take + + 4) Alice sends an encrypted message to Bob. + + +----------------+ +------------+ +----------+ +--------------+ + | Alice's Device | | Alice's HS | | Bob's HS | | Bob's Device | + +----------------+ +------------+ +----------+ +--------------+ + | | | | + |----------------->|-------------->|------------->| + /send/ + + +Client Behaviour +---------------- + +Uploading Keys +~~~~~~~~~~~~~~ + +Keys are uploaded as a signed JSON object. The JSON object must include an +ed25519 key and must be signed by that key. A device may only have one ed25519 +signing key. This key is used as the fingerprint for a device by other clients. + + +.. code:: http + + POST /_matrix/client/v2_alpha/keys/upload/ HTTP/1.1 + Content-Type: application/json + + { + "device_keys": { + "user_id": "", + "device_id": "", + "valid_after_ts": 1234567890123, + "valid_until_ts": 2345678901234, + "algorithms": [ + "", + ], + "keys": { + ":": "", + }, + "signatures:" { + "" { + ":": "" + } } }, + "one_time_keys": { + ":": "" + }, + } + + +Downloading Keys +~~~~~~~~~~~~~~~~ + +Keys are downloaded a collection of signed JSON objects. There +will be JSON object per device per user. If one of the user's +devices doesn't support end-to-end encryption then their +homeserver will synthesise a JSON object without any device keys +for that device. + +The JSON must be signed by both the homeserver of +the user querying the keys and by the homeserver of the device +being queried. This provides an audit trail if either homeserver +lies about the keys a user owns. + +.. code:: http + + POST /keys/query HTTP/1.1 + Content-Type: application/json + + { + "device_keys": { + "": [""] + } } + + +.. code:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "device_keys": { + "": { + "": { + "user_id": "", + "device_id": "", + "valid_after_ts": 1234567890123, + "valid_until_ts": 2345678901234, + "algorithms": [ + "", + ], + "keys": { + ":": "", + }, + "signatures:" { + "": { + ":": "" + }, + "": { + ":": "" + }, + "": { + ":": "" + } } } } } } + + +Taking One Time Keys +~~~~~~~~~~~~~~~~~~~~ + +Some algorithms require one time keys to improve their secrecy and deniability. +Theses keys are used once during session establishment, and are then thrown +away. In order for these keys to be useful for improving deniability they +must not be signed using the ed25519 key for a device. + +A device will generate a number of these keys and publish them onto their +homeserver. A device will periodically check how many one time keys their +homeserver still has. If the number has become too small then the device will +generate new one time keys and upload them to the homeserver. + +Devices will store the private part of each one time key they upload. They can +discard the private part of the one time key when they receive a message using +that key. However one-keys given out by a homeserver may never end up being +used. Therefore a device may end up trying to store too many private keys. A +device that is trying to store too many private keys may discard keys starting +with the oldest. + +A homeserver should ratelimit the number of one time keys that a given user or +remote server can take. A homeserver should discard the public part of a one +time key once it has given that key to another user. + + +.. code:: http + + POST /keys/take HTTP/1.1 + Content-Type: application/json + + { + "one_time_keys": { + "": { + "": "" + } } } + +.. code:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "one_time_keys": { + "": { + "": { + ":": "" + } } } } + + +Sending a Message +~~~~~~~~~~~~~~~~~ + +Encrypted messages are sent in the form. + +.. code:: json + + { + "type": "m.room.message" + "content": {} + "encrypted": { + "algorithm": "" + } + } + + +.. code:: json + + { + "type": "m.room.message" + "content": {} + "encrypted": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "ciphertexts": { + "" { + ": { + "type": 0, + "body": "" + } } } } } From 01927cee9b6d8152579eaa24d4ae6572933e9811 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Jul 2015 09:21:25 +0100 Subject: [PATCH 002/989] Rename "take" to "claim". Hyphenate "one-time". --- specification/41_end_to_end_encryption.rst | 46 +++++++++++----------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index a09c8fc60ce..f30fc09d937 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -37,14 +37,14 @@ Overview |=================>|==============>| /keys/query - 3) Alice selects an algorithm takes any one time keys needed. + 3) Alice selects an algorithm claims any one-time keys needed. +----------------+ +------------+ +----------+ | Alice's Device | | Alice's HS | | Bob's HS | +----------------+ +------------+ +----------+ | | | |=================>|==============>| - /keys/take + /keys/claim 4) Alice sends an encrypted message to Bob. @@ -97,7 +97,7 @@ signing key. This key is used as the fingerprint for a device by other clients. Downloading Keys ~~~~~~~~~~~~~~~~ -Keys are downloaded a collection of signed JSON objects. There +Keys are downloaded as a collection of signed JSON objects. There will be JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their homeserver will synthesise a JSON object without any device keys @@ -150,34 +150,35 @@ lies about the keys a user owns. } } } } } } -Taking One Time Keys +Claiming One Time Keys ~~~~~~~~~~~~~~~~~~~~ -Some algorithms require one time keys to improve their secrecy and deniability. -Theses keys are used once during session establishment, and are then thrown +Some algorithms require one-time keys to improve their secrecy and deniability. +These keys are used once during session establishment, and are then thrown away. In order for these keys to be useful for improving deniability they must not be signed using the ed25519 key for a device. -A device will generate a number of these keys and publish them onto their -homeserver. A device will periodically check how many one time keys their -homeserver still has. If the number has become too small then the device will -generate new one time keys and upload them to the homeserver. - -Devices will store the private part of each one time key they upload. They can -discard the private part of the one time key when they receive a message using -that key. However one-keys given out by a homeserver may never end up being -used. Therefore a device may end up trying to store too many private keys. A -device that is trying to store too many private keys may discard keys starting -with the oldest. - -A homeserver should ratelimit the number of one time keys that a given user or -remote server can take. A homeserver should discard the public part of a one +A device must generate a number of these keys and publish them onto their +homeserver. A device must periodically check how many one-time keys their +homeserver still has. If the number has become too small then the device must +generate new one-time keys and upload them to the homeserver. + +Devices must store the private part of each one-time key they upload. They can +discard the private part of the one-time key when they receive a message using +that key. However it's possible that a one-time key given out by a homeserver +will never be used, so the device that generates the key will never know that +it can discard the key. Therefore a device could end up trying to store too +many private keys. A device that is trying to store too many private keys may +discard keys starting with the oldest. + +A homeserver should ratelimit the number of one-time keys that a given user or +remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. .. code:: http - POST /keys/take HTTP/1.1 + POST /keys/claim HTTP/1.1 Content-Type: application/json { @@ -211,8 +212,7 @@ Encrypted messages are sent in the form. "content": {} "encrypted": { "algorithm": "" - } - } + } } .. code:: json From 42ad1f861240998658fb643c568db70cc2d86c2e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Jul 2015 09:38:08 +0100 Subject: [PATCH 003/989] Add a link to signing JSON section of the spec. Fixup the markup a bit --- specification/41_end_to_end_encryption.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index f30fc09d937..19ee0cf1de3 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -66,6 +66,8 @@ Keys are uploaded as a signed JSON object. The JSON object must include an ed25519 key and must be signed by that key. A device may only have one ed25519 signing key. This key is used as the fingerprint for a device by other clients. +The JSON object is signed using the process given by `Signing JSON`_. + .. code:: http @@ -90,17 +92,16 @@ signing key. This key is used as the fingerprint for a device by other clients. } } }, "one_time_keys": { ":": "" - }, - } + } } Downloading Keys ~~~~~~~~~~~~~~~~ Keys are downloaded as a collection of signed JSON objects. There -will be JSON object per device per user. If one of the user's +will be a JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their -homeserver will synthesise a JSON object without any device keys +homeserver must synthesise a JSON object without any device keys for that device. The JSON must be signed by both the homeserver of @@ -151,7 +152,7 @@ lies about the keys a user owns. Claiming One Time Keys -~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~ Some algorithms require one-time keys to improve their secrecy and deniability. These keys are used once during session establishment, and are then thrown @@ -228,3 +229,4 @@ Encrypted messages are sent in the form. "type": 0, "body": "" } } } } } + From 6f69707c71c50f783d1327c55179f2007129cbdb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Jul 2015 19:30:36 +0100 Subject: [PATCH 004/989] Update e2e spec: Group ciphertext by device key rather than device id, add return to docs for /keys/upload, Use "m.room.encrypted" for now, rather than trying to add an encrypted content to arbitrary event types --- specification/41_end_to_end_encryption.rst | 36 ++++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 19ee0cf1de3..bf762a3d2a1 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -94,6 +94,17 @@ The JSON object is signed using the process given by `Signing JSON`_. ":": "" } } +.. code:: http + + 200 OK + Content-Type: application/json + + { + "one_time_key_counts": { + "": 50 + } + } + Downloading Keys ~~~~~~~~~~~~~~~~ @@ -209,10 +220,10 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.message" - "content": {} - "encrypted": { + "type": "m.room.encrypted" + "content": { "algorithm": "" + } } } @@ -220,13 +231,18 @@ Encrypted messages are sent in the form. { "type": "m.room.message" - "content": {} - "encrypted": { + "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": , "ciphertexts": { - "" { - ": { - "type": 0, - "body": "" - } } } } } + ": { + "type": 0, + "body": "" + } } } } + + +The plaintext payload is of the form: + +.. code:: json + TODO: SPEC the JSON plaintext format From 41d204e72c22b768ae4ac0b5dcda59cdcaa09acd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 09:43:46 +0100 Subject: [PATCH 005/989] Name the key 'ciphertext' rather than 'ciphertexts' --- specification/41_end_to_end_encryption.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index bf762a3d2a1..553664f2edc 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -223,8 +223,7 @@ Encrypted messages are sent in the form. "type": "m.room.encrypted" "content": { "algorithm": "" - } - } } + } } } .. code:: json @@ -234,7 +233,7 @@ Encrypted messages are sent in the form. "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": , - "ciphertexts": { + "ciphertext": { ": { "type": 0, "body": "" @@ -245,4 +244,5 @@ The plaintext payload is of the form: .. code:: json - TODO: SPEC the JSON plaintext format + { + } From 6597aaa448023b31fa62e23183e4a4eeeae0e22d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 11:21:14 +0100 Subject: [PATCH 006/989] Start describing the plaintext payload format for encrypted messages, add the exact URLs used for key queries from clients and for key queries for federation --- specification/41_end_to_end_encryption.rst | 57 ++++++++++++++++------ 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 553664f2edc..4c64e92fe5c 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -96,7 +96,7 @@ The JSON object is signed using the process given by `Signing JSON`_. .. code:: http - 200 OK + HTTP/1.1 200 OK Content-Type: application/json { @@ -162,6 +162,12 @@ lies about the keys a user owns. } } } } } } +Clients use ``/_matrix/client/v2_alpha/keys/query`` on their own homeservers to +claim keys for any user they wish to contact. Homeservers will respond with the +keys for their local users and forward requests for remote users to +``/_matrix/federation/v1/user/keys/query``. + + Claiming One Time Keys ~~~~~~~~~~~~~~~~~~~~~~ @@ -212,6 +218,11 @@ time key once it has given that key to another user. } } } } +Clients use ``/_matrix/client/v2_alpha/keys/claim`` on their own homeservers to +claim keys for any user they wish to contact. Homeservers will respond with the +keys for their local users and forward requests for remote users to +``/_matrix/federation/v1/user/keys/claim``. + Sending a Message ~~~~~~~~~~~~~~~~~ @@ -220,24 +231,27 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.encrypted" - "content": { - "algorithm": "" - } } } + "type": "m.room.encrypted" + "content": { + "algorithm": "" + } } +Using Olm +######### + .. code:: json { - "type": "m.room.message" - "content": { - "algorithm": "m.olm.v1.curve25519-aes-sha2", - "sender_key": , - "ciphertext": { - ": { - "type": 0, - "body": "" - } } } } + "type": "m.room.encrypted" + "content": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "", + "ciphertext": { + "": { + "type": 0, + "body": "" + } } } } The plaintext payload is of the form: @@ -245,4 +259,19 @@ The plaintext payload is of the form: .. code:: json { + "type": "", + "content": "", + "room_id": "", + "fingerprint": "" } + +The type and content of the plaintext message event are given in the payload. +Encyrpting state events is not supported. + +We include the room ID in the payload, because otherwise the homeserver would +be able to change the room a message was sent in. We include a hash of the +participating keys so that clients can detect if another device is unexpectedly +included in the conversation. + +Clients must confirm that the ``sender_key`` actually belongs to the device +that sent the message. From c83e8480e83db63e76ffcbe62734de773794fb8a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 15:55:21 +0100 Subject: [PATCH 007/989] Fix JSON syntax --- specification/41_end_to_end_encryption.rst | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 4c64e92fe5c..5f486f66a6b 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -86,8 +86,8 @@ The JSON object is signed using the process given by `Signing JSON`_. "keys": { ":": "", }, - "signatures:" { - "" { + "signatures": { + "": { ":": "" } } }, "one_time_keys": { @@ -150,7 +150,7 @@ lies about the keys a user owns. "keys": { ":": "", }, - "signatures:" { + "signatures": { "": { ":": "" }, @@ -163,9 +163,10 @@ lies about the keys a user owns. Clients use ``/_matrix/client/v2_alpha/keys/query`` on their own homeservers to -claim keys for any user they wish to contact. Homeservers will respond with the +query keys for any user they wish to contact. Homeservers will respond with the keys for their local users and forward requests for remote users to -``/_matrix/federation/v1/user/keys/query``. +``/_matrix/federation/v1/user/keys/query`` over federation to the remote +server. Claiming One Time Keys @@ -221,7 +222,9 @@ time key once it has given that key to another user. Clients use ``/_matrix/client/v2_alpha/keys/claim`` on their own homeservers to claim keys for any user they wish to contact. Homeservers will respond with the keys for their local users and forward requests for remote users to -``/_matrix/federation/v1/user/keys/claim``. +``/_matrix/federation/v1/user/keys/claim`` over federation to the remote +server. + Sending a Message ~~~~~~~~~~~~~~~~~ @@ -231,7 +234,7 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.encrypted" + "type": "m.room.encrypted", "content": { "algorithm": "" } } @@ -243,7 +246,7 @@ Using Olm .. code:: json { - "type": "m.room.encrypted" + "type": "m.room.encrypted", "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": "", @@ -273,5 +276,5 @@ be able to change the room a message was sent in. We include a hash of the participating keys so that clients can detect if another device is unexpectedly included in the conversation. -Clients must confirm that the ``sender_key`` actually belongs to the device -that sent the message. +Clients must confirm that the ``sender_key`` belongs to the user that sent the +message. From e1f201f9e636fad8cd3ec2b031a58a0fd7018729 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 16:07:03 +0100 Subject: [PATCH 008/989] Add description of the olm type and body JSON keys --- specification/41_end_to_end_encryption.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 5f486f66a6b..1e60fad96ac 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -256,6 +256,21 @@ Using Olm "body": "" } } } } +The ciphertext is a mapping from device curve25519 key to an encypted payload +for that device. The ``body`` is a base64 encoded message body. The type is an +integer indicating the type of the message body: 0 for the intial pre-key +message, 1 for ordinary messages. + +Olm sessions will generate messages with a type of 0 until they receive a +message. Once a session has decrypted a message it will produce messages with +a type of 1. + +When a client receives a message with a type of 0 it must first check if it +already has a matching session. If it does then it will use that session to +try to decrypt the message. If there is no existing session then the client +must create a new session and use the new session to decrypt the message. A +client must not persist a session or remove one-time keys used by a session +until it has sucessfully decrypted a message using that session. The plaintext payload is of the form: From d06580a481f5396c53574ff86e4830b0cb47f7b4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 16:09:14 +0100 Subject: [PATCH 009/989] Spelling --- specification/41_end_to_end_encryption.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 1e60fad96ac..f0d8a50003c 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -258,7 +258,7 @@ Using Olm The ciphertext is a mapping from device curve25519 key to an encypted payload for that device. The ``body`` is a base64 encoded message body. The type is an -integer indicating the type of the message body: 0 for the intial pre-key +integer indicating the type of the message body: 0 for the initial pre-key message, 1 for ordinary messages. Olm sessions will generate messages with a type of 0 until they receive a @@ -270,7 +270,7 @@ already has a matching session. If it does then it will use that session to try to decrypt the message. If there is no existing session then the client must create a new session and use the new session to decrypt the message. A client must not persist a session or remove one-time keys used by a session -until it has sucessfully decrypted a message using that session. +until it has successfully decrypted a message using that session. The plaintext payload is of the form: @@ -284,7 +284,7 @@ The plaintext payload is of the form: } The type and content of the plaintext message event are given in the payload. -Encyrpting state events is not supported. +Encrypting state events is not supported. We include the room ID in the payload, because otherwise the homeserver would be able to change the room a message was sent in. We include a hash of the From 88176ef14845af72cc968d04ad093d8927bdbc1b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Aug 2015 15:05:51 +0100 Subject: [PATCH 010/989] Add notes on algorithm naming. Fix some typos --- specification/41_end_to_end_encryption.rst | 77 +++++++++++++++++----- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index f0d8a50003c..a2b4ff39a8b 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -15,7 +15,7 @@ may be found at https://lwn.net/Articles/634144/ Overview -======== +-------- .. code:: @@ -37,7 +37,7 @@ Overview |=================>|==============>| /keys/query - 3) Alice selects an algorithm claims any one-time keys needed. + 3) Alice selects an algorithm and claims any one-time keys needed. +----------------+ +------------+ +----------+ | Alice's Device | | Alice's HS | | Bob's HS | @@ -56,6 +56,45 @@ Overview /send/ +Algorithms +---------- + +There are two kinds of algorithms: messaging algorithms and key algorithms. +Messaging algorithms are used to securely send messages between devices. +Key algorithms are used for key agreement and digital signatures. + +Messaging Algorithm Names +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Messaging algorithm names use the extensible naming scheme used throughout this +specification. Algorithm names that start with "m." are reserved for algorithms +defined by this specification. Implementations wanting to experiment with new +algorithms are encouraged to pick algorithm names that start with their +domain to reduce the risk of collisions. + +The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm +ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for +ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the +hash ratchet, and HKDF-SHA-256, AES-256, and 8 byte truncated HMAC-SHA-256 +for authenticated encryption. + +Algorithm names should be short and meaningful. A name of "m.olm.v1" is too +short. However a name of +"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-hmac64sha256" +is too long despite giving a more precise description of the algorithm. + +Algorithm names should list the primitives used by the algorithm so that it +easier to see if the algorithm is using a broken primitive. + +Key Algorithms +~~~~~~~~~~~~~~ + +The name "ed25519" corresponds to the Ed25519 signature algorithm. The key is +a Base64 encoded 32-byte Ed25519 public key. + +The name "curve25519" corresponds to the Curve25519 ECDH algorithm. The key is +a Base64 encoded 32-byte Curve25519 public key. + Client Behaviour ---------------- @@ -81,17 +120,17 @@ The JSON object is signed using the process given by `Signing JSON`_. "valid_after_ts": 1234567890123, "valid_until_ts": 2345678901234, "algorithms": [ - "", + "", ], "keys": { - ":": "", + ":": "", }, "signatures": { "": { - ":": "" + ":": "" } } }, "one_time_keys": { - ":": "" + ":": "" } } .. code:: http @@ -101,7 +140,7 @@ The JSON object is signed using the process given by `Signing JSON`_. { "one_time_key_counts": { - "": 50 + "": 50 } } @@ -145,20 +184,20 @@ lies about the keys a user owns. "valid_after_ts": 1234567890123, "valid_until_ts": 2345678901234, "algorithms": [ - "", + "", ], "keys": { ":": "", }, "signatures": { "": { - ":": "" + ":": "" }, "": { - ":": "" + ":": "" }, "": { - ":": "" + ":": "" } } } } } } @@ -190,7 +229,7 @@ it can discard the key. Therefore a device could end up trying to store too many private keys. A device that is trying to store too many private keys may discard keys starting with the oldest. -A homeserver should ratelimit the number of one-time keys that a given user or +A homeserver should rate-limit the number of one-time keys that a given user or remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. @@ -203,7 +242,7 @@ time key once it has given that key to another user. { "one_time_keys": { "": { - "": "" + "": "" } } } .. code:: http @@ -215,7 +254,7 @@ time key once it has given that key to another user. "one_time_keys": { "": { "": { - ":": "" + ":": "" } } } } @@ -225,7 +264,6 @@ keys for their local users and forward requests for remote users to ``/_matrix/federation/v1/user/keys/claim`` over federation to the remote server. - Sending a Message ~~~~~~~~~~~~~~~~~ @@ -236,13 +274,18 @@ Encrypted messages are sent in the form. { "type": "m.room.encrypted", "content": { - "algorithm": "" + "algorithm": "", + "": "" } } Using Olm ######### +Devices that support olm must include "m.olm.v1.curve25519-aes-sha2" in their +list of supported chat algorithms, must list a Curve25519 device key, and +must publish Curve25519 one-time keys. + .. code:: json { @@ -256,7 +299,7 @@ Using Olm "body": "" } } } } -The ciphertext is a mapping from device curve25519 key to an encypted payload +The ciphertext is a mapping from device curve25519 key to an encrypted payload for that device. The ``body`` is a base64 encoded message body. The type is an integer indicating the type of the message body: 0 for the initial pre-key message, 1 for ordinary messages. From 7d805f105e1123aa9fbfb3fe3ba2b8f7fd80950a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Aug 2015 15:08:52 +0100 Subject: [PATCH 011/989] Mention that Olm uses AES in CBC mode --- specification/41_end_to_end_encryption.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index a2b4ff39a8b..023a568482c 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -75,12 +75,12 @@ domain to reduce the risk of collisions. The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the -hash ratchet, and HKDF-SHA-256, AES-256, and 8 byte truncated HMAC-SHA-256 -for authenticated encryption. +hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated +HMAC-SHA-256 for authenticated encryption. Algorithm names should be short and meaningful. A name of "m.olm.v1" is too short. However a name of -"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-hmac64sha256" +"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256" is too long despite giving a more precise description of the algorithm. Algorithm names should list the primitives used by the algorithm so that it From 458383585f6e58b588171d37584daccba49ff5bb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 15:03:55 +0100 Subject: [PATCH 012/989] Stub sections --- specification/modules/instant_messaging.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 43a06aa1374..619226df7c8 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -3,6 +3,9 @@ Instant Messaging .. _module:im: +This module adds support for sending human-readable messages to a room. It also +adds human-readable information to the room itself such as a room name and topic. + Events ------ @@ -15,7 +18,7 @@ Events {{m_room_topic_event}} m.room.message msgtypes ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ .. TODO-spec How a client should handle unknown message types. @@ -27,3 +30,13 @@ outlined below. {{msgtype_events}} + +Client behaviour +---------------- + +Server behaviour +---------------- + +Security considerations +----------------------- + From 8e5c832ff9fa8d8e674346e3776c08e659c306c9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 5 Oct 2015 13:45:23 +0100 Subject: [PATCH 013/989] Flesh out more of the IM module --- .../schema/v1/m.room.message.feedback | 2 +- specification/modules/instant_messaging.rst | 59 +++++++++++++++++-- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/event-schemas/schema/v1/m.room.message.feedback b/event-schemas/schema/v1/m.room.message.feedback index 1bbfc1ba962..2eaed9990da 100644 --- a/event-schemas/schema/v1/m.room.message.feedback +++ b/event-schemas/schema/v1/m.room.message.feedback @@ -1,7 +1,7 @@ { "type": "object", "title": "MessageFeedback", - "description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.", + "description": "**NB: Usage of this event is discouraged in favour of the** `receipts module`_. **Most clients will not recognise this event.** Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged.", "allOf": [{ "$ref": "core-event-schema/room_event.json" }], diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 619226df7c8..7934b80a24a 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -4,15 +4,37 @@ Instant Messaging .. _module:im: This module adds support for sending human-readable messages to a room. It also -adds human-readable information to the room itself such as a room name and topic. +adds support for associating human-readable information with the room itself +such as a room name and topic. Events ------ {{m_room_message_event}} + +.. admonition:: Rationale + + Not all clients can display all message types. The most commonly supported + message type is raw text. As a result, we chose to have a textual fallback + display method represented by the ``body`` key. This means that even if the + client cannot display a particular ``msgtype``, they can still display + *something*, even if it is just plain text. + {{m_room_message_feedback_event}} +Usage of this event is discouraged for several reasons: + - The number of feedback events will grow very quickly with the number of users + in the room. This event provides no way to "batch" feedback, unlike the + `receipts module`_. + - Pairing feedback to messages gets complicated when paginating as feedback + arrives before the message it is acknowledging. + - There are no guarantees that the client has seen the event ID being + acknowledged. + + +.. _`receipts module`: `module:receipts`_ + {{m_room_name_event}} {{m_room_topic_event}} @@ -20,13 +42,10 @@ Events m.room.message msgtypes ~~~~~~~~~~~~~~~~~~~~~~~ -.. TODO-spec - How a client should handle unknown message types. - - Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type of message being sent. Each type has their own required and optional keys, as -outlined below. +outlined below. If a client cannot display the given ``msgtype`` then it MUST +display the fallback plain text ``body`` key instead. {{msgtype_events}} @@ -34,9 +53,37 @@ outlined below. Client behaviour ---------------- +Events which have attachments (e.g. ``m.image``, ``m.file``) are advised to be +uploaded using the `content repository module`_ where available. The +resulting ``mxc://`` URI can then be used in the ``url`` key. The +attachment SHOULD be uploaded *prior* to sending the event in order to stop a +race condition where the recipient receives a link to a non-existent attachment. + +.. _`content repository module`: `module:content`_ + +Recommendations when sending messages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- Advise using idempotent PUTs to send messages (and why) +- Retries (exp backoff, giveup eventually allowing manual retry) +- Queueing (bucket per room) + +Implementing local echo +~~~~~~~~~~~~~~~~~~~~~~~ +- Local echo (document bug with races) - sending state. Pairing returned event ID. + +Displaying membership information with messages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Member name linking (incl. pagination aka historical display names) + Server behaviour ---------------- +- SHOULD enforce the body/msgtype keys are present (can 400 them) + Security considerations ----------------------- +- Not encrypted, link to E2E module. +- XSS: Should sanitise ALL KEYS before injecting as unsafe HTML (name/topic/body/etc) + From 91ca36509b3112a83680a97a0d672a428b4d7fef Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 7 Oct 2015 11:51:49 +0100 Subject: [PATCH 014/989] Flesh out IM module --- specification/0-intro.rst | 2 + specification/modules/instant_messaging.rst | 89 ++++++++++++++++++--- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/specification/0-intro.rst b/specification/0-intro.rst index 445b32ef28a..7c27e0d7812 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -434,6 +434,8 @@ Some requests have unique error codes: :``M_LOGIN_EMAIL_URL_NOT_YET``: Encountered when polling for an email link which has not been clicked yet. +.. _sect:txn_ids: + The C-S API typically uses ``HTTP POST`` to submit requests. This means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to make requests idempotent. In order to use a ``PUT``, paths should be suffixed with diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 7934b80a24a..426189b580a 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -53,7 +53,14 @@ display the fallback plain text ``body`` key instead. Client behaviour ---------------- -Events which have attachments (e.g. ``m.image``, ``m.file``) are advised to be +Clients SHOULD verify the structure of incoming events to ensure that the +expected keys exist and that they are of the right type. Clients can discard +malformed events or display a placeholder message to the user. Redacted +``m.room.message`` events MUST be removed from the client. This can either be +replaced with placeholder text (e.g. "[REDACTED]") or the redacted message can +be removed entirely from the messages view. + +Events which have attachments (e.g. ``m.image``, ``m.file``) SHOULD be uploaded using the `content repository module`_ where available. The resulting ``mxc://`` URI can then be used in the ``url`` key. The attachment SHOULD be uploaded *prior* to sending the event in order to stop a @@ -63,27 +70,87 @@ race condition where the recipient receives a link to a non-existent attachment. Recommendations when sending messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Advise using idempotent PUTs to send messages (and why) -- Retries (exp backoff, giveup eventually allowing manual retry) -- Queueing (bucket per room) -Implementing local echo -~~~~~~~~~~~~~~~~~~~~~~~ -- Local echo (document bug with races) - sending state. Pairing returned event ID. +Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use +``PUT`` requests with `transaction IDs`_ to make requests idempotent. This +ensures that messages are sent exactly once even under poor network conditions. +Clients SHOULD retry requests using an exponential-backoff algorithm for a +certain amount of time T. It is recommended that T is no longer than 5 minutes. +After this time, the client should stop retrying and mark the message as "unsent". +Users should be able to manually resend unsent messages. + +Users may type several messages at once and send them all in quick succession. +Clients SHOULD preserve the order in which they were sent by the user. This +means that clients should wait for the response to the previous request before +sending the next request. This can lead to head-of-line blocking. In order to +reduce the impact of head-of-line blocking, clients should use a queue per room +rather than a global queue, as ordering is only relevant within a single room +rather than between rooms. + +.. _`transaction IDs`: `sect:txn_ids`_ + +Local echo +~~~~~~~~~~ + +Messages SHOULD appear immediately in the message view when a user presses the +"send" button. This should occur even if the message is still sending. This is +referred to as "local echo". Clients SHOULD implement "local echo" of messages. + +Clients need to be able to pair up the "remote echo" from the server with the +"local echo" to prevent duplicate messages being displayed. Ideally this pairing +would occur transparently to the user: the UI would not flicker as it transitions +from local to remote. Flickering cannot be fully avoided in version 1 of the +client-server API. Two scenarios need to be considered: + +- The client sends a message and the remote echo arrives on the event stream + *after* the request to send the message completes. +- The client sends a message and the remote echo arrives on the event stream + *before* the request to send the message completes. + +In the first scenario, the client will receive an event ID when the request to +send the message completes. This ID can be used to identify the duplicate event +when it arrives on the event stream. However, in the second scenario, the event +arrives before the client has obtained an event ID. This makes it impossible to +identify it as a duplicate event. This results in the client displaying the +message twice for a fraction of a second before the the original request to send +the message completes. Once it completes, the client can take remedial actions +to remove the duplicate event by looking for duplicate event IDs. Version 2 of +the client-server API resolves this by attaching the transaction ID of the +sending request to the event itself. + Displaying membership information with messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Member name linking (incl. pagination aka historical display names) +Clients may wish to show the display name and avatar URL of the room member who +sent a message. This can be achieved by inspecting the ``m.room.member`` event +for that user ID. + +When a user paginates the message history, clients may wish to show the +**historical** display name and avatar URL for a room member. This is possible +because older ``m.room.member`` events are returned when paginating. This can +be implemented efficiently by keeping two sets of room state: old and current. +As new events arrive and/or the user paginates back in time, these two sets of +state diverge from each other. New events update the current state and paginated +events update the old state. When paginated events are processed sequentially, +the old state represents the state of the room *at the time the event was sent*. +This can then be used to set the historical display name and avatar URL. Server behaviour ---------------- -- SHOULD enforce the body/msgtype keys are present (can 400 them) +Homeservers SHOULD enforce that ``m.room.message`` events have textual ``body`` +and ``msgtype`` keys by 400ing the request to send a message. Security considerations ----------------------- -- Not encrypted, link to E2E module. -- XSS: Should sanitise ALL KEYS before injecting as unsafe HTML (name/topic/body/etc) +Messages sent using this module are not encrypted. Encryption can be layered +over the top of this module: where the plaintext format is an ``m.room.message`` +conforming to this module. This can be achieved using the `E2E module`_. + +Clients should sanitise **all keys** for unsafe HTML to prevent Cross-Site +Scripting (XSS) attacks. This includes room names and topics. + +.. _`E2E module`: `module:e2e`_ From e378de8379abcb85db8048b7dbd80b3851b4ac51 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 9 Oct 2015 11:04:11 +0100 Subject: [PATCH 015/989] Shuffle history visibility module to be in the module format --- specification/0-events.rst | 2 - specification/modules/history_visibility.rst | 44 +++++++++++++++----- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/specification/0-events.rst b/specification/0-events.rst index 16948462c82..e2373f96b5e 100644 --- a/specification/0-events.rst +++ b/specification/0-events.rst @@ -26,8 +26,6 @@ prefixed with ``m.`` {{m_room_create_event}} -{{m_room_history_visibility_event}} - {{m_room_join_rules_event}} {{m_room_member_event}} diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst index 371282bd8d9..4f0c8976a19 100644 --- a/specification/modules/history_visibility.rst +++ b/specification/modules/history_visibility.rst @@ -1,21 +1,36 @@ Room History Visibility ------------------------ +======================= .. _module:history-visibility: -Whether a member of a room can see the events that happened in a room from -before they joined the room is controlled by the ``history_visibility`` key -of the ``m.room.history_visibility`` state event. The valid values for -``history_visibility`` are: +This module adds support for controlling the visibility of previous events in a +room. Whether a member of a room can see the events that happened in a room from +before they joined the room is controlled by the ``m.room.history_visibility`` +event outlined below. Visibility can take the form of one of three options: + +- ``shared`` - Previous events are always shown to newly joined members. +- ``invited`` - Previous events are shown to newly joined members from the point + they were invited onwards. +- ``joined`` - Previous events are shown to newly joined members from the point + they joined the room onwards. + +Events +------ + +{{m_room_history_visibility_event}} -- ``shared`` -- ``invited`` -- ``joined`` +Client behaviour +---------------- -By default if no ``history_visibility`` is set it is assumed to be ``shared``. +Clients that implement this module MUST present to the user the possible options +for setting history visibility when creating a room. -The rules governing whether a user is allowed to see an event depend solely on -the state of the room at that event: +Server behaviour +---------------- + +By default if no ``history_visibility`` is set the visibility is assumed to be +``shared``. The rules governing whether a user is allowed to see an event depend +solely on the state of the room *at that event*: 1. If the user was joined, allow. 2. If the user was invited and the ``history_visibility`` was set to @@ -24,3 +39,10 @@ the state of the room at that event: was set to ``shared``, allow. 4. Otherwise, deny. +Security considerations +----------------------- + +The default value for ``history_visibility`` is ``shared`` for +backwards-compatibility reasons. Clients need to be aware that by not setting +this event they are exposing all of their room history to anyone in the room. + From 173d00cea04e3c607992396b7395741971578e35 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 13 Oct 2015 11:08:19 +0100 Subject: [PATCH 016/989] Review comments --- specification/modules/history_visibility.rst | 24 ++++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst index 4f0c8976a19..12ea7b22d95 100644 --- a/specification/modules/history_visibility.rst +++ b/specification/modules/history_visibility.rst @@ -6,13 +6,27 @@ Room History Visibility This module adds support for controlling the visibility of previous events in a room. Whether a member of a room can see the events that happened in a room from before they joined the room is controlled by the ``m.room.history_visibility`` -event outlined below. Visibility can take the form of one of three options: - -- ``shared`` - Previous events are always shown to newly joined members. +event outlined below. In all cases, the member still needs to be joined to the +room to receive events for that room. The visibility option simply determines +which subset of events in the room are presented to the client. Visibility can +take the form of one of three options: + +- ``shared`` - Previous events are always shown to newly joined members. All + events in the room are shown, even those sent when the member was not a part + of the room. - ``invited`` - Previous events are shown to newly joined members from the point - they were invited onwards. + they were invited onwards. Events stop being shown when the member's state + changes to something other than ``invite`` or ``join``. - ``joined`` - Previous events are shown to newly joined members from the point - they joined the room onwards. + they joined the room onwards. Events stop being shown when the members state + changes to something other than ``join``. + +.. WARNING:: + These options are applied at the point an event is *sent*. Checks are + performed with the state of the ``m.room.history_visibility`` event when the + event in question is added to the DAG. This means clients cannot + retrospectively choose to show or hide history to new users if the setting at + that time was more restrictive. Events ------ From 4170dbd5cf065f7e1c7fde32f4c5a6e622c98055 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 13 Oct 2015 11:29:54 +0100 Subject: [PATCH 017/989] Review comments --- event-schemas/schema/v1/m.room.message | 2 +- specification/modules/instant_messaging.rst | 44 +++++++++------------ 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/event-schemas/schema/v1/m.room.message b/event-schemas/schema/v1/m.room.message index 27b7e925e30..91c04b7fe97 100644 --- a/event-schemas/schema/v1/m.room.message +++ b/event-schemas/schema/v1/m.room.message @@ -1,7 +1,7 @@ { "type": "object", "title": "Message", - "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.", + "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message. This allows clients to display *something* even if it is just plain text.", "allOf": [{ "$ref": "core-event-schema/room_event.json" }], diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 426189b580a..487240de539 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -12,15 +12,6 @@ Events {{m_room_message_event}} - -.. admonition:: Rationale - - Not all clients can display all message types. The most commonly supported - message type is raw text. As a result, we chose to have a textual fallback - display method represented by the ``body`` key. This means that even if the - client cannot display a particular ``msgtype``, they can still display - *something*, even if it is just plain text. - {{m_room_message_feedback_event}} Usage of this event is discouraged for several reasons: @@ -44,7 +35,7 @@ m.room.message msgtypes Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type of message being sent. Each type has their own required and optional keys, as -outlined below. If a client cannot display the given ``msgtype`` then it MUST +outlined below. If a client cannot display the given ``msgtype`` then it SHOULD display the fallback plain text ``body`` key instead. {{msgtype_events}} @@ -62,9 +53,7 @@ be removed entirely from the messages view. Events which have attachments (e.g. ``m.image``, ``m.file``) SHOULD be uploaded using the `content repository module`_ where available. The -resulting ``mxc://`` URI can then be used in the ``url`` key. The -attachment SHOULD be uploaded *prior* to sending the event in order to stop a -race condition where the recipient receives a link to a non-existent attachment. +resulting ``mxc://`` URI can then be used in the ``url`` key. .. _`content repository module`: `module:content`_ @@ -95,12 +84,15 @@ Local echo Messages SHOULD appear immediately in the message view when a user presses the "send" button. This should occur even if the message is still sending. This is referred to as "local echo". Clients SHOULD implement "local echo" of messages. - -Clients need to be able to pair up the "remote echo" from the server with the -"local echo" to prevent duplicate messages being displayed. Ideally this pairing -would occur transparently to the user: the UI would not flicker as it transitions -from local to remote. Flickering cannot be fully avoided in version 1 of the -client-server API. Two scenarios need to be considered: + +Clients need to be able to match the message they are sending with the same +message which they receive from the event stream. The echo of the same message +from the event stream is referred to as "remote echo". Both echoes need to be +identified as the same message in order to prevent duplicate messages being +displayed. Ideally this pairing would occur transparently to the user: the UI +would not flicker as it transitions from local to remote. Flickering cannot be +fully avoided in version 1 of the client-server API. Two scenarios need to be +considered: - The client sends a message and the remote echo arrives on the event stream *after* the request to send the message completes. @@ -123,8 +115,8 @@ Displaying membership information with messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Clients may wish to show the display name and avatar URL of the room member who -sent a message. This can be achieved by inspecting the ``m.room.member`` event -for that user ID. +sent a message. This can be achieved by inspecting the ``m.room.member`` state +event for that user ID. When a user paginates the message history, clients may wish to show the **historical** display name and avatar URL for a room member. This is possible @@ -139,15 +131,15 @@ This can then be used to set the historical display name and avatar URL. Server behaviour ---------------- -Homeservers SHOULD enforce that ``m.room.message`` events have textual ``body`` -and ``msgtype`` keys by 400ing the request to send a message. +Homeservers SHOULD reject ``m.room.message`` events which don't have a +``msgtype`` key, or who don't have a textual ``body`` key, with an HTTP status +code of 400. Security considerations ----------------------- -Messages sent using this module are not encrypted. Encryption can be layered -over the top of this module: where the plaintext format is an ``m.room.message`` -conforming to this module. This can be achieved using the `E2E module`_. +Messages sent using this module are not encrypted. Messages can be encrypted +using the `E2E module`_. Clients should sanitise **all keys** for unsafe HTML to prevent Cross-Site Scripting (XSS) attacks. This includes room names and topics. From e561a663d30810c0e58ca30b89bc3bc759e92aec Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 13 Oct 2015 16:53:27 +0100 Subject: [PATCH 018/989] Tweaks to intro, start using swagger APIs in the CS API section. --- specification/0-intro.rst | 46 ++++++++++++--------------- specification/1-client_server_api.rst | 44 ++----------------------- 2 files changed, 22 insertions(+), 68 deletions(-) diff --git a/specification/0-intro.rst b/specification/0-intro.rst index 445b32ef28a..1c18a4f07d7 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -99,23 +99,23 @@ Architecture ------------ Matrix defines APIs for synchronising extensible JSON objects known as -``events`` between compatible clients, servers and services. Clients are +"events" between compatible clients, servers and services. Clients are typically messaging/VoIP applications or IoT devices/hubs and communicate by -synchronising communication history with their ``homeserver`` using the -``Client-Server API``. Each homeserver stores the communication history and +synchronising communication history with their "homeserver" using the +"Client-Server API". Each homeserver stores the communication history and account information for all of its clients, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers and their clients. Clients typically communicate with each other by emitting events in the -context of a virtual ``room``. Room data is replicated across *all of the +context of a virtual "room". Room data is replicated across *all of the homeservers* whose users are participating in a given room. As such, *no single homeserver has control or ownership over a given room*. Homeservers model communication history as a partially ordered graph of events known as -the room's ``event graph``, which is synchronised with eventual consistency -between the participating servers using the ``Server-Server API``. This process +the room's "event graph", which is synchronised with eventual consistency +between the participating servers using the "Server-Server API". This process of synchronising shared conversation history between homeservers run by -different parties is called ``Federation``. Matrix optimises for the the +different parties is called "Federation". Matrix optimises for the the Availability and Partitioned properties of CAP theorem at the expense of Consistency. @@ -151,13 +151,13 @@ Users ~~~~~ Each client is associated with a user account, which is identified in Matrix -using a unique "User ID". This ID is namespaced to the home server which +using a unique "User ID". This ID is namespaced to the homeserver which allocated the account and has the form:: @localpart:domain The ``localpart`` of a user ID may be a user name, or an opaque ID identifying -this user. They are case-insensitive. +this user. The ``domain`` of a user ID is the domain of the homeserver. .. TODO-spec - Need to specify precise grammar for Matrix IDs @@ -183,9 +183,9 @@ Event Graphs .. _sect:event-graph: Events exchanged in the context of a room are stored in a directed acyclic graph -(DAG) called an ``event graph``. The partial ordering of this graph gives the +(DAG) called an "event graph". The partial ordering of this graph gives the chronological ordering of events within the room. Each event in the graph has a -list of zero or more ``parent`` events, which refer to any preceding events +list of zero or more "parent" events, which refer to any preceding events which have no chronological successor from the perspective of the homeserver which created the event. @@ -292,11 +292,10 @@ Each room can also have multiple "Room Aliases", which look like:: A room alias "points" to a room ID and is the human-readable label by which rooms are publicised and discovered. The room ID the alias is pointing to can -be obtained by visiting the domain specified. They are case-insensitive. Note -that the mapping from a room alias to a room ID is not fixed, and may change -over time to point to a different room ID. For this reason, Clients SHOULD -resolve the room alias to a room ID once and then use that ID on subsequent -requests. +be obtained by visiting the domain specified. Note that the mapping from a room +alias to a room ID is not fixed, and may change over time to point to a +different room ID. For this reason, Clients SHOULD resolve the room alias to a +room ID once and then use that ID on subsequent requests. When resolving a room alias the server will also respond with a list of servers that are in the room that can be used to join via. @@ -339,12 +338,9 @@ Profiles ~~~~~~~~ Users may publish arbitrary key/value data associated with their account - such -as a human readable ``display name``, a profile photo URL, contact information +as a human readable display name, a profile photo URL, contact information (email address, phone numbers, website URLs etc). -In Client-Server API v2, profile data is typed using namespaced keys for -interoperability, much like events - e.g. ``m.profile.display_name``. - .. TODO Actually specify the different types of data - e.g. what format are display names allowed to be? @@ -431,13 +427,11 @@ Some requests have unique error codes: :``M_BAD_PAGINATION``: Encountered when specifying bad pagination query parameters. -:``M_LOGIN_EMAIL_URL_NOT_YET``: - Encountered when polling for an email link which has not been clicked yet. -The C-S API typically uses ``HTTP POST`` to submit requests. This means these -requests are not idempotent. The C-S API also allows ``HTTP PUT`` to make -requests idempotent. In order to use a ``PUT``, paths should be suffixed with -``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which +The Client-Server API typically uses ``HTTP POST`` to submit requests. This +means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to +make requests idempotent. In order to use a ``PUT``, paths should be suffixed +with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which identifies the request, and is scoped to a given Client (identified by that client's ``access_token``). Crucially, it **only** serves to identify new requests from retransmits. After the request has finished, the ``{txnId}`` diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 59e6b68eec4..94d1a1845d9 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -528,7 +528,7 @@ Room events are split into two categories: :Message events: These are events which describe transient "once-off" activity in a room: typically communication such as sending an instant message or setting up a - VoIP call. These used to be called 'non-state' events. + VoIP call. This specification outlines several events, all with the event type prefix ``m.``. However, applications may wish to add their own type of event, and this @@ -600,9 +600,6 @@ See `Room Events`_ for the ``m.`` event specification. Syncing rooms ~~~~~~~~~~~~~ -.. NOTE:: - This section is a work in progress. - When a client logs in, they may have a list of rooms which they have already joined. These rooms may also have a list of events associated with them. The purpose of 'syncing' is to present the current room and event information in a @@ -620,45 +617,8 @@ presence events will also be returned. A single syncing API is provided: onwards. The event stream cannot do this for a single room currently. As a result, commenting room-scoped initial sync at this time. -The |initialSync|_ API contains the following keys: - -``presence`` - Description: - Contains a list of presence information for users the client is interested - in. - Format: - A JSON array of ``m.presence`` events. - -``end`` - Description: - Contains an event stream token which can be used with the `Event Stream`_. - Format: - A string containing the event stream token. -``rooms`` - Description: - Contains a list of room information for all rooms the client has joined, - and limited room information on rooms the client has been invited to. - Format: - A JSON array containing Room Information JSON objects. - -Room Information: - Description: - Contains all state events for the room, along with a limited amount of - the most recent events, configured via the ``limit`` query - parameter. Also contains additional keys with room metadata, such as the - ``room_id`` and the client's ``membership`` to the room. - Format: - A JSON object with the following keys: - ``room_id`` - A string containing the ID of the room being described. - ``membership`` - A string representing the client's membership status in this room. - ``messages`` - An event stream JSON object containing a ``chunk`` of recent - events (both state events and non-state events), along with an ``end`` token. - ``state`` - A JSON array containing all the current state events for this room. +{{sync_http_api}} Getting events for a room ~~~~~~~~~~~~~~~~~~~~~~~~~ From e716e819635babb0216c971940f9e7a8cdbeacc4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 09:58:03 +0100 Subject: [PATCH 019/989] Add `dir` parameter to pagination. Remove path references This section needs a lot of work to not lie. --- specification/1-client_server_api.rst | 30 +++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 94d1a1845d9..a4f62b4fa99 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -348,11 +348,13 @@ Pagination Querying large datasets in Matrix always uses the same pagination API pattern to to give clients a consistent way of selecting subsets of a potentially changing -dataset. Requests pass in ``from``, ``to`` and ``limit`` parameters which describe -where to read from the stream. ``from`` and ``to`` are opaque textual 'stream -tokens' which describe positions in the dataset. The response returns new -``start`` and ``end`` stream token values which can then be passed to subsequent -requests to continue pagination. +dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters +which describe where to read from the stream. ``from`` and ``to`` are opaque +textual 'stream tokens' which describe the current position in the dataset. +The ``dir`` parameter is an enum representing the direction of events to return: +either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and +``end`` stream token values which can then be passed to subsequent requests to +continue pagination. Pagination Request Query Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -367,24 +369,26 @@ Query parameters: limit: integer - An integer representing the maximum number of items to return. + dir: + f|b - The direction to return events in. Typically this is ``b`` to paginate + backwards in time. 'START' and 'END' are placeholder values used in these examples to describe the start and end of the dataset respectively. -Unless specified, the default pagination parameters are from=START, to=END, -without a limit set. This allows you to hit an API like -/events without any query parameters to get everything. +Unless specified, the default pagination parameters are ``from=START``, +``to=END``, without a limit set. -For example, the event stream has events E1 -> E15. The client wants the last 5 +For example, if an endpoint had events E1 -> E15. The client wants the last 5 events and doesn't know any previous events:: S E |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| | | | | _____| | - |__________________ | ___________________| - | | | - GET /events?to=START&limit=5&from=END + |__________________ | _______________| + | | | + GET /somepath?to=START&limit=5&from=END Returns: E15,E14,E13,E12,E11 @@ -401,7 +405,7 @@ now show page 3 (rooms R11 -> 15):: Currently | viewing | | - GET /rooms/list?from=9&to=END&limit=5 + GET /roomslist?from=9&to=END&limit=5 Returns: R11,R12,R13,R14,R15 Note that tokens are treated in an *exclusive*, not inclusive, manner. The end From 7bdb71b1c9697eb9ded20e028723a500b1d71a02 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 10:25:39 +0100 Subject: [PATCH 020/989] Tweak the syncing section Bring together disparate sections to make it more cohesive. --- api/client-server/v1/sync.yaml | 6 +- specification/1-client_server_api.rst | 99 +++++++++++---------------- 2 files changed, 43 insertions(+), 62 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 7c1d43f3e9c..398b0773a50 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -30,7 +30,9 @@ paths: - in: query type: string name: from - description: The token to stream from. + description: |- + The token to stream from. This token is either from the a previous + request to this API or from the initial sync API. required: false x-example: "s3456_9_0" - in: query @@ -40,7 +42,7 @@ paths: required: false x-example: "35000" - in: query - type: string + type: boolean name: archived description: |- Whether to include rooms that the user has left. If absent then diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index a4f62b4fa99..fefdf511e8e 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1,9 +1,6 @@ Client-Server API ================= -Overview --------- - The client-server API provides a simple lightweight API to let clients send messages, control rooms and synchronise conversation history. It is designed to support both lightweight clients which store no state and lazy-load data from @@ -31,7 +28,7 @@ return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or ``M_UNKNOWN_TOKEN`` respectively. User-Interactive Authentication API ------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This section refers to API Version 2. Some API endpoints such as ``login`` or ``register`` require authentication that @@ -159,7 +156,7 @@ absence of that login stage type in the 'completed' array indicating whether that stage is complete. Example -~~~~~~~ ++++++++ At a high level, the requests made for an API call completing an auth flow with three stages will resemble the following diagram:: @@ -201,7 +198,7 @@ This specification defines the following login types: - ``m.login.dummy`` Password-based -~~~~~~~~~~~~~~ +++++++++++++++ :Type: ``m.login.password`` :Description: @@ -216,7 +213,7 @@ To respond to this type, reply with an auth dict as follows:: } Google ReCaptcha -~~~~~~~~~~~~~~~~ +++++++++++++++++ :Type: ``m.login.recaptcha`` :Description: @@ -230,7 +227,7 @@ To respond to this type, reply with an auth dict as follows:: } Token-based -~~~~~~~~~~~ ++++++++++++ :Type: ``m.login.token`` :Description: @@ -261,7 +258,7 @@ newly provisioned access_token). The ``token`` must be a macaroon. OAuth2-based -~~~~~~~~~~~~ +++++++++++++ :Type: ``m.login.oauth2`` :Description: @@ -285,7 +282,7 @@ the OAuth flow has completed, the client retries the request with the session only, as above. Email-based (identity server) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ++++++++++++++++++++++++++++++ :Type: ``m.login.email.identity`` :Description: @@ -310,7 +307,7 @@ To respond to this type, reply with an auth dict as follows:: } Dummy Auth -~~~~~~~~~~ +++++++++++ :Type: ``m.login.dummy`` :Description: @@ -327,7 +324,7 @@ if provided:: Fallback -~~~~~~~~ +++++++++ Clients cannot be expected to be able to know how to process every single login type. If a client does not know how to handle a given login type, it can direct the user to a web browser with the URL of a fallback page which will allow the @@ -346,7 +343,15 @@ the authentication has been completed. Pagination ---------- -Querying large datasets in Matrix always uses the same pagination API pattern to +.. NOTE:: + The paths referred to in this section are not actual endpoints. They only + serve as examples to explain how pagination functions. + +Pagination is the process of dividing a dataset into multiple discrete pages. +Matrix makes use of pagination to allow clients to view extremely large datasets. +These datasets are not limited to events in a room (for example clients may want +to paginate rooms in addition to events within those rooms). Regardless of *what* +is being paginated, there is a common underlying API which is used to to give clients a consistent way of selecting subsets of a potentially changing dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters which describe where to read from the stream. ``from`` and ``to`` are opaque @@ -354,7 +359,8 @@ textual 'stream tokens' which describe the current position in the dataset. The ``dir`` parameter is an enum representing the direction of events to return: either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and ``end`` stream token values which can then be passed to subsequent requests to -continue pagination. +continue pagination. Not all endpoints will make use of all the parameters +outlined here: see the specific endpoint in question for more information. Pagination Request Query Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -385,10 +391,10 @@ events and doesn't know any previous events:: S E |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| | | | - | _____| | - |__________________ | _______________| - | | | - GET /somepath?to=START&limit=5&from=END + | _____| <--backwards-- | + |__________________ | | ________| + | | | | + GET /somepath?to=START&limit=5&dir=b&from=END Returns: E15,E14,E13,E12,E11 @@ -433,9 +439,6 @@ Events .. _sect:events: -Overview -~~~~~~~~ - The model of conversation history exposed by the client-server API can be considered as a list of events. The server 'linearises' the eventually-consistent event graph of events into an 'event stream' at any given @@ -463,7 +466,7 @@ You can visualise the range of events being returned as:: | | start: '1-2-3' end: 'a-b-c' -Now, to receive future events in real-time on the eventstream, you simply GET +Now, to receive future events in real-time on the event stream, you simply GET $PREFIX/events with a ``from`` parameter of 'a-b-c': in other words passing in the ``end`` token returned by initial sync. The request blocks until new events are available or until your specified timeout elapses, and then returns a @@ -493,31 +496,30 @@ To continue paginating backwards, one calls the /messages API again, supplying the new ``start`` value as the ``from`` parameter. -Receiving live updates on a client -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Syncing +~~~~~~~ -Clients receive new events by long-polling the home server via the -$PREFIX/events API, specifying a timeout in milliseconds in the timeout -parameter. This will hold open the HTTP connection for a short period of time -waiting for new events, returning early if an event occurs. This is called the -`Event Stream`_. All events which are visible to the client will appear in the -event stream. When the request returns, an ``end`` token is included in the +Clients receive new events by "long-polling" the home server via the events API. +This involves specifying a timeout in the request which will hold +open the HTTP connection for a short period of time waiting for new events, +returning early if an event occurs. Only the events API supports long-polling. +All events which are visible to the client will appear in the +events API. When the request returns, an ``end`` token is included in the response. This token can be used in the next request to continue where the -last request left off. - -All events must be de-duplicated based on their event ID. +last request left off. Multiple events can be returned per long-poll. All events +must be de-duplicated based on their event ID. .. TODO is deduplication actually a hard requirement in CS v2? .. TODO-spec - Do we ever return multiple events in a single request? - Don't we get lots of request setup RTT latency if we only do one event per request? Do we ever support streaming requests? Why not websockets? When the client first logs in, they will need to initially synchronise with -their home server. This is achieved via the |initialSync|_ API. This API also -returns an ``end`` token which can be used with the event stream. See the 'Room Sync' section below. +their home server. This is achieved via the initial sync API. This API also +returns an ``end`` token which can be used with the event stream. + +{{sync_http_api}} Events in a room ~~~~~~~~~~~~~~~~ @@ -601,29 +603,6 @@ example:: See `Room Events`_ for the ``m.`` event specification. -Syncing rooms -~~~~~~~~~~~~~ - -When a client logs in, they may have a list of rooms which they have already -joined. These rooms may also have a list of events associated with them. The -purpose of 'syncing' is to present the current room and event information in a -convenient, compact manner. The events returned are not limited to room events; -presence events will also be returned. A single syncing API is provided: - - - |initialSync|_ : A global sync which will present room and event information - for all rooms the user has joined. - -.. TODO-spec room-scoped initial sync - - |/rooms//initialSync|_ : A sync scoped to a single room. Presents - room and event information for this room only. - - Room-scoped initial sync is Very Tricky because typically people would - want to sync the room then listen for any new content from that point - onwards. The event stream cannot do this for a single room currently. - As a result, commenting room-scoped initial sync at this time. - - -{{sync_http_api}} - Getting events for a room ~~~~~~~~~~~~~~~~~~~~~~~~~ From 07d7a3fa3a4d0c9d7e4db70d62a24bc266824dbb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 11:03:53 +0100 Subject: [PATCH 021/989] Shuffle pagination section around --- specification/1-client_server_api.rst | 201 +++++++++++++------------- 1 file changed, 103 insertions(+), 98 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index fefdf511e8e..d8b9299adfd 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -340,100 +340,6 @@ This MUST return an HTML page which can perform this authentication stage. This page must attempt to call the JavaScript function ``window.onAuthDone`` when the authentication has been completed. -Pagination ----------- - -.. NOTE:: - The paths referred to in this section are not actual endpoints. They only - serve as examples to explain how pagination functions. - -Pagination is the process of dividing a dataset into multiple discrete pages. -Matrix makes use of pagination to allow clients to view extremely large datasets. -These datasets are not limited to events in a room (for example clients may want -to paginate rooms in addition to events within those rooms). Regardless of *what* -is being paginated, there is a common underlying API which is used to -to give clients a consistent way of selecting subsets of a potentially changing -dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters -which describe where to read from the stream. ``from`` and ``to`` are opaque -textual 'stream tokens' which describe the current position in the dataset. -The ``dir`` parameter is an enum representing the direction of events to return: -either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and -``end`` stream token values which can then be passed to subsequent requests to -continue pagination. Not all endpoints will make use of all the parameters -outlined here: see the specific endpoint in question for more information. - -Pagination Request Query Parameters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Query parameters: - from: - $streamtoken - The opaque token to start streaming from. - to: - $streamtoken - The opaque token to end streaming at. Typically, - clients will not know the item of data to end at, so this will usually be - omitted. - limit: - integer - An integer representing the maximum number of items to - return. - dir: - f|b - The direction to return events in. Typically this is ``b`` to paginate - backwards in time. - -'START' and 'END' are placeholder values used in these examples to describe the -start and end of the dataset respectively. - -Unless specified, the default pagination parameters are ``from=START``, -``to=END``, without a limit set. - -For example, if an endpoint had events E1 -> E15. The client wants the last 5 -events and doesn't know any previous events:: - - S E - |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| - | | | - | _____| <--backwards-- | - |__________________ | | ________| - | | | | - GET /somepath?to=START&limit=5&dir=b&from=END - Returns: - E15,E14,E13,E12,E11 - - -Another example: a public room list has rooms R1 -> R17. The client is showing 5 -rooms at a time on screen, and is on page 2. They want to -now show page 3 (rooms R11 -> 15):: - - S E - | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token - |-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room - |____________| |________________| - | | - Currently | - viewing | - | - GET /roomslist?from=9&to=END&limit=5 - Returns: R11,R12,R13,R14,R15 - -Note that tokens are treated in an *exclusive*, not inclusive, manner. The end -token from the initial request was '9' which corresponded to R10. When the 2nd -request was made, R10 did not appear again, even though from=9 was specified. If -you know the token, you already have the data. - -Pagination Response -~~~~~~~~~~~~~~~~~~~ - -Responses to pagination requests MUST follow the format:: - - { - "chunk": [ ... , Responses , ... ], - "start" : $streamtoken, - "end" : $streamtoken - } - -Where $streamtoken is an opaque token which can be used in another query to -get the next set of results. The "start" and "end" keys can only be omitted if -the complete dataset is provided in "chunk". - Events ------ @@ -521,8 +427,8 @@ returns an ``end`` token which can be used with the event stream. {{sync_http_api}} -Events in a room -~~~~~~~~~~~~~~~~ +Types of room events +~~~~~~~~~~~~~~~~~~~~ Room events are split into two categories: @@ -544,7 +450,7 @@ convention, e.g. ``com.example.myapp.event``. This ensures event types are suitably namespaced for each application and reduces the risk of clashes. State events -~~~~~~~~~~~~ +++++++++++++ State events can be sent by ``PUT`` ing to |/rooms//state//|_. These events will be @@ -587,7 +493,7 @@ In some cases, there may be no need for a ``state_key``, so it can be omitted:: See `Room Events`_ for the ``m.`` event specification. Message events -~~~~~~~~~~~~~~ +++++++++++++++ Message events can be sent by sending a request to |/rooms//send/|_. These requests *can* use transaction @@ -655,6 +561,105 @@ The redaction event should be added under the key ``redacted_because``. When a client receives a redaction event it should change the redacted event in the same way a server does. +Pagination +---------- + +.. NOTE:: + The paths referred to in this section are not actual endpoints. They only + serve as examples to explain how pagination functions. + +Pagination is the process of dividing a dataset into multiple discrete pages. +Matrix makes use of pagination to allow clients to view extremely large datasets. +These datasets are not limited to events in a room (for example clients may want +to paginate rooms in addition to events within those rooms). Regardless of *what* +is being paginated, there is a common underlying API which is used to +to give clients a consistent way of selecting subsets of a potentially changing +dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters +which describe where to read from the stream. ``from`` and ``to`` are opaque +textual 'stream tokens' which describe the current position in the dataset. +The ``dir`` parameter is an enum representing the direction of events to return: +either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and +``end`` stream token values which can then be passed to subsequent requests to +continue pagination. Not all endpoints will make use of all the parameters +outlined here: see the specific endpoint in question for more information. + +Pagination Request Query Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Query parameters: + from: + $streamtoken - The opaque token to start streaming from. + to: + $streamtoken - The opaque token to end streaming at. Typically, + clients will not know the item of data to end at, so this will usually be + omitted. + limit: + integer - An integer representing the maximum number of items to + return. + dir: + f|b - The direction to return events in. Typically this is ``b`` to paginate + backwards in time. + +'START' and 'END' are placeholder values used in these examples to describe the +start and end of the dataset respectively. + +Unless specified, the default pagination parameters are ``from=START``, +``to=END``, without a limit set. + +For example, if an endpoint had events E1 -> E15. The client wants the last 5 +events and doesn't know any previous events:: + + S E + |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| + | | | + | _____| <--backwards-- | + |__________________ | | ________| + | | | | + GET /somepath?to=START&limit=5&dir=b&from=END + Returns: + E15,E14,E13,E12,E11 + + +Another example: a public room list has rooms R1 -> R17. The client is showing 5 +rooms at a time on screen, and is on page 2. They want to +now show page 3 (rooms R11 -> 15):: + + S E + | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token + |-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room + |____________| |________________| + | | + Currently | + viewing | + | + GET /roomslist?from=9&to=END&limit=5 + Returns: R11,R12,R13,R14,R15 + +Note that tokens are treated in an *exclusive*, not inclusive, manner. The end +token from the initial request was '9' which corresponded to R10. When the 2nd +request was made, R10 did not appear again, even though from=9 was specified. If +you know the token, you already have the data. + +Pagination Response +~~~~~~~~~~~~~~~~~~~ + +Responses to pagination requests MUST follow the format:: + + { + "chunk": [ ... , Responses , ... ], + "start" : $streamtoken, + "end" : $streamtoken + } + +Where $streamtoken is an opaque token which can be used in another query to +get the next set of results. The "start" and "end" keys can only be omitted if +the complete dataset is provided in "chunk". + +Pagination APIs +~~~~~~~~~~~~~~~ + +{{message_pagination_http_api}} + Rooms ----- From 41fb0645a975d5bfb4e6394db1f7c4fef59463eb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 11:16:58 +0100 Subject: [PATCH 022/989] Add message pagination API --- api/client-server/v1/message_pagination.yaml | 131 +++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 api/client-server/v1/message_pagination.yaml diff --git a/api/client-server/v1/message_pagination.yaml b/api/client-server/v1/message_pagination.yaml new file mode 100644 index 00000000000..b3317c13a34 --- /dev/null +++ b/api/client-server/v1/message_pagination.yaml @@ -0,0 +1,131 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Rooms API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/messages": + get: + summary: Get a list of events for this room + description: |- + This API returns a list of message and state events for a room. It uses + pagination query parameters to paginate history in the room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to get events from. + required: true + x-example: "!636q39766251:example.com" + - in: query + type: string + name: from + description: |- + The token to start returning events from. This token can be obtained + from the initial sync API. + required: true + x-example: "s345_678_333" + - in: query + type: string + enum: ["b", "f"] + name: dir + description: |- + The direction to return events from. + required: true + x-example: "b" + - in: query + type: integer + name: limit + description: |- + The maximum number of events to return. Default: 10. + x-example: "3" + responses: + 200: + description: A list of messages with a new token to request more. + schema: + type: object + description: A list of messages with a new token to request more. + properties: + start: + type: string + description: |- + The token to start paginating from. If ``dir=b`` this will be + the token supplied in ``from``. + end: + type: string + description: |- + The token the pagination ends at. If ``dir=b`` this token should + be used again to request even earlier events. + chunk: + type: array + description: |- + A list of room events. + items: + type: object + title: RoomEvent + examples: + application/json: |- + { + "start": "t47429-4392820_219380_26003_2265", + "end": "t47409-4357353_219380_26003_2265", + "chunk": [ + { + "origin_server_ts": 1444812213737, + "user_id": "@alice:example.com", + "event_id": "$1444812213350496Caaaa:example.com", + "content": { + "body": "hello world", + "msgtype":"m.text" + }, + "room_id":"!Xq3620DUiqCaoxq:example.com", + "type":"m.room.message", + "age": 1042 + }, + { + "origin_server_ts": 1444812194656 , + "user_id": "@bob:example.com", + "event_id": "$1444812213350496Cbbbb:example.com", + "content": { + "body": "the world is big", + "msgtype":"m.text" + }, + "room_id":"!Xq3620DUiqCaoxq:example.com", + "type":"m.room.message", + "age": 20123 + }, + { + "origin_server_ts": 1444812163990, + "user_id": "@bob:example.com", + "event_id": "$1444812213350496Ccccc:example.com", + "content": { + "name": "New room name" + }, + "prev_content": { + "name": "Old room name" + }, + "state_key": "", + "room_id":"!Xq3620DUiqCaoxq:example.com", + "type":"m.room.name", + "age": 50789 + } + ] + } + 403: + description: > + You aren't a member of the room. \ No newline at end of file From 6f6861a11d45b86c5cbcad7b864fd527a77f5724 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 13:17:51 +0100 Subject: [PATCH 023/989] Swaggerify the createRoom API --- api/client-server/v1/create_room.yaml | 148 ++++++++++++++++++++++++++ specification/1-client_server_api.rst | 135 ++--------------------- 2 files changed, 155 insertions(+), 128 deletions(-) create mode 100644 api/client-server/v1/create_room.yaml diff --git a/api/client-server/v1/create_room.yaml b/api/client-server/v1/create_room.yaml new file mode 100644 index 00000000000..5bb4e17c245 --- /dev/null +++ b/api/client-server/v1/create_room.yaml @@ -0,0 +1,148 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Creation API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/createRoom": + post: + summary: Create a new room + description: |- + Create a new room with various configuration options. + security: + - accessToken: [] + parameters: + - in: body + name: body + description: The desired room configuration. + schema: + type: object + example: |- + { + "preset": "public_chat", + "room_alias_name": "thepub", + "name": "The Grand Duke Pub", + "topic": "All about happy hour", + "creation_content": { + "m.federate": false + } + } + properties: + visibility: + type: string + enum: ["public", "private"] + description: |- + A ``public`` visibility indicates that the room will be shown + in the published room list. A ``private`` visibility will hide + the room from the published room list. Rooms default to + ``private`` visibility if this key is not included. NB: This + should not be confused with ``join_rules`` which also uses the + word ``public``. + room_alias_name: + type: string + description: |- + The desired room alias **local part**. If this is included, a + room alias will be created and mapped to the newly created + room. The alias will belong on the *same* home server which + created the room. For example, if this was set to "foo" and + sent to the homeserver "example.com" the complete room alias + would be ``#foo:example.com``. + name: + type: string + description: |- + If this is included, an ``m.room.name`` event will be sent + into the room to indicate the name of the room. See Room + Events for more information on ``m.room.name``. + topic: + type: string + description: |- + If this is included, an ``m.room.topic`` event will be sent + into the room to indicate the topic for the room. See Room + Events for more information on ``m.room.topic``. + invite: + type: array + description: |- + A list of user IDs to invite to the room. This will tell the + server to invite everyone in the list to the newly created room. + items: + type: string + creation_content: + title: CreationContent + type: object + description: |- + Extra keys to be added to the content of the ``m.room.create``. + The server will clober the following keys: ``creator``. Future + versions of the specification may allow the server to clobber + other keys. + initial_state: + type: array + description: |- + A list of state events to set in the new room. This allows + the user to override the default state events set in the new + room. The expected format of the state events are an object + with type, state_key and content keys set. + Takes precedence over events set by ``presets``, but gets + overriden by ``name`` and ``topic`` keys. + items: + type: object + title: StateEvent + properties: + type: + type: string + state_key: + type: string + content: + type: string + preset: + type: string + enum: ["private_chat", "public_chat", "trusted_private_chat"] + description: |- + Convenience parameter for setting various default state events + based on a preset. Must be either: + + ``private_chat`` => + ``join_rules`` is set to ``invite``. + ``history_visibility`` is set to ``shared``. + + ``trusted_private_chat`` => + ``join_rules`` is set to ``invite``. + ``history_visibility`` is set to ``shared``. + All invitees are given the same power level as the room creator. + + ``public_chat``: => + ``join_rules`` is set to ``public``. + ``history_visibility`` is set to ``shared``. + + responses: + 200: + description: Information about the newly created room. + schema: + type: object + description: Information about the newly created room. + properties: + room_id: + type: string + description: |- + The created room's ID. + examples: + application/json: |- + { + "room_id": "!sefiuhWgwghwWgh:example.com" + } + 400: + description: > + The request body is malformed or the room alias specified is already taken. \ No newline at end of file diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index d8b9299adfd..f27e4d42419 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -665,140 +665,19 @@ Rooms Creation ~~~~~~~~ -To create a room, a client has to use the |createRoom|_ API. There are various -options which can be set when creating a room: - -``visibility`` - Type: - String - Optional: - Yes - Value: - Either ``public`` or ``private``. - Description: - A ``public`` visibility indicates that the room will be shown in the public - room list. A ``private`` visibility will hide the room from the public room - list. Rooms default to ``private`` visibility if this key is not included. - -``room_alias_name`` - Type: - String - Optional: - Yes - Value: - The room alias localpart. - Description: - If this is included, a room alias will be created and mapped to the newly - created room. The alias will belong on the same home server which created - the room, e.g. ``!qadnasoi:domain.com >>> #room_alias_name:domain.com`` - -``name`` - Type: - String - Optional: - Yes - Value: - The ``name`` value for the ``m.room.name`` state event. - Description: - If this is included, an ``m.room.name`` event will be sent into the room to - indicate the name of the room. See `Room Events`_ for more information on - ``m.room.name``. - -``topic`` - Type: - String - Optional: - Yes - Value: - The ``topic`` value for the ``m.room.topic`` state event. - Description: - If this is included, an ``m.room.topic`` event will be sent into the room - to indicate the topic for the room. See `Room Events`_ for more information - on ``m.room.topic``. - -``invite`` - Type: - List - Optional: - Yes - Value: - A list of user ids to invite. - Description: - This will tell the server to invite everyone in the list to the newly - created room. - -``creation_content`` - Type: - Object - Optional: - Yes - Value: - Extra keys to be added to the content of the ``m.room.create``. The server - will clober the following keys: ``creator``. Future versions of this - spec may allow the server to clobber other keys if required. - Description: - Allows clients to add keys to the content of ``m.room.create``. - -``preset`` - Type: - String - Optional: - Yes - Value: - ``private_chat``, ``trusted_private_chat`` or ``public_chat`` - Description: - Convenience parameter for setting various default state events based on a - preset. - - Three presets are defined: - - - ``private_chat``: Sets the ``join_rules`` to ``invite`` and - ``history_visibility`` to ``shared`` - - ``trusted_private_chat``: Set the ``join_rules`` to ``invite``, - ``history_visibility`` to ``shared`` and gives all invitees the same - power level as the creator. - - ``public_chat``: Sets the ``join_rules`` to ``public`` and - ``history_visibility`` to ``shared`` - -``initial_state`` - Type: - List - Optional: - Yes - Value: - A list of state events to set in the new room. - Description: - Allows the user to override the default state events set in the new room. - - The expected format of the state events are an object with ``type``, - ``state_key`` and ``content`` keys set. - - Takes precedence over events set by ``presets``, but gets overriden by - ``name`` and ``topic`` keys. - -Example:: - - { - "preset": "public_chat", - "room_alias_name": "thepub", - "name": "The Grand Duke Pub", - "topic": "All about happy hour", - "creation_content": { - "m.federate": false - } - } - -The home server will create a ``m.room.create`` event when the room is created, -which serves as the root of the PDU graph for this room. This event also has a +The home server will create a ``m.room.create`` event when a room is created, +which serves as the root of the event graph for this room. This event also has a ``creator`` key which contains the user ID of the room creator. It will also generate several other events in order to manage permissions in this room. This includes: - - ``m.room.power_levels`` : Sets the power levels of users and required power - levels. + - ``m.room.power_levels`` : Sets the power levels of users and required power levels. - ``m.room.join_rules`` : Whether the room is "invite-only" or not. -See `Room Events`_ for more information on these events. +See `Room Events`_ for more information on these events. To create a room, a +client has to use the the following API. + +{{create_room_http_api}} Room aliases ~~~~~~~~~~~~ From 34bd8edec5395a8accde330be4b04dbca41f770b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 13:26:58 +0100 Subject: [PATCH 024/989] Move related auth sections together --- specification/0-events.rst | 4 +- specification/1-client_server_api.rst | 274 +++++++++++++------------- 2 files changed, 138 insertions(+), 140 deletions(-) diff --git a/specification/0-events.rst b/specification/0-events.rst index 16948462c82..640075684c4 100644 --- a/specification/0-events.rst +++ b/specification/0-events.rst @@ -1,5 +1,5 @@ -Events -====== +Event Structure +=============== All communication in Matrix is expressed in the form of data objects called Events. These are the fundamental building blocks common to the client-server, diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index f27e4d42419..319fed91d5a 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -340,6 +340,141 @@ This MUST return an HTML page which can perform this authentication stage. This page must attempt to call the JavaScript function ``window.onAuthDone`` when the authentication has been completed. +Registration +~~~~~~~~~~~~ +This section refers to API Version 2. These API calls currently use the prefix +``/_matrix/client/v2_alpha``. + +Registering for a user account is done using the request:: + + POST $V2PREFIX/register + +This API endpoint uses the User-Interactive Authentication API. +This API endpoint does not require an access token. + +The body of the POST request is a JSON object containing: + +username + Optional. This is the local part of the desired Matrix ID. If omitted, the + Home Server must generate a Matrix ID local part. +password + Required. The desired password for the account. +bind_email + Optional. If ``true``, the server binds the email used for authentication to + the Matrix ID with the ID Server. + +On success, this returns a JSON object with keys: + +user_id + The fully-qualified Matrix ID that has been registered. +access_token + An access token for the new account. +home_server + The hostname of the Home Server on which the account has been registered. + +This endpoint may also return the following error codes: + +M_USER_IN_USE + If the Matrix ID is already in use +M_EXCLUSIVE + If the requested Matrix ID is in the exclusive namespace of an application + service. + +Home Servers MUST perform the relevant checks and return these codes before +performing User-Interactive Authentication, although they may also return +them after authentication is completed if, for example, the requested user ID +was registered whilst the client was performing authentication. + +Old V1 API docs: |register|_ + +{{login_http_api}} + +Changing Password ++++++++++++++++++ +This section refers to API Version 2. These API calls currently use the prefix +``/_matrix/client/v2_alpha``. + +Request:: + + POST $V2PREFIX/account/password + +This API endpoint uses the User-Interactive Authentication API. An access token +should be submitted to this endpoint if the client has an active session. The +Home Server may change the flows available depending on whether a valid access +token is provided. + +The body of the POST request is a JSON object containing: + +new_password + The new password for the account. + +On success, an empty JSON object is returned. + +The error code M_NOT_FOUND is returned if the user authenticated with a third +party identifier but the Home Server could not find a matching account in its +database. + +Adding a Third Party Identifier ++++++++++++++++++++++++++++++++ +This section refers to API Version 2. These API calls currently use the prefix +``/_matrix/client/v2_alpha``. + +Request:: + + POST $V2PREFIX/account/3pid + +Used to add a third party identifier to the user's account. + +The body of the POST request is a JSON object containing: + +threePidCreds + An object containing third party identifier credentials. +bind + Optional. A boolean indicating whether the Home Server should also bind this + third party identifier to the account's matrix ID with the Identity Server. If + supplied and true, the Home Server must bind the 3pid accordingly. + +The third party identifier credentials object comprises: + +id_server + The colon-separated hostname and port of the Identity Server used to + authenticate the third party identifier. If the port is the default, it and the + colon should be omitted. +sid + The session ID given by the Identity Server +client_secret + The client secret used in the session with the Identity Server. + +On success, the empty JSON object is returned. + +May also return error codes: + +M_THREEPID_AUTH_FAILED + If the credentials provided could not be verified with the ID Server. + +Fetching Currently Associated Third Party Identifiers ++++++++++++++++++++++++++++++++++++++++++++++++++++++ +This section refers to API Version 2. These API calls currently use the prefix +``/_matrix/client/v2_alpha``. + +Request:: + + GET $V2PREFIX/account/3pid + +This returns a list of third party identifiers that the Home Server has +associated with the user's account. This is *not* the same as the list of third +party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers +in this list may be used by the Home Server as, for example, identifiers that it +will accept to reset the user's account password. + +Returns a JSON object with the key ``threepids`` whose contents is an array of +objects with the following keys: + +medium + The medium of the 3pid (eg, ``email``) +address + The textual address of the 3pid, eg. the email address + Events ------ @@ -742,7 +877,7 @@ certain operations such as kicking, banning and sending state events. See `m.room.power_levels`_ for more information. Joining rooms -------------- +~~~~~~~~~~~~~ Users need to be a member of a room in order to send and receive events in that room. There are several states in which a user may be, in relation to a room: @@ -821,143 +956,6 @@ member's state, by making a request to "membership": "ban" } - -Registration ------------- -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Registering for a user account is done using the request:: - - POST $V2PREFIX/register - -This API endpoint uses the User-Interactive Authentication API. -This API endpoint does not require an access token. - -The body of the POST request is a JSON object containing: - -username - Optional. This is the local part of the desired Matrix ID. If omitted, the - Home Server must generate a Matrix ID local part. -password - Required. The desired password for the account. -bind_email - Optional. If ``true``, the server binds the email used for authentication to - the Matrix ID with the ID Server. - -On success, this returns a JSON object with keys: - -user_id - The fully-qualified Matrix ID that has been registered. -access_token - An access token for the new account. -home_server - The hostname of the Home Server on which the account has been registered. - -This endpoint may also return the following error codes: - -M_USER_IN_USE - If the Matrix ID is already in use -M_EXCLUSIVE - If the requested Matrix ID is in the exclusive namespace of an application - service. - -Home Servers MUST perform the relevant checks and return these codes before -performing User-Interactive Authentication, although they may also return -them after authentication is completed if, for example, the requested user ID -was registered whilst the client was performing authentication. - -Old V1 API docs: |register|_ - -{{login_http_api}} - -Changing Password -~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - POST $V2PREFIX/account/password - -This API endpoint uses the User-Interactive Authentication API. An access token -should be submitted to this endpoint if the client has an active session. The -Home Server may change the flows available depending on whether a valid access -token is provided. - -The body of the POST request is a JSON object containing: - -new_password - The new password for the account. - -On success, an empty JSON object is returned. - -The error code M_NOT_FOUND is returned if the user authenticated with a third -party identifier but the Home Server could not find a matching account in its -database. - -Adding a Third Party Identifier -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - POST $V2PREFIX/account/3pid - -Used to add a third party identifier to the user's account. - -The body of the POST request is a JSON object containing: - -threePidCreds - An object containing third party identifier credentials. -bind - Optional. A boolean indicating whether the Home Server should also bind this - third party identifier to the account's matrix ID with the Identity Server. If - supplied and true, the Home Server must bind the 3pid accordingly. - -The third party identifier credentials object comprises: - -id_server - The colon-separated hostname and port of the Identity Server used to - authenticate the third party identifier. If the port is the default, it and the - colon should be omitted. -sid - The session ID given by the Identity Server -client_secret - The client secret used in the session with the Identity Server. - -On success, the empty JSON object is returned. - -May also return error codes: - -M_THREEPID_AUTH_FAILED - If the credentials provided could not be verified with the ID Server. - -Fetching Currently Associated Third Party Identifiers -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - GET $V2PREFIX/account/3pid - -This returns a list of third party identifiers that the Home Server has -associated with the user's account. This is *not* the same as the list of third -party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers -in this list may be used by the Home Server as, for example, identifiers that it -will accept to reset the user's account password. - -Returns a JSON object with the key ``threepids`` whose contents is an array of -objects with the following keys: - -medium - The medium of the 3pid (eg, ``email``) -address - The textual address of the 3pid, eg. the email address - - Profiles -------- From 4a9db39277a6f61bb4efc27bb6a14815a89acd05 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 14 Oct 2015 13:38:28 +0100 Subject: [PATCH 025/989] Clean up some untruths in the login api doc add "type", and "username" -> "user" --- api/client-server/v1/login.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/api/client-server/v1/login.yaml b/api/client-server/v1/login.yaml index 3d415c2930a..bbab46dfa8b 100644 --- a/api/client-server/v1/login.yaml +++ b/api/client-server/v1/login.yaml @@ -33,17 +33,21 @@ paths: type: object example: |- { - "username": "cheeky_monkey", + "type": "m.login.pasword", + "user": "cheeky_monkey", "password": "ilovebananas" } properties: - username: + type: + type: string + description: The login type being used. Currently only "m.login.password" is supported. + user: type: string description: The fully qualified user ID or just local part of the user ID, to log in. password: type: string description: The user's password. - required: ["username", "password"] + required: ["type", "user", "password"] responses: 200: description: The user has been authenticated. @@ -78,6 +82,15 @@ paths: home_server: type: string description: The hostname of the Home Server on which the account has been registered. + 400: + description: |- + Part of the request was invalid. For example, the login type may not be recognised. + examples: + application/json: |- + { + "errcode": "M_UNKNOWN", + "error": "Bad login type." + } 403: description: |- The login attempt failed. For example, the password may have been incorrect. From e42c8b5b63a637eae7863285f342938fad6adc18 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 14:19:11 +0100 Subject: [PATCH 026/989] Add search API --- api/client-server/v1/search.yaml | 126 +++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 api/client-server/v1/search.yaml diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml new file mode 100644 index 00000000000..c9e9b14b98a --- /dev/null +++ b/api/client-server/v1/search.yaml @@ -0,0 +1,126 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Search API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/search": + post: + summary: Search server side for things. + description: |- + Performs a full text search across different categories. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "search_categories": { + "room_events": { + "keys": [ + "content.body" + ], + "search_term": "martians and men" + } + } + } + properties: + search_categories: + type: object + description: Describes which categories to search in and + their creteria. + properties: + room_events: + type: object + description: Mapping of category name to search criteria. + properties: + search_term: + type: string + description: The string to search events for + keys: + type: array + items: + type: string + enum: ["content.body", "content.name", "content.topic"] + description: The keys to search. Defaults to all. + required: ["search_term"] + required: ["search_categories"] + responses: + 200: + description: Results of the search. + schema: + type: object + required: ["search_categories"] + properties: + search_categories: + type: object + description: Describes which categories to search in and + their creteria. + properties: + room_events: + type: object + description: Mapping of category name to search criteria. + properties: + count: + type: number + description: Total number of results found + results: + type: object + description: Mapping of event_id to result. + additionalProperties: + type: object + desciption: The result object. + properties: + rank: + type: number + description: A number that describes how closely + this result matches the search. Higher is + closer. + result: + type: object + description: The event that matched. + allOf: + - "$ref": "core-event-schema/room_event.json" + examples: + application/json: |- + { + "search_categories": { + "room_events": { + "count": 24, + "results": { + "$144429830826TWwbB:localhost": { + "rank": 0.00424866, + "result": { + "age": 526228296, + "content": { + "body": "Test content", + "msgtype": "m.text" + }, + "event_id": "$144429830826TWwbB:localhost", + "origin_server_ts": 1444298308034, + "room_id": "!qPewotXpIctQySfjSy:localhost", + "type": "m.room.message", + "user_id": "@test:localhost" + } + } + } + } + } + } From 270e1d7aa81b8692dbffe4604df81c96de99d6cb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 14 Oct 2015 14:29:36 +0100 Subject: [PATCH 027/989] Add richvdh to list of trusted pushers for spectulator richvdh is awesome, he should have all the permissions. --- scripts/speculator/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index d641d7da426..dab624dfba8 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -364,6 +364,7 @@ func main() { "illicitonion": true, "Kegsay": true, "NegativeMjark": true, + "richvdh": true, } rand.Seed(time.Now().Unix()) masterCloneDir, err := gitClone(matrixDocCloneURL, false) From a7a5cb088b8b84d93d46f83b10fdf5d222c8987c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 14:35:08 +0100 Subject: [PATCH 028/989] Add titles --- api/client-server/v1/search.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index c9e9b14b98a..fa6ab2a6085 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -67,15 +67,18 @@ paths: description: Results of the search. schema: type: object + title: Results required: ["search_categories"] properties: search_categories: type: object + title: Categories description: Describes which categories to search in and their creteria. properties: room_events: type: object + title: Room Event Results description: Mapping of category name to search criteria. properties: count: @@ -83,9 +86,11 @@ paths: description: Total number of results found results: type: object + title: Results description: Mapping of event_id to result. additionalProperties: type: object + title: Result desciption: The result object. properties: rank: From 013b6db8958b7b02000044ee6829ccbdbd24a08f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 14 Oct 2015 14:42:52 +0100 Subject: [PATCH 029/989] Add a note on building the speculator ... because the wiki says you need to read the README on how to build it. --- scripts/speculator/README | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/speculator/README b/scripts/speculator/README index 0a9f53fd2b8..c54e0f2b014 100644 --- a/scripts/speculator/README +++ b/scripts/speculator/README @@ -8,3 +8,7 @@ It serves the following HTTP endpoints: To run it, you must install the `go` tool, and run: `go run main.go` + +To build the binary (which is necessary for deployment to the matrix.org +servers), you must again install `go`, and then run: + `go build` From 3608f3a282d9d9eae775dafb30eb22b813d319a6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 14:44:09 +0100 Subject: [PATCH 030/989] SPEC-144 : Use https examples not http. --- specification/modules/push.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 6e9c8536a8b..525e730d604 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -353,14 +353,14 @@ Examples To create a rule that suppresses notifications for the room with ID ``!dj234r78wl45Gh4D:matrix.org``:: - curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \ + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \ '{ "actions" : ["dont_notify"] }' To suppress notifications for the user ``@spambot:matrix.org``:: - curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \ + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \ '{ "actions" : ["dont_notify"] }' @@ -368,7 +368,7 @@ To suppress notifications for the user ``@spambot:matrix.org``:: To always notify for messages that contain the work 'cake' and set a specific sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``):: - curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \ + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \ '{ "pattern": "cake", "actions" : ["notify", {"set_sound":"cakealarm.wav"}] @@ -377,7 +377,7 @@ sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``):: To add a rule suppressing notifications for messages starting with 'cake' but ending with 'lie', superseding the previous rule:: - curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \ + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \ '{ "pattern": "cake*lie", "actions" : ["notify"] @@ -387,7 +387,7 @@ To add a custom sound for notifications messages containing the word 'beer' in any rooms with 10 members or fewer (with greater importance than the room, sender and content rules):: - curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \ + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \ '{ "conditions": [ {"kind": "event_match", "key": "content.body", "pattern": "beer" }, From ca9e44baaca52882fc9b661edf1ee71428f3713c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 15:01:21 +0100 Subject: [PATCH 031/989] Moar titles --- api/client-server/v1/search.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index fa6ab2a6085..e62f2e8e358 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -44,11 +44,13 @@ paths: properties: search_categories: type: object + title: "Categories" description: Describes which categories to search in and their creteria. properties: room_events: type: object + title: "Room Events" description: Mapping of category name to search criteria. properties: search_term: From 392a1c5ad821941ccedac27052b7022a62fd2b2c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 15:06:16 +0100 Subject: [PATCH 032/989] Another title --- api/client-server/v1/search.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index e62f2e8e358..3efef9f5db4 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -102,6 +102,7 @@ paths: closer. result: type: object + title: Event description: The event that matched. allOf: - "$ref": "core-event-schema/room_event.json" From cd34d4e19b334c3a8165bb48deaa5adc7085fecd Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 15:12:15 +0100 Subject: [PATCH 033/989] SPEC-182: Make it clearer how ASes are supposed to create users by linkifying the right text. --- specification/0-feature_profiles.rst | 2 +- specification/3-application_service_api.rst | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index b9f12b74bbd..6c11dd2f229 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -1,7 +1,7 @@ Feature Profiles ================ -.. sect:feature-profiles: +.. _sect:feature-profiles: Matrix supports many different kinds of clients: from embedded IoT devices to desktop clients. Not all clients can provide the same feature sets as other diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index a6e82137e10..d1799158f43 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -94,7 +94,7 @@ API called when: - HS receives an event for an unknown user ID in the AS's namespace, e.g. an invite event to a room. Notes: - - When the AS receives this request, if the user exists, it must create the user via + - When the AS receives this request, if the user exists, it must `create the user`_ via the CS API. - It can also set arbitrary information about the user (e.g. display name, join rooms, etc) using the CS API. @@ -226,6 +226,9 @@ Ordering notes: ] } + +.. _create the user: `sect:asapi-permissions`_ + Client-Server v2 API Extensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -291,6 +294,9 @@ Notes: Server admin style permissions ++++++++++++++++++++++++++++++ + +.. _sect:asapi-permissions: + The home server needs to give the application service *full control* over its namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as From 45b1d08c7e338586d1a716e447e3f025a960aaa6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 15:16:12 +0100 Subject: [PATCH 034/989] SPEC-207: Add a warning about needing AS tokens to be unique and why --- specification/3-application_service_api.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index a6e82137e10..557e16aac13 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -54,6 +54,11 @@ An example HS configuration required to pass traffic to the AS is: aliases: [] # Namespaces of room aliases which should be delegated to the AS rooms: [] # Namespaces of room ids which should be delegated to the AS +.. WARNING:: + If the homeserver in question has multiple application services, each + ``as_token`` MUST be unique per application service as this token is used to + identify the application service. + - An application service can state whether they should be the only ones who can manage a specified namespace. This is referred to as an "exclusive" namespace. An exclusive namespace prevents humans and other application From 44b19a8b0e16a25950899c4f5d90b2ea99b4de69 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 15:27:56 +0100 Subject: [PATCH 035/989] SPEC-165: Outline directory traversal attacks on MXC URIs --- specification/modules/content_repo.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index 9ac5e1991df..52937eada09 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -52,7 +52,17 @@ The HTTP GET endpoint does not require any authentication. Knowing the URL of the content is sufficient to retrieve the content, even if the entity isn't in the room. -Homeservers have additional concerns: +MXC URIs are vulnerable to directory traversal attacks such as +``mxc://127.0.0.1/../../../some_service/etc/passwd``. This would cause the target +homeserver to try to access and return this file. As such, homeservers MUST +sanitise MXC URIs by allowing only alphanumeric (``A-Za-z0-9``), ``_`` +and ``-`` characters in the ``server-name`` and ``media-id`` values. This set +of whitelisted characters allows URL-safe base64 encodings specified in RFC 4648. +Applying this character whitelist is preferable to blacklisting ``.`` and ``/`` +as there are techniques around blacklisted characters (percent-encoded characters, +UTF-8 encoded traversals, etc). + +Homeservers have additional content-specific concerns: - Clients may try to upload very large files. Homeservers should not store files that are too large and should not serve them to clients. From 2b9484cf48249a4c34e1487ddb11863ad1f00bdf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 15:35:39 +0100 Subject: [PATCH 036/989] Spell things correctly --- api/client-server/v1/search.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index 3efef9f5db4..a40d49c062d 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -93,7 +93,7 @@ paths: additionalProperties: type: object title: Result - desciption: The result object. + description: The result object. properties: rank: type: number From f0d80529517eddac9432daab9984c38784528a76 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 15:42:44 +0100 Subject: [PATCH 037/989] SPEC-205: Warn about password strengths in m.login.password section --- specification/1-client_server_api.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 59e6b68eec4..a63c114a839 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -215,6 +215,12 @@ To respond to this type, reply with an auth dict as follows:: "password": "" } +.. WARNING:: + Clients SHOULD enforce that the password provided is suitably complex. The + password SHOULD include a lower-case letter, an upper-case letter, a number + and a symbol and be at a minimum 8 characters in length. Servers MAY reject + weak passwords with an error code ``M_WEAK_PASSWORD``. + Google ReCaptcha ~~~~~~~~~~~~~~~~ :Type: From d8bc0c9315f1dcddb4a819d2249bdae7ee0c494a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 16:13:43 +0100 Subject: [PATCH 038/989] Add search module --- specification/modules/search.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 specification/modules/search.rst diff --git a/specification/modules/search.rst b/specification/modules/search.rst new file mode 100644 index 00000000000..c571d68dfc1 --- /dev/null +++ b/specification/modules/search.rst @@ -0,0 +1,14 @@ +Server Side Search +================== + +.. _module:search: + +TODO: Add summary. + +Client behaviour +---------------- +{{search_http_api}} + +Security considerations +----------------------- +The server must only return results that the user has permission to see. From 6a0595bc46c3751090d5d24e2ab94647c9573915 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 16:15:55 +0100 Subject: [PATCH 039/989] Add in size limits as per SPEC-222 --- .../v1/core-event-schema/state_event.json | 2 +- event-schemas/schema/v1/m.room.name | 2 +- specification/0-events.rst | 17 +++++++++++++++++ specification/0-intro.rst | 2 +- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/v1/core-event-schema/state_event.json index f70118f49c7..88b4900ae3f 100644 --- a/event-schemas/schema/v1/core-event-schema/state_event.json +++ b/event-schemas/schema/v1/core-event-schema/state_event.json @@ -8,7 +8,7 @@ "properties": { "state_key": { "type": "string", - "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event." + "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. The key MUST NOT start with '_'." }, "prev_content": { "type": "object", diff --git a/event-schemas/schema/v1/m.room.name b/event-schemas/schema/v1/m.room.name index b43f02ccfc8..0e0b25be23e 100644 --- a/event-schemas/schema/v1/m.room.name +++ b/event-schemas/schema/v1/m.room.name @@ -11,7 +11,7 @@ "properties": { "name": { "type": "string", - "description": "The name of the room." + "description": "The name of the room. This MUST NOT exceed 255 bytes." } }, "required": ["name"] diff --git a/specification/0-events.rst b/specification/0-events.rst index 16948462c82..ae5a698ad83 100644 --- a/specification/0-events.rst +++ b/specification/0-events.rst @@ -12,6 +12,23 @@ server-server and application-service APIs, and are described below. {{common_state_event_fields}} +Size limits +----------- + +The total size of any event MUST NOT exceed 65 KB. There are additional +restrictions on sizes per key: + + - ``user_id`` MUST NOT exceed 255 bytes (including domain). + - ``room_id`` MUST NOT exceed 255 bytes. + - ``state_key`` MUST NOT exceed 255 bytes. + - ``type`` MUST NOT exceed 255 bytes. + - ``event_id`` MUST NOT exceed 255 bytes. + - ``user_id`` MUST NOT exceed 255 bytes. + +Some event types have additional sizes restrictions which are specified in +the description of the event. Additional keys have no limit other than that +implied by the total 65 KB limit on events. + Room Events ----------- .. NOTE:: diff --git a/specification/0-intro.rst b/specification/0-intro.rst index 445b32ef28a..e5cf1ac8e47 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -296,7 +296,7 @@ be obtained by visiting the domain specified. They are case-insensitive. Note that the mapping from a room alias to a room ID is not fixed, and may change over time to point to a different room ID. For this reason, Clients SHOULD resolve the room alias to a room ID once and then use that ID on subsequent -requests. +requests. Room aliases MUST NOT exceed 255 bytes. When resolving a room alias the server will also respond with a list of servers that are in the room that can be used to join via. From 0efa3fd98128d9e7657a884cb59d5dc987de2245 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 16:17:04 +0100 Subject: [PATCH 040/989] Add module --- specification/0-feature_profiles.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index b9f12b74bbd..ec234a0e973 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -26,6 +26,7 @@ Summary `Content Repository`_ Required Required Required Optional Optional `Managing History Visibility`_ Required Required Required Required Optional `End-to-End Encryption`_ Optional Optional Optional Optional Optional + `Search`_ Optional Optional Optional Optional Optional ===================================== ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* @@ -39,6 +40,7 @@ Summary .. _VoIP: `module:voip`_ .. _Content Repository: `module:content`_ .. _Managing History Visibility: `module:history-visibility`_ +.. _Search: `module:search`_ Clients ------- From 38fcc563dac266b782c296363d3e62f75fe3767b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 16:20:08 +0100 Subject: [PATCH 041/989] Spell out full title --- specification/0-feature_profiles.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index ec234a0e973..128dc9b442d 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -26,7 +26,7 @@ Summary `Content Repository`_ Required Required Required Optional Optional `Managing History Visibility`_ Required Required Required Required Optional `End-to-End Encryption`_ Optional Optional Optional Optional Optional - `Search`_ Optional Optional Optional Optional Optional + `Server Side Search`_ Optional Optional Optional Optional Optional ===================================== ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* @@ -40,7 +40,7 @@ Summary .. _VoIP: `module:voip`_ .. _Content Repository: `module:content`_ .. _Managing History Visibility: `module:history-visibility`_ -.. _Search: `module:search`_ +.. _Server Side Search: `module:search`_ Clients ------- From 5e48b0b79a5f7263a54fd27962da4f08616e8fa4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 16:21:12 +0100 Subject: [PATCH 042/989] Add search to targets --- specification/targets.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/targets.yaml b/specification/targets.yaml index c2b6eeda0fa..1cccf6482b4 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -24,6 +24,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/history_visibility.rst - modules/push.rst - modules/third_party_invites.rst + - modules/search.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 007bb1a69a6d528acfb0f743ac8599592290c403 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 16:26:14 +0100 Subject: [PATCH 043/989] Add extra new line --- specification/modules/search.rst | 1 + specification/targets.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index c571d68dfc1..054719592e3 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -12,3 +12,4 @@ Client behaviour Security considerations ----------------------- The server must only return results that the user has permission to see. + diff --git a/specification/targets.yaml b/specification/targets.yaml index 1cccf6482b4..b549ca536cb 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -26,6 +26,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/third_party_invites.rst - modules/search.rst + title_styles: ["=", "-", "~", "+", "^", "`"] # The templating system doesn't know the right title style to use when generating From 70704240dcc16ab8e4c4a982ab5b73a0cc047d44 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 16:29:18 +0100 Subject: [PATCH 044/989] gendoc should just add newlines rather than complaining about the lack thereof --- scripts/gendoc.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 8db604c4e2e..74a6c1da1c5 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -128,11 +128,8 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): ) else: rst = f.read() - if rst[-2:] != "\n\n": - raise Exception( - ("File %s should end with TWO new-line characters to ensure " + - "file concatenation works correctly.") % (file_info,) - ) + + rst += "\n\n" return rst # dicts look like {0: filepath, 1: filepath} where the key is the title level elif isinstance(file_info, dict): From 84af5776d7487473aae3d8679120e8d3992b4d4e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 17:02:39 +0100 Subject: [PATCH 045/989] Newlines --- api/client-server/v1/create_room.yaml | 2 +- api/client-server/v1/message_pagination.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v1/create_room.yaml b/api/client-server/v1/create_room.yaml index 5bb4e17c245..051c4b84380 100644 --- a/api/client-server/v1/create_room.yaml +++ b/api/client-server/v1/create_room.yaml @@ -145,4 +145,4 @@ paths: } 400: description: > - The request body is malformed or the room alias specified is already taken. \ No newline at end of file + The request body is malformed or the room alias specified is already taken. diff --git a/api/client-server/v1/message_pagination.yaml b/api/client-server/v1/message_pagination.yaml index b3317c13a34..d2bc0554dd8 100644 --- a/api/client-server/v1/message_pagination.yaml +++ b/api/client-server/v1/message_pagination.yaml @@ -128,4 +128,4 @@ paths: } 403: description: > - You aren't a member of the room. \ No newline at end of file + You aren't a member of the room. From 1f2f14dc08a8b4dbed83e37dfc667a0777e4accb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 17:07:25 +0100 Subject: [PATCH 046/989] YAML tweaks --- api/client-server/v1/sync.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 398b0773a50..50af8546b3d 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -31,7 +31,7 @@ paths: type: string name: from description: |- - The token to stream from. This token is either from the a previous + The token to stream from. This token is either from a previous request to this API or from the initial sync API. required: false x-example: "s3456_9_0" @@ -45,10 +45,10 @@ paths: type: boolean name: archived description: |- - Whether to include rooms that the user has left. If absent then + Whether to include rooms that the user has left. If ``false`` then only rooms that the user has been invited to or has joined are - included. If set to "true" then rooms that the user has left are - included as well. + included. If set to ``true`` then rooms that the user has left are + included as well. By default this is ``false``. required: false x-example: "true" responses: From 728b5a79512c1f3c35d3c2f54f50f60eeb009d50 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 17:20:00 +0100 Subject: [PATCH 047/989] Add very short summary --- specification/modules/search.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 054719592e3..d6bfd3919b2 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -3,7 +3,8 @@ Server Side Search .. _module:search: -TODO: Add summary. +The search API allows clients to perform full text search across events in +their rooms without having to download the entire histories. Client behaviour ---------------- From e6eb19c46115f1e324fa3c474341c627b4219cf9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Oct 2015 17:34:25 +0100 Subject: [PATCH 048/989] Review comments --- specification/1-client_server_api.rst | 37 +++++++++++++++++---------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 319fed91d5a..ecd5a3e210c 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -29,6 +29,9 @@ return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or User-Interactive Authentication API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _sect:auth-api: + This section refers to API Version 2. Some API endpoints such as ``login`` or ``register`` require authentication that @@ -349,9 +352,11 @@ Registering for a user account is done using the request:: POST $V2PREFIX/register -This API endpoint uses the User-Interactive Authentication API. +This API endpoint uses the `User-Interactive Authentication API`_. This API endpoint does not require an access token. +.. _User-Interactive Authentication API: `sect:auth-api`_ + The body of the POST request is a JSON object containing: username @@ -371,6 +376,9 @@ access_token An access token for the new account. home_server The hostname of the Home Server on which the account has been registered. +refresh_token + A token that may be exchanged for a new ``access_token`` using the + ``/tokenrefresh`` API endpoint. This endpoint may also return the following error codes: @@ -381,10 +389,12 @@ M_EXCLUSIVE service. Home Servers MUST perform the relevant checks and return these codes before -performing User-Interactive Authentication, although they may also return +performing `User-Interactive Authentication`_, although they may also return them after authentication is completed if, for example, the requested user ID was registered whilst the client was performing authentication. +.. _User-Interactive Authentication: `sect:auth-api`_ + Old V1 API docs: |register|_ {{login_http_api}} @@ -414,8 +424,8 @@ The error code M_NOT_FOUND is returned if the user authenticated with a third party identifier but the Home Server could not find a matching account in its database. -Adding a Third Party Identifier -+++++++++++++++++++++++++++++++ +Adding Account Administrative Contact Information ++++++++++++++++++++++++++++++++++++++++++++++++++ This section refers to API Version 2. These API calls currently use the prefix ``/_matrix/client/v2_alpha``. @@ -423,18 +433,18 @@ Request:: POST $V2PREFIX/account/3pid -Used to add a third party identifier to the user's account. +Used to add contact information to the user's account. The body of the POST request is a JSON object containing: threePidCreds - An object containing third party identifier credentials. + An object containing contact information. bind Optional. A boolean indicating whether the Home Server should also bind this third party identifier to the account's matrix ID with the Identity Server. If supplied and true, the Home Server must bind the 3pid accordingly. -The third party identifier credentials object comprises: +The contact information object comprises: id_server The colon-separated hostname and port of the Identity Server used to @@ -452,8 +462,8 @@ May also return error codes: M_THREEPID_AUTH_FAILED If the credentials provided could not be verified with the ID Server. -Fetching Currently Associated Third Party Identifiers -+++++++++++++++++++++++++++++++++++++++++++++++++++++ +Fetching Currently Associated Contact Information ++++++++++++++++++++++++++++++++++++++++++++++++++ This section refers to API Version 2. These API calls currently use the prefix ``/_matrix/client/v2_alpha``. @@ -706,8 +716,8 @@ Pagination Pagination is the process of dividing a dataset into multiple discrete pages. Matrix makes use of pagination to allow clients to view extremely large datasets. These datasets are not limited to events in a room (for example clients may want -to paginate rooms in addition to events within those rooms). Regardless of *what* -is being paginated, there is a common underlying API which is used to +to paginate a list of rooms in addition to events within those rooms). Regardless +of *what* is being paginated, there is a common underlying API which is used to to give clients a consistent way of selecting subsets of a potentially changing dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters which describe where to read from the stream. ``from`` and ``to`` are opaque @@ -800,13 +810,14 @@ Rooms Creation ~~~~~~~~ -The home server will create a ``m.room.create`` event when a room is created, +The home server will create an ``m.room.create`` event when a room is created, which serves as the root of the event graph for this room. This event also has a ``creator`` key which contains the user ID of the room creator. It will also generate several other events in order to manage permissions in this room. This includes: - - ``m.room.power_levels`` : Sets the power levels of users and required power levels. + - ``m.room.power_levels`` : Sets the power levels of users and required power + levels for various actions within the room such as sending events. - ``m.room.join_rules`` : Whether the room is "invite-only" or not. See `Room Events`_ for more information on these events. To create a room, a From c4d1b568435aca0216762e1331a6d42f49bf5f68 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 09:57:21 +0100 Subject: [PATCH 049/989] Add a comment about enforcement. --- specification/3-application_service_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index 557e16aac13..8d0efdefc51 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -57,7 +57,7 @@ An example HS configuration required to pass traffic to the AS is: .. WARNING:: If the homeserver in question has multiple application services, each ``as_token`` MUST be unique per application service as this token is used to - identify the application service. + identify the application service. The homeserver MUST enforce this. - An application service can state whether they should be the only ones who can manage a specified namespace. This is referred to as an "exclusive" From c82c07ddd26055cb832d00e8df920a571ead9074 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 09:58:39 +0100 Subject: [PATCH 050/989] Review comments --- specification/0-events.rst | 3 +-- specification/0-intro.rst | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/specification/0-events.rst b/specification/0-events.rst index ae5a698ad83..f24ad9844e3 100644 --- a/specification/0-events.rst +++ b/specification/0-events.rst @@ -23,9 +23,8 @@ restrictions on sizes per key: - ``state_key`` MUST NOT exceed 255 bytes. - ``type`` MUST NOT exceed 255 bytes. - ``event_id`` MUST NOT exceed 255 bytes. - - ``user_id`` MUST NOT exceed 255 bytes. -Some event types have additional sizes restrictions which are specified in +Some event types have additional size restrictions which are specified in the description of the event. Additional keys have no limit other than that implied by the total 65 KB limit on events. diff --git a/specification/0-intro.rst b/specification/0-intro.rst index e5cf1ac8e47..90ec94ef74b 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -296,7 +296,7 @@ be obtained by visiting the domain specified. They are case-insensitive. Note that the mapping from a room alias to a room ID is not fixed, and may change over time to point to a different room ID. For this reason, Clients SHOULD resolve the room alias to a room ID once and then use that ID on subsequent -requests. Room aliases MUST NOT exceed 255 bytes. +requests. Room aliases MUST NOT exceed 255 bytes (including the domain). When resolving a room alias the server will also respond with a list of servers that are in the room that can be used to join via. From f51ee706330be5904077b85b595d209794f290d3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 10:19:51 +0100 Subject: [PATCH 051/989] Review comments round 2 --- specification/modules/instant_messaging.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 487240de539..0b4e3e64d93 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -84,6 +84,9 @@ Local echo Messages SHOULD appear immediately in the message view when a user presses the "send" button. This should occur even if the message is still sending. This is referred to as "local echo". Clients SHOULD implement "local echo" of messages. +Clients MAY display messages in a different format to indicate that the server +has not processed the message. This format should be removed when the server +responds. Clients need to be able to match the message they are sending with the same message which they receive from the event stream. The echo of the same message @@ -91,7 +94,7 @@ from the event stream is referred to as "remote echo". Both echoes need to be identified as the same message in order to prevent duplicate messages being displayed. Ideally this pairing would occur transparently to the user: the UI would not flicker as it transitions from local to remote. Flickering cannot be -fully avoided in version 1 of the client-server API. Two scenarios need to be +fully avoided in the current client-server API. Two scenarios need to be considered: - The client sends a message and the remote echo arrives on the event stream @@ -106,8 +109,8 @@ arrives before the client has obtained an event ID. This makes it impossible to identify it as a duplicate event. This results in the client displaying the message twice for a fraction of a second before the the original request to send the message completes. Once it completes, the client can take remedial actions -to remove the duplicate event by looking for duplicate event IDs. Version 2 of -the client-server API resolves this by attaching the transaction ID of the +to remove the duplicate event by looking for duplicate event IDs. A future version +of the client-server API will resolve this by attaching the transaction ID of the sending request to the event itself. @@ -132,7 +135,7 @@ Server behaviour ---------------- Homeservers SHOULD reject ``m.room.message`` events which don't have a -``msgtype`` key, or who don't have a textual ``body`` key, with an HTTP status +``msgtype`` key, or which don't have a textual ``body`` key, with an HTTP status code of 400. Security considerations @@ -141,7 +144,7 @@ Security considerations Messages sent using this module are not encrypted. Messages can be encrypted using the `E2E module`_. -Clients should sanitise **all keys** for unsafe HTML to prevent Cross-Site +Clients should sanitise **all displayed keys** for unsafe HTML to prevent Cross-Site Scripting (XSS) attacks. This includes room names and topics. .. _`E2E module`: `module:e2e`_ From 507cb29e39c0dec1f0ad51095f350d4e5f37e4e7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 10:37:15 +0100 Subject: [PATCH 052/989] Review comments round deux --- specification/modules/history_visibility.rst | 30 +++++++++++--------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst index 12ea7b22d95..0405b3f1791 100644 --- a/specification/modules/history_visibility.rst +++ b/specification/modules/history_visibility.rst @@ -4,21 +4,25 @@ Room History Visibility .. _module:history-visibility: This module adds support for controlling the visibility of previous events in a -room. Whether a member of a room can see the events that happened in a room from -before they joined the room is controlled by the ``m.room.history_visibility`` -event outlined below. In all cases, the member still needs to be joined to the -room to receive events for that room. The visibility option simply determines -which subset of events in the room are presented to the client. Visibility can -take the form of one of three options: - -- ``shared`` - Previous events are always shown to newly joined members. All - events in the room are shown, even those sent when the member was not a part +room. + +In all cases, a user needs to join a room to view events in that room. Once they +have joined a room, they will gain access to a subset of events in the room. How +this subset is chosen is controlled by the ``m.room.history_visibility`` event +outlined below. After a user has left a room, they may seen any events which they +were allowed to see before they left the room, but no events received after they +left. + +The three options for this event are: + +- ``shared`` - Previous events are always accessible to newly joined members. All + events in the room are accessible, even those sent when the member was not a part of the room. -- ``invited`` - Previous events are shown to newly joined members from the point - they were invited onwards. Events stop being shown when the member's state +- ``invited`` - Previous events are accessible to newly joined members from the point + they were invited onwards. Events stop being accessible when the member's state changes to something other than ``invite`` or ``join``. -- ``joined`` - Previous events are shown to newly joined members from the point - they joined the room onwards. Events stop being shown when the members state +- ``joined`` - Previous events are accessible to newly joined members from the point + they joined the room onwards. Events stop being accessible when the members state changes to something other than ``join``. .. WARNING:: From 5aad67f74d7fe611981aa75c3b42d7c17726b4de Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 11:01:11 +0100 Subject: [PATCH 053/989] Review comments round III --- specification/modules/history_visibility.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst index 0405b3f1791..f26d1b592cc 100644 --- a/specification/modules/history_visibility.rst +++ b/specification/modules/history_visibility.rst @@ -13,7 +13,7 @@ outlined below. After a user has left a room, they may seen any events which the were allowed to see before they left the room, but no events received after they left. -The three options for this event are: +The three options for the ``m.room.history_visibility`` event are: - ``shared`` - Previous events are always accessible to newly joined members. All events in the room are accessible, even those sent when the member was not a part @@ -22,7 +22,7 @@ The three options for this event are: they were invited onwards. Events stop being accessible when the member's state changes to something other than ``invite`` or ``join``. - ``joined`` - Previous events are accessible to newly joined members from the point - they joined the room onwards. Events stop being accessible when the members state + they joined the room onwards. Events stop being accessible when the member's state changes to something other than ``join``. .. WARNING:: From 96a4996c767fe4dc8337c248296d1e7789511079 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 11:02:37 +0100 Subject: [PATCH 054/989] Mention search categories --- specification/modules/search.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index d6bfd3919b2..9fee097caf3 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -10,6 +10,13 @@ Client behaviour ---------------- {{search_http_api}} +Search Categories +~~~~~~~~~~~~~~~~~ + +The search API allows clients to search in different categories of items. +Currently, the only specified category is ``room_events``, which searches for +events in rooms the user had joined. + Security considerations ----------------------- The server must only return results that the user has permission to see. From af347baa686b069fa7d77668506d96a66a0fb388 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 11:06:15 +0100 Subject: [PATCH 055/989] Better phrasing --- specification/modules/search.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 9fee097caf3..87116c2c5a5 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -14,7 +14,7 @@ Search Categories ~~~~~~~~~~~~~~~~~ The search API allows clients to search in different categories of items. -Currently, the only specified category is ``room_events``, which searches for +Currently the only specified category is ``room_events``, which include events in rooms the user had joined. Security considerations From c47a94658ae0701e52ea912861697460784f7638 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 11:12:00 +0100 Subject: [PATCH 056/989] Mention supported keys --- specification/modules/search.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 87116c2c5a5..665709c8e10 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -11,11 +11,22 @@ Client behaviour {{search_http_api}} Search Categories -~~~~~~~~~~~~~~~~~ +----------------- The search API allows clients to search in different categories of items. -Currently the only specified category is ``room_events``, which include -events in rooms the user had joined. +Currently the only specified category is ``room_events``. + +``room_events`` +~~~~~~~~~~~~~~~ + +This category covers all events in rooms that the user had joined. The search +is performed on certain keys of certain event types. + +The supported keys to search over are: + +- ``content.body`` in ``m.room.message`` +- ``content.name`` in ``m.room.name`` +- ``content.topic`` in ``m.room.topic`` Security considerations ----------------------- From 6b72ddfb8cbf8628f2ea2efe24d05a609588a211 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 13:36:43 +0100 Subject: [PATCH 057/989] Spelling and typos --- specification/3-application_service_api.rst | 19 ++++++++++--------- specification/4-server_server_api.rst | 2 +- specification/6-appendices.rst | 6 +++--- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index 8d0efdefc51..cf4942af9ee 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -322,7 +322,7 @@ including the AS token on a ``/register`` request, along with a login type of Application services which attempt to create users or aliases *outside* of their defined namespaces will receive an error code ``M_EXCLUSIVE``. Similarly, -normal users who attempt to create users or alises *inside* an application +normal users who attempt to create users or aliases *inside* an application service-defined namespace will receive the same ``M_EXCLUSIVE`` error code, but only if the application service has defined the namespace as ``exclusive``. @@ -375,9 +375,10 @@ an API is exposed. Room Aliases ++++++++++++ We may want to expose some 3P network rooms so Matrix users can join them directly, -e.g. IRC rooms. We don't want to expose every 3P network room though, e.g. mailto, -tel. Rooms which are publicly accessible (e.g. IRC rooms) can be exposed as an alias by -the application service. Private rooms (e.g. sending an email to someone) should not +e.g. IRC rooms. We don't want to expose every 3P network room though, e.g. +``mailto``, ``tel``. Rooms which are publicly accessible (e.g. IRC rooms) can be +exposed as an alias by the application service. Private rooms +(e.g. sending an email to someone) should not be exposed in this way, but should instead operate using normal invite/join semantics. Therefore, the ID conventions discussed below are only valid for public rooms which expose room aliases. @@ -397,9 +398,9 @@ SHOULD be mapped in the same way as "user" URIs. Event fields ++++++++++++ -We recommend that any gatewayed events should include an ``external_url`` field -in their content to provide a way for Matrix clients to link into the 'native' -client from which the event originated. For instance, this could contain the -message-ID for emails/nntp posts, or a link to a blog comment when gatewaying -blog comment traffic in & out of matrix +We recommend that any events that originated from a remote network should +include an ``external_url`` field in their content to provide a way for Matrix +clients to link into the 'native' client from which the event originated. +For instance, this could contain the message-ID for emails/nntp posts, or a link +to a blog comment when bridging blog comment traffic in & out of Matrix. diff --git a/specification/4-server_server_api.rst b/specification/4-server_server_api.rst index c5ff2b877d3..66367cb0a13 100644 --- a/specification/4-server_server_api.rst +++ b/specification/4-server_server_api.rst @@ -630,7 +630,7 @@ because HTTP services like Matrix are often deployed behind load balancers that handle the TLS and these load balancers make it difficult to check TLS client certificates. -A home server may provide a TLS client certficate and the receiving home server +A home server may provide a TLS client certificate and the receiving home server may check that the client certificate matches the certificate of the origin home server. diff --git a/specification/6-appendices.rst b/specification/6-appendices.rst index de1ac290560..c45aa0a5195 100644 --- a/specification/6-appendices.rst +++ b/specification/6-appendices.rst @@ -24,7 +24,7 @@ Threat: Unrecoverable Consistency Violations ++++++++++++++++++++++++++++++++++++++++++++ An attacker could send messages which created an unrecoverable "split-brain" -state in the cluster such that the victim's servers could no longer dervive a +state in the cluster such that the victim's servers could no longer derive a consistent view of the chatroom state. Threat: Bad History @@ -63,7 +63,7 @@ Spoofing An attacker could try to send a message claiming to be from the victim without the victim having sent the message in order to: -* Impersonate the victim while performing illict activity. +* Impersonate the victim while performing illicit activity. * Obtain privileges of the victim. Threat: Altering Message Contents @@ -81,7 +81,7 @@ with a phony "origin" field. Spamming ~~~~~~~~ -The attacker could try to send a high volume of solicicted or unsolicted +The attacker could try to send a high volume of solicited or unsolicited messages to the victim in order to: * Find victims for scams. From 3d087df5387f59c1d18f3d0020d0aa26914d2208 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 13:40:05 +0100 Subject: [PATCH 058/989] Kill number prefixes for good --- .../{6-appendices.rst => appendices.rst} | 0 ...ce_api.rst => application_service_api.rst} | 0 ...t_server_api.rst => client_server_api.rst} | 0 ...{0-event_signing.rst => event_signing.rst} | 0 specification/{0-events.rst => events.rst} | 0 ...ture_profiles.rst => feature_profiles.rst} | 0 ...ntity_servers.rst => identity_servers.rst} | 0 specification/{0-intro.rst => intro.rst} | 0 specification/{2-modules.rst => modules.rst} | 0 ...r_server_api.rst => server_server_api.rst} | 0 specification/targets.yaml | 20 +++++++++---------- 11 files changed, 10 insertions(+), 10 deletions(-) rename specification/{6-appendices.rst => appendices.rst} (100%) rename specification/{3-application_service_api.rst => application_service_api.rst} (100%) rename specification/{1-client_server_api.rst => client_server_api.rst} (100%) rename specification/{0-event_signing.rst => event_signing.rst} (100%) rename specification/{0-events.rst => events.rst} (100%) rename specification/{0-feature_profiles.rst => feature_profiles.rst} (100%) rename specification/{5-identity_servers.rst => identity_servers.rst} (100%) rename specification/{0-intro.rst => intro.rst} (100%) rename specification/{2-modules.rst => modules.rst} (100%) rename specification/{4-server_server_api.rst => server_server_api.rst} (100%) diff --git a/specification/6-appendices.rst b/specification/appendices.rst similarity index 100% rename from specification/6-appendices.rst rename to specification/appendices.rst diff --git a/specification/3-application_service_api.rst b/specification/application_service_api.rst similarity index 100% rename from specification/3-application_service_api.rst rename to specification/application_service_api.rst diff --git a/specification/1-client_server_api.rst b/specification/client_server_api.rst similarity index 100% rename from specification/1-client_server_api.rst rename to specification/client_server_api.rst diff --git a/specification/0-event_signing.rst b/specification/event_signing.rst similarity index 100% rename from specification/0-event_signing.rst rename to specification/event_signing.rst diff --git a/specification/0-events.rst b/specification/events.rst similarity index 100% rename from specification/0-events.rst rename to specification/events.rst diff --git a/specification/0-feature_profiles.rst b/specification/feature_profiles.rst similarity index 100% rename from specification/0-feature_profiles.rst rename to specification/feature_profiles.rst diff --git a/specification/5-identity_servers.rst b/specification/identity_servers.rst similarity index 100% rename from specification/5-identity_servers.rst rename to specification/identity_servers.rst diff --git a/specification/0-intro.rst b/specification/intro.rst similarity index 100% rename from specification/0-intro.rst rename to specification/intro.rst diff --git a/specification/2-modules.rst b/specification/modules.rst similarity index 100% rename from specification/2-modules.rst rename to specification/modules.rst diff --git a/specification/4-server_server_api.rst b/specification/server_server_api.rst similarity index 100% rename from specification/4-server_server_api.rst rename to specification/server_server_api.rst diff --git a/specification/targets.yaml b/specification/targets.yaml index c2b6eeda0fa..4ac34ae3f34 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,17 +1,17 @@ targets: main: # arbitrary name to identify this build target files: # the sort order of files to cat - - 0-intro.rst - - 1-client_server_api.rst - - { 1: 0-events.rst } - - { 1: 0-event_signing.rst } - - 2-modules.rst - - { 1: 0-feature_profiles.rst } + - intro.rst + - client_server_api.rst + - { 1: events.rst } + - { 1: event_signing.rst } + - modules.rst + - { 1: feature_profiles.rst } - { 1: "group:modules" } # reference a group of files - - 3-application_service_api.rst - - 4-server_server_api.rst - - 5-identity_servers.rst - - 6-appendices.rst + - application_service_api.rst + - server_server_api.rst + - identity_servers.rst + - appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - modules/instant_messaging.rst From 55cc5c5bb5c63900ca1e496d09a8b78f5373a3d8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 14:53:03 +0100 Subject: [PATCH 059/989] Swaggerify application services --- api/client-server/v1/application_service.yaml | 201 ++++++++++++++++ specification/3-application_service_api.rst | 219 +++++++----------- 2 files changed, 282 insertions(+), 138 deletions(-) create mode 100644 api/client-server/v1/application_service.yaml diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/v1/application_service.yaml new file mode 100644 index 00000000000..5e99562f819 --- /dev/null +++ b/api/client-server/v1/application_service.yaml @@ -0,0 +1,201 @@ +swagger: '2.0' +info: + title: "Matrix Application Service API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: "" +consumes: + - application/json +produces: + - application/json +paths: + "/transactions/{txnId}": + put: + summary: Send some events to the application service. + description: |- + This API is called by the HS when the HS wants to push an event (or + batch of events) to the AS. + parameters: + - in: path + name: txnId + type: string + description: |- + The transaction ID for this set of events. Homeservers generate + these IDs and they are used to ensure idempotency of requests. + required: true + x-example: "35" + - in: body + name: body + description: A list of events + schema: + type: object + example: |- + { + "events": [ + { + "age": 32, + "content": { + "body": "incoming message", + "msgtype": "m.text" + }, + "event_id": "$14328055551tzaee:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + }, + { + "age": 1984, + "content": { + "body": "another incoming message", + "msgtype": "m.text" + }, + "event_id": "$1228055551ffsef:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + } + ] + } + description: "Transaction informations" + properties: + events: + type: array + description: A list of events + items: + type: object + title: Event + required: ["events"] + responses: + 200: + description: The transaction was processed successfully. + examples: + application/json: |- + {} + schema: + type: object + + "/rooms/{roomAlias}": + get: + summary: Query if a room alias should exist on the application service. + description: |- + This endpoint is invoked by the homeserver on an application service to query + the existence of a given room alias. The homeserver will only query room + aliases inside the application service's ``aliases`` namespace. The + homeserver will send this request when it receives a request to join a + room alias within the application services's namespace. + parameters: + - in: path + name: roomAlias + type: string + description: The room alias being queried. + required: true + x-example: "#magicforest:example.com" + responses: + 200: + description: |- + The application service indicates that this room alias exists. The + application service MUST have created a room and associated it with + the queried room alias using the client-server API. Additional + information about the room such as its' name and topic can be set + before responding. + examples: + application/json: |- + {} + schema: + type: object + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: |- + { + "errcode": "ORG.MATRIX.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: |- + { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: |- + The application service indicates that this room alias does not exist. + Optional error information can be included in the body of this response. + examples: + application/json: |- + { + "errcode": "ORG.MATRIX.MYAPPSERVICE_NOT_FOUND" + } + schema: + type: object + "/users/{userId}": + get: + summary: Query if a user should exist on the application service. + description: |- + This endpoint is invoked by the homeserver on an application service to query + the existence of a given user ID. The homeserver will only query user IDs + inside the application service's ``users`` namespace. The homeserver will + send this request when it receives an *event* for an unknown user ID in + the application service's namespace. + parameters: + - in: path + name: userId + type: string + description: The user ID being queried. + required: true + x-example: "@alice:example.com" + responses: + 200: + description: |- + The application service indicates that this user exists. The application + service MUST create the user using the client-server API. + examples: + application/json: |- + {} + schema: + type: object + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: |- + { + "errcode": "ORG.MATRIX.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: |- + { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: |- + The application service indicates that this user does not exist. + Optional error information can be included in the body of this response. + examples: + application/json: |- + { + "errcode": "ORG.MATRIX.MYAPPSERVICE_NOT_FOUND" + } + schema: + type: object + diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index 8d0efdefc51..c4bedf3f12d 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -4,7 +4,7 @@ Application Service API The Matrix client-server API and server-server APIs provide the means to implement a consistent self-contained federated messaging fabric. However, they provide limited means of implementing custom server-side behaviour in Matrix -(e.g. gateways, filters, extensible hooks etc). The Application Service API +(e.g. gateways, filters, extensible hooks etc). The Application Service API (AS API) defines a standard API to allow such extensible functionality to be implemented irrespective of the underlying homeserver implementation. @@ -20,7 +20,10 @@ They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the application service. This is achieved by manually configuring the homeserver -with information about the AS. +with information about the application service (AS). + +Registration +~~~~~~~~~~~~ .. NOTE:: Previously, application services could register with a homeserver via HTTP @@ -38,7 +41,30 @@ with information about the AS. A better solution would be to somehow mandate that the API done to avoid abuse. -An example HS configuration required to pass traffic to the AS is: +Application services register "namespaces" of user IDs, room aliases and room IDs. +These namespaces are represented as regular expressions. An application service +is said to be "interested" in a given event if one of the IDs in the event match +the regular expression provided by the application service. An application +service can also state whether they should be the only ones who +can manage a specified namespace. This is referred to as an "exclusive" +namespace. An exclusive namespace prevents humans and other application +services from creating/deleting entities in that namespace. Typically, +exclusive namespaces are used when the rooms represent real rooms on +another service (e.g. IRC). Non-exclusive namespaces are used when the +application service is merely augmenting the room itself (e.g. providing +logging or searching facilities). Namespaces are represented by POSIX extended +regular expressions and look like: + +.. code-block:: yaml + + users: + - exclusive: true + regex: @irc.freenode.net_.* + + +The registration is represented by a series of key-value pairs, which this +specification will present as YAML. An example HS configuration required to pass +traffic to the AS is: .. code-block:: yaml @@ -59,121 +85,59 @@ An example HS configuration required to pass traffic to the AS is: ``as_token`` MUST be unique per application service as this token is used to identify the application service. The homeserver MUST enforce this. -- An application service can state whether they should be the only ones who - can manage a specified namespace. This is referred to as an "exclusive" - namespace. An exclusive namespace prevents humans and other application - services from creating/deleting entities in that namespace. Typically, - exclusive namespaces are used when the rooms represent real rooms on - another service (e.g. IRC). Non-exclusive namespaces are used when the - application service is merely augmenting the room itself (e.g. providing - logging or searching facilities). -- Namespaces are represented by POSIX extended regular expressions, - e.g: - -.. code-block:: yaml - - users: - - exclusive: true - regex: @irc.freenode.net_.* - Home Server -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This contains application service APIs which are used by the home server. All -application services MUST implement these APIs. -User Query -++++++++++ +Pushing events +++++++++++++++ -This API is called by the HS to query the existence of a user on the Application -Service's namespace. +The application service API provides a transaction API for sending a list of +events. Each list of events includes a transaction ID, which works as follows: -Inputs: - - User ID - - HS Credentials -Output: - - Whether the user exists. -Side effects: - - User is created on the HS by the AS via CS APIs during the processing of this request. -API called when: - - HS receives an event for an unknown user ID in the AS's namespace, e.g. an - invite event to a room. -Notes: - - When the AS receives this request, if the user exists, it must create the user via - the CS API. - - It can also set arbitrary information about the user (e.g. display name, join rooms, etc) - using the CS API. - - When this setup is complete, the AS should respond to the HS request. This means the AS - blocks the HS until the user is created. - - This is deemed more flexible than alternative methods (e.g. returning a JSON blob with the - user's display name and get the HS to provision the user). -Retry notes: - - The home server cannot respond to the client's request until the response to - this API is obtained from the AS. - - Recommended that home servers try a few times then time out, returning a - 408 Request Timeout to the client. - :: - GET /users/$user_id?access_token=$hs_token - - Returns: - 200 : User is recognised. - 404 : User not found. - 401 : Credentials need to be supplied. - 403 : HS credentials rejected. - - - 200 OK response format - - {} - -Room Alias Query -++++++++++++++++ -This API is called by the HS to query the existence of a room alias on the -Application Service's namespace. + Typical + HS ---> AS : Home server sends events with transaction ID T. + <--- : AS sends back 200 OK. + + AS ACK Lost + HS ---> AS : Home server sends events with transaction ID T. + <-/- : AS 200 OK is lost. + HS ---> AS : Home server retries with the same transaction ID of T. + <--- : AS sends back 200 OK. If the AS had processed these events + already, it can NO-OP this request (and it knows if it is the same + events based on the transaction ID). + +The events sent to the application service should be linearised, as if they were +from the event stream. The homeserver MUST maintain a queue of transactions to +send to the AS. If the application service cannot be reached, the homeserver +SHOULD backoff exponentially until the application service is reachable again. +As application services cannot *modify* the events in any way, these requests can +be made without blocking other aspects of the homeserver. Homeservers MUST NOT +alter (e.g. add more) events they were going to send within that transaction ID +on retries, as the AS may have already processed the events. + +Querying +++++++++ -Inputs: - - Room alias - - HS Credentials -Output: - - Whether the room exists. -Side effects: - - Room is created on the HS by the AS via CS APIs during the processing of - this request. -API called when: - - HS receives an event to join a room alias in the AS's namespace. -Notes: - - When the AS receives this request, if the room exists, it must create the room via - the CS API. - - It can also set arbitrary information about the room (e.g. name, topic, etc) - using the CS API. - - It can send messages as other users in order to populate scrollback. - - When this setup is complete, the AS should respond to the HS request. This means the AS - blocks the HS until the room is created and configured. - - This is deemed more flexible than alternative methods (e.g. returning an initial sync - style JSON blob and get the HS to provision the room). It also means that the AS knows - the room ID -> alias mapping. -Retry notes: - - The home server cannot respond to the client's request until the response to - this API is obtained from the AS. - - Recommended that home servers try a few times then time out, returning a - 408 Request Timeout to the client. - -:: +The application service API includes two querying APIs: for room aliases and for +user IDs. The application service SHOULD create the queried entity if it desires. +During this process, the application service is blocking the homeserver until the +entity is created and configured. If the homeserver does not receive a response +to this request, the homeserver should retry several times before timing out. This +should result in an HTTP status 408 "Request Timeout" on the client which initiated +this request (e.g. to join a room alias). + +.. admonition:: Rationale + + Blocking the homeserver and expecting the application service to create the entity + using the client-server API is simpler and more flexible than alternative methods + such as returning an initial sync style JSON blob and get the HS to provision + the room/user. This also meant that there didn't need to be a "backchannel" to inform + the application service about information about the entity such as room ID to + room alias mappings. - GET /rooms/$room_alias?access_token=$hs_token - - Returns: - 200 : Room is recognised. - 404 : Room not found. - 401 : Credentials need to be supplied. - 403 : HS credentials rejected. - - - 200 OK response format - - {} Pushing +++++++ @@ -187,38 +151,9 @@ Inputs: Output: - None. -Data flows: - -:: - - Typical - HS ---> AS : Home server sends events with transaction ID T. - <--- : AS sends back 200 OK. - - AS ACK Lost - HS ---> AS : Home server sends events with transaction ID T. - <-/- : AS 200 OK is lost. - HS ---> AS : Home server retries with the same transaction ID of T. - <--- : AS sends back 200 OK. If the AS had processed these events - already, it can NO-OP this request (and it knows if it is the same - events based on the transacton ID). - - -Retry notes: - - If the HS fails to pass on the events to the AS, it must retry the request. - - Since ASes by definition cannot alter the traffic being passed to it (unlike - say, a Policy Server), these requests can be done in parallel to general HS - processing; the HS doesn't need to block whilst doing this. - - Home servers should use exponential backoff as their retry algorithm. - - Home servers MUST NOT alter (e.g. add more) events they were going to - send within that transaction ID on retries, as the AS may have already - processed the events. Ordering notes: - - The events sent to the AS should be linearised, as they are from the event - stream. - - The home server will need to maintain a queue of transactions to send to - the AS. + - :: @@ -231,6 +166,14 @@ Ordering notes: ] } +HTTP APIs ++++++++++ + +This contains application service APIs which are used by the home server. All +application services MUST implement these APIs. These APIs are defined below. + +{{application_service_http_api}} + Client-Server v2 API Extensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From f20faa80e57a20b8c8512154bb267ef45d17fc60 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 14:58:00 +0100 Subject: [PATCH 060/989] Swagger validation --- api/client-server/v1/application_service.yaml | 2 +- templating/matrix_templates/units.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/v1/application_service.yaml index 5e99562f819..22b4dfc5a5d 100644 --- a/api/client-server/v1/application_service.yaml +++ b/api/client-server/v1/application_service.yaml @@ -6,7 +6,7 @@ host: localhost:8008 schemes: - https - http -basePath: "" +basePath: "/" consumes: - application/json produces: diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 7b871057b75..37d005c7c17 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -153,7 +153,7 @@ def _load_swagger_meta(self, api, group_name): for path in api["paths"]: for method in api["paths"][path]: single_api = api["paths"][path][method] - full_path = api.get("basePath", "") + path + full_path = api.get("basePath", "").rstrip("/") + path endpoint = { "title": single_api.get("summary", ""), "desc": single_api.get("description", single_api.get("summary", "")), @@ -299,7 +299,7 @@ def _load_swagger_meta(self, api, group_name): ) ] if len(params_missing_examples) == 0: - path_template = api.get("basePath", "") + path + path_template = api.get("basePath", "").rstrip("/") + path qps = {} body = "" for param in single_api.get("parameters", []): @@ -398,7 +398,7 @@ def _load_swagger_meta(self, api, group_name): }) return { - "base": api.get("basePath"), + "base": api.get("basePath").rstrip("/"), "group": group_name, "endpoints": endpoints, } From 906c59385f19b7fab34135146663872e3d0ce15a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 15:49:58 +0100 Subject: [PATCH 061/989] s/had/has/ --- specification/modules/search.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 665709c8e10..785ce26adae 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -19,7 +19,7 @@ Currently the only specified category is ``room_events``. ``room_events`` ~~~~~~~~~~~~~~~ -This category covers all events in rooms that the user had joined. The search +This category covers all events in rooms that the user has joined. The search is performed on certain keys of certain event types. The supported keys to search over are: From cfca4a6c09a5ea31fb8193c70f61c6fa9ca055ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 15:51:02 +0100 Subject: [PATCH 062/989] s/creteria/criteria/ --- api/client-server/v1/search.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index a40d49c062d..cdb53cf2872 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -46,7 +46,7 @@ paths: type: object title: "Categories" description: Describes which categories to search in and - their creteria. + their criteria. properties: room_events: type: object @@ -76,7 +76,7 @@ paths: type: object title: Categories description: Describes which categories to search in and - their creteria. + their criteria. properties: room_events: type: object From 770cfbc5dee292498a2aaffc5144f70cc7f2f456 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 16:08:24 +0100 Subject: [PATCH 063/989] Be explicit about the events the search is performed over --- specification/modules/search.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 785ce26adae..9b88cc69c42 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -3,8 +3,9 @@ Server Side Search .. _module:search: -The search API allows clients to perform full text search across events in -their rooms without having to download the entire histories. +The search API allows clients to perform full text search across events in all +rooms that the user has been in, including those that they have left. Only +events that the user is allowed to see will be searched. Client behaviour ---------------- @@ -19,8 +20,9 @@ Currently the only specified category is ``room_events``. ``room_events`` ~~~~~~~~~~~~~~~ -This category covers all events in rooms that the user has joined. The search -is performed on certain keys of certain event types. +This category covers all events that the user is allowed to see, including +events in rooms that they have left. The search is performed on certain keys of +certain event types. The supported keys to search over are: From 24e36adbe3da45f35113d5712f9ee9fa71ccefb4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 16:09:11 +0100 Subject: [PATCH 064/989] Mention e2e encryption --- specification/modules/search.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 9b88cc69c42..62c78e8fefa 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -30,6 +30,8 @@ The supported keys to search over are: - ``content.name`` in ``m.room.name`` - ``content.topic`` in ``m.room.topic`` +The search will *not* include rooms that are end to end encrypted. + Security considerations ----------------------- The server must only return results that the user has permission to see. From eca98af89693ec09eb68c8ab7f230d9852606367 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2015 16:38:12 +0100 Subject: [PATCH 065/989] Swaggerify /register endpoint Need to move registration/login/auth sections around once #94 lands. --- api/client-server/v2_alpha/registration.yaml | 95 ++++++++++++++++++++ specification/1-client_server_api.rst | 49 ++-------- 2 files changed, 102 insertions(+), 42 deletions(-) create mode 100644 api/client-server/v2_alpha/registration.yaml diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml new file mode 100644 index 00000000000..ad591206aa4 --- /dev/null +++ b/api/client-server/v2_alpha/registration.yaml @@ -0,0 +1,95 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 Registration API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v2_alpha +consumes: + - application/json +produces: + - application/json +paths: + "/register": + post: + summary: Register for an account on this homeserver. + description: |- + Register for an account on this homeserver. + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "username": "cheeky_monkey", + "password": "ilovebananas", + "bind_email": false + } + properties: + bind_email: + type: boolean + description: |- + If true, the server binds the email used for authentication to + the Matrix ID with the ID Server. + username: + type: string + description: |- + The local part of the desired Matrix ID. If omitted, + the homeserver MUST generate a Matrix ID local part. + password: + type: string + description: The desired password for the account. + required: ["password"] + responses: + 200: + description: The account has been registered. + examples: + application/json: |- + { + "user_id": "@cheeky_monkey:matrix.org", + "access_token": "abc123", + "home_server": "matrix.org", + "refresh_token": "def456" + } + schema: + type: object + properties: + user_id: + type: string + description: The fully-qualified Matrix ID that has been registered. + access_token: + type: string + description: |- + An access token for the account. + This access token can then be used to authorize other requests. + The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``. + There is no specific error message to indicate that a request has failed because + an access token has expired; instead, if a client has reason to believe its + access token is valid, and it receives an auth error, they should attempt to + refresh for a new token on failure, and retry the request with the new token. + refresh_token: + type: string + # TODO: Work out how to linkify /tokenrefresh + description: |- + (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint. + home_server: + type: string + description: The hostname of the Home Server on which the account has been registered. + 400: + description: |- + Part of the request was invalid. For example, this user ID may be taken. This may + be returned at any stage of the registration process, including after authentication + if the requested user ID was registered whilst the client was performing authentication. + examples: + application/json: |- + { + "errcode": "M_USER_IN_USE", + "error": "Desired user ID is already taken." + } + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index a63c114a839..84de17da0a6 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1001,52 +1001,17 @@ member's state, by making a request to } -Registration ------------- -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Registering for a user account is done using the request:: - - POST $V2PREFIX/register +Account operations +------------------ +Registration +~~~~~~~~~~~~ This API endpoint uses the User-Interactive Authentication API. -This API endpoint does not require an access token. -The body of the POST request is a JSON object containing: +{{v2_registration_http_api}} -username - Optional. This is the local part of the desired Matrix ID. If omitted, the - Home Server must generate a Matrix ID local part. -password - Required. The desired password for the account. -bind_email - Optional. If ``true``, the server binds the email used for authentication to - the Matrix ID with the ID Server. - -On success, this returns a JSON object with keys: - -user_id - The fully-qualified Matrix ID that has been registered. -access_token - An access token for the new account. -home_server - The hostname of the Home Server on which the account has been registered. - -This endpoint may also return the following error codes: - -M_USER_IN_USE - If the Matrix ID is already in use -M_EXCLUSIVE - If the requested Matrix ID is in the exclusive namespace of an application - service. - -Home Servers MUST perform the relevant checks and return these codes before -performing User-Interactive Authentication, although they may also return -them after authentication is completed if, for example, the requested user ID -was registered whilst the client was performing authentication. - -Old V1 API docs: |register|_ +Login +~~~~~ {{login_http_api}} From a6cb2e4339a9661dc822057666f92edc4adc5008 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 16:40:21 +0100 Subject: [PATCH 066/989] Mention rank --- specification/modules/search.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 62c78e8fefa..493c2b7c208 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -32,6 +32,9 @@ The supported keys to search over are: The search will *not* include rooms that are end to end encrypted. +The results include a ``rank`` key that can be used to sort the results by +revelancy. The higher the ``rank`` the more relevant the result is. + Security considerations ----------------------- The server must only return results that the user has permission to see. From eb59b8e9d1ac096865bf433c803b69d1e25c9419 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 10:16:40 +0100 Subject: [PATCH 067/989] Mention count --- specification/modules/search.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 493c2b7c208..03d2475fbb6 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -35,6 +35,10 @@ The search will *not* include rooms that are end to end encrypted. The results include a ``rank`` key that can be used to sort the results by revelancy. The higher the ``rank`` the more relevant the result is. +The value of ``count`` may not match the number of results. For example due to +the search query matching 1000s of results and the server truncating the +response. + Security considerations ----------------------- The server must only return results that the user has permission to see. From 716c5b7a8b6d4a4109e0fee16929a88fae04bda4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 10:26:14 +0100 Subject: [PATCH 068/989] Add 400 and 429 response codes --- api/client-server/v1/search.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index cdb53cf2872..885e62b3cc3 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -132,3 +132,9 @@ paths: } } } + 400: + description: Part of the request was invalid. + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" From 20b11281ea822f672f7f24d139a7e170efb4cbf3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 16 Oct 2015 13:10:19 +0100 Subject: [PATCH 069/989] Remove lies about OAuth which doesn't exist --- specification/3-application_service_api.rst | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index 8d0efdefc51..23fc126f9a6 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -249,11 +249,8 @@ additional permissions granting the AS permission to masquerade as a matrix user Inputs: - Application service token (``access_token``) + - User ID in the AS namespace to act as. - Either: - - User ID in the AS namespace to act as. - Or: - - OAuth2 token of real user (which may end up being an access token) Notes: - This will apply on all aspects of the CS API, except for Account Management. - The ``as_token`` is inserted into ``access_token`` which is usually where the @@ -268,12 +265,6 @@ Notes: access_token: The application service token user_id: The desired user ID to act as. - /path?access_token=$token&user_token=$token - - Query Parameters: - access_token: The application service token - user_token: The token granted to the AS by the real user - Timestamp massaging +++++++++++++++++++ The application service may want to inject events at a certain time (reflecting From 6770d6b2d64530113525729061387ce64090f4eb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 16 Oct 2015 14:05:18 +0100 Subject: [PATCH 070/989] Review comments --- api/client-server/v1/application_service.yaml | 4 +- specification/3-application_service_api.rst | 37 +++---------------- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/v1/application_service.yaml index 22b4dfc5a5d..a7400bd405e 100644 --- a/api/client-server/v1/application_service.yaml +++ b/api/client-server/v1/application_service.yaml @@ -87,7 +87,7 @@ paths: the existence of a given room alias. The homeserver will only query room aliases inside the application service's ``aliases`` namespace. The homeserver will send this request when it receives a request to join a - room alias within the application services's namespace. + room alias within the application service's namespace. parameters: - in: path name: roomAlias @@ -101,7 +101,7 @@ paths: The application service indicates that this room alias exists. The application service MUST have created a room and associated it with the queried room alias using the client-server API. Additional - information about the room such as its' name and topic can be set + information about the room such as its name and topic can be set before responding. examples: application/json: |- diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index c4bedf3f12d..decb6f81b26 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -12,10 +12,10 @@ irrespective of the underlying homeserver implementation. Add in Client-Server services? Overview of bots? Seems weird to be in the spec given it is VERY implementation specific. -Passive Application Services ----------------------------- -"Passive" application services can only observe events from a given home server, -and inject events into a room they are participating in. +Application Services +-------------------- +Application services are passive and can only observe events from a given +homeserver. They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the @@ -139,33 +139,6 @@ this request (e.g. to join a room alias). room alias mappings. -Pushing -+++++++ -This API is called by the HS when the HS wants to push an event (or batch of -events) to the AS. - -Inputs: - - HS Credentials - - Event(s) to give to the AS - - HS-generated transaction ID -Output: - - None. - - -Ordering notes: - - - -:: - - PUT /transactions/$transaction_id?access_token=$hs_token - - Request format - { - events: [ - ... - ] - } - HTTP APIs +++++++++ @@ -177,7 +150,7 @@ application services MUST implement these APIs. These APIs are defined below. Client-Server v2 API Extensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Passive application services can utilise a more powerful version of the +Application services can utilise a more powerful version of the client-server API by identifying itself as an application service to the home server. From e0fe3c42c8cb58c994396310279a18b9fa8b9f01 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 16 Oct 2015 14:19:00 +0100 Subject: [PATCH 071/989] Review comments --- api/client-server/v2_alpha/registration.yaml | 12 +++++++++--- specification/1-client_server_api.rst | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml index ad591206aa4..9973d0b43d0 100644 --- a/api/client-server/v2_alpha/registration.yaml +++ b/api/client-server/v2_alpha/registration.yaml @@ -80,9 +80,15 @@ paths: description: The hostname of the Home Server on which the account has been registered. 400: description: |- - Part of the request was invalid. For example, this user ID may be taken. This may - be returned at any stage of the registration process, including after authentication - if the requested user ID was registered whilst the client was performing authentication. + Part of the request was invalid. This may include one of the following error codes: + + * ``M_USER_IN_USE`` : The desired user ID is already taken. + * ``M_EXCLUSIVE`` : The desired user ID is in the exclusive namespace + claimed by an application service. + + This errors may be returned at any stage of the registration process, + including after authentication if the requested user ID was registered + whilst the client was performing authentication. examples: application/json: |- { diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 84de17da0a6..331f951a9e7 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1006,7 +1006,7 @@ Account operations Registration ~~~~~~~~~~~~ -This API endpoint uses the User-Interactive Authentication API. +This API endpoint uses the `User-Interactive Authentication API`_. {{v2_registration_http_api}} From 4bb042daeb3b6bce59f75c308f059910517bd9b0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 16 Oct 2015 15:22:50 +0100 Subject: [PATCH 072/989] Review comments round II --- api/client-server/v1/sync.yaml | 30 ++-- specification/1-client_server_api.rst | 210 +++++++++++++------------- 2 files changed, 122 insertions(+), 118 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 50af8546b3d..d07e9399c5d 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -41,16 +41,6 @@ paths: description: The maximum time in milliseconds to wait for an event. required: false x-example: "35000" - - in: query - type: boolean - name: archived - description: |- - Whether to include rooms that the user has left. If ``false`` then - only rooms that the user has been invited to or has joined are - included. If set to ``true`` then rooms that the user has left are - included as well. By default this is ``false``. - required: false - x-example: "true" responses: 200: description: "The events received, which may be none." @@ -80,19 +70,19 @@ paths: start: type: string description: |- - A token which correlates to the first value in ``chunk``. Used - for pagination. + A token which correlates to the first value in ``chunk``. This + is usually the same token supplied to ``from=``. end: type: string description: |- - A token which correlates to the last value in ``chunk``. Used - for pagination. + A token which correlates to the last value in ``chunk``. This + token should be used in the next request to ``/events``. chunk: type: array description: "An array of events." items: type: object - title: RoomEvent + title: Event allOf: - "$ref": "core-event-schema/room_event.json" 400: @@ -112,6 +102,16 @@ paths: description: The maximum number of messages to return for each room. required: false x-example: "2" + - in: query + type: boolean + name: archived + description: |- + Whether to include rooms that the user has left. If ``false`` then + only rooms that the user has been invited to or has joined are + included. If set to ``true`` then rooms that the user has left are + included as well. By default this is ``false``. + required: false + x-example: "true" responses: 200: description: The user's current state. diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index b96b4bb3c43..96f62c9becc 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -491,6 +491,100 @@ medium address The textual address of the 3pid, eg. the email address +Pagination +---------- + +.. NOTE:: + The paths referred to in this section are not actual endpoints. They only + serve as examples to explain how pagination functions. + +Pagination is the process of dividing a dataset into multiple discrete pages. +Matrix makes use of pagination to allow clients to view extremely large datasets. +These datasets are not limited to events in a room (for example clients may want +to paginate a list of rooms in addition to events within those rooms). Regardless +of *what* is being paginated, there is a common underlying API which is used to +to give clients a consistent way of selecting subsets of a potentially changing +dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters +which describe where to read from the stream. ``from`` and ``to`` are opaque +textual 'stream tokens' which describe the current position in the dataset. +The ``dir`` parameter is an enum representing the direction of events to return: +either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and +``end`` stream token values which can then be passed to subsequent requests to +continue pagination. Not all endpoints will make use of all the parameters +outlined here: see the specific endpoint in question for more information. + +Pagination Request Query Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Query parameters: + from: + $streamtoken - The opaque token to start streaming from. + to: + $streamtoken - The opaque token to end streaming at. Typically, + clients will not know the item of data to end at, so this will usually be + omitted. + limit: + integer - An integer representing the maximum number of items to + return. + dir: + f|b - The direction to return events in. Typically this is ``b`` to paginate + backwards in time. + +'START' and 'END' are placeholder values used in these examples to describe the +start and end of the dataset respectively. + +Unless specified, the default pagination parameters are ``from=START``, +``to=END``, without a limit set. + +For example, if an endpoint had events E1 -> E15. The client wants the last 5 +events and doesn't know any previous events:: + + S E + |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| + | | | + | _____| <--backwards-- | + |__________________ | | ________| + | | | | + GET /somepath?to=START&limit=5&dir=b&from=END + Returns: + E15,E14,E13,E12,E11 + + +Another example: a public room list has rooms R1 -> R17. The client is showing 5 +rooms at a time on screen, and is on page 2. They want to +now show page 3 (rooms R11 -> 15):: + + S E + | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token + |-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room + |____________| |________________| + | | + Currently | + viewing | + | + GET /roomslist?from=9&to=END&limit=5 + Returns: R11,R12,R13,R14,R15 + +Note that tokens are treated in an *exclusive*, not inclusive, manner. The end +token from the initial request was '9' which corresponded to R10. When the 2nd +request was made, R10 did not appear again, even though from=9 was specified. If +you know the token, you already have the data. + +Pagination Response +~~~~~~~~~~~~~~~~~~~ + +Responses to pagination requests MUST follow the format:: + + { + "chunk": [ ... , Responses , ... ], + "start" : $streamtoken, + "end" : $streamtoken + } + +Where $streamtoken is an opaque token which can be used in another query to +get the next set of results. The "start" and "end" keys can only be omitted if +the complete dataset is provided in "chunk". + Events ------ @@ -563,8 +657,14 @@ returning early if an event occurs. Only the events API supports long-polling. All events which are visible to the client will appear in the events API. When the request returns, an ``end`` token is included in the response. This token can be used in the next request to continue where the -last request left off. Multiple events can be returned per long-poll. All events -must be de-duplicated based on their event ID. +last request left off. Multiple events can be returned per long-poll. + +.. Warning:: + Events are ordered in this API according to the arrival time of the event on + the homeserver. This can conflict with other APIs which order events based on + their partial ordering in the event graph. This can result in duplicate events + being received (once per API call). Clients SHOULD de-duplicate events based + on the event ID when this happens. .. TODO is deduplication actually a hard requirement in CS v2? @@ -573,8 +673,8 @@ must be de-duplicated based on their event ID. Do we ever support streaming requests? Why not websockets? When the client first logs in, they will need to initially synchronise with -their home server. This is achieved via the initial sync API. This API also -returns an ``end`` token which can be used with the event stream. +their home server. This is achieved via the initial sync API described below. +This API also returns an ``end`` token which can be used with the event stream. {{sync_http_api}} @@ -667,6 +767,9 @@ There are several APIs provided to ``GET`` events for a room: {{rooms_http_api}} + +{{message_pagination_http_api}} + Redactions ~~~~~~~~~~ Since events are extensible it is possible for malicious users and/or servers @@ -712,105 +815,6 @@ The redaction event should be added under the key ``redacted_because``. When a client receives a redaction event it should change the redacted event in the same way a server does. -Pagination ----------- - -.. NOTE:: - The paths referred to in this section are not actual endpoints. They only - serve as examples to explain how pagination functions. - -Pagination is the process of dividing a dataset into multiple discrete pages. -Matrix makes use of pagination to allow clients to view extremely large datasets. -These datasets are not limited to events in a room (for example clients may want -to paginate a list of rooms in addition to events within those rooms). Regardless -of *what* is being paginated, there is a common underlying API which is used to -to give clients a consistent way of selecting subsets of a potentially changing -dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters -which describe where to read from the stream. ``from`` and ``to`` are opaque -textual 'stream tokens' which describe the current position in the dataset. -The ``dir`` parameter is an enum representing the direction of events to return: -either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and -``end`` stream token values which can then be passed to subsequent requests to -continue pagination. Not all endpoints will make use of all the parameters -outlined here: see the specific endpoint in question for more information. - -Pagination Request Query Parameters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Query parameters: - from: - $streamtoken - The opaque token to start streaming from. - to: - $streamtoken - The opaque token to end streaming at. Typically, - clients will not know the item of data to end at, so this will usually be - omitted. - limit: - integer - An integer representing the maximum number of items to - return. - dir: - f|b - The direction to return events in. Typically this is ``b`` to paginate - backwards in time. - -'START' and 'END' are placeholder values used in these examples to describe the -start and end of the dataset respectively. - -Unless specified, the default pagination parameters are ``from=START``, -``to=END``, without a limit set. - -For example, if an endpoint had events E1 -> E15. The client wants the last 5 -events and doesn't know any previous events:: - - S E - |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| - | | | - | _____| <--backwards-- | - |__________________ | | ________| - | | | | - GET /somepath?to=START&limit=5&dir=b&from=END - Returns: - E15,E14,E13,E12,E11 - - -Another example: a public room list has rooms R1 -> R17. The client is showing 5 -rooms at a time on screen, and is on page 2. They want to -now show page 3 (rooms R11 -> 15):: - - S E - | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token - |-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room - |____________| |________________| - | | - Currently | - viewing | - | - GET /roomslist?from=9&to=END&limit=5 - Returns: R11,R12,R13,R14,R15 - -Note that tokens are treated in an *exclusive*, not inclusive, manner. The end -token from the initial request was '9' which corresponded to R10. When the 2nd -request was made, R10 did not appear again, even though from=9 was specified. If -you know the token, you already have the data. - -Pagination Response -~~~~~~~~~~~~~~~~~~~ - -Responses to pagination requests MUST follow the format:: - - { - "chunk": [ ... , Responses , ... ], - "start" : $streamtoken, - "end" : $streamtoken - } - -Where $streamtoken is an opaque token which can be used in another query to -get the next set of results. The "start" and "end" keys can only be omitted if -the complete dataset is provided in "chunk". - -Pagination APIs -~~~~~~~~~~~~~~~ - -{{message_pagination_http_api}} - Rooms ----- From 62d53b4f3321d27cb0b3bc95e5b7625c1efe454d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 16 Oct 2015 15:58:11 +0100 Subject: [PATCH 073/989] Review comments round III --- specification/client_server_api.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 96f62c9becc..ee70d9d3370 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -663,11 +663,8 @@ last request left off. Multiple events can be returned per long-poll. Events are ordered in this API according to the arrival time of the event on the homeserver. This can conflict with other APIs which order events based on their partial ordering in the event graph. This can result in duplicate events - being received (once per API call). Clients SHOULD de-duplicate events based - on the event ID when this happens. - -.. TODO - is deduplication actually a hard requirement in CS v2? + being received (once per distinct API called). Clients SHOULD de-duplicate + events based on the event ID when this happens. .. TODO-spec Do we ever support streaming requests? Why not websockets? From d39a9082a0feabf926a94ac458f6e480bfcece11 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 16 Oct 2015 16:43:33 +0100 Subject: [PATCH 074/989] Add invite_room_state to spec. Flesh out info. --- event-schemas/schema/v1/m.room.member | 17 +++++++++++------ .../matrix_templates/templates/events.tmpl | 10 +++++----- templating/matrix_templates/units.py | 8 ++++++++ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 912f6cf30c1..9414a1741bc 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. The ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -24,7 +24,7 @@ }, "third_party_invite": { "type": "object", - "title": "invite", + "title": "Invite", "properties": { "token": { "type": "string", @@ -65,17 +65,22 @@ "description": "A subset of the state of the room at the time of the invite, if ``membership`` is ``invite``", "items": { "type": "object", - "title": "StateEvent", + "title": "StrippedState", "description": "A stripped down state event, with only the ``type``, ``state_key`` and ``content`` keys.", "properties": { "type": { - "type": "string" + "type": "string", + "description": "The ``type`` for the event.", + "enum": ["``m.room.join_rules``", "``m.room.canonical_alias``", "``m.room.avatar``", "``m.room.name``"] }, "state_key": { - "type": "string" + "type": "string", + "description": "The ``state_key`` for the event." }, "content": { - "type": "object" + "title": "EventContent", + "type": "object", + "description": "The ``content`` for the event." } } } diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index ffae59e0fcf..9dcc468e629 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -7,18 +7,18 @@ {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== - {{table.title or "Content"}} Key Type Description -================== ================= =========================================== +======================= ================= =========================================== + {{table.title or "Content"}} Key Type Description +======================= ================= =========================================== {% for row in table.rows -%} {# -#} {# Row type needs to prepend spaces to line up with the type column (19 ch) -#} {# Desc needs to prepend the required text (maybe) and prepend spaces too -#} {# It also needs to then wrap inside the desc col (43 ch width) -#} {# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} +{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(42)}} {% endfor -%} -================== ================= =========================================== +======================= ================= =========================================== {% endfor %} Example:: diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 7b871057b75..33b5e6b44b6 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -532,6 +532,14 @@ def load_event_schemas(self): Units.prop(json_schema, "properties/content") ) + # this is horrible + if schema["type"] == "m.room.member": + invite_room_state = get_json_schema_object_fields( + json_schema["properties"]["invite_room_state"]["items"] + ) + schema["content_fields"].extend(invite_room_state) + + # grab msgtype if it is the right kind of event msgtype = Units.prop( json_schema, "properties/content/properties/msgtype/enum" From a8d8412068c934acb19bfcd1b3423d35bb01c5bc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 16 Oct 2015 16:48:18 +0100 Subject: [PATCH 075/989] Add invite_room_state example. --- event-schemas/examples/v1/m.room.member | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index a5ab79b5cb3..078cc5f10d0 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -12,6 +12,22 @@ "sender": "@zun:zun.soft" } }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], "state_key": "@alice:localhost", "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", From 83d21484df4f01ee3ce8dd10820b7c7778686612 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 16 Oct 2015 16:54:40 +0100 Subject: [PATCH 076/989] Oopsie --- event-schemas/schema/v1/m.room.member | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 9414a1741bc..54ae910010c 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -71,7 +71,7 @@ "type": { "type": "string", "description": "The ``type`` for the event.", - "enum": ["``m.room.join_rules``", "``m.room.canonical_alias``", "``m.room.avatar``", "``m.room.name``"] + "enum": ["m.room.join_rules", "m.room.canonical_alias", "m.room.avatar", "m.room.name"] }, "state_key": { "type": "string", From 6161a920b6b3b54b6806c01011acf374ca45f180 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 16 Oct 2015 18:29:20 +0100 Subject: [PATCH 077/989] Update 3pid invite section to reflect signed property --- event-schemas/examples/v1/m.room.member | 10 +++++++++- event-schemas/schema/v1/m.room.member | 20 ++++++++++++++++--- specification/modules/third_party_invites.rst | 12 ++++++----- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index a5ab79b5cb3..4d7896f6407 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -8,7 +8,15 @@ "token": "pc98", "public_key": "abc123", "key_validity_url": "https://magic.forest/verifykey", - "signature": "q1w2e3", + "signed": { + "mxid": "@alice:localhost", + "token": "pc98", + "signatures": { + "magic.forest": { + "ed25519:0": "poi098" + } + } + }, "sender": "@zun:zun.soft" } }, diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 912f6cf30c1..205c426ea0e 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -38,9 +38,23 @@ "type": "string", "description": "A base64-encoded ed25519 key with which token must be signed." }, - "signature": { - "type": "string", - "description": "A base64-encoded signature of token with public_key." + "signed": { + "type": "object", + "title": "signed_third_party_invite", + "properties": { + "mxid": { + "type": "string", + "description": "The invited matrix user ID. Must be equal to the user_id property of the event." + }, + "token": { + "type": "string", + "description": "The token property of the containing third_party_invite object.", + }, + "signatures": { + "type": "object", + "description": "A single signature from the verifying server, in the format specified by the Signing Events section." + } + } }, "sender": { "type": "string", diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index a9883db5317..ebca59cf303 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -36,7 +36,8 @@ A client asks a server to invite a user by their third party identifier. Server behaviour ---------------- -All homeservers MUST verify that sig(``token``, ``public_key``) = ``signature``. +All homeservers MUST verify the signature in the ``signed`` property of the +``third_party_invite`` property in the ``content`` the event. If a client of the current homeserver is joining by an ``m.room.third_party_invite``, that homesever MUST validate that the public @@ -93,11 +94,12 @@ For example: When the third party user validates their identity, they are told about the invite, and ask their homeserver, H3, to join the room. - H3 validates that sign(``token``, ``public_key``) = ``signature``, and may check - ``key_validity_url``. + H3 validates that signature in the ``signed`` property of the + ``third_party_invite`` property of the ``content`` property of the event, + and may check ``key_validity_url``. - H3 then asks H1 to join it to the room. H1 *must* validate that - sign(``token``, ``public_key``) = ``signature`` *and* check ``key_validity_url``. + H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` + property *and* check ``key_validity_url``. Having validated these things, H1 writes the join event to the room, and H3 begins participating in the room. H2 *must* accept this event. From a38e0862cf0b325c7ab57b4624d9c9c90a337388 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 16 Oct 2015 18:56:07 +0100 Subject: [PATCH 078/989] Fix required field names --- event-schemas/schema/v1/m.room.member | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 205c426ea0e..c52d0d4c25f 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -54,14 +54,15 @@ "type": "object", "description": "A single signature from the verifying server, in the format specified by the Signing Events section." } - } + }, + "required": ["mxid", "signatures", "token"] }, "sender": { "type": "string", "description": "The matrix user ID of the user who send the invite which is being used." } }, - "required": ["token", "key_validity_url", "public_key", "signature", "sender"] + "required": ["token", "key_validity_url", "public_key", "sender", "signed"] } }, "required": ["membership"] From 29720cd152bd12f08b155895fddfc0ad65eaca7a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 16 Oct 2015 19:00:36 +0100 Subject: [PATCH 079/989] Initial attempt at describing the event redaction algorithm --- specification/event_signing.rst | 60 +++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/specification/event_signing.rst b/specification/event_signing.rst index bde58f0cd7f..2f61566ecdb 100644 --- a/specification/event_signing.rst +++ b/specification/event_signing.rst @@ -190,9 +190,63 @@ in the event JSON in a ``hash`` object under a ``sha256`` key. event_json_object["unsigned"] = unsigned return event_json_object -Then all non-essential keys are stripped from the event object, and the -resulting object which included the ``hash`` key is signed using the JSON -signing algorithm +The event is then stripped of all non-essential keys both at the top level and +within the ``content`` object. Essential top-level keys given below; any +top-level keys not in this list are removed. + +.. code:: + + auth_events + depth + event_id + hashes + membership + origin + origin_server_ts + prev_events + prev_state + room_id + sender + signatures + state_key + type + +A new ``content`` object is constructed for the resulting event that contains +only the essential keys of the original event. If the original event lacked a +``content`` object at all, a new blank one is created for it. + +The keys that are considered essential for the ``content`` object depend on the +the ``type`` of the event. These are: + +.. code:: + + type is "m.room.aliases": + aliases + + type is "m.room.create": + creator + + type is "m.room.history_visibility": + history_visibility + + type is "m.room.join_rules": + join_rule + + type is "m.room.member": + membership + + type is "m.room.power_levels": + ban + events + events_default + kick + redact + state_default + users + users_default + +The resulting stripped object with the new ``content`` object and the original +``hashes`` key is then signed using the JSON signing algorithm. .. code:: python From c70067dd1e7eb26c5f1e17ccab5f3a7fbc786ae0 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 19 Oct 2015 10:44:26 +0100 Subject: [PATCH 080/989] Simplify language --- specification/modules/third_party_invites.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index ebca59cf303..85538c31484 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -36,8 +36,8 @@ A client asks a server to invite a user by their third party identifier. Server behaviour ---------------- -All homeservers MUST verify the signature in the ``signed`` property of the -``third_party_invite`` property in the ``content`` the event. +All homeservers MUST verify the signature in the event's +``content.third_party_invite.signed`` object. If a client of the current homeserver is joining by an ``m.room.third_party_invite``, that homesever MUST validate that the public @@ -94,9 +94,8 @@ For example: When the third party user validates their identity, they are told about the invite, and ask their homeserver, H3, to join the room. - H3 validates that signature in the ``signed`` property of the - ``third_party_invite`` property of the ``content`` property of the event, - and may check ``key_validity_url``. + H3 validates the signature in the event's + ``content.third_party_invite.signed`` object. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` property *and* check ``key_validity_url``. From ae90d15b3c6955779356466a6edb6e7e6d48c105 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 19 Oct 2015 10:46:07 +0100 Subject: [PATCH 081/989] Add m.room.avatar --- event-schemas/examples/v1/m.room.avatar | 18 ++++++ event-schemas/schema/v1/m.room.avatar | 64 +++++++++++++++++++++ specification/modules/instant_messaging.rst | 2 + 3 files changed, 84 insertions(+) create mode 100644 event-schemas/examples/v1/m.room.avatar create mode 100644 event-schemas/schema/v1/m.room.avatar diff --git a/event-schemas/examples/v1/m.room.avatar b/event-schemas/examples/v1/m.room.avatar new file mode 100644 index 00000000000..9fb1189c1ef --- /dev/null +++ b/event-schemas/examples/v1/m.room.avatar @@ -0,0 +1,18 @@ +{ + "age": 242352, + "content": { + "info": { + "h": 398, + "w": 394, + "mimetype": "image/jpeg", + "size": 31037 + }, + "url": "mxc://localhost/JWEIFJgwEIhweiWJE" + }, + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.avatar", + "state_key": "", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/schema/v1/m.room.avatar b/event-schemas/schema/v1/m.room.avatar new file mode 100644 index 00000000000..276c975c0c0 --- /dev/null +++ b/event-schemas/schema/v1/m.room.avatar @@ -0,0 +1,64 @@ +{ + "title": "RoomAvatar", + "description": "A picture that is associated with the room. This can be displayed alongside the room information.", + "type": "object", + "allOf": [{ + "$ref": "core-event-schema/state_event.json" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to the image." + }, + "thumbnail_url": { + "type": "string", + "description": "The URL to the thumbnail of the image." + }, + "thumbnail_info": { + "type": "object", + "title": "ImageInfo", + "description": "Metadata about the image referred to in ``thumbnail_url``.", + "allOf": [{ + "$ref": "core-event-schema/msgtype_infos/image_info.json" + }] + }, + "info": { + "type": "object", + "title": "ImageInfo", + "description": "Metadata about the image referred to in ``url``.", + "properties": { + "size": { + "type": "integer", + "description": "Size of the image in bytes." + }, + "w": { + "type": "integer", + "description": "The width of the image in pixels." + }, + "h": { + "type": "integer", + "description": "The height of the image in pixels." + }, + "mimetype": { + "type": "string", + "description": "The mimetype of the image, e.g. ``image/jpeg``." + } + } + } + }, + "required": ["url"] + }, + "state_key": { + "type": "string", + "description": "A zero-length string.", + "pattern": "^$" + }, + "type": { + "type": "string", + "enum": ["m.room.avatar"] + } + } +} diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 0b4e3e64d93..09fcb843d35 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -30,6 +30,8 @@ Usage of this event is discouraged for several reasons: {{m_room_topic_event}} +{{m_room_avatar_event}} + m.room.message msgtypes ~~~~~~~~~~~~~~~~~~~~~~~ From f51e0310668b32ec766f1d8a28076e4affe123c0 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 19 Oct 2015 12:36:10 +0100 Subject: [PATCH 082/989] Fix JSON --- event-schemas/schema/v1/m.room.member | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index c52d0d4c25f..e0ad673c0c5 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -40,7 +40,7 @@ }, "signed": { "type": "object", - "title": "signed_third_party_invite", + "title": "signed", "properties": { "mxid": { "type": "string", @@ -48,11 +48,12 @@ }, "token": { "type": "string", - "description": "The token property of the containing third_party_invite object.", + "description": "The token property of the containing third_party_invite object." }, "signatures": { "type": "object", - "description": "A single signature from the verifying server, in the format specified by the Signing Events section." + "description": "A single signature from the verifying server, in the format specified by the Signing Events section.", + "title": "Signatures" } }, "required": ["mxid", "signatures", "token"] From b2aae762fc5740a6e471745e85d1f0287ddabb8b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 19 Oct 2015 12:36:49 +0100 Subject: [PATCH 083/989] Give useful error if z-schema is missing --- event-schemas/check.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/event-schemas/check.sh b/event-schemas/check.sh index fac36cf85be..a6d03b5a44e 100755 --- a/event-schemas/check.sh +++ b/event-schemas/check.sh @@ -1,5 +1,11 @@ #!/bin/bash -e # Runs z-schema over all of the schema files (looking for matching examples) + +if ! which z-schema; then + echo >&2 "Need to install z-schema; run: sudo npm install -g z-schema" + exit 1 +fi + find schema/v1/m.* | while read line do split_path=(${line///// }) From bbd3f8072cccf3d029448f43290c2326cfb6276b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 19 Oct 2015 13:28:44 +0100 Subject: [PATCH 084/989] Review comments --- event-schemas/schema/v1/m.room.member | 1 + templating/matrix_templates/units.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 54ae910010c..121f144a4f7 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -67,6 +67,7 @@ "type": "object", "title": "StrippedState", "description": "A stripped down state event, with only the ``type``, ``state_key`` and ``content`` keys.", + "required": ["type", "state_key", "content"], "properties": { "type": { "type": "string", diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 33b5e6b44b6..9a17dd5b2ed 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -532,7 +532,8 @@ def load_event_schemas(self): Units.prop(json_schema, "properties/content") ) - # this is horrible + # This is horrible because we're special casing a key on m.room.member. + # We need to do this because we want to document a non-content object. if schema["type"] == "m.room.member": invite_room_state = get_json_schema_object_fields( json_schema["properties"]["invite_room_state"]["items"] From f450fc1db6adb9cb76925e90aff8f3782081bb07 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 19 Oct 2015 13:32:43 +0100 Subject: [PATCH 085/989] Typo --- api/client-server/v2_alpha/registration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml index 9973d0b43d0..5222b3f3fc0 100644 --- a/api/client-server/v2_alpha/registration.yaml +++ b/api/client-server/v2_alpha/registration.yaml @@ -86,7 +86,7 @@ paths: * ``M_EXCLUSIVE`` : The desired user ID is in the exclusive namespace claimed by an application service. - This errors may be returned at any stage of the registration process, + These errors may be returned at any stage of the registration process, including after authentication if the requested user ID was registered whilst the client was performing authentication. examples: From 111ca99519a32ba0412aed423517774250184567 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 19 Oct 2015 13:46:30 +0100 Subject: [PATCH 086/989] Clarify that this doesn't include events that occurred after you left the room. --- specification/modules/search.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 03d2475fbb6..6a545aa091d 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -5,7 +5,8 @@ Server Side Search The search API allows clients to perform full text search across events in all rooms that the user has been in, including those that they have left. Only -events that the user is allowed to see will be searched. +events that the user is allowed to see will be searched, e.g. it won't include +evnets in rooms that happened after you left. Client behaviour ---------------- From bfa9937bdc1c7025cb5265da3bf5d22b3a78d152 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 17:28:31 +0100 Subject: [PATCH 087/989] Minor wording updates --- specification/event_signing.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specification/event_signing.rst b/specification/event_signing.rst index 2f61566ecdb..9f33c796eae 100644 --- a/specification/event_signing.rst +++ b/specification/event_signing.rst @@ -191,8 +191,8 @@ in the event JSON in a ``hash`` object under a ``sha256`` key. return event_json_object The event is then stripped of all non-essential keys both at the top level and -within the ``content`` object. Essential top-level keys given below; any -top-level keys not in this list are removed. +within the ``content`` object. Any top-level keys not in he following list MUST +be removed: .. code:: @@ -212,8 +212,9 @@ top-level keys not in this list are removed. type A new ``content`` object is constructed for the resulting event that contains -only the essential keys of the original event. If the original event lacked a -``content`` object at all, a new blank one is created for it. +only the essential keys of the original ``content`` object. If the original +event lacked a ``content`` object at all, a new empty JSON object is created +for it. The keys that are considered essential for the ``content`` object depend on the the ``type`` of the event. These are: @@ -246,7 +247,7 @@ the ``type`` of the event. These are: users_default The resulting stripped object with the new ``content`` object and the original -``hashes`` key is then signed using the JSON signing algorithm. +``hashes`` key is then signed using the JSON signing algorithm outlined below: .. code:: python From 9252b3e1a29c4c105aaa169e4c03a2ff6ac3a1d8 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 17:44:09 +0100 Subject: [PATCH 088/989] Fix typo s/he/the/ --- specification/event_signing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/event_signing.rst b/specification/event_signing.rst index 9f33c796eae..3ad3d8a7ca1 100644 --- a/specification/event_signing.rst +++ b/specification/event_signing.rst @@ -191,8 +191,8 @@ in the event JSON in a ``hash`` object under a ``sha256`` key. return event_json_object The event is then stripped of all non-essential keys both at the top level and -within the ``content`` object. Any top-level keys not in he following list MUST -be removed: +within the ``content`` object. Any top-level keys not in the following list +MUST be removed: .. code:: From 228acc59aa2e21fcd74aeec189493e5f1e6ccfc2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 18:26:52 +0100 Subject: [PATCH 089/989] Capture cryptographic test vectors for JSON or event signing from source code --- specification/appendices.rst | 157 +++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/specification/appendices.rst b/specification/appendices.rst index c45aa0a5195..53119006b5d 100644 --- a/specification/appendices.rst +++ b/specification/appendices.rst @@ -128,3 +128,160 @@ Threat: Disclosure to Servers Within Chatroom An attacker could take control of a server within a chatroom to expose message contents or metadata for messages in that room. + +Cryptographic Test Vectors +-------------------------- + +To assist in the development of compatible implementations, the following test +values may be useful for verifying the cryptograph event signing code. + +Signing Key +~~~~~~~~~~~ + +The following test vectors all use the 32-byte value given by the following +Base64-encoded string as the seed for generating the ``ed25519`` signing key: + +.. code:: + + SIGNING_KEY_SEED = decode_base64( + "YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1" + ) + +In each case, the server name and key ID are as follows: + +.. code:: + + SERVER_NAME = "domain" + + KEY_ID = "ed25519:1" + +JSON Signing +~~~~~~~~~~~~ + +Given an empty JSON object: + +.. code:: json + + {} + +The JSON signing algorithm should emit the following signed data: + +.. code:: json + + { + "signatures": { + "domain": { + "ed25519:1": "K8280/U9SSy9IVtjBuVeLr+HpOB4BQFWbg+UZaADMtTdGYI7Geitb76LTrr5QV/7Xg4ahLwYGYZzuHGZKM5ZAQ" + } + } + } + +Given the following JSON object with data values in it: + +.. code:: json + + { + "one": 1, + "two": "Two" + } + +The JSON signing algorithm should emit the following signed JSON: + +.. code:: json + + { + "one": 1, + "signatures": { + "domain": { + "ed25519:1": "KqmLSbO39/Bzb0QIYE82zqLwsA+PDzYIpIRA2sRQ4sL53+sN6/fpNSoqE7BP7vBZhG6kYdD13EIMJpvhJI+6Bw" + } + }, + "two": "Two" + } + +Event Signing +~~~~~~~~~~~~~ + +Given the following minimally-sized event: + +.. code:: json + + { + "event_id": "$0:domain", + "origin": "domain", + "origin_server_ts": 1000000, + "signatures": {}, + "type": "X", + "unsigned": { + "age_ts": 1000000 + } + } + +The event signing algorithm should emit the following signed event: + +.. code:: json + + { + "event_id": "$0:domain", + "hashes": { + "sha256": "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI" + }, + "origin": "domain", + "origin_server_ts": 1000000, + "signatures": { + "domain": { + "ed25519:1": "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA" + } + }, + "type": "X", + "unsigned": { + "age_ts": 1000000 + } + } + +Given the following event containing redactable content: + +.. code:: json + + { + "content": { + "body": "Here is the message content", + }, + "event_id": "$0:domain", + "origin": "domain", + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:domain", + "sender": "@u:domain", + "signatures": {}, + "unsigned": { + "age_ts": 1000000 + } + } + +The event signing algorithm should emit the following signed event: + +.. code:: json + + { + "content": { + "body": "Here is the message content", + }, + "event_id": "$0:domain", + "hashes": { + "sha256": "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g" + }, + "origin": "domain", + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:domain", + "sender": "@u:domain", + "signatures": { + "domain": { + "ed25519:1": "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUwu6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA" + } + }, + "unsigned": { + "age_ts": 1000000 + } + } From e38e4788f24e4f98bcf59592417f9fa6d37a6c0e Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 18:29:37 +0100 Subject: [PATCH 090/989] Spell 'cryptographic' correctly --- specification/appendices.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/appendices.rst b/specification/appendices.rst index 53119006b5d..2868277698e 100644 --- a/specification/appendices.rst +++ b/specification/appendices.rst @@ -133,7 +133,7 @@ Cryptographic Test Vectors -------------------------- To assist in the development of compatible implementations, the following test -values may be useful for verifying the cryptograph event signing code. +values may be useful for verifying the cryptographic event signing code. Signing Key ~~~~~~~~~~~ From 74d42f01423acefdb88e3727d61530f371b108bc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 20 Oct 2015 11:08:22 +0100 Subject: [PATCH 091/989] Allow rejecting invitations --- specification/client_server_api.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index c25248f868a..791c1f4ae0f 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -905,6 +905,11 @@ room. There are several states in which a user may be, in relation to a room: - Joined (the user can send and receive events in the room) - Banned (the user is not allowed to join the room) +There is an exception to the requirement that a user join a room before sending +events to it: users may send an ``m.room.member`` event to a room with +``content.membership`` set to ``leave`` to reject an invitation if they have +currently been invited to a room but have not joined it. + Some rooms require that users be invited to it before they can join; others allow anyone to join. Whether a given room is an "invite-only" room is determined by the room config key ``m.room.join_rules``. It can have one of the @@ -931,8 +936,11 @@ Leaving rooms A user can leave a room to stop receiving events for that room. A user must -have joined the room before they are eligible to leave the room. If the room is -an "invite-only" room, they will need to be re-invited before they can re-join +have been invited to or have joined the room before they are eligible to leave +the room. Leaving a room to which the user has been invited rejects the invite. + +Whether or not they actually joined the room, if the room is +an "invite-only" room they will need to be re-invited before they can re-join the room. To leave a room, a request should be made to |/rooms//leave|_ with:: From bd539c72b414864cd2e759c8dcbcebfc00d0dae2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 21 Oct 2015 09:52:44 +0100 Subject: [PATCH 092/989] More review comments --- api/client-server/v1/application_service.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/v1/application_service.yaml index a7400bd405e..557d5a70bc6 100644 --- a/api/client-server/v1/application_service.yaml +++ b/api/client-server/v1/application_service.yaml @@ -115,7 +115,7 @@ paths: examples: application/json: |- { - "errcode": "ORG.MATRIX.MYAPPSERVICE_UNAUTHORIZED" + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: type: object @@ -147,7 +147,7 @@ paths: This endpoint is invoked by the homeserver on an application service to query the existence of a given user ID. The homeserver will only query user IDs inside the application service's ``users`` namespace. The homeserver will - send this request when it receives an *event* for an unknown user ID in + send this request when it receives an event for an unknown user ID in the application service's namespace. parameters: - in: path From 4c3e1b9ed37d69b488d7c215b2ca287b6f0a2f94 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 21 Oct 2015 09:53:25 +0100 Subject: [PATCH 093/989] Find replace fail --- api/client-server/v1/application_service.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/v1/application_service.yaml index 557d5a70bc6..1a5ec8382c8 100644 --- a/api/client-server/v1/application_service.yaml +++ b/api/client-server/v1/application_service.yaml @@ -136,7 +136,7 @@ paths: examples: application/json: |- { - "errcode": "ORG.MATRIX.MYAPPSERVICE_NOT_FOUND" + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: type: object @@ -173,7 +173,7 @@ paths: examples: application/json: |- { - "errcode": "ORG.MATRIX.MYAPPSERVICE_UNAUTHORIZED" + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: type: object @@ -194,7 +194,7 @@ paths: examples: application/json: |- { - "errcode": "ORG.MATRIX.MYAPPSERVICE_NOT_FOUND" + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: type: object From 82d4ea199f507e7dbdc9a960d50ab831e69f26b9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 22 Oct 2015 16:09:01 +0100 Subject: [PATCH 094/989] Trust Leo --- scripts/speculator/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index dab624dfba8..e6992d4a73a 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -365,6 +365,7 @@ func main() { "Kegsay": true, "NegativeMjark": true, "richvdh": true, + "leonerd": true, } rand.Seed(time.Now().Unix()) masterCloneDir, err := gitClone(matrixDocCloneURL, false) From 4a558ad63b0724a3b4f715df864cf698fbbcb1b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Oct 2015 10:19:25 +0100 Subject: [PATCH 095/989] Typo --- specification/modules/search.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 6a545aa091d..089953d2256 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -6,7 +6,7 @@ Server Side Search The search API allows clients to perform full text search across events in all rooms that the user has been in, including those that they have left. Only events that the user is allowed to see will be searched, e.g. it won't include -evnets in rooms that happened after you left. +events in rooms that happened after you left. Client behaviour ---------------- From e3e8026025c3dbc02092958f957f1731b14f2853 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Oct 2015 10:51:31 +0100 Subject: [PATCH 096/989] Fix list formatting so that we aren't including everything in blockquotes --- api/client-server/v1/membership.yaml | 20 ++--- api/client-server/v2_alpha/registration.yaml | 6 +- specification/client_server_api.rst | 48 ++++++------ specification/events.rst | 10 +-- specification/modules/content_repo.rst | 25 +++---- specification/modules/presence.rst | 32 ++++---- specification/modules/push.rst | 78 ++++++++++---------- specification/modules/voip_events.rst | 20 ++--- 8 files changed, 120 insertions(+), 119 deletions(-) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index efc82578601..fb0737e59ed 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -52,8 +52,8 @@ paths: 403: description: |- You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: - - The room is invite-only and the user was not invited. - - The user has been banned from the room. + - The room is invite-only and the user was not invited. + - The user has been banned from the room. examples: application/json: |- {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} @@ -119,10 +119,10 @@ paths: 403: description: |- You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: - - The invitee has been banned from the room. - - The invitee is already a member of the room. - - The inviter is not currently in the room. - - The inviter's power level is insufficient to invite users to the room. + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. examples: application/json: |- {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} @@ -164,13 +164,13 @@ paths: user ID, the identity server will give the user a list of pending invitations, each containing: - - The room ID to which they were invited + - The room ID to which they were invited - - The token given to the homeserver + - The token given to the homeserver - - A signature of the token, signed with the identity server's private key + - A signature of the token, signed with the identity server's private key - - The matrix user ID who invited them to the room + - The matrix user ID who invited them to the room If a token is requested from the identity server, the home server will append a ``m.room.third_party_invite`` event to the room. diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml index 5222b3f3fc0..b531c6cfc1d 100644 --- a/api/client-server/v2_alpha/registration.yaml +++ b/api/client-server/v2_alpha/registration.yaml @@ -82,9 +82,9 @@ paths: description: |- Part of the request was invalid. This may include one of the following error codes: - * ``M_USER_IN_USE`` : The desired user ID is already taken. - * ``M_EXCLUSIVE`` : The desired user ID is in the exclusive namespace - claimed by an application service. + * ``M_USER_IN_USE`` : The desired user ID is already taken. + * ``M_EXCLUSIVE`` : The desired user ID is in the exclusive namespace + claimed by an application service. These errors may be returned at any stage of the registration process, including after authentication if the requested user ID was registered diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 791c1f4ae0f..fcbc58a9847 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -787,23 +787,23 @@ it to be redacted, which may include a reason. Upon receipt of a redaction event, the server should strip off any keys not in the following list: - - ``event_id`` - - ``type`` - - ``room_id`` - - ``user_id`` - - ``state_key`` - - ``prev_state`` - - ``content`` +- ``event_id`` +- ``type`` +- ``room_id`` +- ``user_id`` +- ``state_key`` +- ``prev_state`` +- ``content`` The content object should also be stripped of all keys, unless it is one of one of the following event types: - - ``m.room.member`` allows key ``membership`` - - ``m.room.create`` allows key ``creator`` - - ``m.room.join_rules`` allows key ``join_rule`` - - ``m.room.power_levels`` allows keys ``ban``, ``events``, ``events_default``, +- ``m.room.member`` allows key ``membership`` +- ``m.room.create`` allows key ``creator`` +- ``m.room.join_rules`` allows key ``join_rule`` +- ``m.room.power_levels`` allows keys ``ban``, ``events``, ``events_default``, ``kick``, ``redact``, ``state_default``, ``users``, ``users_default``. - - ``m.room.aliases`` allows key ``aliases`` +- ``m.room.aliases`` allows key ``aliases`` .. TODO Need to update m.room.power_levels to reflect new power levels formatting @@ -823,9 +823,9 @@ which serves as the root of the event graph for this room. This event also has a generate several other events in order to manage permissions in this room. This includes: - - ``m.room.power_levels`` : Sets the power levels of users and required power +- ``m.room.power_levels`` : Sets the power levels of users and required power levels for various actions within the room such as sending events. - - ``m.room.join_rules`` : Whether the room is "invite-only" or not. +- ``m.room.join_rules`` : Whether the room is "invite-only" or not. See `Room Events`_ for more information on these events. To create a room, a client has to use the the following API. @@ -899,11 +899,11 @@ Joining rooms Users need to be a member of a room in order to send and receive events in that room. There are several states in which a user may be, in relation to a room: - - Unrelated (the user cannot send or receive events in the room) - - Invited (the user has been invited to participate in the room, but is not - yet participating) - - Joined (the user can send and receive events in the room) - - Banned (the user is not allowed to join the room) +- Unrelated (the user cannot send or receive events in the room) +- Invited (the user has been invited to participate in the room, but is not + yet participating) +- Joined (the user can send and receive events in the room) +- Banned (the user is not allowed to join the room) There is an exception to the requirement that a user join a room before sending events to it: users may send an ``m.room.member`` event to a room with @@ -1095,11 +1095,11 @@ many places of a client's display, changes to these fields cause an automatic propagation event to occur, informing likely-interested parties of the new values. This change is conveyed using two separate mechanisms: - - a ``m.room.member`` event is sent to every room the user is a member of, - to update the ``displayname`` and ``avatar_url``. - - a ``m.presence`` presence status update is sent, again containing the new - values of the ``displayname`` and ``avatar_url`` keys, in addition to the - required ``presence`` key containing the current presence state of the user. +- a ``m.room.member`` event is sent to every room the user is a member of, + to update the ``displayname`` and ``avatar_url``. +- a ``m.presence`` presence status update is sent, again containing the new + values of the ``displayname`` and ``avatar_url`` keys, in addition to the + required ``presence`` key containing the current presence state of the user. Both of these should be done automatically by the home server when a user successfully changes their display name or avatar URL fields. diff --git a/specification/events.rst b/specification/events.rst index 0eb4d933eb9..7d64882bf7c 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -18,11 +18,11 @@ Size limits The total size of any event MUST NOT exceed 65 KB. There are additional restrictions on sizes per key: - - ``user_id`` MUST NOT exceed 255 bytes (including domain). - - ``room_id`` MUST NOT exceed 255 bytes. - - ``state_key`` MUST NOT exceed 255 bytes. - - ``type`` MUST NOT exceed 255 bytes. - - ``event_id`` MUST NOT exceed 255 bytes. +- ``user_id`` MUST NOT exceed 255 bytes (including domain). +- ``room_id`` MUST NOT exceed 255 bytes. +- ``state_key`` MUST NOT exceed 255 bytes. +- ``type`` MUST NOT exceed 255 bytes. +- ``event_id`` MUST NOT exceed 255 bytes. Some event types have additional size restrictions which are specified in the description of the event. Additional keys have no limit other than that diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index 52937eada09..29dae059f89 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -64,21 +64,20 @@ UTF-8 encoded traversals, etc). Homeservers have additional content-specific concerns: - - Clients may try to upload very large files. Homeservers should not store files - that are too large and should not serve them to clients. +- Clients may try to upload very large files. Homeservers should not store files + that are too large and should not serve them to clients. - - Clients may try to upload very large images. Homeservers should not attempt to - generate thumbnails for images that are too large. +- Clients may try to upload very large images. Homeservers should not attempt to + generate thumbnails for images that are too large. - - Remote homeservers may host very large files or images. Homeservers should not - proxy or thumbnail large files or images from remote homeservers. +- Remote homeservers may host very large files or images. Homeservers should not + proxy or thumbnail large files or images from remote homeservers. - - Clients may try to upload a large number of files. Homeservers should limit the - number and total size of media that can be uploaded by clients. +- Clients may try to upload a large number of files. Homeservers should limit the + number and total size of media that can be uploaded by clients. - - Clients may try to access a large number of remote files through a homeserver. - Homeservers should restrict the number and size of remote files that it caches. - - - Clients or remote homeservers may try to upload malicious files targeting - vulnerabilities in either the homeserver thumbnailing or the client decoders. +- Clients may try to access a large number of remote files through a homeserver. + Homeservers should restrict the number and size of remote files that it caches. +- Clients or remote homeservers may try to upload malicious files targeting + vulnerabilities in either the homeserver thumbnailing or the client decoders. diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index 79151c4fe62..3602d105bb0 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -5,10 +5,10 @@ Presence Each user has the concept of presence information. This encodes: - * Whether the user is currently online - * How recently the user was last active (as seen by the server) - * Whether a given client considers the user to be currently idle - * Arbitrary information about the user's current status (e.g. "in a meeting"). +* Whether the user is currently online +* How recently the user was last active (as seen by the server) +* Whether a given client considers the user to be currently idle +* Arbitrary information about the user's current status (e.g. "in a meeting"). This information is collated from both per-device (``online``, ``idle``, ``last_active``) and per-user (status) data, aggregated by the user's homeserver @@ -24,14 +24,14 @@ who must accept the invitation. User's presence state is represented by the ``presence`` key, which is an enum of one of the following: - - ``online`` : The default state when the user is connected to an event - stream. - - ``unavailable`` : The user is not reachable at this time e.g. they are - idle. - - ``offline`` : The user is not connected to an event stream or is - explicitly suppressing their profile information from being sent. - - ``free_for_chat`` : The user is generally willing to receive messages - moreso than default. +- ``online`` : The default state when the user is connected to an event + stream. +- ``unavailable`` : The user is not reachable at this time e.g. they are + idle. +- ``offline`` : The user is not connected to an event stream or is + explicitly suppressing their profile information from being sent. +- ``free_for_chat`` : The user is generally willing to receive messages + moreso than default. Events ------ @@ -94,10 +94,10 @@ pro-active event from the user. A pro-active event may be sending a message to a room or changing presence state to a higher level of availability. Levels of availability are defined from low to high as follows: - - ``offline`` - - ``unavailable`` - - ``online`` - - ``free_for_chat`` +- ``offline`` +- ``unavailable`` +- ``online`` +- ``free_for_chat`` Based on this list, changing state from ``unavailable`` to ``online`` counts as a pro-active event, whereas ``online`` to ``unavailable`` does not. This diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 525e730d604..893e948154c 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -37,19 +37,19 @@ The above diagram shows the flow of push notifications being sent to a handset where push notifications are submitted via the handset vendor, such as Apple's APNS or Google's GCM. This happens as follows: - 1. The client app signs in to a homeserver. - 2. The client app registers with its vendor's Push Provider and - obtains a routing token of some kind. - 3. The mobile app uses the Client/Server API to add a 'pusher', providing the - URL of a specific Push Gateway which is configured for that - application. It also provides the routing token it has acquired from the - Push Provider. - 4. The homeserver starts sending HTTP requests to the Push Gateway using the - supplied URL. The Push Gateway relays this notification to - the Push Provider, passing the routing token along with any - necessary private credentials the provider requires to send push - notifications. - 5. The Push Provider sends the notification to the device. +1. The client app signs in to a homeserver. +2. The client app registers with its vendor's Push Provider and + obtains a routing token of some kind. +3. The mobile app uses the Client/Server API to add a 'pusher', providing the + URL of a specific Push Gateway which is configured for that + application. It also provides the routing token it has acquired from the + Push Provider. +4. The homeserver starts sending HTTP requests to the Push Gateway using the + supplied URL. The Push Gateway relays this notification to + the Push Provider, passing the routing token along with any + necessary private credentials the provider requires to send push + notifications. +5. The Push Provider sends the notification to the device. Definitions for terms used in this section are below: @@ -127,16 +127,16 @@ affect delivery of notifications via pushers with a matching ``profile_tag``. All device-specific rules have a higher priority than global rules. This means that the full list of rule kinds, in descending priority order, is as follows: - * Device-specific Override - * Device-specific Content - * Device-specific Room - * Device-specific Sender - * Device-specific Underride - * Global Override - * Global Content - * Global Room - * Global Sender - * Global Underride +* Device-specific Override +* Device-specific Content +* Device-specific Room +* Device-specific Sender +* Device-specific Underride +* Global Override +* Global Content +* Global Room +* Global Sender +* Global Underride Rules with the same ``kind`` can specify an ordering priority. This determines which rule is selected in the event of multiple matches. For example, a rule @@ -315,16 +315,17 @@ rule determines its behaviour. The following conditions are defined: ``event_match`` This is a glob pattern match on a field of the event. Parameters: - * ``key``: The dot-separated field of the event to match, e.g. ``content.body`` - * ``pattern``: The glob-style pattern to match against. Patterns with no - special glob characters should be treated as having asterisks - prepended and appended when testing the condition. + + * ``key``: The dot-separated field of the event to match, e.g. ``content.body`` + * ``pattern``: The glob-style pattern to match against. Patterns with no + special glob characters should be treated as having asterisks + prepended and appended when testing the condition. ``profile_tag`` Matches the ``profile_tag`` of the device that the notification would be delivered to. Parameters: - * ``profile_tag``: The profile_tag to match with. + * ``profile_tag``: The profile_tag to match with. ``contains_display_name`` This matches unencrypted messages where ``content.body`` contains the owner's @@ -334,10 +335,11 @@ rule determines its behaviour. The following conditions are defined: ``room_member_count`` This matches the current number of members in the room. Parameters: - * ``is``: A decimal integer optionally prefixed by one of, ``==``, ``<``, - ``>``, ``>=`` or ``<=``. A prefix of ``<`` matches rooms where the member - count is strictly less than the given number and so forth. If no prefix is - present, this parameter defaults to ``==``. + + * ``is``: A decimal integer optionally prefixed by one of, ``==``, ``<``, + ``>``, ``>=`` or ``<=``. A prefix of ``<`` matches rooms where the member + count is strictly less than the given number and so forth. If no prefix is + present, this parameter defaults to ``==``. Push Rules: API ~~~~~~~~~~~~~~~ @@ -418,12 +420,12 @@ client app and its' push gateway to agree on. As APNS requires that the sender has a private key owned by the app developer, each app must have its own push gateway. It is recommended that: - * The APNS token be base64 encoded and used as the pushkey. - * A different app_id be used for apps on the production and sandbox - APS environments. - * APNS push gateways do not attempt to wait for errors from the APNS - gateway before returning and instead to store failures and return - 'rejected' responses next time that pushkey is used. +* The APNS token be base64 encoded and used as the pushkey. +* A different app_id be used for apps on the production and sandbox + APS environments. +* APNS push gateways do not attempt to wait for errors from the APNS + gateway before returning and instead to store failures and return + 'rejected' responses next time that pushkey is used. Security considerations ----------------------- diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst index a7b02538c84..6945a09e240 100644 --- a/specification/modules/voip_events.rst +++ b/specification/modules/voip_events.rst @@ -62,19 +62,19 @@ As calls are "placed" to rooms rather than users, the glare resolution algorithm outlined below is only considered for calls which are to the same room. The algorithm is as follows: - - If an ``m.call.invite`` to a room is received whilst the client is - **preparing to send** an ``m.call.invite`` to the same room: +- If an ``m.call.invite`` to a room is received whilst the client is + **preparing to send** an ``m.call.invite`` to the same room: - * the client should cancel its outgoing call and instead - automatically accept the incoming call on behalf of the user. + * the client should cancel its outgoing call and instead + automatically accept the incoming call on behalf of the user. - - If an ``m.call.invite`` to a room is received **after the client has sent** - an ``m.call.invite`` to the same room and is waiting for a response: +- If an ``m.call.invite`` to a room is received **after the client has sent** + an ``m.call.invite`` to the same room and is waiting for a response: - * the client should perform a lexicographical comparison of the call IDs of - the two calls and use the *lesser* of the two calls, aborting the - greater. If the incoming call is the lesser, the client should accept - this call on behalf of the user. + * the client should perform a lexicographical comparison of the call IDs of + the two calls and use the *lesser* of the two calls, aborting the + greater. If the incoming call is the lesser, the client should accept + this call on behalf of the user. The call setup should appear seamless to the user as if they had simply placed From cc146209329e7369ecf164bf4108ed4863250e01 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Oct 2015 11:36:07 +0100 Subject: [PATCH 097/989] Add a newline before the list --- api/client-server/v1/membership.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index fb0737e59ed..13bfb6bbcb1 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -119,6 +119,7 @@ paths: 403: description: |- You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + - The invitee has been banned from the room. - The invitee is already a member of the room. - The inviter is not currently in the room. @@ -221,10 +222,11 @@ paths: 403: description: |- You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: - - The invitee has been banned from the room. - - The invitee is already a member of the room. - - The inviter is not currently in the room. - - The inviter's power level is insufficient to invite users to the room. + + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. examples: application/json: |- {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} From 01064369a450210ea7780fb1c4b4819125aaa160 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Oct 2015 11:57:15 +0100 Subject: [PATCH 098/989] Add some clarification on the difference between 'scale' and 'crop' resize methods --- specification/modules/content_repo.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index 52937eada09..fd797b2c324 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -36,6 +36,10 @@ width and height are close to the requested size and the aspect matches the requested size. The client should scale the image if it needs to fit within a given rectangle. +In summary: + * "scale" maintains the original aspect ratio of the image + * "crop" provides an image in the aspect ratio of the sizes given in the request + Server behaviour ---------------- From df443c10ac973686a57f302629d5b8151f4aa324 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Oct 2015 11:58:06 +0100 Subject: [PATCH 099/989] Move css into a css subdirectory. Add css to put a bar next to blockquotes. --- scripts/{ => css}/basic.css | 0 scripts/css/blockquote.css | 5 +++++ scripts/{ => css}/codehighlight.css | 0 scripts/{ => css}/nature.css | 0 scripts/gendoc.py | 2 +- 5 files changed, 6 insertions(+), 1 deletion(-) rename scripts/{ => css}/basic.css (100%) create mode 100644 scripts/css/blockquote.css rename scripts/{ => css}/codehighlight.css (100%) rename scripts/{ => css}/nature.css (100%) diff --git a/scripts/basic.css b/scripts/css/basic.css similarity index 100% rename from scripts/basic.css rename to scripts/css/basic.css diff --git a/scripts/css/blockquote.css b/scripts/css/blockquote.css new file mode 100644 index 00000000000..151d3bce082 --- /dev/null +++ b/scripts/css/blockquote.css @@ -0,0 +1,5 @@ +blockquote { + margin: 20px 0 30px; + border-left: 5px solid; + padding-left: 20px; +} diff --git a/scripts/codehighlight.css b/scripts/css/codehighlight.css similarity index 100% rename from scripts/codehighlight.css rename to scripts/css/codehighlight.css diff --git a/scripts/nature.css b/scripts/css/nature.css similarity index 100% rename from scripts/nature.css rename to scripts/css/nature.css diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 74a6c1da1c5..ed36a5a7826 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -15,7 +15,7 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) stylesheets = { - "stylesheet_path": ["basic.css", "nature.css", "codehighlight.css"] + "stylesheet_path": glob.glob("css/*.css"), } VERBOSE = False From d18d406c41e0658b376555d666dc3690749fab5e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 23 Oct 2015 12:49:17 +0100 Subject: [PATCH 100/989] Add the room send and state APIs to the spec --- api/client-server/v1/room_send.yaml | 125 +++++++++++++++++++++++++++ api/client-server/v1/room_state.yaml | 78 +++++++++++++++++ specification/client_server_api.rst | 64 +++++--------- 3 files changed, 227 insertions(+), 40 deletions(-) create mode 100644 api/client-server/v1/room_send.yaml create mode 100644 api/client-server/v1/room_state.yaml diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/v1/room_send.yaml new file mode 100644 index 00000000000..9c9273dfe5c --- /dev/null +++ b/api/client-server/v1/room_send.yaml @@ -0,0 +1,125 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 message event send API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/send/{eventType}/{txnId}": + put: + summary: Send a message event to the given room. + description: |- + This endpoint is used to send a message event to a room. Message events + allow access to historical events and pagination, making them suited + for "once-off" activity in a room. + + The body of the request should be the content object of the event; the + fields in this object will vary depending on the type of event. See + `Room Events`_ for the m. event specification. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to send the event to. + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventType + description: The type of event to send. + required: true + x-example: "m.room.message" + - in: path + name: txnId + type: string + description: |- + The transaction ID for this event. Clients should generate a + unique ID; it will be used by the server to ensure idempotency of requests. + required: true + x-example: "35" + - in: body + name: body + schema: + type: object + example: |- + { + "msgtype": "m.text", + "body": "hello" + } + responses: + 200: + description: "An ID for the sent event." + examples: + application/json: |- + { + "event_id": "YUwRidLecu" + } + schema: + type: object + properties: + event_id: + type: string + description: |- + A unique identifier for the event. + "/rooms/{roomId}/send/{eventType}": + post: + summary: Send a message event to the given room. + description: |- + This endpoint can be used to send a message event to a room; however + the lack of a transaction ID means that it is possible to cause message + duplication if events are resent on error, so it is preferable to use + `PUT /_matrix/client/api/v1/rooms/{roomId}/send/{eventType}/{txnId}`_. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to send the event to. + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventType + description: The type of event to send. + required: true + x-example: "m.room.message" + - in: body + name: body + schema: + type: object + example: |- + { + "msgtype": "m.text", + "body": "hello" + } + responses: + 200: + description: "An ID for the sent event." + examples: + application/json: |- + { + "event_id": "YUwRidLecu" + } + schema: + type: object + properties: + event_id: + type: string + description: |- + A unique identifier for the event. diff --git a/api/client-server/v1/room_state.yaml b/api/client-server/v1/room_state.yaml new file mode 100644 index 00000000000..0c0e76e6908 --- /dev/null +++ b/api/client-server/v1/room_state.yaml @@ -0,0 +1,78 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 state event send API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/state/{eventType}/{stateKey}": + put: + summary: Send a message event to the given room. + description: | + State events can be sent using this endpoint. These events will be + overwritten if ````, ```` and ```` all + match. If the state event has no ``state_key``, it can be omitted from + the path. These requests **cannot use transaction IDs** like other + ``PUT`` paths because they cannot be differentiated from the + ``state_key``. Furthermore, ``POST`` is unsupported on state paths. + + The body of the request should be the content object of the event; the + fields in this object will vary depending on the type of event. See + `Room Events`_ for the ``m.`` event specification. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to set the state in + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventType + description: The type of event to send. + required: true + x-example: "m.room.name" + - in: path + type: string + name: stateKey + description: The state_key for the state to send. Defaults to the empty string. + required: false + x-example: "" + - in: body + name: body + schema: + type: object + example: | + { + "name": "The room name" + } + responses: + 200: + description: "An ID for the sent event." + examples: + application/json: |- + { + "event_id": "YUwRidLecu" + } + schema: + type: object + properties: + event_id: + type: string + description: |- + A unique identifier for the event. diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 791c1f4ae0f..4df821e0a1d 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -697,22 +697,32 @@ events are added, the event ``type`` key SHOULD follow the Java package naming convention, e.g. ``com.example.myapp.event``. This ensures event types are suitably namespaced for each application and reduces the risk of clashes. -State events -++++++++++++ +Getting events for a room +~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are several APIs provided to ``GET`` events for a room: + +{{rooms_http_api}} + + +{{message_pagination_http_api}} + + +Sending events to a room +~~~~~~~~~~~~~~~~~~~~~~~~ + +{{room_state_http_api}} + + +**Examples** -State events can be sent by ``PUT`` ing to -|/rooms//state//|_. These events will be -overwritten if ````, ```` and ```` all match. -If the state event has no ``state_key``, it can be omitted from the path. These -requests **cannot use transaction IDs** like other ``PUT`` paths because they -cannot be differentiated from the ``state_key``. Furthermore, ``POST`` is -unsupported on state paths. Valid requests look like:: +Valid requests look like:: - PUT /rooms/!roomid:domain/state/m.example.event - { "key" : "without a state key" } + PUT /rooms/!roomid:domain/state/m.example.event + { "key" : "without a state key" } - PUT /rooms/!roomid:domain/state/m.another.example.event/foo - { "key" : "with 'foo' as the state key" } + PUT /rooms/!roomid:domain/state/m.another.example.event/foo + { "key" : "with 'foo' as the state key" } In contrast, these requests are invalid:: @@ -738,34 +748,8 @@ In some cases, there may be no need for a ``state_key``, so it can be omitted:: PUT /rooms/!roomid:domain/state/m.room.bgd.color { "color": "red", "hex": "#ff0000" } -See `Room Events`_ for the ``m.`` event specification. +{{room_send_http_api}} -Message events -++++++++++++++ - -Message events can be sent by sending a request to -|/rooms//send/|_. These requests *can* use transaction -IDs and ``PUT``/``POST`` methods. Message events allow access to historical -events and pagination, making it best suited for sending messages. For -example:: - - POST /rooms/!roomid:domain/send/m.custom.example.message - { "text": "Hello world!" } - - PUT /rooms/!roomid:domain/send/m.custom.example.message/11 - { "text": "Goodbye world!" } - -See `Room Events`_ for the ``m.`` event specification. - -Getting events for a room -~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are several APIs provided to ``GET`` events for a room: - -{{rooms_http_api}} - - -{{message_pagination_http_api}} Redactions ~~~~~~~~~~ From d2bbc461e4b2bb473c4a190aacbf4ecf4796da4b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 23 Oct 2015 13:25:12 +0100 Subject: [PATCH 101/989] mark stateKey as required in room/{id}/state Swagger insists that path params be mandatory. --- api/client-server/v1/room_state.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/v1/room_state.yaml b/api/client-server/v1/room_state.yaml index 0c0e76e6908..bd8e29468cf 100644 --- a/api/client-server/v1/room_state.yaml +++ b/api/client-server/v1/room_state.yaml @@ -51,15 +51,15 @@ paths: type: string name: stateKey description: The state_key for the state to send. Defaults to the empty string. - required: false + required: true x-example: "" - in: body name: body schema: type: object - example: | + example: |- { - "name": "The room name" + "name": "New name for the room" } responses: 200: From 67880686d588eb54ef2b3f3a75d4ba4f984977b5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 23 Oct 2015 14:02:53 +0100 Subject: [PATCH 102/989] Allow history to be noted as world-readable SPEC-237 --- event-schemas/schema/v1/m.room.history_visibility | 4 ++-- specification/modules/history_visibility.rst | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/event-schemas/schema/v1/m.room.history_visibility b/event-schemas/schema/v1/m.room.history_visibility index bc33f4b3164..9abcf5b8411 100644 --- a/event-schemas/schema/v1/m.room.history_visibility +++ b/event-schemas/schema/v1/m.room.history_visibility @@ -1,7 +1,7 @@ { "type": "object", "title": "Controls visibility of history.", - "description": "This event controls whether a member of a room can see the events that happened in a room from before they joined.", + "description": "This event controls whether a user can see the events that happened in a room from before they joined.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -12,7 +12,7 @@ "history_visibility": { "type": "string", "description": "Who can see the room history.", - "enum": ["invited","joined","shared"] + "enum": ["invited","joined","shared","world_readable"] } }, "required": ["history_visibility"] diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst index f26d1b592cc..112e2e21955 100644 --- a/specification/modules/history_visibility.rst +++ b/specification/modules/history_visibility.rst @@ -6,14 +6,14 @@ Room History Visibility This module adds support for controlling the visibility of previous events in a room. -In all cases, a user needs to join a room to view events in that room. Once they +In all cases except ``world_readable``, a user needs to join a room to view events in that room. Once they have joined a room, they will gain access to a subset of events in the room. How this subset is chosen is controlled by the ``m.room.history_visibility`` event outlined below. After a user has left a room, they may seen any events which they were allowed to see before they left the room, but no events received after they left. -The three options for the ``m.room.history_visibility`` event are: +The four options for the ``m.room.history_visibility`` event are: - ``shared`` - Previous events are always accessible to newly joined members. All events in the room are accessible, even those sent when the member was not a part @@ -24,6 +24,9 @@ The three options for the ``m.room.history_visibility`` event are: - ``joined`` - Previous events are accessible to newly joined members from the point they joined the room onwards. Events stop being accessible when the member's state changes to something other than ``join``. +- ``world_readable`` - All events while this is the ``m.room.history_visibility`` value + may be shared by any participating homeserver with anyone, regardless of whether + they have ever joined the room. .. WARNING:: These options are applied at the point an event is *sent*. Checks are @@ -43,10 +46,13 @@ Client behaviour Clients that implement this module MUST present to the user the possible options for setting history visibility when creating a room. +Clients may want to display a notice that their events may be read by non-joined +people if the value is set to ``world_readable``. + Server behaviour ---------------- -By default if no ``history_visibility`` is set the visibility is assumed to be +By default if no ``history_visibility`` is set, or if the value is not understood, the visibility is assumed to be ``shared``. The rules governing whether a user is allowed to see an event depend solely on the state of the room *at that event*: From 28ab643a4e767f11d5afaacc10e689eb9e9ceca9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 23 Oct 2015 15:54:19 +0100 Subject: [PATCH 103/989] Remove duplicated registration/login APIs Currently the spec duplicates all of the account-management APIs. There's still work to be done here, but the complete duplication is confusing. --- api/client-server/v2_alpha/registration.yaml | 5 + specification/client_server_api.rst | 161 +------------------ 2 files changed, 9 insertions(+), 157 deletions(-) diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml index 5222b3f3fc0..a53d4f411a4 100644 --- a/api/client-server/v2_alpha/registration.yaml +++ b/api/client-server/v2_alpha/registration.yaml @@ -89,6 +89,11 @@ paths: These errors may be returned at any stage of the registration process, including after authentication if the requested user ID was registered whilst the client was performing authentication. + + Home Servers MUST perform the relevant checks and return these codes before + performing `User-Interactive Authentication`_, although they may also return + them after authentication is completed if, for example, the requested user ID + was registered whilst the client was performing authentication. examples: application/json: |- { diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 791c1f4ae0f..89b6666a750 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -349,67 +349,21 @@ This MUST return an HTML page which can perform this authentication stage. This page must attempt to call the JavaScript function ``window.onAuthDone`` when the authentication has been completed. -Registration -~~~~~~~~~~~~ +API calls using the User-Interactive Authentication mechanism +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This section refers to API Version 2. These API calls currently use the prefix ``/_matrix/client/v2_alpha``. -Registering for a user account is done using the request:: - - POST $V2PREFIX/register - -This API endpoint uses the `User-Interactive Authentication API`_. -This API endpoint does not require an access token. - -.. _User-Interactive Authentication API: `sect:auth-api`_ - -The body of the POST request is a JSON object containing: - -username - Optional. This is the local part of the desired Matrix ID. If omitted, the - Home Server must generate a Matrix ID local part. -password - Required. The desired password for the account. -bind_email - Optional. If ``true``, the server binds the email used for authentication to - the Matrix ID with the ID Server. - -On success, this returns a JSON object with keys: - -user_id - The fully-qualified Matrix ID that has been registered. -access_token - An access token for the new account. -home_server - The hostname of the Home Server on which the account has been registered. -refresh_token - A token that may be exchanged for a new ``access_token`` using the - ``/tokenrefresh`` API endpoint. - -This endpoint may also return the following error codes: - -M_USER_IN_USE - If the Matrix ID is already in use -M_EXCLUSIVE - If the requested Matrix ID is in the exclusive namespace of an application - service. - -Home Servers MUST perform the relevant checks and return these codes before -performing `User-Interactive Authentication`_, although they may also return -them after authentication is completed if, for example, the requested user ID -was registered whilst the client was performing authentication. - .. _User-Interactive Authentication: `sect:auth-api`_ +{{v2_registration_http_api}} + Old V1 API docs: |register|_ {{login_http_api}} Changing Password +++++++++++++++++ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - Request:: POST $V2PREFIX/account/password @@ -432,9 +386,6 @@ database. Adding Account Administrative Contact Information +++++++++++++++++++++++++++++++++++++++++++++++++ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - Request:: POST $V2PREFIX/account/3pid @@ -470,9 +421,6 @@ M_THREEPID_AUTH_FAILED Fetching Currently Associated Contact Information +++++++++++++++++++++++++++++++++++++++++++++++++ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - Request:: GET $V2PREFIX/account/3pid @@ -982,107 +930,6 @@ member's state, by making a request to "membership": "ban" } -Account operations ------------------- - -Registration -~~~~~~~~~~~~ -This API endpoint uses the `User-Interactive Authentication API`_. - -{{v2_registration_http_api}} - -Login -~~~~~ - -{{login_http_api}} - -Changing Password -~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - POST $V2PREFIX/account/password - -This API endpoint uses the User-Interactive Authentication API. An access token -should be submitted to this endpoint if the client has an active session. The -Home Server may change the flows available depending on whether a valid access -token is provided. - -The body of the POST request is a JSON object containing: - -new_password - The new password for the account. - -On success, an empty JSON object is returned. - -The error code M_NOT_FOUND is returned if the user authenticated with a third -party identifier but the Home Server could not find a matching account in its -database. - -Adding a Third Party Identifier -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - POST $V2PREFIX/account/3pid - -Used to add a third party identifier to the user's account. - -The body of the POST request is a JSON object containing: - -threePidCreds - An object containing third party identifier credentials. -bind - Optional. A boolean indicating whether the Home Server should also bind this - third party identifier to the account's matrix ID with the Identity Server. If - supplied and true, the Home Server must bind the 3pid accordingly. - -The third party identifier credentials object comprises: - -id_server - The colon-separated hostname and port of the Identity Server used to - authenticate the third party identifier. If the port is the default, it and the - colon should be omitted. -sid - The session ID given by the Identity Server -client_secret - The client secret used in the session with the Identity Server. - -On success, the empty JSON object is returned. - -May also return error codes: - -M_THREEPID_AUTH_FAILED - If the credentials provided could not be verified with the ID Server. - -Fetching Currently Associated Third Party Identifiers -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - GET $V2PREFIX/account/3pid - -This returns a list of third party identifiers that the Home Server has -associated with the user's account. This is *not* the same as the list of third -party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers -in this list may be used by the Home Server as, for example, identifiers that it -will accept to reset the user's account password. - -Returns a JSON object with the key ``threepids`` whose contents is an array of -objects with the following keys: - -medium - The medium of the 3pid (eg, ``email``) -address - The textual address of the 3pid, eg. the email address - - Profiles -------- From 637718108ef8ddadef779b904a9042d8d1c2d525 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Oct 2015 16:52:37 +0100 Subject: [PATCH 104/989] Add newline before list --- api/client-server/v1/membership.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index 13bfb6bbcb1..f8dfdea5554 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -52,6 +52,7 @@ paths: 403: description: |- You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: + - The room is invite-only and the user was not invited. - The user has been banned from the room. examples: From 19456974567dbb52189ea247406c71fd64f111b2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 23 Oct 2015 16:58:55 +0100 Subject: [PATCH 105/989] Further tweaks to the room send and state APIs - fix confusion re empty/absent state_keys - move 'types of room events' section earlier in the 'Events' section - remove some redundant anchors --- api/client-server/v1/room_state.yaml | 8 +++-- specification/client_server_api.rst | 51 +++++++++++++--------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/api/client-server/v1/room_state.yaml b/api/client-server/v1/room_state.yaml index bd8e29468cf..ec30fd3476a 100644 --- a/api/client-server/v1/room_state.yaml +++ b/api/client-server/v1/room_state.yaml @@ -24,9 +24,11 @@ paths: description: | State events can be sent using this endpoint. These events will be overwritten if ````, ```` and ```` all - match. If the state event has no ``state_key``, it can be omitted from - the path. These requests **cannot use transaction IDs** like other - ``PUT`` paths because they cannot be differentiated from the + match. If the state event has an empty ``state_key``, it can be + omitted from the path. + + Requests to this endpoint **cannot use transaction IDs** + like other ``PUT`` paths because they cannot be differentiated from the ``state_key``. Furthermore, ``POST`` is unsupported on state paths. The body of the request should be the content object of the event; the diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 4df821e0a1d..9e89ecf4913 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -647,6 +647,30 @@ To continue paginating backwards, one calls the /messages API again, supplying the new ``start`` value as the ``from`` parameter. +Types of room events +~~~~~~~~~~~~~~~~~~~~ + +Room events are split into two categories: + +:State Events: + These are events which update the metadata state of the room (e.g. room topic, + room membership etc). State is keyed by a tuple of event ``type`` and a + ``state_key``. State in the room with the same key-tuple will be overwritten. + +:Message events: + These are events which describe transient "once-off" activity in a room: + typically communication such as sending an instant message or setting up a + VoIP call. + +This specification outlines several events, all with the event type prefix +``m.``. (See `Room Events`_ for the m. event specification.) However, +applications may wish to add their own type of event, and this can be achieved +using the REST API detailed in the following sections. If new events are added, +the event ``type`` key SHOULD follow the Java package naming convention, +e.g. ``com.example.myapp.event``. This ensures event types are suitably +namespaced for each application and reduces the risk of clashes. + + Syncing ~~~~~~~ @@ -675,27 +699,6 @@ This API also returns an ``end`` token which can be used with the event stream. {{sync_http_api}} -Types of room events -~~~~~~~~~~~~~~~~~~~~ - -Room events are split into two categories: - -:State Events: - These are events which update the metadata state of the room (e.g. room topic, - room membership etc). State is keyed by a tuple of event ``type`` and a - ``state_key``. State in the room with the same key-tuple will be overwritten. - -:Message events: - These are events which describe transient "once-off" activity in a room: - typically communication such as sending an instant message or setting up a - VoIP call. - -This specification outlines several events, all with the event type prefix -``m.``. However, applications may wish to add their own type of event, and this -can be achieved using the REST API detailed in the following sections. If new -events are added, the event ``type`` key SHOULD follow the Java package naming -convention, e.g. ``com.example.myapp.event``. This ensures event types are -suitably namespaced for each application and reduces the risk of clashes. Getting events for a room ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1144,12 +1147,6 @@ have to wait in milliseconds before they can try again. .. |/rooms//state| replace:: ``/rooms//state`` .. _/rooms//state: /docs/api/client-server/#!/-rooms/get_state_events -.. |/rooms//send/| replace:: ``/rooms//send/`` -.. _/rooms//send/: /docs/api/client-server/#!/-rooms/send_non_state_event - -.. |/rooms//state//| replace:: ``/rooms//state//`` -.. _/rooms//state//: /docs/api/client-server/#!/-rooms/send_state_event - .. |/rooms//invite| replace:: ``/rooms//invite`` .. _/rooms//invite: /docs/api/client-server/#!/-rooms/invite From 36634df3555c72456ac7aa2b8968e96a5921f896 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Oct 2015 10:18:40 +0000 Subject: [PATCH 106/989] Add filter param --- api/client-server/v1/search.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index 885e62b3cc3..cc2990564a8 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -62,6 +62,12 @@ paths: type: string enum: ["content.body", "content.name", "content.topic"] description: The keys to search. Defaults to all. + filter: + type: object + title: Filter + description: |- + The filter to apply to search results. + This has the same format as v2 filter API. required: ["search_term"] required: ["search_categories"] responses: From 1aeddac3199e045a69ff163cbfc1bd4c4fc553a4 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 26 Oct 2015 10:28:14 +0000 Subject: [PATCH 107/989] Cache spec output Does not cache inability to generate spec. Does not cache diffs or anything fancy. Just the raw spec generation. --- scripts/speculator/README | 8 ++++++-- scripts/speculator/main.go | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts/speculator/README b/scripts/speculator/README index c54e0f2b014..82dc3d36e44 100644 --- a/scripts/speculator/README +++ b/scripts/speculator/README @@ -6,9 +6,13 @@ It serves the following HTTP endpoints: - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. - /diff/html/123 which gives a diff of the spec's HTML at pull request 123. -To run it, you must install the `go` tool, and run: +The build or run, you need a working `go` installation. +Then fetch dependencies: + ` go get github.com/hashicorp/golang-lru` + +To run it, then run: `go run main.go` To build the binary (which is necessary for deployment to the matrix.org -servers), you must again install `go`, and then run: +servers), you must again install `go` and dependencies, and then run: `go build` diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e6992d4a73a..85fb2596587 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -25,6 +25,8 @@ import ( "strings" "syscall" "time" + + "github.com/hashicorp/golang-lru" ) type PullRequest struct { @@ -53,6 +55,7 @@ type User struct { var ( port = flag.Int("port", 9000, "Port on which to listen for HTTP") allowedMembers map[string]bool + specCache *lru.Cache // string -> []byte ) func (u *User) IsTrusted() bool { @@ -196,6 +199,10 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { } sha = pr.Head.SHA } + if cached, ok := specCache.Get(sha); ok { + w.Write(cached.([]byte)) + return + } dst, err := s.generateAt(sha) defer os.RemoveAll(dst) @@ -210,6 +217,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } w.Write(b) + specCache.Add(sha, b) } func checkAuth(pr *PullRequest) error { @@ -367,6 +375,9 @@ func main() { "richvdh": true, "leonerd": true, } + if err := initCache(); err != nil { + log.Fatal(err) + } rand.Seed(time.Now().Unix()) masterCloneDir, err := gitClone(matrixDocCloneURL, false) if err != nil { @@ -388,3 +399,9 @@ func serveText(s string) func(http.ResponseWriter, *http.Request) { io.WriteString(w, s) } } + +func initCache() error { + c, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + specCache = c + return err +} From 3b7585cbda5e588bb1de47230d8d9a39a40eb1cd Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 26 Oct 2015 11:17:19 +0000 Subject: [PATCH 108/989] Make nested request objects display correctly (search API) This now displays search_categories.room_events.filter and co correctly. Also make arrays of enums display correctly. --- .../matrix_templates/templates/http-api.tmpl | 12 +++++----- templating/matrix_templates/units.py | 24 ++++++++++++++++++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index d1633513c4d..2812cf7b5da 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -13,17 +13,17 @@ Request format: -=================================== ================= =========================================== - Parameter Value Description -=================================== ================= =========================================== +=========================================== ================= =========================================== + Parameter Value Description +=========================================== ================= =========================================== {% for loc in endpoint.req_param_by_loc -%} *{{loc}} parameters* -------------------------------------------------------------------------------------------------- +--------------------------------------------------------------------------------------------------------- {% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(36-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(54)}} +{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} {% endfor -%} {% endfor -%} -=================================== ================= =========================================== +=========================================== ================= =========================================== {% if endpoint.res_tables|length > 0 -%} Response format: diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 78d1b3678ee..c5cf00443fc 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -121,6 +121,17 @@ def get_json_schema_object_fields(obj, enforce_title=False): tables += nested_object else: value_type = "[%s]" % props[key_name]["items"]["type"] + array_enums = props[key_name]["items"].get("enum") + if array_enums: + if len(array_enums) > 1: + value_type = "[enum]" + desc += ( + " One of: %s" % json.dumps(array_enums) + ) + else: + desc += ( + " Must be '%s'." % array_enums[0] + ) else: value_type = props[key_name]["type"] if props[key_name].get("enum"): @@ -252,13 +263,24 @@ def _load_swagger_meta(self, api, group_name): req_obj = req_obj["items"] req_tables = get_json_schema_object_fields(req_obj) + + if req_tables > 1: + for table in req_tables[1:]: + nested_key_name = [ + s["key"] for s in req_tables[0]["rows"] if + s["type"] == ("{%s}" % table["title"]) + ][0] + for row in table["rows"]: + row["key"] = "%s.%s" % (nested_key_name, row["key"]) + key_sep = "[0]." if is_array else "." for table in req_tables: if table.get("no-table"): continue for row in table["rows"]: + nested_key = key + key_sep + row["key"] endpoint["req_params"].append({ - "key": key + key_sep + row["key"], + "key": nested_key, "loc": "JSON body", "type": row["type"], "desc": row["req_str"] + row["desc"] From 88c459d5490e3c74118d5cfd34038869418d3d98 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Oct 2015 13:40:24 +0000 Subject: [PATCH 109/989] Add back in login v1 fallback API --- specification/client_server_api.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 9e23033499a..9266002f564 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -362,6 +362,18 @@ Old V1 API docs: |register|_ {{login_http_api}} +Fallback +++++++++ + +Clients cannot be expected to be able to know how to process every single login +type. If a client determines it does not know how to handle a given login type, +it should request a login fallback page: + + GET /_matrix/client/api/v1/login/fallback + +This returns an HTML and JavaScript page which can perform the entire login +process. + Changing Password +++++++++++++++++ Request:: From 0529b4a9a2018feb509459403d2c0f2a26029b40 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Oct 2015 13:45:36 +0000 Subject: [PATCH 110/989] Reword --- specification/client_server_api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 9266002f564..da6f13a7161 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -365,14 +365,14 @@ Old V1 API docs: |register|_ Fallback ++++++++ -Clients cannot be expected to be able to know how to process every single login -type. If a client determines it does not know how to handle a given login type, -it should request a login fallback page: +If a client does not recognize all or any login flows, it can use the fallback +login API:: GET /_matrix/client/api/v1/login/fallback This returns an HTML and JavaScript page which can perform the entire login -process. +process. The page will attempt to call the JavaScript function +``window.onLogin`` when login has been successfully completed. Changing Password +++++++++++++++++ From 98dbf4d60b7e79d7b29fb0aa17e7aaa9bf7d0273 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Oct 2015 13:46:41 +0000 Subject: [PATCH 111/989] Better flow --- specification/client_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index da6f13a7161..fac2e8ff9ba 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -365,7 +365,7 @@ Old V1 API docs: |register|_ Fallback ++++++++ -If a client does not recognize all or any login flows, it can use the fallback +If a client does not recognize any or all login flows it can use the fallback login API:: GET /_matrix/client/api/v1/login/fallback From 9a5bd6d21f9306ff1673155a41cc055de6024665 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Oct 2015 13:56:07 +0000 Subject: [PATCH 112/989] Use static --- specification/client_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index fac2e8ff9ba..9f942c94033 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -368,7 +368,7 @@ Fallback If a client does not recognize any or all login flows it can use the fallback login API:: - GET /_matrix/client/api/v1/login/fallback + GET /_matrix/static/client/login/ This returns an HTML and JavaScript page which can perform the entire login process. The page will attempt to call the JavaScript function From 30d46a19d51e8ed64ef9743ee7f6f451c034a723 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 26 Oct 2015 14:01:27 +0000 Subject: [PATCH 113/989] Review comments --- templating/matrix_templates/units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index c5cf00443fc..c434214116c 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -268,7 +268,7 @@ def _load_swagger_meta(self, api, group_name): for table in req_tables[1:]: nested_key_name = [ s["key"] for s in req_tables[0]["rows"] if - s["type"] == ("{%s}" % table["title"]) + s["type"] == ("{%s}" % (table["title"],)) ][0] for row in table["rows"]: row["key"] = "%s.%s" % (nested_key_name, row["key"]) From 3cf431cac1e3ed81578ba821f894ca6146487bf6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Oct 2015 14:10:02 +0000 Subject: [PATCH 114/989] Add link --- specification/client_server_api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 9f942c94033..8988acb4f89 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -365,6 +365,8 @@ Old V1 API docs: |register|_ Fallback ++++++++ +.. _sect:login_fallback: + If a client does not recognize any or all login flows it can use the fallback login API:: From 696f75d80b07fc16b01e238f3a377d345e02ca6e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Oct 2015 14:14:13 +0000 Subject: [PATCH 115/989] Rename section --- specification/client_server_api.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 8988acb4f89..09c6765c38e 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -362,10 +362,8 @@ Old V1 API docs: |register|_ {{login_http_api}} -Fallback -++++++++ - -.. _sect:login_fallback: +Login Fallback +++++++++++++++ If a client does not recognize any or all login flows it can use the fallback login API:: From 13eddd456f8277c9aec1743f5541d43f38b016a1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 26 Oct 2015 15:52:45 +0000 Subject: [PATCH 116/989] Proposal for adding full_state param to v2 sync --- api/client-server/v2_alpha/sync.yaml | 20 ++++++++++++++++++-- specification/client_server_api.rst | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 266f27bc4c7..5acbf583dfb 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -22,7 +22,7 @@ paths: summary: Synchronise the client's state and receive new messages. description: |- Synchronise the client's state with the latest state on the server. - Client's use this API when they first log in to get an initial snapshot + Clients use this API when they first log in to get an initial snapshot of the state on the server, and then continue to call this API to get incremental deltas to the state, and to receive new messages. security: @@ -40,6 +40,22 @@ paths: description: |- A point in time to continue a sync from. x-example: "s72594_4483_1934" + - in: query + name: full_state + type: boolean + description: |- + Controls whether to include the full state for all rooms the user + is a member of. + + If this is set to ``true``, then all state events will be returned, + even if ``since`` is non-empty. The timeline will still be limited + by the ``since`` parameter. + + If ``false``, and ``since`` is non-empty, only state which has + changed since the point indicated by ``since`` will be returned. + + By default, this is ``false``. + x-example: "false" - in: query name: set_presence type: string @@ -135,7 +151,7 @@ paths: type: object description: |- The state of a room that the user has been invited - to. These state events may only have the `sender``, + to. These state events may only have the ``sender``, ``type``, ``state_key`` and ``content`` keys present. These events do not replace any state that the client already has for the room, for example if diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 09c6765c38e..b02dbf28cc9 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -659,6 +659,8 @@ This API also returns an ``end`` token which can be used with the event stream. {{sync_http_api}} +{{v2_sync_http_api}} + Getting events for a room ~~~~~~~~~~~~~~~~~~~~~~~~~ From 12e33a3b09ac85af785d077f80543984e1f71dc5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 16:14:24 +0000 Subject: [PATCH 117/989] Document a v2 api for setting tags on rooms --- api/client-server/v2_alpha/tags.yaml | 148 +++++++++++++++++++++++++ event-schemas/examples/v1/m.tag | 8 ++ event-schemas/schema/v1/m.tag | 23 ++++ specification/modules/room_tagging.rst | 21 ++++ 4 files changed, 200 insertions(+) create mode 100644 api/client-server/v2_alpha/tags.yaml create mode 100644 event-schemas/examples/v1/m.tag create mode 100644 event-schemas/schema/v1/m.tag create mode 100644 specification/modules/room_tagging.rst diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml new file mode 100644 index 00000000000..91b81212474 --- /dev/null +++ b/api/client-server/v2_alpha/tags.yaml @@ -0,0 +1,148 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server tag API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/rooms/{roomId}/tags": + get: + summary: List the tags for a room. + description: |- + List the tags set by a user on a room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to get tags for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to get tags for. + x-example: "!726s6s6q:example.com" + responses: + 200: + description: + The list of tags for the user for the room. + schema: + type: object + properties: + tags: + type: array + items: + type: string + examples: + application/json: |- + { + "tags": [ + "work", + "pinned" + ] + } + "/user/{userId}/rooms/{roomId}/tags/{tag}": + put: + summary: Add a tag to a room. + description: |- + Add a tag to the room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to add a tag for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to add a tag to. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: tag + required: true + description: |- + The tag to add. + x-example: "work" + - in: body + name: body + required: true + description: |- + An empty JSON object. + schema: + type: object + example: |- + {} + responses: + 200: + description: + The tag was successfully added. + schema: + type: object + examples: + application/json: |- + {} + delete: + summary: Remove a tag from the room. + description: |- + Remove a tag from the room. + security: + - access_token: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to remove a tag for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to remove a tag from. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: tag + required: true + description: |- + The tag to remove. + x-example: "work" + responses: + 200: + description: + The tag was successfully removed + schema: + type: object + examples: + application/json: |- + {} diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag new file mode 100644 index 00000000000..28431b73abf --- /dev/null +++ b/event-schemas/examples/v1/m.tag @@ -0,0 +1,8 @@ +{ + "type": "m.tag", + "content": { + "tags": [ + "work" + ] + } +} diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag new file mode 100644 index 00000000000..235d442d28f --- /dev/null +++ b/event-schemas/schema/v1/m.tag @@ -0,0 +1,23 @@ +{ + "type": "object", + "title": "Tag Event", + "description": "Informs the client of tags on a room.", + "properties": { + "type": { + "type": "string", + "enum": ["m.tag"] + }, + "content": { + "type": "object", + "properties": { + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "required": ["type", "content"] +} diff --git a/specification/modules/room_tagging.rst b/specification/modules/room_tagging.rst new file mode 100644 index 00000000000..76fff125190 --- /dev/null +++ b/specification/modules/room_tagging.rst @@ -0,0 +1,21 @@ +Room Tagging +============ + +.. _module:tagging: + +Users can add tags to rooms. Tags are short strings used to label rooms, e.g. +"work", "familly". A room may have multiple tags. Tags are only visible to the +user that set them but are shared across all their devices. + +Events +------ + +The tags on a room are passed as single ``m.tag`` event in the ``ephemeral`` +section of a v2 room sync. + +{{m_tag_event}} + +Client Behaviour +---------------- + +{{v2_tags_http_api}} From 9b0d20315ab6cc5e13cb6a72d02c0cb328b2eb9d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 16:22:04 +0000 Subject: [PATCH 118/989] Add the tags module to the specification targets --- specification/modules/{room_tagging.rst => tags.rst} | 0 specification/targets.yaml | 1 + 2 files changed, 1 insertion(+) rename specification/modules/{room_tagging.rst => tags.rst} (100%) diff --git a/specification/modules/room_tagging.rst b/specification/modules/tags.rst similarity index 100% rename from specification/modules/room_tagging.rst rename to specification/modules/tags.rst diff --git a/specification/targets.yaml b/specification/targets.yaml index 2482dcfda7f..2c6a8f2051e 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -25,6 +25,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/push.rst - modules/third_party_invites.rst - modules/search.rst + - modules/tags.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 3f1cbb22bb5a143b7318794429d7e5dce6c0e024 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 16:27:57 +0000 Subject: [PATCH 119/989] Enable syntax highlighting for the JSON event examples --- templating/matrix_templates/templates/events.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index 9dcc468e629..fb876440cc8 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -21,6 +21,8 @@ ======================= ================= =========================================== {% endfor %} -Example:: +Example: + +.. code:: json {{example | jsonify(4, 4)}} From 71874870c8b66aa1f71247b74d9bb6453880e015 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 17:25:33 +0000 Subject: [PATCH 120/989] Enable syntax highlighting for example http requests --- scripts/css/codehighlight.css | 14 ++++++++++++-- .../matrix_templates/templates/http-api.tmpl | 4 +++- templating/matrix_templates/units.py | 12 +++++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/scripts/css/codehighlight.css b/scripts/css/codehighlight.css index 5c9b0c36f7d..fafc43f4758 100644 --- a/scripts/css/codehighlight.css +++ b/scripts/css/codehighlight.css @@ -1,6 +1,16 @@ pre.code .comment, code .comment { color: green } pre.code .keyword, code .keyword { color: darkred; font-weight: bold } pre.code .name.builtin, code .name.builtin { color: darkred; font-weight: bold } -pre.code .literal.number, code .literal.number { color: blue } pre.code .name.tag, code .name.tag { color: darkgreen } -pre.code .literal.string, code .literal.string { color: darkblue } +pre.code .literal, code .literal { color: darkblue } +pre.code .literal.number, code .literal.number { color: blue } + + +/* HTTP Methods have class "name function" */ +pre.code.http .name.function, code.http .name.function { color: black; font-weight: bold } +/* HTTP Paths have class "name namespace" */ +pre.code.http .name.namespace, code.http .name.namespace { color: darkgreen } +/* HTTP "HTTP" strings have class "keyword reserved" */ +pre.code.http .keyword.reserved, code.http .keyword.reserved { color: black; font-weight: bold } +/* HTTP Header names have class "name attribute" */ +pre.code.http .name.attribute, code.http .name.attribute { color: black; font-weight: bold } diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 2812cf7b5da..86eacb145c9 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -47,7 +47,9 @@ Response format: {% endfor %} {% endif -%} -Example request:: +Example request: + +.. code:: http {{endpoint.example.req | indent_block(2)}} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index c434214116c..be113b9c047 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -336,9 +336,15 @@ def _load_swagger_meta(self, api, group_name): elif param["in"] == "query": qps[param["name"]] = param["x-example"] query_string = "" if len(qps) == 0 else "?"+urllib.urlencode(qps) - endpoint["example"]["req"] = "%s %s%s\n%s" % ( - method.upper(), path_template, query_string, body - ) + if body: + endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % ( + method.upper(), path_template, query_string, body + ) + else: + endpoint["example"]["req"] = "%s %s%s HTTP/1.1\n\n" % ( + method.upper(), path_template, query_string + ) + else: self.log( "The following parameters are missing examples :( \n %s" % From 65066a76b36de8595bf5b67e075c631c8e77d5b1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 18:30:04 +0000 Subject: [PATCH 121/989] Add the m.tags event to a ``private_user_data`` key rather than including it under the ``ephemeral`` key --- api/client-server/v2_alpha/sync.yaml | 17 ++++++++++++++++- specification/modules/tags.rst | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 266f27bc4c7..3a1d79db6be 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -121,6 +121,14 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" + private_user_data: + title: Private User Data + type: object + description: |- + The private data that this user has attached to + this room. + allOf: + - $ref: "definitions/event_batch.json" invited: title: Invited type: object @@ -253,11 +261,18 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } ] + }, + "private_user_data": { + "events": [ + { + "type": "m.tags", + "content": {"tags": ["work"]} + } + ] } } }, diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 76fff125190..1a364a4346a 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -10,8 +10,8 @@ user that set them but are shared across all their devices. Events ------ -The tags on a room are passed as single ``m.tag`` event in the ``ephemeral`` -section of a v2 room sync. +The tags on a room are passed as single ``m.tag`` event in the +``private_user_data`` section of a room in v2 sync. {{m_tag_event}} From ece42688d089efbbe69e4d6c7b07bb2d878ec7b3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 26 Oct 2015 19:20:58 +0000 Subject: [PATCH 122/989] Clarify the interaction between full_state and timeout. --- api/client-server/v2_alpha/sync.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 5acbf583dfb..013084058c4 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -49,7 +49,9 @@ paths: If this is set to ``true``, then all state events will be returned, even if ``since`` is non-empty. The timeline will still be limited - by the ``since`` parameter. + by the ``since`` parameter. In this case, the ``timeout`` parameter + will be ignored and the query will return immediately, possibly with + an empty timeline. If ``false``, and ``since`` is non-empty, only state which has changed since the point indicated by ``since`` will be returned. From 81a60a25ccd78b413d5fc0a25e6267dcf05db23d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 14:46:10 +0000 Subject: [PATCH 123/989] Update 3pid spec based on new implementation --- event-schemas/examples/v1/m.room.member | 15 ---------- event-schemas/schema/v1/m.room.member | 20 ++----------- specification/modules/third_party_invites.rst | 29 ++++++++++--------- 3 files changed, 18 insertions(+), 46 deletions(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index 2c174e8a6e2..d7605dcd199 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -4,21 +4,6 @@ "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", "displayname": "Alice Margatroid", - "third_party_invite": { - "token": "pc98", - "public_key": "abc123", - "key_validity_url": "https://magic.forest/verifykey", - "signed": { - "mxid": "@alice:localhost", - "token": "pc98", - "signatures": { - "magic.forest": { - "ed25519:0": "poi098" - } - } - }, - "sender": "@zun:zun.soft" - } }, "invite_room_state": [ { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 45f2ad70571..1d09a0dd875 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -26,18 +26,6 @@ "type": "object", "title": "Invite", "properties": { - "token": { - "type": "string", - "description": "A token which must be correctly signed, in order to join the room." - }, - "key_validity_url": { - "type": "string", - "description": "A URL which can be fetched, with querystring ``public_key=public_key``, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." - }, - "public_key": { - "type": "string", - "description": "A base64-encoded ed25519 key with which token must be signed." - }, "signed": { "type": "object", "title": "signed", @@ -57,13 +45,9 @@ } }, "required": ["mxid", "signatures", "token"] - }, - "sender": { - "type": "string", - "description": "The matrix user ID of the user who send the invite which is being used." } }, - "required": ["token", "key_validity_url", "public_key", "sender", "signed"] + "required": ["signed"] } }, "required": ["membership"] diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 85538c31484..140bab86db7 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -15,13 +15,15 @@ The homeserver asks the identity server whether a Matrix user ID is known for that identifier. If it is, an invite is simply issued for that user. If it is not, the homeserver asks the identity server to record the details of -the invitation, and to notify the client of this pending invitation if it gets +the invitation, and to notify the invitee's homeserver of this pending invitation if it gets a binding for this identifier in the future. The identity server returns a token -and public key to the homeserver. +and public key to the inviting homeserver. -If a client then tries to join the room in the future, it will be allowed to if -it presents both the token, and a signature of that token from the identity -server which can be verified with the public key. +When the invitee's homeserver receives the notification of the binding, it +should insert an ``m.room.member`` event into the room's graph for that user, +with ``content.membership`` = ``invite``, as well as a +``content.third_party_invite`` property whichi contains proof that the invitee +does indeed own that third party identifier. Events ------ @@ -39,9 +41,10 @@ Server behaviour All homeservers MUST verify the signature in the event's ``content.third_party_invite.signed`` object. -If a client of the current homeserver is joining by an -``m.room.third_party_invite``, that homesever MUST validate that the public -key used for signing is still valid, by checking ``key_validity_url``. It does +When a homeserver inserts an ``m.room.member`` ``invite`` event into the graph +because of an ``m.room.third_party_invite`` event, +that homesever MUST validate that the public +key used for signing is still valid, by checking ``key_validity_url`` from the ``m.room.third_party_invite``. It does this by making an HTTP GET request to ``key_validity_url``: .. TODO: Link to identity server spec when it exists @@ -91,16 +94,16 @@ For example: H1 asks the identity server for a binding to a Matrix user ID, and has none, so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, they are told about the - invite, and ask their homeserver, H3, to join the room. + When the third party user validates their identity, their homeserver, H3, + is notified, and attempts to issue an ``m.room.member`` event to participate + in the room. - H3 validates the signature in the event's - ``content.third_party_invite.signed`` object. + H3 validates the signature given to it by the identity server. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` property *and* check ``key_validity_url``. - Having validated these things, H1 writes the join event to the room, and H3 + Having validated these things, H1 writes the invite event to the room, and H3 begins participating in the room. H2 *must* accept this event. The reason that no other homeserver may reject the event based on checking From b92a0f2b4de47093b229e9f0521aec2bb008435c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 14:49:22 +0000 Subject: [PATCH 124/989] Remove extra trailing comma --- event-schemas/examples/v1/m.room.member | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index d7605dcd199..e2ca56685e8 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -3,7 +3,7 @@ "content": { "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", - "displayname": "Alice Margatroid", + "displayname": "Alice Margatroid" }, "invite_room_state": [ { From 745e60757768b296c3136b761b244e1167c32a2b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 28 Oct 2015 14:49:50 +0000 Subject: [PATCH 125/989] Use 'room' or 'room ID' instead of 'context' when describing federation protocol --- specification/server_server_api.rst | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 66367cb0a13..26e040ca2dc 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -15,9 +15,9 @@ There are three main kinds of communication that occur between home servers: Persisted Data Units (PDUs): These events are broadcast from one home server to any others that have - joined the same "context" (namely, a Room ID). They are persisted in + joined the same room (identified by Room ID). They are persisted in long-term storage and record the history of messages and state for a - context. + room. Like email, it is the responsibility of the originating server of a PDU to deliver that event to its recipient servers. However PDUs are signed @@ -26,7 +26,7 @@ Persisted Data Units (PDUs): Ephemeral Data Units (EDUs): These events are pushed between pairs of home servers. They are not - persisted and are not part of the history of a "context", nor does the + persisted and are not part of the history of a room, nor does the receiving home server have to reply to them. Queries: @@ -338,11 +338,11 @@ PDUs All PDUs have: -- An ID -- A context +- An ID to identify the PDU itself +- A room ID that it relates to - A declaration of their type -- A list of other PDU IDs that have been seen recently on that context - (regardless of which origin sent them) +- A list of other PDU IDs that have been seen recently in that room (regardless + of which origin sent them) Required PDU Fields @@ -351,7 +351,7 @@ Required PDU Fields ==================== ================== ======================================= Key Type Description ==================== ================== ======================================= -``context`` String Event context identifier +``context`` String Room identifier ``user_id`` String The ID of the user sending the PDU ``origin`` String DNS name of homeserver that created this PDU @@ -363,7 +363,7 @@ Required PDU Fields ``content`` Object The content of the PDU. ``prev_pdus`` List of (String, The originating homeserver, PDU ids and String, Object) hashes of the most recent PDUs the - Triplets homeserver was aware of for the context + Triplets homeserver was aware of for the room when it made this PDU ``depth`` Integer The maximum depth of the previous PDUs plus one @@ -440,7 +440,7 @@ keys exist to support this: EDUs ---- -EDUs, by comparison to PDUs, do not have an ID, a context, or a list of +EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of "previous" IDs. The only mandatory fields for these are the type, origin and destination home server names, and the actual nested content. @@ -491,23 +491,23 @@ Retrieves a given PDU from the server. The response will contain a single new Transaction, inside which will be the requested PDU. -To fetch all the state of a given context:: +To fetch all the state of a given room:: - GET .../state// + GET .../state// Response: JSON encoding of a single Transaction containing multiple PDUs -Retrieves a snapshot of the entire current state of the given context. The +Retrieves a snapshot of the entire current state of the given room. The response will contain a single Transaction, inside which will be a list of PDUs that encode the state. -To backfill events on a given context:: +To backfill events on a given room:: - GET .../backfill// + GET .../backfill// Query args: v, limit Response: JSON encoding of a single Transaction containing multiple PDUs Retrieves a sliding-window history of previous PDUs that occurred on the given -context. Starting from the PDU ID(s) given in the "v" argument, the PDUs that +room. Starting from the PDU ID(s) given in the "v" argument, the PDUs that preceded it are retrieved, up to a total number given by the "limit" argument. These are then returned in a new Transaction containing all of the PDUs. From 176f919fc89f5d01669aea17003aef8da903ce4e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 15:00:53 +0000 Subject: [PATCH 126/989] Show multiple examples where present --- .../v1/m.room.member#third_party_invite | 41 +++++++++++++++++++ templating/matrix_templates/sections.py | 2 +- .../matrix_templates/templates/events.tmpl | 4 +- templating/matrix_templates/units.py | 11 +++-- 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 event-schemas/examples/v1/m.room.member#third_party_invite diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/v1/m.room.member#third_party_invite new file mode 100644 index 00000000000..4ef571ec523 --- /dev/null +++ b/event-schemas/examples/v1/m.room.member#third_party_invite @@ -0,0 +1,41 @@ +{ + "age": 242352, + "content": { + "membership": "join", + "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "displayname": "Alice Margatroid", + "third_party_invite": { + "signed": { + "mxid": "@alice:localhost", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + }, + "token": "abc123" + } + } + }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], + "state_key": "@alice:localhost", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.member", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index e75a75af868..81b2bb3cc12 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -35,7 +35,7 @@ def _render_events(self, filterFn, sortFn): if not filterFn(event_name): continue sections.append(template.render( - example=examples[event_name], + examples=examples[event_name], event=schemas[event_name], title_kind=subtitle_title_char )) diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fb876440cc8..fbefe17f3c9 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -21,8 +21,10 @@ ======================= ================= =========================================== {% endfor %} -Example: +Example{% if examples | length > 1 %}s{% endif %}: +{% for example in examples %} .. code:: json {{example | jsonify(4, 4)}} +{% endfor %} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index c434214116c..d33ba81dfe8 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -495,9 +495,14 @@ def load_event_examples(self): if not filename.startswith("m."): continue with open(os.path.join(path, filename), "r") as f: - examples[filename] = json.loads(f.read()) - if filename == "m.room.message#m.text": - examples["m.room.message"] = examples[filename] + event_name = filename.split("#")[0] + example = json.loads(f.read()) + + examples[filename] = examples.get(event_name, []) + examples[filename].append(example) + if filename != event_name: + examples[event_name] = examples.get(event_name, []) + examples[event_name].append(example) return examples def load_event_schemas(self): From 810922bb381a0f7162941605ec631f00927071ef Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 15:06:44 +0000 Subject: [PATCH 127/989] Fix schema validator for multiple examples --- event-schemas/check_examples.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index e54d3a1cae6..b10b2d0efae 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -60,6 +60,8 @@ def check_example_dir(exampledir, schemadir): continue examplepath = os.path.join(root, filename) schemapath = examplepath.replace(exampledir, schemadir) + if schemapath.find("#") >= 0: + schemapath = schemapath[:schemapath.find("#")] try: check_example_file(examplepath, schemapath) except Exception as e: From 5b6f858802332e8fcee368e9db6cdfac7533c995 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 28 Oct 2015 20:28:49 +0000 Subject: [PATCH 128/989] Some initial notes by way of the remote join handshake; with several TODOs and unanswered questions --- specification/server_server_api.rst | 149 +++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 26e040ca2dc..f02391286c0 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -533,6 +533,150 @@ part of the path specifies the kind of query being made, and its query arguments have a meaning specific to that kind of query. The response is a JSON-encoded object whose meaning also depends on the kind of query. + +To join a room:: + + GET .../make_join// + Response: JSON encoding of a join proto-event + + PUT .../send_join// + Response: JSON encoding of the state of the room at the time of the event + +Performs the room join handshake. For more information, see "Joining Rooms" +below. + +Joining Rooms +------------- + +When a new user wishes to join room that the user's homeserver already knows +about, the homeserver can immediately determine if this is allowable by +inspecting the state of the room, and if it is acceptable, it can generate, +sign, and emit a new ``m.room.member`` state event adding the user into that +room. When the homeserver does not yet know about the room it cannot do this +directly. Instead, it must take a longer multi-stage handshaking process by +which it first selects a remote homeserver which is already participating in +that room, and uses it to assist in the joining procss. This is the remote join +handshake. + +This handshake involves the homeserver of the new member wishing to join +(referred to here as the "joining" server), the directory server hosting the +room alias the user is requesting to join with, and a homeserver where existing +room members are already present (referred to as the "resident" server). + +In summary, the remote join handshake consists of the joining server querying +the directory server for information about the room alias; receiving a room ID +and a list of join candidates. The joining server then requests information +about the current state of the room from one of the residents. It uses this +information to construct a ``m.room.member`` event which it finally sends to +a resident server. + +Conceptually these are three different roles of homeserver. In practice the +directory server is likely to be resident in the room, and so may be selected +by the joining server to be the assisting resident. Likewise, it is likely that +the joining server picks the same candidate resident for both phases of event +construction, though in principle any valid candidate may be used at each time. +Thus, any join handshake can potentially involve anywhere from two to four +homeservers, though most in practice will use just two. + +.. TODO-doc + - Consider drawing a request/response diagram here + +The first part of the handshake involves using the directory server to request +the room ID and join candidates. This is covered in more detail on the +directory server documentation, below. + +Once the joining server has the room ID and the join candidates, it then needs +to obtain enough of the current state of the room to fill in the required +fields of the ``m.room.member`` event. It obtains this by selecting a resident +from the candidate list, and requesting the ``make_join`` endpoint using a +``GET`` request, specifying the room ID and the user ID of the new member who +is attempting to join. + +The resident server replies to this request with a JSON-encoded object having a +single key called ``event``; within this is an object whose fields contain some +of the information that the joining server will need. Despite its name, this +object is not a full object; notably it does not need to be hashed or signed by +the assisting resident. The required fields are: + +==================== ======== ============ + Key Type Description +==================== ======== ============ +``type`` String The value ``m.room.member`` +``auth_events`` String An event-reference list containing the + authorization events that would allow this member + to join +``content`` Object The event content +``depth`` Integer (this field must be present but is ignored; it + may be 0) +``event_id`` String A new event ID specified by the assisting + resident +``origin`` String The name of the assisting resident homeserver +``origin_server_ts`` Integer A timestamp added by the resident homeserver +``prev_events`` List (TODO(paul): ? - I notice these can be blank) +``prev_state`` List (TODO(paul): ? - I notice these can be blank) +``room_id`` String The room ID of the room +``sender`` String The user ID of the joining member +``state_key`` String The user ID of the joining member +==================== ======== ============ + +The ``content`` field itself must be an object, containing: + +============== ====== ============ + Key Type Description +============== ====== ============ +``membership`` String The value ``join`` +============== ====== ============ + +The joining server now has sufficient information to construct the real join +event from these protoevent fields. It copies the values of most of them, +adding (or replacing) the following fields: + +==================== ======= ============ + Key Type Description +==================== ======= ============ +``event_id`` String A new event ID specified by the joining homeserver +``origin`` String The name of the joining homeserver +``origin_server_ts`` Integer A timestamp added by the joining homeserver +==================== ======= ============ + +.. TODO-spec + - Why does the protoevent have an event_id, only for the real event to ignore + it and specify a different one? We should definitely pick one or the other. + +This will be a true event, so the joining server should apply the event-signing +algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` +fields. + +To complete the join handshake, the joining server must now submit this new +event to an assisting resident, by using the ``send_join`` endpoint. This is +invoked using the room ID and the event ID of the new member event. + +The assisting resident then accepts this event into the room's event graph, and +responds to the joining server with the full set of state for the newly-joined +room. This is returned as a two-element list, whose first element is the +integer 200, and whose second element contains the following keys: + +.. TODO-spec + - This is likely an implementation bug; see SYN-490. This should probably + actually just return the object directly + +============== ===== ============ + Key Type Description +============== ===== ============ +``auth_chain`` List A list of events giving the authorization chain for this + join event +``state`` List A complete list of the prevailing state events at the + instant just before accepting the new ``m.room.member`` + event +============== ===== ============ + +.. TODO-spec + - (paul) I don't really understand why the full auth_chain events are given + here. What purpose does it serve expanding them out in full, when surely + they'll appear in the state anyway? + - (paul) the state seems to be entirely ignored by synapse, so I'm not really + sure what ought to be there. + Backfilling ----------- .. NOTE:: @@ -763,6 +907,7 @@ Querying directory information:: servers: list of strings giving the join candidates The list of join candidates is a list of server names that are likely to hold -the given room; these are servers that the requesting server may wish to try -joining with. This list may or may not include the server answering the query. +the given room; these are servers that the requesting server may wish to use as +assisting resident servers as part of the remote join handshake. This list may +or may not include the server answering the query. From 6c1df04b4b99e027ba0750af785a335b08133fc8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 29 Oct 2015 02:11:47 +0000 Subject: [PATCH 129/989] typo --- api/client-server/v2_alpha/filter.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 37a0a3aaac4..0c2761b7bab 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -56,7 +56,7 @@ paths: "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] }, - "emphemeral": { + "ephemeral": { "types": ["m.receipt", "m.typing"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] @@ -120,7 +120,7 @@ paths: "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] }, - "emphemeral": { + "ephemeral": { "types": ["m.receipt", "m.typing"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] From fa2fe2ddd9e290ff8e339be6666f496c8ba06e7c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Oct 2015 12:43:36 +0000 Subject: [PATCH 130/989] Add event context api swagger --- api/client-server/v1/event_context.yaml | 152 ++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 api/client-server/v1/event_context.yaml diff --git a/api/client-server/v1/event_context.yaml b/api/client-server/v1/event_context.yaml new file mode 100644 index 00000000000..808055eb313 --- /dev/null +++ b/api/client-server/v1/event_context.yaml @@ -0,0 +1,152 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Rooms API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/context/{eventId}": + get: + summary: Get events and state around the specified event. + description: |- + This API returns a number of events that happened just before and + after the specified event. This allows clients to get the context of + an event. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to get events from. + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventId + description: The event to get context around. + required: true + x-example: "!636q39766251:example.com" + - in: query + type: integer + name: limit + description: |- + The maximum number of events to return. Default: 10. + x-example: "3" + responses: + 200: + description: The events and state surrounding the requested event. + schema: + type: object + description: The events and state surrounding the requested event. + properties: + start: + type: string + description: |- + A token that can be used to paginate backwards with. + end: + type: string + description: |- + A token that can be used to paginate forwards with. + events_before: + type: array + description: |- + A list of room events that happened just before the + requested event. + items: + type: object + title: RoomEvent + events_after: + type: array + description: |- + A list of room events that happened just after the + requested event. + items: + type: object + title: RoomEvent + state: + type: array + description: |- + The state of the room at the last event returned. + items: + type: object + title: RoomEvent + examples: + application/json: |- + { + "end": "t29-57_2_0_2", + "events_after": [ + { + "age": 91911336, + "content": { + "body": "7", + "msgtype": "m.text" + }, + "event_id": "$14460306086CiUaL:localhost:8480", + "origin_server_ts": 1446030608551, + "room_id": "!sCDvXTtzjpiPxaqkkt:localhost:8480", + "type": "m.room.message", + "user_id": "@test:localhost:8480" + } + ], + "events_before": [ + { + "age": 91911903, + "content": { + "body": "5", + "msgtype": "m.text" + }, + "event_id": "$14460306074UYTlh:localhost:8480", + "origin_server_ts": 1446030607984, + "room_id": "!sCDvXTtzjpiPxaqkkt:localhost:8480", + "type": "m.room.message", + "user_id": "@test:localhost:8480" + } + ], + "start": "t27-54_2_0_2", + "state": [ + { + "age": 3123715284, + "content": { + "creator": "@test:localhost:8480" + }, + "event_id": "$14429988040dgQAE:localhost:8480", + "origin_server_ts": 1442998804603, + "room_id": "!sCDvXTtzjpiPxaqkkt:localhost:8480", + "state_key": "", + "type": "m.room.create", + "user_id": "@test:localhost:8480" + }, + { + "age": 2067105053, + "content": { + "avatar_url": "mxc://localhost:8480/tVWZTAIIfqtXMZZtmGCkVjTD#auto", + "displayname": "Bob2", + "membership": "join" + }, + "event_id": "$14440554144URDbf:localhost:8480", + "origin_server_ts": 1444055414834, + "replaces_state": "$14440552472PgiGk:localhost:8480", + "room_id": "!sCDvXTtzjpiPxaqkkt:localhost:8480", + "state_key": "@test:localhost:8480", + "type": "m.room.member", + "user_id": "@test:localhost:8480" + } + ] + } + 403: + description: > + You aren't a member of the room. From cddfc110edc9018e44ce1a3690f3f6431a0ea9b7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 29 Oct 2015 12:48:04 +0000 Subject: [PATCH 131/989] Review comments --- specification/modules/third_party_invites.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 140bab86db7..6d3f8f6a551 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -22,7 +22,7 @@ and public key to the inviting homeserver. When the invitee's homeserver receives the notification of the binding, it should insert an ``m.room.member`` event into the room's graph for that user, with ``content.membership`` = ``invite``, as well as a -``content.third_party_invite`` property whichi contains proof that the invitee +``content.third_party_invite`` property which contains proof that the invitee does indeed own that third party identifier. Events @@ -94,8 +94,8 @@ For example: H1 asks the identity server for a binding to a Matrix user ID, and has none, so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, their homeserver, H3, - is notified, and attempts to issue an ``m.room.member`` event to participate + When the third party user validates their identity, their homeserver H3 + is notified and attempts to issue an ``m.room.member`` event to participate in the room. H3 validates the signature given to it by the identity server. From d067e50af5990177199a6f3adb0617c2a8ee8836 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:38:33 +0000 Subject: [PATCH 132/989] Document the differences in event formatting between the v1 and v2 client APIs --- specification/events.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/specification/events.rst b/specification/events.rst index 7d64882bf7c..d2428b6bc2e 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -12,6 +12,41 @@ server-server and application-service APIs, and are described below. {{common_state_event_fields}} +Differences between /v1 and /v2 events +-------------------------------------- + +There are a few differences between how events are formatted for sending +between servers over federation and how they are formatted for sending between +a server and its clients. + +Additionally there are a few differences between the format of events in the +responses to client APIs with a /v1 prefix and responses APIs with a /v2 +prefix. + +Events in responses for APIs with the /v2 prefix are generated from an event +formatted for federation by: + +* Removing the following keys: + ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, + ``origin``, ``prev_state``. +* Adding an ``age`` to the ``unsigned`` object which gives the time in + milliseconds that has ellapsed since the event was sent. +* Adding a ``prev_content`` to the ``unsigned`` object if the event is + a ``state event`` which gives previous content of that state key. +* Adding a ``redacted_because`` to the ``unsigned`` object if the event was + redacted which gives the event that redacted it. +* Adding a ``transaction_id`` if the event was sent by the client requesting it. + +Events in responses for APIs with the /v1 prefix are generated from an event +formatted for the /v2 prefix by: + +* Moving the folling keys from the ``unsigned`` object to the top level event + object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. +* Removing the ``unsigned`` object. +* If the event was an ``m.room.member`` with ``membership`` set to ``invite`` + then adding a ``invite_room_state`` key to the top level event object. + + Size limits ----------- From 451801bf38159aacc5ee2e05d0230a5b89382712 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:40:05 +0000 Subject: [PATCH 133/989] Add an example of ``prev_content`` in ``unsigned`` to v2 /sync --- api/client-server/v2_alpha/sync.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 013084058c4..a2d5a2b892d 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -241,6 +241,9 @@ paths: "type": "m.room.member", "state_key": "@bob:example.com", "content": {"membership": "join"}, + "unsigned": { + "prev_content": {"membership": "invite"} + }, "origin_server_ts": 1417731086795 }, "$74686972643033:example.com": { From d297d83151cf44b785ca5d671ddf4bc8de5f4974 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:45:53 +0000 Subject: [PATCH 134/989] Mention that sender is renamed to user_id in v1 --- specification/events.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/events.rst b/specification/events.rst index d2428b6bc2e..a1aece1c41c 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -43,6 +43,7 @@ formatted for the /v2 prefix by: * Moving the folling keys from the ``unsigned`` object to the top level event object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. * Removing the ``unsigned`` object. +* Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From 91eb25b76d12b7936583e7a3c482d5b3c066245e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 15:45:46 +0000 Subject: [PATCH 135/989] Include the full schema for an http API in the docs by resolving references to other files --- templating/matrix_templates/units.py | 66 +++++++++++++++++++++------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index be113b9c047..74aee26cab9 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -28,7 +28,25 @@ STATE_EVENT = "core-event-schema/state_event.json" -def get_json_schema_object_fields(obj, enforce_title=False): +def resolve_references(path, schema): + if isinstance(schema, dict): + result = {} + for key, value in schema.items(): + if key == "$ref": + path = os.path.join(os.path.dirname(path), value) + with open(path) as f: + schema = json.load(f) + return resolve_references(path, schema) + else: + result[key] = resolve_references(path, value) + return result + elif isinstance(schema, list): + return [resolve_references(path, value) for value in schema] + else: + return schema + + +def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) if obj.get("type") != "object": @@ -36,9 +54,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): "get_json_schema_object_fields: Object %s isn't an object." % obj ) if enforce_title and not obj.get("title"): - raise Exception( - "get_json_schema_object_fields: Nested object %s doesn't have a title." % obj - ) + # Force a default titile of "NO_TITLE" to make it obvious in the + # specification output which parts of the schema are missing a title + obj["title"] = 'NO_TITLE' required_keys = obj.get("required") if not required_keys: @@ -73,9 +91,15 @@ def get_json_schema_object_fields(obj, enforce_title=False): "Object %s has no properties or parents." % obj ) if not props: # parents only + if include_parents: + if obj["title"] == "NO_TITLE" and parents[0].get("title"): + obj["title"] = parents[0].get("title") + props = parents[0].get("properties") + + if not props: return [{ "title": obj["title"], - "parent": parents[0]["$ref"], + "parent": parents[0].get("$ref"), "no-table": True }] @@ -91,7 +115,8 @@ def get_json_schema_object_fields(obj, enforce_title=False): if prop_val == "object": nested_object = get_json_schema_object_fields( props[key_name]["additionalProperties"], - enforce_title=True + enforce_title=True, + include_parents=include_parents, ) key = props[key_name]["additionalProperties"].get( "x-pattern", "string" @@ -103,8 +128,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): value_type = "{string: %s}" % prop_val else: nested_object = get_json_schema_object_fields( - props[key_name], - enforce_title=True + props[key_name], + enforce_title=True, + include_parents=include_parents, ) value_type = "{%s}" % nested_object[0]["title"] @@ -114,8 +140,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): # if the items of the array are objects then recurse if props[key_name]["items"]["type"] == "object": nested_object = get_json_schema_object_fields( - props[key_name]["items"], - enforce_title=True + props[key_name]["items"], + enforce_title=True, + include_parents=include_parents, ) value_type = "[%s]" % nested_object[0]["title"] tables += nested_object @@ -159,7 +186,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): class MatrixUnits(Units): - def _load_swagger_meta(self, api, group_name): + def _load_swagger_meta(self, filepath, api, group_name): endpoints = [] for path in api["paths"]: for method in api["paths"][path]: @@ -262,7 +289,10 @@ def _load_swagger_meta(self, api, group_name): if is_array_of_objects: req_obj = req_obj["items"] - req_tables = get_json_schema_object_fields(req_obj) + req_tables = get_json_schema_object_fields( + resolve_references(filepath, req_obj), + include_parents=True, + ) if req_tables > 1: for table in req_tables[1:]: @@ -379,7 +409,10 @@ def _load_swagger_meta(self, api, group_name): elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] - res_tables = get_json_schema_object_fields(schema) + res_tables = get_json_schema_object_fields( + resolve_references(filepath, schema), + include_parents=True, + ) for table in res_tables: if "no-table" not in table: endpoint["res_tables"].append(table) @@ -445,13 +478,16 @@ def load_swagger_apis(self): if not filename.endswith(".yaml"): continue self.log("Reading swagger API: %s" % filename) - with open(os.path.join(path, filename), "r") as f: + filepath = os.path.join(path, filename) + with open(filepath, "r") as f: # strip .yaml group_name = filename[:-5].replace("-", "_") if is_v2: group_name = "v2_" + group_name api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta(api, group_name) + api["__meta"] = self._load_swagger_meta( + filepath, api, group_name + ) apis[group_name] = api return apis From 9f4d81308d3eca3870f54c196335d2bb5176bd44 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 30 Oct 2015 16:21:50 +0000 Subject: [PATCH 136/989] Pull out separate invite_room_state example --- .../v1/m.room.member#invite_room_state | 30 +++++++++++++++++++ .../v1/m.room.member#third_party_invite | 16 ---------- 2 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 event-schemas/examples/v1/m.room.member#invite_room_state diff --git a/event-schemas/examples/v1/m.room.member#invite_room_state b/event-schemas/examples/v1/m.room.member#invite_room_state new file mode 100644 index 00000000000..e2ca56685e8 --- /dev/null +++ b/event-schemas/examples/v1/m.room.member#invite_room_state @@ -0,0 +1,30 @@ +{ + "age": 242352, + "content": { + "membership": "join", + "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "displayname": "Alice Margatroid" + }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], + "state_key": "@alice:localhost", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.member", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/v1/m.room.member#third_party_invite index 4ef571ec523..2457302ac89 100644 --- a/event-schemas/examples/v1/m.room.member#third_party_invite +++ b/event-schemas/examples/v1/m.room.member#third_party_invite @@ -16,22 +16,6 @@ } } }, - "invite_room_state": [ - { - "type": "m.room.name", - "state_key": "", - "content": { - "name": "Forest of Magic" - } - }, - { - "type": "m.room.join_rules", - "state_key": "", - "content": { - "join_rules": "invite" - } - } - ], "state_key": "@alice:localhost", "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", From b49472e3b09c385d6a3071c31bdfe425c16174f4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 16:52:34 +0000 Subject: [PATCH 137/989] Add private_user_data to v1 /initialSync --- api/client-server/v1/sync.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index d07e9399c5d..8517ae4d3a8 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -245,7 +245,12 @@ paths: "user_id": "@alice:localhost" } ], - "visibility": "private" + "visibility": "private", + "private_user_data": [{ + "type": "m.tag", + "content": {"tags": ["work"]}, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost" + }] } ] } @@ -332,6 +337,16 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." + private_user_data: + type: array + description: |- + The private data that this user has attached to + this room. + items: + title: Event + type: object + allOf: + - "$ref": "core-event-schema/event.json" required: ["room_id", "membership"] required: ["end", "rooms", "presence"] 404: From ad86426e9506a9d0807267acf50e617497ebbd96 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 16:55:18 +0000 Subject: [PATCH 138/989] Add private_user_data to v1 room /initialSync --- api/client-server/v1/rooms.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 9300d7d10e5..4627360319a 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -311,7 +311,12 @@ paths: "user_id": "@alice:example.com" } ], - "visibility": "private" + "visibility": "private", + "private_user_data": [{ + "type": "m.tag", + "content": {"tags": ["work"]}, + "room_id": "!636q39766251:example.com" + }] } schema: title: RoomInfo @@ -371,6 +376,15 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." + private_user_data: + type: array + description: |- + The private data that this user has attached to this room. + items: + title: Event + type: object + allOf: + - "$ref": "core-event-schema/event.json" required: ["room_id", "membership"] 403: description: > From f557e698606a2d33f61100cc63bd15a43d7494ba Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 10:13:47 +0000 Subject: [PATCH 139/989] Note that m.tag events can appear in v1 initialSync and /events as well as v2 sync. Only add the room_id for v1 /events since it is redundant in v1 /initialSync --- api/client-server/v1/sync.yaml | 3 +-- specification/modules/tags.rst | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 8517ae4d3a8..0f11a98f297 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,8 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]}, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost" + "content": {"tags": ["work"]} }] } ] diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 1a364a4346a..567e5085422 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -10,8 +10,13 @@ user that set them but are shared across all their devices. Events ------ -The tags on a room are passed as single ``m.tag`` event in the -``private_user_data`` section of a room in v2 sync. +The tags on a room are receieved as single ``m.tag`` event in the +``private_user_data`` section of a room in a v2 /sync. + +The ``m.tag`` can also be received in a v1 /events response or in the +``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +events appearing in v1 /events will have a ``room_id`` with the room +the tags are for. {{m_tag_event}} From 52f55e05422024e6701b8d211258e1b6a4011a00 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 13:31:25 +0000 Subject: [PATCH 140/989] Allow room tags to have asssociated content, and return that content in the m.tag events --- api/client-server/v2_alpha/tags.yaml | 17 ++++++++--------- event-schemas/examples/v1/m.tag | 6 +++--- event-schemas/schema/v1/m.tag | 7 ++++--- specification/modules/tags.rst | 3 +++ 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml index 91b81212474..009e6000ce8 100644 --- a/api/client-server/v2_alpha/tags.yaml +++ b/api/client-server/v2_alpha/tags.yaml @@ -49,16 +49,15 @@ paths: type: object properties: tags: - type: array - items: - type: string + title: Tags + type: object examples: application/json: |- { - "tags": [ - "work", - "pinned" - ] + "tags": { + "work": {"order": 1}, + "pinned": {} + } } "/user/{userId}/rooms/{roomId}/tags/{tag}": put: @@ -94,11 +93,11 @@ paths: name: body required: true description: |- - An empty JSON object. + Extra data for the tag, e.g. ordering. schema: type: object example: |- - {} + {"order": 1} responses: 200: description: diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag index 28431b73abf..1fd3158eda0 100644 --- a/event-schemas/examples/v1/m.tag +++ b/event-schemas/examples/v1/m.tag @@ -1,8 +1,8 @@ { "type": "m.tag", "content": { - "tags": [ - "work" - ] + "tags": { + "work": {} + } } } diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index 235d442d28f..2ec1a55f965 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -11,9 +11,10 @@ "type": "object", "properties": { "tags": { - "type": "array", - "items": { - "type": "string" + "type": "object", + "additionalProperties": { + "title": "Tag", + "type": "object" } } } diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 567e5085422..c2b65b3e17f 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -18,6 +18,9 @@ The ``m.tag`` can also be received in a v1 /events response or in the events appearing in v1 /events will have a ``room_id`` with the room the tags are for. +Each tag has an associated JSON object with information about the tag, e.g how +to order the tags. + {{m_tag_event}} Client Behaviour From 149890227aa62cfd0c01b94b0c532b7a5f2a1b36 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 13:35:11 +0000 Subject: [PATCH 141/989] Fix wording --- specification/modules/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index c2b65b3e17f..e9b4435e0f8 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -19,7 +19,7 @@ events appearing in v1 /events will have a ``room_id`` with the room the tags are for. Each tag has an associated JSON object with information about the tag, e.g how -to order the tags. +to order the rooms with a given tag. {{m_tag_event}} From 40fa339cf7790837f550cf1ac7fa580a8442ad80 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 2 Nov 2015 15:00:18 +0000 Subject: [PATCH 142/989] Draw a pretty (well at least I think it's pretty) ASCII diagram of the remote join handshake --- specification/server_server_api.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f02391286c0..66633376255 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -578,8 +578,23 @@ construction, though in principle any valid candidate may be used at each time. Thus, any join handshake can potentially involve anywhere from two to four homeservers, though most in practice will use just two. -.. TODO-doc - - Consider drawing a request/response diagram here +:: + + Client Joining Directory Resident + Server Server Server + + join request --> + | + directory request -------> + <---------- directory response + | + make_join request -----------------------> + <------------------------------- make_join response + | + send_join request -----------------------> + <------------------------------- send_join response + | + <---------- join response The first part of the handshake involves using the directory server to request the room ID and join candidates. This is covered in more detail on the From f6c55979e0cdf6881ca4211745b9fc5cd6f6fbb7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 2 Nov 2015 15:17:18 +0000 Subject: [PATCH 143/989] Remove TODO comment about SYN-490 as it's unlikely to matter for v1; we'll fix it in v2 --- specification/server_server_api.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 66633376255..d26cc26ec2c 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -671,10 +671,6 @@ responds to the joining server with the full set of state for the newly-joined room. This is returned as a two-element list, whose first element is the integer 200, and whose second element contains the following keys: -.. TODO-spec - - This is likely an implementation bug; see SYN-490. This should probably - actually just return the object directly - ============== ===== ============ Key Type Description ============== ===== ============ From 2e3a0b4e00798b672fe3282e24cc1e1c34cd8b0b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 2 Nov 2015 15:26:06 +0000 Subject: [PATCH 144/989] Specify guest accounts --- api/client-server/v2_alpha/registration.yaml | 17 +++++++++++++++++ templating/build.py | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml index 2bd86e73010..681654ae72e 100644 --- a/api/client-server/v2_alpha/registration.yaml +++ b/api/client-server/v2_alpha/registration.yaml @@ -17,7 +17,24 @@ paths: summary: Register for an account on this homeserver. description: |- Register for an account on this homeserver. + + There are two kinds of user account: + + - `user` accounts. These accounts may use the full API described in this specification. + + - `guest` accounts. These accounts may have limited permissions and may not be supported by all servers. + parameters: + - in: query + name: kind + type: string + x-example: guest + required: false + default: user + enum: + - guest + - user + description: The kind of account to register. Defaults to `user`. - in: body name: body schema: diff --git a/templating/build.py b/templating/build.py index b91e1da209f..a35d8a0853c 100755 --- a/templating/build.py +++ b/templating/build.py @@ -84,6 +84,13 @@ def wrap(input, wrap=80, initial_indent=""): input_lines = input.split('\n\n') wrapper = TextWrapper(initial_indent=initial_indent, width=wrap) output_lines = [wrapper.fill(line) for line in input_lines] + + for i in range(len(output_lines)): + line = output_lines[i] + in_bullet = line.startswith("- ") + if in_bullet: + output_lines[i] = line.replace("\n", "\n " + initial_indent) + return '\n\n'.join(output_lines) # make Jinja aware of the templates and filters From e9d361841b8be0f3f4edc2e042439cc77c57225f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:42:58 +0000 Subject: [PATCH 145/989] Fix tag examples --- api/client-server/v1/rooms.yaml | 3 +-- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 4627360319a..e7b93ad9552 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,8 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]}, - "room_id": "!636q39766251:example.com" + "content": {"tags": {"work": {}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 0f11a98f297..89b9400e97c 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]} + "content": {"tags": {"work": {}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 3a1d79db6be..b5465601cb1 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -270,7 +270,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": ["work"]} + "content": {"tags": {"work": {}}} } ] } From bcb8fac53c3b6beb47777148a387604d11c71a7d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:46:01 +0000 Subject: [PATCH 146/989] Add a description for the tag event --- event-schemas/schema/v1/m.tag | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index 2ec1a55f965..a36c6a45023 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -12,13 +12,14 @@ "properties": { "tags": { "type": "object", + "description": "The tags on the room and their contents.", "additionalProperties": { - "title": "Tag", + "title": "Tag Contents", "type": "object" } } } } }, - "required": ["type", "content"] + "required": ["type", "content"] } From fba3c04e42113e4d3a079b9945abe737f68dc154 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:48:30 +0000 Subject: [PATCH 147/989] Apparently the spec generator breaks if the title in a schema is too long --- event-schemas/schema/v1/m.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index a36c6a45023..4c5b4fa53d0 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -14,7 +14,7 @@ "type": "object", "description": "The tags on the room and their contents.", "additionalProperties": { - "title": "Tag Contents", + "title": "Tag", "type": "object" } } From 3953006792cca87800af18076e715715322c830b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:54:33 +0000 Subject: [PATCH 148/989] Fix spelling --- specification/modules/tags.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index e9b4435e0f8..3d45975c42c 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -4,13 +4,13 @@ Room Tagging .. _module:tagging: Users can add tags to rooms. Tags are short strings used to label rooms, e.g. -"work", "familly". A room may have multiple tags. Tags are only visible to the +"work", "family". A room may have multiple tags. Tags are only visible to the user that set them but are shared across all their devices. Events ------ -The tags on a room are receieved as single ``m.tag`` event in the +The tags on a room are received as single ``m.tag`` event in the ``private_user_data`` section of a room in a v2 /sync. The ``m.tag`` can also be received in a v1 /events response or in the From d53814097f11ca5f24bb6b1af0b1b9863243f173 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:57:06 +0000 Subject: [PATCH 149/989] Add example content to the tags in example tag events --- api/client-server/v1/rooms.yaml | 2 +- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- event-schemas/examples/v1/m.tag | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index e7b93ad9552..9b8b930f866 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,7 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 89b9400e97c..b52908a99ec 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index b5465601cb1..4c1a7601bad 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -270,7 +270,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} } ] } diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag index 1fd3158eda0..00e81060b07 100644 --- a/event-schemas/examples/v1/m.tag +++ b/event-schemas/examples/v1/m.tag @@ -2,7 +2,7 @@ "type": "m.tag", "content": { "tags": { - "work": {} + "work": {"order": 1} } } } From 30a4f17420f1c6b21648c7e5f729fb4d84468d54 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 3 Nov 2015 16:15:36 +0000 Subject: [PATCH 150/989] Don't mark 'aliases' on m.room.canonical_alias as required, because it isn't. --- event-schemas/schema/v1/m.room.canonical_alias | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/v1/m.room.canonical_alias index 25cd00c0868..49e0e669a52 100644 --- a/event-schemas/schema/v1/m.room.canonical_alias +++ b/event-schemas/schema/v1/m.room.canonical_alias @@ -13,8 +13,7 @@ "type": "string", "description": "The canonical alias." } - }, - "required": ["alias"] + } }, "state_key": { "type": "string", From ebc02371095be8789051ac0a47de99803f961606 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 19:35:44 +0000 Subject: [PATCH 151/989] Add the missing titles to the schema --- .../v1/definitions/push_rule.json | 3 +- .../v1/definitions/push_ruleset.json | 37 +++++++++++-------- .../v2_alpha/definitions/event_batch.json | 1 + .../schema/v1/core-event-schema/event.json | 1 + .../v1/core-event-schema/state_event.json | 1 + event-schemas/schema/v1/m.room.member | 1 + 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/api/client-server/v1/definitions/push_rule.json b/api/client-server/v1/definitions/push_rule.json index 4df93f67b39..d10f11f8e74 100644 --- a/api/client-server/v1/definitions/push_rule.json +++ b/api/client-server/v1/definitions/push_rule.json @@ -1,4 +1,5 @@ { + "title": "PushRule", "type": "object", "properties": { "default": { @@ -17,4 +18,4 @@ "type": "array" } } -} \ No newline at end of file +} diff --git a/api/client-server/v1/definitions/push_ruleset.json b/api/client-server/v1/definitions/push_ruleset.json index e03727012aa..529ebeed091 100644 --- a/api/client-server/v1/definitions/push_ruleset.json +++ b/api/client-server/v1/definitions/push_ruleset.json @@ -1,60 +1,65 @@ { - "type": "object", + "type": "object", "properties": { "content": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "override": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "sender": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "underride": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "room": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" } } -} \ No newline at end of file +} diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index 75762d7585a..395aed13d75 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -5,6 +5,7 @@ "type": "array", "description": "List of events", "items": { + "title": "Event", "type": "object" } } diff --git a/event-schemas/schema/v1/core-event-schema/event.json b/event-schemas/schema/v1/core-event-schema/event.json index e73aec809cc..f9103715e66 100644 --- a/event-schemas/schema/v1/core-event-schema/event.json +++ b/event-schemas/schema/v1/core-event-schema/event.json @@ -5,6 +5,7 @@ "properties": { "content": { "type": "object", + "title": "EventContent", "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body." }, "type": { diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/v1/core-event-schema/state_event.json index 88b4900ae3f..5809cf7f6b8 100644 --- a/event-schemas/schema/v1/core-event-schema/state_event.json +++ b/event-schemas/schema/v1/core-event-schema/state_event.json @@ -11,6 +11,7 @@ "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. The key MUST NOT start with '_'." }, "prev_content": { + "title": "EventContent", "type": "object", "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing." } diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 45f2ad70571..0bd8cd1e9a7 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -8,6 +8,7 @@ "properties": { "content": { "type": "object", + "title": "EventContent", "properties": { "membership": { "type": "string", From 8322151661f8c8a7cb6cfa3fe1d22ab83ce55642 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 19:42:49 +0000 Subject: [PATCH 152/989] Don't put a space when appending the "Must be" strings to the desciption if there isn't a description, otherwise it will mess up the indent --- templating/matrix_templates/units.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 74aee26cab9..9aa9489aa1e 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -164,12 +164,16 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" + if desc: + desc += " " desc += ( - " One of: %s" % json.dumps(props[key_name]["enum"]) + "One of: %s" % json.dumps(props[key_name]["enum"]) ) else: + if desc: + desc += " " desc += ( - " Must be '%s'." % props[key_name]["enum"][0] + "Must be '%s'." % props[key_name]["enum"][0] ) if isinstance(value_type, list): value_type = " or ".join(value_type) From e49ea9015f4ee5f86b2d401190ec39517c5ae49b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:39:36 +0000 Subject: [PATCH 153/989] Deduplicate tables with the same title --- templating/matrix_templates/units.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 9aa9489aa1e..9cc68f49661 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -185,7 +185,17 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "desc": desc, "req_str": "**Required.** " if required else "" }) - return tables + + titles = set() + filtered = [] + for table in tables: + if table.get("title") in titles: + continue + + titles.add(table.get("title")) + filtered.append(table) + + return filtered class MatrixUnits(Units): From 8070489080afdc08179087c64b1f6a4a261c224d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:44:20 +0000 Subject: [PATCH 154/989] Handle lists of types in arrays --- templating/matrix_templates/units.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 9cc68f49661..cec5b83ec82 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -147,7 +147,10 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals value_type = "[%s]" % nested_object[0]["title"] tables += nested_object else: - value_type = "[%s]" % props[key_name]["items"]["type"] + value_type = props[key_name]["items"]["type"] + if isinstance(value_type, list): + value_type = " or ".join(value_type) + value_type = "[%s]" % value_type array_enums = props[key_name]["items"].get("enum") if array_enums: if len(array_enums) > 1: From c7199463704709ca4c0dc8a3b60572210f464c84 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:51:50 +0000 Subject: [PATCH 155/989] Enable syntax highlighting for message type examples --- templating/matrix_templates/templates/msgtypes.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index f78624517c5..18d3492bc85 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -18,6 +18,8 @@ ================== ================= =========================================== {% endfor %} -Example:: +Example: + +.. code:: json {{example | jsonify(4, 4)}} From 2fd5fc39a032dbce80a3aba4361f79f553cc5b85 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 Nov 2015 10:55:11 +0000 Subject: [PATCH 156/989] Add spec for calculating display names for rooms and users Merged from https://github.com/matrix-org/matrix-doc/pull/145 --- specification/modules/instant_messaging.rst | 125 +++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 09fcb843d35..7a2142dd7cc 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -116,12 +116,58 @@ of the client-server API will resolve this by attaching the transaction ID of th sending request to the event itself. +Calculating the display name for a user +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clients may wish to show the human-readable display name of a room member as +part of a membership list, or when they send a message. However, different +members may have conflicting display names. Display names MUST be disambiguated +before showing them to the user, in order to prevent spoofing of other users. + +To ensure this is done consistently across clients, clients SHOULD use the +following algorithm to calculate a disambiguated display name for a given user: + +1. Inspect the ``m.room.member`` state event for the relevant user id. +2. If the ``m.room.member`` state event has no ``displayname`` field, or if + that field has a ``null`` value, use the raw user id as the display + name. Otherwise: +3. If the ``m.room.member`` event has a ``displayname`` which is unique among + members of the room with ``membership: join`` or ``membership: invite``, use + the given ``displayname`` as the user-visible display name. Otherwise: +4. The ``m.room.member`` event has a non-unique ``displayname``. This should be + disambiguated using the user id, for example "display name + (@id:homeserver.org)". + +Developers should take note of the following when implementing the above +algorithm: + +* The user-visible display name of one member can be affected by changes in the + state of another member. For example, if ``@user1:matrix.org`` is present in + a room, with ``displayname: Alice``, then when ``@user2:example.com`` joins + the room, also with ``displayname: Alice``, *both* users must be given + disambiguated display names. Similarly, when one of the users then changes + their display name, there is no longer a clash, and *both* users can be given + their chosen display name. Clients should be alert to this possibility and + ensure that all affected users are correctly renamed. + +* The display name of a room may also be affected by changes in the membership + list. This is due to the room name sometimes being based on user display + names (see `Calculating the display name for a room`_). + +* If the entire membership list is searched for clashing display names, this + leads to an O(N^2) implementation for building the list of room members. This + will be very inefficient for rooms with large numbers of members. It is + recommended that client implementations maintain a hash table mapping from + ``displayname`` to a list of room members using that name. Such a table can + then be used for efficient calculation of whether disambiguation is needed. + + Displaying membership information with messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Clients may wish to show the display name and avatar URL of the room member who sent a message. This can be achieved by inspecting the ``m.room.member`` state -event for that user ID. +event for that user ID (see `Calculating the display name for a user`_). When a user paginates the message history, clients may wish to show the **historical** display name and avatar URL for a room member. This is possible @@ -133,6 +179,83 @@ events update the old state. When paginated events are processed sequentially, the old state represents the state of the room *at the time the event was sent*. This can then be used to set the historical display name and avatar URL. + +Calculating the display name for a room +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clients may wish to show a human-readable name for a room. There are a number +of possibilities for choosing a useful name. To ensure that rooms are named +consistently across clients, clients SHOULD use the following algorithm to +choose a name: + +1. If the room has an `m.room.name`_ state event, use the name given by that + event. + +#. If the room has an `m.room.canonical_alias`_ state event, use the alias + given by that event. + +#. If neither of the above events are present, a name should be composed based + on the members of the room. Clients should consider `m.room.member`_ events + for users other than the logged-in user, with ``membership: join`` or + ``membership: invite``. + + .. _active_members: + + i. If there is only one such event, the display name for the room should be + the `disambiguated display name`_ of the corresponding user. + + #. If there are two such events, they should be lexicographically sorted by + their ``state_key`` (i.e. the corresponding user IDs), and the display + name for the room should be the `disambiguated display name`_ of both + users: " and ", or a localised variant thereof. + + #. If there are three or more such events, the display name for the room + should be based on the disambiguated display name of the user + corresponding to the first such event, under a lexicographical sorting + according to their ``state_key``. The display name should be in the + format " and others" (or a localised variant thereof), where N + is the number of `m.room.member`_ events with ``membership: join`` or + ``membership: invite``, excluding the logged-in user and "user1". + + For example, if Alice joins a room, where Bob (whose user id is + ``@superuser:example.com``), Carol (user id ``@carol:example.com``) and + Dan (user id ``@dan:matrix.org``) are in conversation, Alice's + client should show the room name as "Carol and 2 others". + + .. TODO-spec + Sorting by user_id certainly isn't ideal, as IDs at the start of the + alphabet will end up dominating room names: they will all be called + "Arathorn and 15 others". Furthermore - user_ids are not necessarily + ASCII, which means we need to either specify a collation order, or specify + how to choose one. + + Ideally we might sort by the time when the user was first invited to, or + first joined the room. But we don't have this information. + +#. If the room has no ``m.room.name`` or ``m.room.canonical_alias`` events, and + no active members other than the current user, clients should consider + ``m.room.member`` events with ``membership: leave``. If such events exist, a + display name such as "Empty room (was and others)" (or a + localised variant thereof) should be used, following similar rules as for + active members (see `above `_). + +#. A complete absence of ``m.room.name``, ``m.room.canonical_alias``, and + ``m.room.member`` events is likely to indicate a problem with creating the + room or synchronising the state table; however clients should still handle + this situation. A display name such as "Empty room" (or a localised variant + thereof) should be used in this situation. + +.. _`disambiguated display name`: `Calculating the display name for a user`_ + +Clients SHOULD NOT use `m.room.aliases`_ events as a source for room names, as +it is difficult for clients to agree on the best alias to use, and aliases can +change unexpectedly. + +.. TODO-spec + How can we make this less painful for clients to implement, without forcing + an English-language implementation on them all? + + Server behaviour ---------------- From 8cba11b1cd6f70906aa3f8ce03d4332d469af13f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 Nov 2015 11:06:31 +0000 Subject: [PATCH 157/989] Add some links to spec bugs for display names Just added a couple of TODO comments to useful jira bugs --- specification/modules/instant_messaging.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 7a2142dd7cc..a58c762f387 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -138,6 +138,11 @@ following algorithm to calculate a disambiguated display name for a given user: disambiguated using the user id, for example "display name (@id:homeserver.org)". + .. TODO-spec + what does it mean for a ``displayname`` to be 'unique'? Are we + case-sensitive? Do we care about homograph attacks? See + https://matrix.org/jira/browse/SPEC-221. + Developers should take note of the following when implementing the above algorithm: @@ -232,6 +237,8 @@ choose a name: Ideally we might sort by the time when the user was first invited to, or first joined the room. But we don't have this information. + See https://matrix.org/jira/browse/SPEC-267 for further discussion. + #. If the room has no ``m.room.name`` or ``m.room.canonical_alias`` events, and no active members other than the current user, clients should consider ``m.room.member`` events with ``membership: leave``. If such events exist, a From 161441fa3ae83db985c51c9f1c8a4cb96494bd45 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 18:11:20 +0000 Subject: [PATCH 158/989] Update 3pid spec based on new implementation --- api/client-server/v1/membership.yaml | 110 +-------------- .../v1/third_party_membership.yaml | 127 ++++++++++++++++++ event-schemas/examples/v1/m.room.member | 17 +-- event-schemas/schema/v1/m.room.member | 20 +-- specification/modules/anonymous_access.rst | 50 +++++++ specification/modules/third_party_invites.rst | 85 ++++++++---- 6 files changed, 246 insertions(+), 163 deletions(-) create mode 100644 api/client-server/v1/third_party_membership.yaml create mode 100644 specification/modules/anonymous_access.rst diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index f8dfdea5554..c089e154a9a 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -73,9 +73,12 @@ paths: post: summary: Invite a user to participate in a particular room. description: |- + .. _invite-by-user-id-endpoint: + *Note that there are two forms of this API, which are documented separately. This version of the API requires that the inviter knows the Matrix - identifier of the invitee.* + identifier of the invitee. The other is documented in the* + `third party invites section`_. This API invites a user to participate in a particular room. They do not start participating in the room until they actually join the @@ -86,6 +89,8 @@ paths: If the user was invited to the room, the home server will append a ``m.room.member`` event to the room. + + .. _third party invites section: `invite-by-third-party-id-endpoint`_ security: - accessToken: [] parameters: @@ -132,106 +137,3 @@ paths: description: This request was rate-limited. schema: "$ref": "definitions/error.yaml" - - "/rooms/{roomId}/invite": - post: - summary: Invite a user to participate in a particular room. - description: |- - *Note that there are two forms of this API, which are documented separately. - This version of the API does not require that the inviter know the Matrix - identifier of the invitee, and instead relies on third party identifiers. - The homeserver uses an identity server to perform the mapping from - third party identifier to a Matrix identifier.* - - This API invites a user to participate in a particular room. - They do not start participating in the room until they actually join the - room. - - Only users currently in a particular room can invite other users to - join that room. - - If the identity server did know the Matrix user identifier for the - third party identifier, the home server will append a ``m.room.member`` - event to the room. - - If the identity server does not know a Matrix user identifier for the - passed third party identifier, the homeserver will issue an invitation - which can be accepted upon providing proof of ownership of the third - party identifier. This is achieved by the identity server generating a - token, which it gives to the inviting homeserver. The homeserver will - add an ``m.room.third_party_invite`` event into the graph for the room, - containing that token. - - When the invitee binds the invited third party identifier to a Matrix - user ID, the identity server will give the user a list of pending - invitations, each containing: - - - The room ID to which they were invited - - - The token given to the homeserver - - - A signature of the token, signed with the identity server's private key - - - The matrix user ID who invited them to the room - - If a token is requested from the identity server, the home server will - append a ``m.room.third_party_invite`` event to the room. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room identifier (not alias) to which to invite the user. - required: true - x-example: "!d41d8cd:matrix.org" - - in: body - name: body - required: true - schema: - type: object - example: |- - { - "id_server": "matrix.org", - "medium": "email", - "address": "cheeky@monkey.com", - "display_name": "A very cheeky monkey" - } - properties: - id_server: - type: string - description: The hostname+port of the identity server which should be used for third party identifier lookups. - medium: - type: string - # TODO: Link to identity service spec when it eixsts - description: The kind of address being passed in the address field, for example ``email``. - address: - type: string - description: The invitee's third party identifier. - display_name: - type: string - description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. - required: ["id_server", "medium", "address", "display_name"] - responses: - 200: - description: The user has been invited to join the room. - examples: - application/json: |- - {} - schema: - type: object - 403: - description: |- - You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: - - - The invitee has been banned from the room. - - The invitee is already a member of the room. - - The inviter is not currently in the room. - - The inviter's power level is insufficient to invite users to the room. - examples: - application/json: |- - {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} - 429: - description: This request was rate-limited. - schema: - "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/v1/third_party_membership.yaml new file mode 100644 index 00000000000..90b167b443a --- /dev/null +++ b/api/client-server/v1/third_party_membership.yaml @@ -0,0 +1,127 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Membership API for third party identifiers" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/invite": + post: + summary: Invite a user to participate in a particular room. + description: |- + .. _invite-by-third-party-id-endpoint: + + *Note that there are two forms of this API, which are documented separately. + This version of the API does not require that the inviter know the Matrix + identifier of the invitee, and instead relies on third party identifiers. + The homeserver uses an identity server to perform the mapping from + third party identifier to a Matrix identifier. The other is documented in the* + `joining rooms section`_. + + This API invites a user to participate in a particular room. + They do not start participating in the room until they actually join the + room. + + Only users currently in a particular room can invite other users to + join that room. + + If the identity server did know the Matrix user identifier for the + third party identifier, the home server will append a ``m.room.member`` + event to the room. + + If the identity server does not know a Matrix user identifier for the + passed third party identifier, the homeserver will issue an invitation + which can be accepted upon providing proof of ownership of the third + party identifier. This is achieved by the identity server generating a + token, which it gives to the inviting homeserver. The homeserver will + add an ``m.room.third_party_invite`` event into the graph for the room, + containing that token. + + When the invitee binds the invited third party identifier to a Matrix + user ID, the identity server will give the user a list of pending + invitations, each containing: + + - The room ID to which they were invited + + - The token given to the homeserver + + - A signature of the token, signed with the identity server's private key + + - The matrix user ID who invited them to the room + + If a token is requested from the identity server, the home server will + append a ``m.room.third_party_invite`` event to the room. + + .. _joining rooms section: `invite-by-user-id-endpoint`_ + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) to which to invite the user. + required: true + x-example: "!d41d8cd:matrix.org" + - in: body + name: body + required: true + schema: + type: object + example: |- + { + "id_server": "matrix.org", + "medium": "email", + "address": "cheeky@monkey.com", + "display_name": "A very cheeky monkey" + } + properties: + id_server: + type: string + description: The hostname+port of the identity server which should be used for third party identifier lookups. + medium: + type: string + # TODO: Link to identity service spec when it eixsts + description: The kind of address being passed in the address field, for example ``email``. + address: + type: string + description: The invitee's third party identifier. + display_name: + type: string + description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. + required: ["id_server", "medium", "address", "display_name"] + responses: + 200: + description: The user has been invited to join the room. + examples: + application/json: |- + {} + schema: + type: object + 403: + description: |- + You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index 2c174e8a6e2..e2ca56685e8 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -3,22 +3,7 @@ "content": { "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", - "displayname": "Alice Margatroid", - "third_party_invite": { - "token": "pc98", - "public_key": "abc123", - "key_validity_url": "https://magic.forest/verifykey", - "signed": { - "mxid": "@alice:localhost", - "token": "pc98", - "signatures": { - "magic.forest": { - "ed25519:0": "poi098" - } - } - }, - "sender": "@zun:zun.soft" - } + "displayname": "Alice Margatroid" }, "invite_room_state": [ { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 0bd8cd1e9a7..81057049c2e 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -27,18 +27,6 @@ "type": "object", "title": "Invite", "properties": { - "token": { - "type": "string", - "description": "A token which must be correctly signed, in order to join the room." - }, - "key_validity_url": { - "type": "string", - "description": "A URL which can be fetched, with querystring ``public_key=public_key``, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." - }, - "public_key": { - "type": "string", - "description": "A base64-encoded ed25519 key with which token must be signed." - }, "signed": { "type": "object", "title": "signed", @@ -58,13 +46,9 @@ } }, "required": ["mxid", "signatures", "token"] - }, - "sender": { - "type": "string", - "description": "The matrix user ID of the user who send the invite which is being used." } }, - "required": ["token", "key_validity_url", "public_key", "sender", "signed"] + "required": ["signed"] } }, "required": ["membership"] diff --git a/specification/modules/anonymous_access.rst b/specification/modules/anonymous_access.rst new file mode 100644 index 00000000000..5a4211875e6 --- /dev/null +++ b/specification/modules/anonymous_access.rst @@ -0,0 +1,50 @@ +Guest access +================ + +.. _module:guest-access: + +It may be desirable to allow users without a fully registered user account to +ephemerally access Matrix rooms. This module specifies limited ways of doing so. + +Note that this is not currently a complete anonymous access solution; in +particular, it only allows servers to provided anonymous access to rooms in +which they are already participating, and relies on individual homeservers to +adhere to the conventions which this module sets, rather than allowing all +participating homeservers to enforce them. + +Events +------ + +{{m_room_guest_accessibility}} + +Client behaviour +---------------- +A client can register for guest access using the FOO endpoint. From that point +on, they can interact with a limited subset of the existing client-server API, +as if they were a fully registered user, using the access token granted to them +by the server. + +These users are only allowed to make calls in relation to rooms which have the +``m.room.history_visibility`` event set to ``world_readable``. + +The APIs they are allowed to hit are: + +/rooms/{roomId}/messages +/rooms/{roomId}/state +/rooms/{roomId}/state/{eventType}/{stateKey} +/events + +Server behaviour +---------------- +Does the server need to handle any of the new events in a special way (e.g. +typing timeouts, presence). Advice on how to persist events and/or requests are +recommended to aid implementation. Federation-specific logic should be included +here. + +Security considerations +----------------------- +This includes privacy leaks: for example leaking presence info. How do +misbehaving clients or servers impact this module? This section should always be +included, if only to say "we've thought about it but there isn't anything to do +here". + diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 85538c31484..4d268631d8c 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -1,27 +1,30 @@ Third party invites =================== -.. _module:third_party_invites: +.. _module:third-party-invites: This module adds in support for inviting new members to a room where their Matrix user ID is not known, instead addressing them by a third party identifier such as an email address. - There are two flows here; one if a Matrix user ID is known for the third party identifier, and one if not. Either way, the client calls ``/invite`` with the details of the third party identifier. The homeserver asks the identity server whether a Matrix user ID is known for -that identifier. If it is, an invite is simply issued for that user. +that identifier: + +- If it is, an invite is simply issued for that user. -If it is not, the homeserver asks the identity server to record the details of -the invitation, and to notify the client of this pending invitation if it gets -a binding for this identifier in the future. The identity server returns a token -and public key to the homeserver. +- If it is not, the homeserver asks the identity server to record the details of + the invitation, and to notify the invitee's homeserver of this pending invitation if it gets + a binding for this identifier in the future. The identity server returns a token + and public key to the inviting homeserver. -If a client then tries to join the room in the future, it will be allowed to if -it presents both the token, and a signature of that token from the identity -server which can be verified with the public key. +When the invitee's homeserver receives the notification of the binding, it +should insert an ``m.room.member`` event into the room's graph for that user, +with ``content.membership`` = ``invite``, as well as a +``content.third_party_invite`` property which contains proof that the invitee +does indeed own that third party identifier. Events ------ @@ -33,15 +36,18 @@ Client behaviour A client asks a server to invite a user by their third party identifier. +{{third_party_membership_http_api}} + Server behaviour ---------------- All homeservers MUST verify the signature in the event's ``content.third_party_invite.signed`` object. -If a client of the current homeserver is joining by an -``m.room.third_party_invite``, that homesever MUST validate that the public -key used for signing is still valid, by checking ``key_validity_url``. It does +When a homeserver inserts an ``m.room.member`` ``invite`` event into the graph +because of an ``m.room.third_party_invite`` event, +that homesever MUST validate that the public +key used for signing is still valid, by checking ``key_validity_url`` from the ``m.room.third_party_invite``. It does this by making an HTTP GET request to ``key_validity_url``: .. TODO: Link to identity server spec when it exists @@ -84,24 +90,24 @@ membership is questionable. For example: - If room R has two participating homeservers, H1, H2 +#. Room R has two participating homeservers, H1, H2 - And user A on H1 invites a third party identifier to room R +#. User A on H1 invites a third party identifier to room R - H1 asks the identity server for a binding to a Matrix user ID, and has none, - so issues an ``m.room.third_party_invite`` event to the room. +#. H1 asks the identity server for a binding to a Matrix user ID, and has none, + so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, they are told about the - invite, and ask their homeserver, H3, to join the room. +#. When the third party user validates their identity, their homeserver H3 + is notified and attempts to issue an ``m.room.member`` event to participate + in the room. - H3 validates the signature in the event's - ``content.third_party_invite.signed`` object. +#. H3 validates the signature given to it by the identity server. - H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` - property *and* check ``key_validity_url``. +#. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` + property *and* check ``key_validity_url``. - Having validated these things, H1 writes the join event to the room, and H3 - begins participating in the room. H2 *must* accept this event. +#. Having validated these things, H1 writes the invite event to the room, and H3 + begins participating in the room. H2 *must* accept this event. The reason that no other homeserver may reject the event based on checking ``key_validity_url`` is that we must ensure event acceptance is deterministic. @@ -112,3 +118,32 @@ This relies on participating servers trusting each other, but that trust is already implied by the server-server protocol. Also, the public key signature verification must still be performed, so the attack surface here is minimized. +Security considerations +----------------------- + +There are a number of privary and trust implications to this module. + +It is important for user privacy that leaking the mapping between a matrix user +ID and a third party identifier is hard. In particular, being able to look up +all third party identifiers from a matrix user ID (and accordingly, being able +to link each third party identifier) should be avoided wherever possible. +To this end, when implementing this API care should be taken to avoid +adding links between these two identifiers as room events. This mapping can be +unintentionally created by specifying the third party identifier in the +``display_name`` field of the ``m.room.third_party_invite`` event, and then +observing which matrix user ID joins the room using that invite. Clients SHOULD +set ``display_name`` to a value other than the third party identifier, e.g. the +invitee's common name. + +Homeservers are not required to trust any particular identity server(s). It is +generally a client's responsibility to decide which identity servers it trusts, +not a homeserver's. Accordingly, this API takes identity servers as input from +end users, and doesn't have any specific trusted set. It is possible some +homeservers may want to supply defaults, or reject some identity servers for +*its* users, but no homeserver is allowed to dictate which identity servers +*other* homeservers' users trust. + +There is some risk of denial of service attacks by flooding homeservers or +identity servers with many requests, or much state to store. Defending against +these is left to the implementer's discretion. + From 559747e77a1bac4cd104f9526bfcaa84217651fb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 19:18:28 +0000 Subject: [PATCH 159/989] speculator: Sent Content-Type: text/html header Go is auto-detecting that this is XML (because for some reason we generate XHTML), and serving it with a Content-Type header text/xml. This causes the browser to render it as XHTML, which gives interesting quirks like extra newlines. This forces the browser to interpret it as HTML. What we should probably do instead of stop generating XHTML and start generating HTML. But in the mean time, this will fix the rendering issues. --- scripts/speculator/main.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 85fb2596587..ef37b931e6c 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -384,9 +384,9 @@ func main() { log.Fatal(err) } s := server{masterCloneDir} - http.HandleFunc("/spec/", s.serveSpec) - http.HandleFunc("/diff/rst/", s.serveRSTDiff) - http.HandleFunc("/diff/html/", s.serveHTMLDiff) + http.HandleFunc("/spec/", forceHTML(s.serveSpec)) + http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) + http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", listPulls) @@ -394,6 +394,13 @@ func main() { log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } +func forceHTML(h func(w http.ResponseWriter, req *http.Request)) func(w http.ResponseWriter, req *http.Request) { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "text/html") + h(w, req) + } +} + func serveText(s string) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, req *http.Request) { io.WriteString(w, s) From 937ff046d866b96d068c3e1424689ebb58de99f5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 19:21:16 +0000 Subject: [PATCH 160/989] Force / to be HTML too --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index ef37b931e6c..7f86bd62819 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -388,7 +388,7 @@ func main() { http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) - http.HandleFunc("/", listPulls) + http.HandleFunc("/", forceHTML(listPulls)) fmt.Printf("Listening on port %d\n", *port) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) From 7f6eafdce52972021643512d09e46eaef88c810d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 14:46:55 +0000 Subject: [PATCH 161/989] continuserv: set Content-Type header --- scripts/continuserv/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 573c2c95529..425a1af9f81 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -113,6 +113,7 @@ func filter(e fsnotify.Event) bool { func serve(w http.ResponseWriter, req *http.Request) { wg.Wait() b := toServe.Load().([]byte) + w.Header().Set("Content-Type", "text/html") w.Write(b) } From 1be5b856bd2447aa557ada4fcde8b8ddf63a3f70 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 16:05:07 +0000 Subject: [PATCH 162/989] Preserve text/plain for errors Newlines are nice --- scripts/continuserv/main.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 425a1af9f81..4613437a938 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -25,7 +25,7 @@ import ( var ( port = flag.Int("port", 8000, "Port on which to serve HTTP") - toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + toServe atomic.Value // Always contains valid bytesOrErr to serve. May be stale unless wg is zero. wg sync.WaitGroup // Indicates how many updates are pending. mu sync.Mutex // Prevent multiple updates in parallel. ) @@ -112,9 +112,14 @@ func filter(e fsnotify.Event) bool { func serve(w http.ResponseWriter, req *http.Request) { wg.Wait() - b := toServe.Load().([]byte) - w.Header().Set("Content-Type", "text/html") - w.Write(b) + b := toServe.Load().(bytesOrErr) + if b.err != nil { + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte(b.err.Error())) + } else { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte(b.bytes)) + } } func populateOnce(dir string) { @@ -127,15 +132,15 @@ func populateOnce(dir string) { cmd.Stderr = &b err := cmd.Run() if err != nil { - toServe.Store([]byte(fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()).Error())) + toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())}) return } specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) if err != nil { - toServe.Store([]byte(fmt.Errorf("error reading spec: %v", err).Error())) + toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) return } - toServe.Store(specBytes) + toServe.Store(bytesOrErr{specBytes, nil}) } func doPopulate(ch chan struct{}, dir string) { @@ -160,3 +165,8 @@ func exists(path string) bool { _, err := os.Stat(path) return !os.IsNotExist(err) } + +type bytesOrErr struct { + bytes []byte + err error +} From bbf9e229a7735c1c1ac295169a9f81b83874cc91 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 16:09:09 +0000 Subject: [PATCH 163/989] continuserv: guard concurrent accesses to wg --- scripts/continuserv/main.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 573c2c95529..2c6072a3340 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -25,9 +25,11 @@ import ( var ( port = flag.Int("port", 8000, "Port on which to serve HTTP") - toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. - wg sync.WaitGroup // Indicates how many updates are pending. - mu sync.Mutex // Prevent multiple updates in parallel. + mu sync.Mutex // Prevent multiple updates in parallel. + toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + + wgMu sync.Mutex // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel. + wg sync.WaitGroup // Indicates how many updates are pending. ) func main() { @@ -111,7 +113,9 @@ func filter(e fsnotify.Event) bool { } func serve(w http.ResponseWriter, req *http.Request) { + wgMu.Lock() wg.Wait() + wgMu.Unlock() b := toServe.Load().([]byte) w.Write(b) } @@ -143,7 +147,9 @@ func doPopulate(ch chan struct{}, dir string) { select { case <-ch: if pending == 0 { + wgMu.Lock() wg.Add(1) + wgMu.Unlock() } pending++ case <-time.After(10 * time.Millisecond): From e72151f2c3f8e2825eb2719b3e4598672c134a03 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 18:15:21 +0000 Subject: [PATCH 164/989] Specify guest room access This was reviewed as PR #150 and merged from daniel/anonymousaccess --- api/client-server/v1/guest_events.yaml | 103 ++++++++++++++++++ event-schemas/examples/v1/m.room.guest_access | 12 ++ event-schemas/schema/v1/m.room.guest_access | 30 +++++ specification/modules/guest_access.rst | 81 ++++++++++++++ specification/targets.yaml | 1 + templating/matrix_templates/units.py | 9 +- 6 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 api/client-server/v1/guest_events.yaml create mode 100644 event-schemas/examples/v1/m.room.guest_access create mode 100644 event-schemas/schema/v1/m.room.guest_access create mode 100644 specification/modules/guest_access.rst diff --git a/api/client-server/v1/guest_events.yaml b/api/client-server/v1/guest_events.yaml new file mode 100644 index 00000000000..bbb5799af23 --- /dev/null +++ b/api/client-server/v1/guest_events.yaml @@ -0,0 +1,103 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Sync Guest API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/events": + get: + summary: Listen on the event stream. + description: |- + This will listen for new events related to a particular room and return + them to the caller. This will block until an event is received, or until + the ``timeout`` is reached. + + This API is the same as the non-guest /events endpoint, but can be + called by guest users. + security: + - accessToken: [] + parameters: + - in: query + type: string + name: from + description: |- + The token to stream from. This token is either from a previous + request to this API or from the initial sync API. + required: false + x-example: "s3456_9_0" + - in: query + type: integer + name: timeout + description: The maximum time in milliseconds to wait for an event. + required: false + x-example: "35000" + - in: query + type: array + items: + type: string + name: room_id + description: |- + The room IDs for which events should be returned. + x-example: + - "!somewhere:over" + - "!the:rainbow" + responses: + 200: + description: "The events received, which may be none." + examples: + application/json: |- + { + "start": "s3456_9_0", + "end": "s3457_9_0", + "chunk": [ + { + "age": 32, + "content": { + "body": "incoming message", + "msgtype": "m.text" + }, + "event_id": "$14328055551tzaee:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + } + ] + } + schema: + type: object + properties: + start: + type: string + description: |- + A token which correlates to the first value in ``chunk``. This + is usually the same token supplied to ``from=``. + end: + type: string + description: |- + A token which correlates to the last value in ``chunk``. This + token should be used in the next request to ``/events``. + chunk: + type: array + description: "An array of events." + items: + type: object + title: Event + allOf: + - "$ref": "core-event-schema/room_event.json" + 400: + description: "Bad pagination ``from`` parameter." diff --git a/event-schemas/examples/v1/m.room.guest_access b/event-schemas/examples/v1/m.room.guest_access new file mode 100644 index 00000000000..8ad4d294d5e --- /dev/null +++ b/event-schemas/examples/v1/m.room.guest_access @@ -0,0 +1,12 @@ +{ + "age": 242353, + "content": { + "guest_access": "can_join" + }, + "state_key": "", + "origin_server_ts": 1431961217938, + "event_id": "$WLGTSEFSEG:localhost", + "type": "m.room.guest_access", + "room_id": "!Cuyf34gef24u:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/schema/v1/m.room.guest_access b/event-schemas/schema/v1/m.room.guest_access new file mode 100644 index 00000000000..a0141f37419 --- /dev/null +++ b/event-schemas/schema/v1/m.room.guest_access @@ -0,0 +1,30 @@ +{ + "type": "object", + "title": "Controls whether guest users are allowed to join rooms.", + "description": "This event controls whether guest users are allowed to join rooms. If this event is absent, servers should act as if it is present and has the guest_access value \"forbidden\".", + "allOf": [{ + "$ref": "core-event-schema/state_event.json" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "guest_access": { + "type": "string", + "description": "Whether guests can join the room.", + "enum": ["can_join", "forbidden"] + } + }, + "required": ["guest_access"] + }, + "state_key": { + "type": "string", + "description": "A zero-length string.", + "pattern": "^$" + }, + "type": { + "type": "string", + "enum": ["m.room.guest_access"] + } + } +} diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst new file mode 100644 index 00000000000..ac373af933f --- /dev/null +++ b/specification/modules/guest_access.rst @@ -0,0 +1,81 @@ +Guest access +============ + +.. _module:guest-access: + +There are times when it is desirable for clients to be able to interact with +rooms without having to fully register for an account on a homeserver or join +the room. This module specifies how these clients should interact with servers +in order to participate in rooms as guests. + +Guest users retrieve access tokens from a homeserver using the ordinary +`register endpoint <#post-matrix-client-api-v2-alpha-register>`_, specifying +the ``kind`` parameter as ``guest``. They may then interact with the +client-server API as any other user would, but will only have access to a subset +of the API as described the Client behaviour subsection below. +Homeservers may choose not to allow this access at all to their local users, but +have no information about whether users on other homeservers are guests or not. + +This module does not fully factor in federation; it relies on individual +homeservers properly adhering to the rules set out in this module, rather than +allowing all homeservers to enforce the rules on each other. + +Events +------ +{{m_room_guest_access_event}} + +Client behaviour +---------------- +The following API endpoints are allowed to be accessed by guest accounts for +retrieving events: + +* `GET /rooms/:room_id/state <#get-matrix-client-api-v1-rooms-roomid-state>`_ +* `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-api-v1-rooms-roomid-state-eventtype-statekey>`_ +* `GET /rooms/:room_id/messages <#get-matrix-client-api-v1-rooms-roomid-messages>`_ + +There is also a special version of the +`GET /events <#get-matrix-client-api-v1-events>`_ endpoint: + +{{guest_events_http_api}} + +They will only return events which happened while the room state had the +``m.room.history_visibility`` state event present with ``history_visibility`` +value ``world_readable``. Guest clients do not need to join rooms in order to +receive events for them. + +The following API endpoints are allowed to be accessed by guest accounts for +sending events: + +* `POST /rooms/:room_id/join <#post-matrix-client-api-v1-rooms-roomid-join>`_ +* `PUT /rooms/:room_id/send/m.room.message/:txn_id <#put-matrix-client-api-v1-rooms-roomid-send-eventtype-txnid>`_ + +Guest clients *do* need to join rooms in order to send events to them. + +Server behaviour +---------------- +Servers are required to only return events to guest accounts for rooms where +the room state at the event had the ``m.room.history_visibility`` state event +present with ``history_visibility`` value ``world_readable``. These events may +be returned even if the anonymous user is not joined to the room. + +Servers MUST only allow guest users to join rooms if the ``m.room.guest_access`` +state event is present on the room, and has the ``guest_access`` value +``can_join``. If the ``m.room.guest_access`` event is changed to stop this from +being the case, the server MUST set those users' ``m.room.member`` state to +``leave``. + +Security considerations +----------------------- +Each homeserver manages its own guest accounts itself, and whether an account +is a guest account or not is not information passed from server to server. +Accordingly, any server participating in a room is trusted to properly enforce +the permissions outlined in this section. + +Clients may wish to display to their users that rooms which are +``world_readable`` *may* be showing messages to non-joined users. There is no +way using this module to find out whether any non-joined guest users *do* see +events in the room, or to list or count any guest users. + +Homeservers may want to enable protections such as captchas for guest +registration to prevent spam, denial of service, and similar attacks. + diff --git a/specification/targets.yaml b/specification/targets.yaml index 2482dcfda7f..8e6a2ce00b2 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -25,6 +25,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/push.rst - modules/third_party_invites.rst - modules/search.rst + - modules/guest_access.rst title_styles: ["=", "-", "~", "+", "^", "`"] diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index cec5b83ec82..adb7f427c43 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -369,7 +369,7 @@ def _load_swagger_meta(self, filepath, api, group_name): ] if len(params_missing_examples) == 0: path_template = api.get("basePath", "").rstrip("/") + path - qps = {} + qps = [] body = "" for param in single_api.get("parameters", []): if param["in"] == "path": @@ -381,7 +381,12 @@ def _load_swagger_meta(self, filepath, api, group_name): elif param["in"] == "body": body = param["schema"]["example"] elif param["in"] == "query": - qps[param["name"]] = param["x-example"] + example = param["x-example"] + if type(example) == list: + for value in example: + qps.append((param["name"], value)) + else: + qps.append((param["name"], example)) query_string = "" if len(qps) == 0 else "?"+urllib.urlencode(qps) if body: endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % ( From 436a35e9f647fcaa5ef27b48ef26b683440fcb40 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 9 Nov 2015 16:04:31 +0000 Subject: [PATCH 165/989] Document macaroon type=login --- drafts/macaroons_caveats.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 93622c3d400..a7c1b03616a 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -25,10 +25,13 @@ Specified caveats: | gen | Generation of the macaroon caveat spec. | 1 | | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | | type | The purpose of this macaroon. | access - used to authorize any action except token refresh | -| refresh - only used to authorize a token refresh | +| | | refresh - only used to authorize a token refresh | +| | | login - issued as a very short-lived token by third party login flows; proves that | +| | | authentication has happened but doesn't grant any privileges other than being able to be | +| | | exchanged for other tokens. | | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | -| Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | -| Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | -| Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| -| Note that exact equality of time is largely meaningless. | +| | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | +| | | Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | +| | | Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| +| | | Note that exact equality of time is largely meaningless. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ From 24c2036a35738ebf9c1ebe3b01cef1943a087702 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 9 Nov 2015 17:30:18 +0000 Subject: [PATCH 166/989] 3pid invites: remove mentions of display_name --- api/client-server/v1/third_party_membership.yaml | 8 ++------ specification/modules/third_party_invites.rst | 11 ++++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/v1/third_party_membership.yaml index 90b167b443a..a10d616722d 100644 --- a/api/client-server/v1/third_party_membership.yaml +++ b/api/client-server/v1/third_party_membership.yaml @@ -84,8 +84,7 @@ paths: { "id_server": "matrix.org", "medium": "email", - "address": "cheeky@monkey.com", - "display_name": "A very cheeky monkey" + "address": "cheeky@monkey.com" } properties: id_server: @@ -98,10 +97,7 @@ paths: address: type: string description: The invitee's third party identifier. - display_name: - type: string - description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. - required: ["id_server", "medium", "address", "display_name"] + required: ["id_server", "medium", "address"] responses: 200: description: The user has been invited to join the room. diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 4d268631d8c..d8e3d4d92f2 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -127,13 +127,10 @@ It is important for user privacy that leaking the mapping between a matrix user ID and a third party identifier is hard. In particular, being able to look up all third party identifiers from a matrix user ID (and accordingly, being able to link each third party identifier) should be avoided wherever possible. -To this end, when implementing this API care should be taken to avoid -adding links between these two identifiers as room events. This mapping can be -unintentionally created by specifying the third party identifier in the -``display_name`` field of the ``m.room.third_party_invite`` event, and then -observing which matrix user ID joins the room using that invite. Clients SHOULD -set ``display_name`` to a value other than the third party identifier, e.g. the -invitee's common name. +To this end, the third party identifier is not put in any event, rather an +opaque display name provided by the identity server is put into the events. +Clients should not remember or display third party identifiers from invites, +other than for the use of the inviter themself. Homeservers are not required to trust any particular identity server(s). It is generally a client's responsibility to decide which identity servers it trusts, From c1866ebebc708d0e8a404edbc3f0bb9b0f3d4297 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:26:06 +0000 Subject: [PATCH 167/989] Fix table formatting --- drafts/macaroons_caveats.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index a7c1b03616a..71c4784e273 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -21,14 +21,17 @@ Specified caveats: +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | Caveat name | Description | Legal Values | -+-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ ++=============+==================================================+================================================================================================+ | gen | Generation of the macaroon caveat spec. | 1 | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | type | The purpose of this macaroon. | access - used to authorize any action except token refresh | | | | refresh - only used to authorize a token refresh | | | | login - issued as a very short-lived token by third party login flows; proves that | | | | authentication has happened but doesn't grant any privileges other than being able to be | | | | exchanged for other tokens. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | | | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | | | | Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | From 51fe4a90b6f3e79428f1e1160b7c7df4f906adb9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:28:27 +0000 Subject: [PATCH 168/989] More formatting fixes --- drafts/macaroons_caveats.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 71c4784e273..11e36e59294 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -9,12 +9,15 @@ Caveats can only be used for reducing the scope of a token, never for increasing Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed. -All caveats must take the form: +All caveats must take the form:: -`key` `operator` `value` -where `key` is a non-empty string drawn from the character set [A-Za-z0-9_] -`operator` is a non-empty string which does not contain whitespace -`value` is a non-empty string + key operator value + +where: + - ``key`` is a non-empty string drawn from the character set [A-Za-z0-9_] + - ``operator`` is a non-empty string which does not contain whitespace + - ``value`` is a non-empty string + And these are joined by single space characters. Specified caveats: From c8f6ed11074f489df5562ce614e3113f48ea77d9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:31:31 +0000 Subject: [PATCH 169/989] More formatting.. --- drafts/macaroons_caveats.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 11e36e59294..de5973fac29 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -29,11 +29,11 @@ Specified caveats: +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ -| type | The purpose of this macaroon. | access - used to authorize any action except token refresh | -| | | refresh - only used to authorize a token refresh | -| | | login - issued as a very short-lived token by third party login flows; proves that | -| | | authentication has happened but doesn't grant any privileges other than being able to be | -| | | exchanged for other tokens. | +| type | The purpose of this macaroon. | - ``access``: used to authorize any action except token refresh | +| | | - ``refresh``: only used to authorize a token refresh | +| | | - ``login``: issued as a very short-lived token by third party login flows; proves that | +| | | authentication has happened but doesn't grant any privileges other than being able to be | +| | | exchanged for other tokens. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | | | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | From c00abe9f2faa80f6cfcf7fba028de870eca676ac Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 10 Nov 2015 15:26:51 +0000 Subject: [PATCH 170/989] Fix msgtype display --- templating/matrix_templates/sections.py | 2 +- templating/matrix_templates/units.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 81b2bb3cc12..78aabca761c 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -136,7 +136,7 @@ def render_msgtype_events(self): if not event_name.startswith("m.room.message#m."): continue sections.append(template.render( - example=examples[event_name], + example=examples[event_name][0], event=schemas[event_name], title_kind=subtitle_title_char )) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 5ae117ca4b0..94435c52ee9 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -562,7 +562,7 @@ def load_event_examples(self): event_name = filename.split("#")[0] example = json.loads(f.read()) - examples[filename] = examples.get(event_name, []) + examples[filename] = examples.get(filename, []) examples[filename].append(example) if filename != event_name: examples[event_name] = examples.get(event_name, []) From 27ffe7bacd15ae559cdfc7bce03a42d00699d7d4 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 10 Nov 2015 15:34:32 +0000 Subject: [PATCH 171/989] Don't serve rst diffs as HTML --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 7f86bd62819..e1898d7ee5d 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -385,7 +385,7 @@ func main() { } s := server{masterCloneDir} http.HandleFunc("/spec/", forceHTML(s.serveSpec)) - http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) + http.HandleFunc("/diff/rst/", s.serveRSTDiff) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", forceHTML(listPulls)) From d7357ef9b75893631e4a8b1e5e40273885cd6709 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 11:39:40 +0000 Subject: [PATCH 172/989] Specify /publicRooms --- api/client-server/v1/list_public_rooms.yaml | 85 +++++++++++++++++++ specification/client_server_api.rst | 5 ++ .../matrix_templates/templates/http-api.tmpl | 5 +- 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 api/client-server/v1/list_public_rooms.yaml diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/v1/list_public_rooms.yaml new file mode 100644 index 00000000000..2f189d2eb6a --- /dev/null +++ b/api/client-server/v1/list_public_rooms.yaml @@ -0,0 +1,85 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Creation API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +paths: + "/publicRooms": + get: + summary: Lists the public rooms on the server. + description: |- + Lists the public rooms on the server. + + This API returns paginated responses. + responses: + 200: + description: A list of the rooms on the server. + schema: + type: object + description: A list of the rooms on the server. + properties: + chunk: + title: "PublicRoomsChunks" + type: array + description: |- + A paginated chunk of public rooms. + items: + type: object + title: "PublicRoomsChunk" + properties: + aliases: + type: array + description: |- + Aliases of the room. May be empty. + items: + type: string + name: + type: string + description: |- + The name of the room, if any. May be null. + num_joined_members: + type: number + description: |- + The number of members joined to the room. + room_id: + type: string + description: |- + The ID of the room. + topic: + type: string + description: |- + The topic of the room, if any. May be null. + start: + type: string + description: |- + A pagination token for the response. + end: + type: string + description: |- + A pagination token for the response. + examples: + application/json: |- + { + "chunk": [ + { + "aliases": ["#murrays:cheese.bar"], + "name": "CHEESE", + "num_joined_members": 37, + "room_id": "!ol19s:bleecker.street", + "topic": "Tasty tasty cheese" + } + ], + "start": "p190q", + "end": "p1902" + } + 400: + description: > + The request body is malformed or the room alias specified is already taken. diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index b02dbf28cc9..05b6ff3c28a 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -931,6 +931,11 @@ member's state, by making a request to "membership": "ban" } +Listing rooms +~~~~~~~~~~~~~ + +{{list_public_rooms_http_api}} + Profiles -------- diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 86eacb145c9..d7258e98cf7 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -12,7 +12,7 @@ {{":Requires auth: Yes." if endpoint.requires_auth else "" }} Request format: - +{% if (endpoint.req_param_by_loc | length) %} =========================================== ================= =========================================== Parameter Value Description =========================================== ================= =========================================== @@ -24,6 +24,9 @@ Request format: {% endfor -%} {% endfor -%} =========================================== ================= =========================================== +{% else %} +`No parameters` +{% endif %} {% if endpoint.res_tables|length > 0 -%} Response format: From dcf54e11b10b07fd30cd18be127d9cde9b198e8a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 11:53:31 +0000 Subject: [PATCH 173/989] Specify /publicRooms world_readable and guest_access Depends on https://github.com/matrix-org/matrix-doc/pull/154 --- api/client-server/v1/list_public_rooms.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/v1/list_public_rooms.yaml index 2f189d2eb6a..23b1c3d0cb4 100644 --- a/api/client-server/v1/list_public_rooms.yaml +++ b/api/client-server/v1/list_public_rooms.yaml @@ -57,6 +57,16 @@ paths: type: string description: |- The topic of the room, if any. May be null. + world_readable: + type: boolean + description: |- + Whether the room may be viewed by guest users without joining. + guest_can_join: + type: boolean + description: |- + Whether guest users may join the room and participate in it. + If they can, they will be subject to ordinary power level + rules like any other user. start: type: string description: |- @@ -71,10 +81,12 @@ paths: "chunk": [ { "aliases": ["#murrays:cheese.bar"], + "guest_can_join": false, "name": "CHEESE", "num_joined_members": 37, "room_id": "!ol19s:bleecker.street", - "topic": "Tasty tasty cheese" + "topic": "Tasty tasty cheese", + "world_readable": true } ], "start": "p190q", From 740cc66a7cccc366ca34eac1ebf65fd7c9af90b3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 14:01:45 +0000 Subject: [PATCH 174/989] speculator: Fetch before deciding head is fresh --- scripts/speculator/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e1898d7ee5d..e467018eb4d 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -178,6 +178,10 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { + if err := gitFetch(s.matrixDocCloneURL); err != nil { + writeError(w, 500, err) + return + } originHead, err := s.getSHAOf("origin/master") if err != nil { writeError(w, 500, err) From 0f0359d9c1049c60dbea365344b40cadc1a50c16 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 15:13:34 +0000 Subject: [PATCH 175/989] speculator: Nessle up some more if statements --- scripts/speculator/main.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e467018eb4d..97e67c8c2fd 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -74,8 +74,7 @@ func gitClone(url string, shared bool) (string, error) { cmd.Args = append(cmd.Args, "--shared") } - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return "", fmt.Errorf("error cloning repo: %v", err) } return directory, nil @@ -92,8 +91,7 @@ func gitFetch(path string) error { func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) } return nil @@ -126,8 +124,7 @@ func generate(dir string) error { cmd.Dir = path.Join(dir, "scripts") var b bytes.Buffer cmd.Stderr = &b - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return nil @@ -167,8 +164,7 @@ func (s *server) getSHAOf(ref string) (string, error) { cmd.Dir = path.Join(s.matrixDocCloneURL) var b bytes.Buffer cmd.Stdout = &b - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return strings.TrimSpace(b.String()), nil From 061105c9dc25d52a01ba260afc9c90bd0158931b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 17:18:58 +0000 Subject: [PATCH 176/989] Guest users are allowed room initialSync --- api/client-server/v1/rooms.yaml | 2 +- specification/modules/guest_access.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 9300d7d10e5..90b51290ab6 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -371,7 +371,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - required: ["room_id", "membership"] + required: ["room_id"] 403: description: > You aren't a member of the room and weren't previously a diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index ac373af933f..ae8d156e875 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -32,6 +32,7 @@ retrieving events: * `GET /rooms/:room_id/state <#get-matrix-client-api-v1-rooms-roomid-state>`_ * `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-api-v1-rooms-roomid-state-eventtype-statekey>`_ * `GET /rooms/:room_id/messages <#get-matrix-client-api-v1-rooms-roomid-messages>`_ +* `GET /rooms/:room_id/initialSync <#get-matrix-client-api-v1-rooms-roomid-initialsync>`_ There is also a special version of the `GET /events <#get-matrix-client-api-v1-events>`_ endpoint: From 4d3175fc8b5c01a41afb802c2e4bc0e191b822db Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 12 Nov 2015 10:45:57 +0000 Subject: [PATCH 177/989] Note that invite_room_state is optional --- event-schemas/schema/v1/m.room.member | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 81057049c2e..25b3010971c 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event may also include an ``invite_room_state`` key **outside the** ``content`` **key**. If present, this contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], From 29d9c8eec6a0a7da49b78c4885b168662fc04f99 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 12 Nov 2015 12:05:02 +0000 Subject: [PATCH 178/989] Guests are allowed to set displaynames --- specification/modules/guest_access.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index ac373af933f..d7359f23621 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -51,6 +51,11 @@ sending events: Guest clients *do* need to join rooms in order to send events to them. +The following API endpoints are allowed to be accessed by guest accounts for +their own account maintenance: + +* `PUT /profile/:user_id/displayname <#put-matrix-client-api-v1-profile-userid-displayname>`_ + Server behaviour ---------------- Servers are required to only return events to guest accounts for rooms where From d7c69fae438410e1b91030e60466f48f28e27360 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:33:34 +0000 Subject: [PATCH 179/989] Fix typo 'process' --- specification/server_server_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index d26cc26ec2c..1c3e761def8 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -555,8 +555,8 @@ sign, and emit a new ``m.room.member`` state event adding the user into that room. When the homeserver does not yet know about the room it cannot do this directly. Instead, it must take a longer multi-stage handshaking process by which it first selects a remote homeserver which is already participating in -that room, and uses it to assist in the joining procss. This is the remote join -handshake. +that room, and uses it to assist in the joining process. This is the remote +join handshake. This handshake involves the homeserver of the new member wishing to join (referred to here as the "joining" server), the directory server hosting the From aac45295ee9dc615cfc7290cc928c011ca0d4e79 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:34:22 +0000 Subject: [PATCH 180/989] Remark that the directory server step could be skipped in an invite case --- specification/server_server_api.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 1c3e761def8..e158ae23d28 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -596,9 +596,12 @@ homeservers, though most in practice will use just two. | <---------- join response -The first part of the handshake involves using the directory server to request -the room ID and join candidates. This is covered in more detail on the -directory server documentation, below. +The first part of the handshake usually involves using the directory server to +request the room ID and join candidates. This is covered in more detail on the +directory server documentation, below. In the case of a new user joining a +room as a result of a received invite, the joining user's homeserver could +optimise this step away by picking the origin server of that invite message as +the join candidate. Once the joining server has the room ID and the join candidates, it then needs to obtain enough of the current state of the room to fill in the required From db5a90edcd7fcc33385d2145b2ad1ed93313182f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:36:26 +0000 Subject: [PATCH 181/989] Avoid using the words 'current state' when talking about the result of the /make_join request --- specification/server_server_api.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e158ae23d28..6dfbd86f1f5 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -566,9 +566,8 @@ room members are already present (referred to as the "resident" server). In summary, the remote join handshake consists of the joining server querying the directory server for information about the room alias; receiving a room ID and a list of join candidates. The joining server then requests information -about the current state of the room from one of the residents. It uses this -information to construct a ``m.room.member`` event which it finally sends to -a resident server. +about the room from one of the residents. It uses this information to construct +a ``m.room.member`` event which it finally sends to a resident server. Conceptually these are three different roles of homeserver. In practice the directory server is likely to be resident in the room, and so may be selected @@ -604,11 +603,11 @@ optimise this step away by picking the origin server of that invite message as the join candidate. Once the joining server has the room ID and the join candidates, it then needs -to obtain enough of the current state of the room to fill in the required -fields of the ``m.room.member`` event. It obtains this by selecting a resident -from the candidate list, and requesting the ``make_join`` endpoint using a -``GET`` request, specifying the room ID and the user ID of the new member who -is attempting to join. +to obtain enough information about the room to fill in the required fields of +the ``m.room.member`` event. It obtains this by selecting a resident from the +candidate list, and requesting the ``make_join`` endpoint using a ``GET`` +request, specifying the room ID and the user ID of the new member who is +attempting to join. The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some From 885dd1e86cc8d6ee818ab32ab246eeec9a1f5ede Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:39:31 +0000 Subject: [PATCH 182/989] Explain the 'prev_events' join protoevent key --- specification/server_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6dfbd86f1f5..ebf55eb4b61 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -629,7 +629,8 @@ the assisting resident. The required fields are: resident ``origin`` String The name of the assisting resident homeserver ``origin_server_ts`` Integer A timestamp added by the resident homeserver -``prev_events`` List (TODO(paul): ? - I notice these can be blank) +``prev_events`` List An event-reference list containing the immediate + predecessor events ``prev_state`` List (TODO(paul): ? - I notice these can be blank) ``room_id`` String The room ID of the room ``sender`` String The user ID of the joining member From 988d77347651ce2504ab1e52196166358dffe7f7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 17:49:22 +0000 Subject: [PATCH 183/989] /make_join protoevent no longer needs the pointless 'prev_state' key (SYN-517) --- specification/server_server_api.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index ebf55eb4b61..9cc10141f88 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -631,7 +631,6 @@ the assisting resident. The required fields are: ``origin_server_ts`` Integer A timestamp added by the resident homeserver ``prev_events`` List An event-reference list containing the immediate predecessor events -``prev_state`` List (TODO(paul): ? - I notice these can be blank) ``room_id`` String The room ID of the room ``sender`` String The user ID of the joining member ``state_key`` String The user ID of the joining member @@ -688,8 +687,6 @@ integer 200, and whose second element contains the following keys: - (paul) I don't really understand why the full auth_chain events are given here. What purpose does it serve expanding them out in full, when surely they'll appear in the state anyway? - - (paul) the state seems to be entirely ignored by synapse, so I'm not really - sure what ought to be there. Backfilling ----------- From cc8ef691fb20331d135a39993e843d42588e0761 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 16:49:22 +0000 Subject: [PATCH 184/989] Rewrite the table templates Allow columns to stretch if they end up with wide content. Apart from the hassle of having to manually update the calculations, having the columns wide enough to hold the widest thing they might ever have leads to tables with lots of whitespace in the results. --- templating/build.py | 22 ++++ .../templates/common-event-fields.tmpl | 15 +-- .../matrix_templates/templates/events.tmpl | 15 +-- .../matrix_templates/templates/http-api.tmpl | 27 +---- .../matrix_templates/templates/msgtypes.tmpl | 15 +-- .../matrix_templates/templates/tables.tmpl | 104 ++++++++++++++++++ 6 files changed, 139 insertions(+), 59 deletions(-) create mode 100644 templating/matrix_templates/templates/tables.tmpl diff --git a/templating/build.py b/templating/build.py index a35d8a0853c..77fecf91f6e 100755 --- a/templating/build.py +++ b/templating/build.py @@ -93,6 +93,27 @@ def wrap(input, wrap=80, initial_indent=""): return '\n\n'.join(output_lines) + def fieldwidths(input, keys, defaults=[], default_width=15): + """ + A template filter to help in the generation of tables. + + Given a list of rows, returns a list giving the maximum length of the + values in each column. + + :param list[dict[str, str]] input: a list of rows. Each row should be a + dict with the keys given in ``keys``. + :param list[str] keys: the keys corresponding to the table columns + :param list[int] defaults: for each column, the default column width. + :param int default_width: if ``defaults`` is shorter than ``keys``, this + will be used as a fallback + """ + def colwidth(key, default): + return reduce(max, (len(row[key]) for row in input), + default if default is not None else default_width) + + results = map(colwidth, keys, defaults) + return results + # make Jinja aware of the templates and filters env = Environment( loader=FileSystemLoader(in_mod.exports["templates"]), @@ -102,6 +123,7 @@ def wrap(input, wrap=80, initial_indent=""): env.filters["indent"] = indent env.filters["indent_block"] = indent_block env.filters["wrap"] = wrap + env.filters["fieldwidths"] = fieldwidths # load up and parse the lowest single units possible: we don't know or care # which spec section will use it, we just need it there in memory for when diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 3f16be3da3b..8d8c8f0ca1b 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,17 +1,8 @@ +{% import 'tables.tmpl' as tables -%} + {{common_event.title}} Fields {{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} -================== ================= =========================================== - Key Type Description -================== ================= =========================================== -{% for row in common_event.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc | indent(18 - (row.type|length)) |wrap(43) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(common_event.rows, ["Key", "Type", "Description"]) }} diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fb876440cc8..324a8f3e0aa 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.type}}`` {{(4 + event.type | length) * title_kind}} *{{event.typeof}}* @@ -7,18 +9,7 @@ {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ================= =========================================== - {{table.title or "Content"}} Key Type Description -======================= ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(42)}} -{% endfor -%} -======================= ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index d7258e98cf7..1253e66ebef 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{endpoint.method}} {{endpoint.path}}`` {{(5 + (endpoint.path | length) + (endpoint.method | length)) * title_kind}} {% if "alias_for_path" in endpoint -%} @@ -13,17 +15,7 @@ Request format: {% if (endpoint.req_param_by_loc | length) %} -=========================================== ================= =========================================== - Parameter Value Description -=========================================== ================= =========================================== -{% for loc in endpoint.req_param_by_loc -%} -*{{loc}} parameters* ---------------------------------------------------------------------------------------------------------- -{% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} -{% endfor -%} -{% endfor -%} -=========================================== ================= =========================================== +{{ tables.split_paramtable(endpoint.req_param_by_loc) }} {% else %} `No parameters` {% endif %} @@ -34,18 +26,7 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ========================= ========================================== - Param Type Description -======================= ========================= ========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (42 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(26 - (row.type|length))) |indent_block(50)}} -{% endfor -%} -======================= ========================= ========================================== +{{ tables.paramtable(table.rows) }} {% endfor %} {% endif -%} diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 18d3492bc85..87cf4a1938a 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,21 +1,12 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.msgtype}}`` {{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== - {{table.title or "Content"}} Key Type Description -================== ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/tables.tmpl b/templating/matrix_templates/templates/tables.tmpl new file mode 100644 index 00000000000..aba6c0c49f1 --- /dev/null +++ b/templating/matrix_templates/templates/tables.tmpl @@ -0,0 +1,104 @@ +{# + # A set of macros for generating RST tables + #} + + +{# + # write a table for a list of parameters. + # + # 'rows' is the list of parameters. Each row should have the keys + # 'key', 'type', and 'desc'. + #} +{% macro paramtable(rows, titles=["Parameter", "Type", "Description"]) -%} +{{ split_paramtable({None: rows}, titles) }} +{% endmacro %} + + +{# + # write a table for the request parameters, split by location. + # 'rows_by_loc' is a map from location to a list of parameters. + # + # As a special case, if a key of 'rows_by_loc' is 'None', no title row is + # written for that location. This is used by the standard 'paramtable' macro. + #} +{% macro split_paramtable(rows_by_loc, + titles=["Parameter", "Type", "Description"]) -%} + +{% set rowkeys = ['key', 'type', 'desc'] %} +{% set titlerow = {'key': titles[0], 'type': titles[1], 'desc': titles[2]} %} + +{# We need the rows flattened into a single list. Abuse the 'sum' filter to + # join arrays instead of add numbers. -#} +{% set flatrows = rows_by_loc.values()|sum(start=[]) -%} + +{# Figure out the widths of the columns. The last column is always 50 characters + # wide; the others default to 10, but stretch if there is wider text in the + # column. -#} +{% set fieldwidths = (([titlerow] + flatrows) | + fieldwidths(rowkeys[0:-1], [10, 10])) + [50] -%} + +{{ tableheader(fieldwidths) }} +{{ tablerow(fieldwidths, titlerow, rowkeys) }} +{{ tableheader(fieldwidths) }} +{% for loc in rows_by_loc -%} + +{% if loc != None -%} +{{ tablespan(fieldwidths, "*" ~ loc ~ " parameters*") }} +{% endif -%} + +{% for row in rows_by_loc[loc] -%} +{{ tablerow(fieldwidths, row, rowkeys) }} +{% endfor -%} +{% endfor -%} + +{{ tableheader(fieldwidths) }} +{% endmacro %} + + + +{# + # Write a table header row, for the given column widths + #} +{% macro tableheader(widths) -%} +{% for arg in widths -%} +{{"="*arg}} {% endfor -%} +{% endmacro %} + + + +{# + # Write a normal table row. Each of 'widths' and 'keys' should be sequences + # of the same length; 'widths' defines the column widths, and 'keys' the + # attributes of 'row' to look up for values to put in the columns. + #} +{% macro tablerow(widths, row, keys) -%} +{% for key in keys -%} +{% set value=row[key] -%} +{% if not loop.last -%} + {# the first few columns need space after them -#} + {{ value }}{{" "*(1+widths[loop.index0]-value|length) -}} +{% else -%} + {# the last column needs wrapping and indenting (by the sum of the widths of + the preceding columns, plus the number of preceding columns (for the + separators)) -#} + {{ value | wrap(widths[loop.index0]) | + indent_block(widths[0:-1]|sum + loop.index0) -}} +{% endif -%} +{% endfor -%} +{% endmacro %} + + + + +{# + # write a tablespan row. This is a single value which spans the entire table. + #} +{% macro tablespan(widths, value) -%} +{{value}} +{# we write a trailing space to stop the separator being misinterpreted + # as a header line. -#} +{{"-"*(widths|sum + widths|length -1)}} {% endmacro %} + + + + From 838af2a23ec623b2a7f2bd18ac0af51a097dd2b5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 17:21:56 +0000 Subject: [PATCH 185/989] Updates to swagger table generation A bunch of related fixes to the code for parsing the state and API yaml files: 1. Some of our objects are {key: {key: value}} - style nested key/value dictionaries. Handle this by refactoring get_json_schema_object_fields so that such objects are handled wherever they appear, rather than when they are just subproperties of a 'proper' object. 2. Fix multi-level inheritance (so an object can have an 'allOf' property which can successfully refer to an object which itself has an 'allOf' property). 3. $ref fields in event schemas weren't being expanded correctly 4. sort type tables breadth-first rather than depth-first so that the ordering in complex structures like the /sync response makes a bit more sense. --- templating/build.py | 4 + templating/matrix_templates/units.py | 200 +++++++++++++++++---------- 2 files changed, 130 insertions(+), 74 deletions(-) diff --git a/templating/build.py b/templating/build.py index a35d8a0853c..eea6ac7acc9 100755 --- a/templating/build.py +++ b/templating/build.py @@ -42,6 +42,7 @@ from argparse import ArgumentParser, FileType import importlib import json +import logging import os import sys from textwrap import TextWrapper @@ -188,6 +189,9 @@ def log(line): ) args = parser.parse_args() + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + if not args.input: raise Exception("Missing [i]nput python module.") diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index adb7f427c43..5acbcd568b7 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -8,6 +8,7 @@ file instead. """ from batesian.units import Units +import logging import inspect import json import os @@ -27,6 +28,7 @@ ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" +logger = logging.getLogger(__name__) def resolve_references(path, schema): if isinstance(schema, dict): @@ -46,6 +48,32 @@ def resolve_references(path, schema): return schema +def inherit_parents(obj): + """ + Recurse through the 'allOf' declarations in the object + """ + logger.debug("inherit_parents %r" % obj) + parents = obj.get("allOf", []) + if not parents: + return obj + + result = {} + + # settings defined in the child take priority over the parents, so we + # iterate through the parents first, and then overwrite with the settings + # from the child. + for p in map(inherit_parents, parents) + [obj]: + for key in ('title', 'type', 'required'): + if p.get(key): + result[key] = p[key] + + for key in ('properties', 'additionalProperties', 'patternProperties'): + if p.get(key): + result.setdefault(key, {}).update(p[key]) + + return result + + def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) @@ -53,22 +81,44 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals raise Exception( "get_json_schema_object_fields: Object %s isn't an object." % obj ) + + obj = inherit_parents(obj) + + logger.debug("Processing object with title '%s'", obj.get("title")) + if enforce_title and not obj.get("title"): # Force a default titile of "NO_TITLE" to make it obvious in the # specification output which parts of the schema are missing a title obj["title"] = 'NO_TITLE' - required_keys = obj.get("required") - if not required_keys: - required_keys = [] + additionalProps = obj.get("additionalProperties") + if additionalProps: + # not "really" an object, just a KV store + logger.debug("%s is a pseudo-object", obj.get("title")) + + key_type = additionalProps.get("x-pattern", "string") + + value_type = additionalProps["type"] + if value_type == "object": + nested_objects = get_json_schema_object_fields( + additionalProps, + enforce_title=True, + include_parents=include_parents, + ) + value_type = nested_objects[0]["title"] + tables = [x for x in nested_objects if not x.get("no-table")] + else: + key_type = "string" + tables = [] - fields = { - "title": obj.get("title"), - "rows": [] - } - tables = [fields] + tables = [{ + "title": "{%s: %s}" % (key_type, value_type), + "no-table": True + }]+tables + + logger.debug("%s done: returning %s", obj.get("title"), tables) + return tables - parents = obj.get("allOf") props = obj.get("properties") if not props: props = obj.get("patternProperties") @@ -79,73 +129,60 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals if pretty_key: props[pretty_key] = props[key_name] del props[key_name] - if not props and not parents: - # Sometimes you just want to specify that a thing is an object without - # doing all the keys. Allow people to do that if they set a 'title'. - if obj.get("title"): - parents = [{ - "$ref": obj.get("title") - }] - if not props and not parents: - raise Exception( - "Object %s has no properties or parents." % obj - ) - if not props: # parents only - if include_parents: - if obj["title"] == "NO_TITLE" and parents[0].get("title"): - obj["title"] = parents[0].get("title") - props = parents[0].get("properties") - if not props: + # Sometimes you just want to specify that a thing is an object without + # doing all the keys. Allow people to do that if they set a 'title'. + if not props and obj.get("title"): return [{ "title": obj["title"], - "parent": parents[0].get("$ref"), "no-table": True }] + if not props: + raise Exception( + "Object %s has no properties and no title" % obj + ) + + required_keys = set(obj.get("required", [])) + + fields = { + "title": obj.get("title"), + "rows": [] + } + + tables = [fields] + for key_name in sorted(props): + logger.debug("Processing property %s.%s", obj.get('title'), key_name) value_type = None required = key_name in required_keys desc = props[key_name].get("description", "") - - if props[key_name]["type"] == "object": - if props[key_name].get("additionalProperties"): - # not "really" an object, just a KV store - prop_val = props[key_name]["additionalProperties"]["type"] - if prop_val == "object": - nested_object = get_json_schema_object_fields( - props[key_name]["additionalProperties"], - enforce_title=True, - include_parents=include_parents, - ) - key = props[key_name]["additionalProperties"].get( - "x-pattern", "string" - ) - value_type = "{%s: %s}" % (key, nested_object[0]["title"]) - if not nested_object[0].get("no-table"): - tables += nested_object - else: - value_type = "{string: %s}" % prop_val - else: - nested_object = get_json_schema_object_fields( - props[key_name], - enforce_title=True, - include_parents=include_parents, - ) - value_type = "{%s}" % nested_object[0]["title"] - - if not nested_object[0].get("no-table"): - tables += nested_object - elif props[key_name]["type"] == "array": + prop_type = props[key_name].get('type') + + if prop_type is None: + raise KeyError("Property '%s' of object '%s' missing 'type' field" + % (key_name, obj)) + logger.debug("%s is a %s", key_name, prop_type) + + if prop_type == "object": + nested_objects = get_json_schema_object_fields( + props[key_name], + enforce_title=True, + include_parents=include_parents, + ) + value_type = nested_objects[0]["title"] + + tables += [x for x in nested_objects if not x.get("no-table")] + elif prop_type == "array": # if the items of the array are objects then recurse if props[key_name]["items"]["type"] == "object": - nested_object = get_json_schema_object_fields( + nested_objects = get_json_schema_object_fields( props[key_name]["items"], enforce_title=True, include_parents=include_parents, ) - value_type = "[%s]" % nested_object[0]["title"] - tables += nested_object + value_type = "[%s]" % nested_objects[0]["title"] + tables += nested_objects else: value_type = props[key_name]["items"]["type"] if isinstance(value_type, list): @@ -163,7 +200,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals " Must be '%s'." % array_enums[0] ) else: - value_type = props[key_name]["type"] + value_type = prop_type if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" @@ -188,15 +225,32 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "desc": desc, "req_str": "**Required.** " if required else "" }) + logger.debug("Done property %s" % key_name) + + return tables + + +def get_tables_for_schema(path, schema, include_parents=False): + resolved_schema = resolve_references(path, schema) + tables = get_json_schema_object_fields(resolved_schema, + include_parents=include_parents, + ) + + # the result may contain duplicates, if objects are referred to more than + # once. Filter them out. + # + # Go through the tables backwards so that we end up with a breadth-first + # rather than depth-first ordering. titles = set() filtered = [] - for table in tables: + for table in reversed(tables): if table.get("title") in titles: continue titles.add(table.get("title")) filtered.append(table) + filtered.reverse() return filtered @@ -306,16 +360,14 @@ def _load_swagger_meta(self, filepath, api, group_name): if is_array_of_objects: req_obj = req_obj["items"] - req_tables = get_json_schema_object_fields( - resolve_references(filepath, req_obj), - include_parents=True, - ) + req_tables = get_tables_for_schema( + filepath, req_obj, include_parents=True) if req_tables > 1: for table in req_tables[1:]: nested_key_name = [ s["key"] for s in req_tables[0]["rows"] if - s["type"] == ("{%s}" % (table["title"],)) + s["type"] == ("%s" % (table["title"],)) ][0] for row in table["rows"]: row["key"] = "%s.%s" % (nested_key_name, row["key"]) @@ -431,8 +483,7 @@ def _load_swagger_meta(self, filepath, api, group_name): elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] - res_tables = get_json_schema_object_fields( - resolve_references(filepath, schema), + res_tables = get_tables_for_schema(filepath, schema, include_parents=True, ) for table in res_tables: @@ -571,8 +622,9 @@ def load_event_schemas(self): for filename in os.listdir(path): if not filename.startswith("m."): continue - self.log("Reading %s" % os.path.join(path, filename)) - with open(os.path.join(path, filename), "r") as f: + filepath = os.path.join(path, filename) + self.log("Reading %s" % filepath) + with open(filepath, "r") as f: json_schema = json.loads(f.read()) schema = { "typeof": None, @@ -614,15 +666,15 @@ def load_event_schemas(self): schema["desc"] = json_schema.get("description", "") # walk the object for field info - schema["content_fields"] = get_json_schema_object_fields( + schema["content_fields"] = get_tables_for_schema(filepath, Units.prop(json_schema, "properties/content") ) # This is horrible because we're special casing a key on m.room.member. # We need to do this because we want to document a non-content object. if schema["type"] == "m.room.member": - invite_room_state = get_json_schema_object_fields( - json_schema["properties"]["invite_room_state"]["items"] + invite_room_state = get_tables_for_schema(filepath, + json_schema["properties"]["invite_room_state"]["items"], ) schema["content_fields"].extend(invite_room_state) From 3f0262081c7a900cd1e5bcd7ca9a2991facb6ef9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 11 Nov 2015 19:22:38 +0000 Subject: [PATCH 186/989] Update sync API defn to reflect SPEC-254 changes State now corresponds to the start of the timeline, not the end. --- api/client-server/v2_alpha/sync.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index a2d5a2b892d..9e921610cdd 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -119,7 +119,11 @@ paths: title: State type: object description: |- - The state updates for the room. + Updates to the state, between the time indicated by + the ``since`` parameter, and the start of the + ``timeline`` (or all state up to the start of the + ``timeline``, if ``since`` is not given, or + ``full_state`` is true). allOf: - $ref: "definitions/room_event_batch.json" timeline: @@ -193,8 +197,7 @@ paths: title: State type: object description: |- - The state updates for the room up to the point when - the user left. + The state updates for the room up to the start of the timeline. allOf: - $ref: "definitions/room_event_batch.json" timeline: @@ -260,7 +263,6 @@ paths: "state": { "events": [ "$66697273743031:example.com", - "$7365636s6r6432:example.com" ] }, "timeline": { From 57995a815a83ec7496283eae5511ebd89ddeb747 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:30:29 +0000 Subject: [PATCH 187/989] Fix a number of untruths in the documentation about /sync. Attempts to make the /sync documentation better represent fact as it currently stands - in particular document the structure of the returned events. --- api/client-server/v2_alpha/core-event-schema | 1 - .../v2_alpha/definitions/event.json | 53 +++++++++++++++++++ .../v2_alpha/definitions/event_batch.json | 4 +- .../v2_alpha/definitions/timeline_batch.json | 4 +- api/client-server/v2_alpha/sync.yaml | 16 +++--- specification/events.rst | 5 +- 6 files changed, 68 insertions(+), 15 deletions(-) delete mode 120000 api/client-server/v2_alpha/core-event-schema create mode 100644 api/client-server/v2_alpha/definitions/event.json diff --git a/api/client-server/v2_alpha/core-event-schema b/api/client-server/v2_alpha/core-event-schema deleted file mode 120000 index b020e6da409..00000000000 --- a/api/client-server/v2_alpha/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -../../../event-schemas/schema/v1/core-event-schema \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json new file mode 100644 index 00000000000..6a01c688522 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "title": "Event", + "properties": { + "content": { + "type": "object", + "title": "EventContent", + "description": "The content of this event. The fields in this object will vary depending on the type of event." + }, + "origin_server_ts": { + "type": "integer", + "format": "int64", + "description": "Timestamp in milliseconds on originating homeserver when this event was sent." + }, + "sender": { + "type": "string", + "description": "The MXID of the user who sent this event." + }, + "state_key": { + "type": "string", + "description": "Optional. This key will only be present for state events. A unique key which defines the overwriting semantics for this piece of room state." + }, + "type": { + "type": "string", + "description": "The type of event." + }, + "unsigned": { + "type": "object", + "title": "Unsigned", + "description": "Information about this event which was not sent by the originating homeserver", + "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "replaces_state": { + "type": "string", + "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "txn_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + } + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index 395aed13d75..7f4894234fe 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -5,8 +5,8 @@ "type": "array", "description": "List of events", "items": { - "title": "Event", - "type": "object" + "type": "object", + "allOf": [{"$ref": "event.json" }] } } } diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index ddf8d341619..15ca1ead64e 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -4,11 +4,11 @@ "properties": { "limited": { "type": "boolean", - "description": "Whether there are more events on the server" + "description": "True if the number of events returned was limited by the ``limit`` on the filter" }, "prev_batch": { "type": "string", - "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve earlier events" } } } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 9e921610cdd..06763a08c7c 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -96,8 +96,10 @@ paths: Updates to rooms. properties: joined: - title: Joined + title: Joined Rooms type: object + description: |- + The rooms that the user has joined. additionalProperties: title: Joined Room type: object @@ -114,7 +116,7 @@ paths: description: An event object. type: object allOf: - - $ref: "core-event-schema/event.json" + - $ref: "definitions/event.json" state: title: State type: object @@ -144,7 +146,7 @@ paths: allOf: - $ref: "definitions/event_batch.json" invited: - title: Invited + title: Invited Rooms type: object description: |- The rooms that the user has been invited to. @@ -171,11 +173,10 @@ paths: allOf: - $ref: "definitions/event_batch.json" archived: - title: Archived + title: Archived rooms type: object description: |- - The rooms that the user has left or been banned from. The - entries in the room_map will lack an ``ephemeral`` key. + The rooms that the user has left or been banned from. additionalProperties: title: Archived Room type: object @@ -192,7 +193,7 @@ paths: description: An event object. type: object allOf: - - $ref: "core-event-schema/event.json" + - $ref: "definitions/event.json" state: title: State type: object @@ -276,7 +277,6 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } diff --git a/specification/events.rst b/specification/events.rst index a1aece1c41c..7bf080125bf 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -30,12 +30,13 @@ formatted for federation by: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, ``origin``, ``prev_state``. * Adding an ``age`` to the ``unsigned`` object which gives the time in - milliseconds that has ellapsed since the event was sent. + milliseconds that has elapsed since the event was sent. * Adding a ``prev_content`` to the ``unsigned`` object if the event is a ``state event`` which gives previous content of that state key. * Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``transaction_id`` if the event was sent by the client requesting it. +* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the + client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: From b41d771c1586a06e387222030f923ed5fc95ecf0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:58:04 +0000 Subject: [PATCH 188/989] Fix typos in /sync example response --- api/client-server/v2_alpha/sync.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 06763a08c7c..2290b0700a8 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -253,7 +253,7 @@ paths: "$74686972643033:example.com": { "sender": "@alice:example.com", "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, + "unsigned": {"age": 124524, "txn_id": "1234"}, "content": { "body": "I am a fish", "msgtype": "m.text" @@ -263,7 +263,7 @@ paths: }, "state": { "events": [ - "$66697273743031:example.com", + "$66697273743031:example.com" ] }, "timeline": { From 29bd4d45ee9155bd80e61ce50ab34745be1fd8a4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:40:41 +0000 Subject: [PATCH 189/989] Flatten the response to /sync Now that we don't expect there to be duplication betwen the 'timeline' and 'state' results, there's no point in having the state map. (That does mean the events themselves need event_id fields though). Also: - move the contents of the 'unsigned' dictionary into the events themselves - replace the state list with two layers of dictionary keyed on type and state_key - rename the children of the 'rooms' dict from "joined/invited/archived" to "join/invite/leave" to match the membership states --- .../v2_alpha/definitions/event.json | 45 +++--- .../definitions/room_event_batch.json | 12 -- .../v2_alpha/definitions/timeline_batch.json | 2 +- api/client-server/v2_alpha/sync.yaml | 134 +++++++----------- specification/events.rst | 17 +-- 5 files changed, 84 insertions(+), 126 deletions(-) delete mode 100644 api/client-server/v2_alpha/definitions/room_event_batch.json diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 6a01c688522..3a15357aa5e 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -2,16 +2,34 @@ "type": "object", "title": "Event", "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, "content": { "type": "object", "title": "EventContent", "description": "The content of this event. The fields in this object will vary depending on the type of event." }, + "event_id": { + "type": "string", + "description": "Globally unique identifier for this event." + }, "origin_server_ts": { "type": "integer", "format": "int64", "description": "Timestamp in milliseconds on originating homeserver when this event was sent." }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "prev_sender": { + "type": "string", + "description": "Optional. The ``sender`` of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there was no previous event for this state, this key will be missing." + }, "sender": { "type": "string", "description": "The MXID of the user who sent this event." @@ -24,30 +42,9 @@ "type": "string", "description": "The type of event." }, - "unsigned": { - "type": "object", - "title": "Unsigned", - "description": "Information about this event which was not sent by the originating homeserver", - "properties": { - "age": { - "type": "integer", - "format": "int64", - "description": "Time in milliseconds since the event was sent." - }, - "prev_content": { - "title": "EventContent", - "type": "object", - "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "replaces_state": { - "type": "string", - "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "txn_id": { - "type": "string", - "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." - } - } + "txn_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json deleted file mode 100644 index fcf148f3699..00000000000 --- a/api/client-server/v2_alpha/definitions/room_event_batch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "properties": { - "events": { - "type": "array", - "description": "List of event ids", - "items": { - "type": "string" - } - } - } -} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index 15ca1ead64e..f27a5746c5a 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref":"definitions/room_event_batch.json"}], + "allOf": [{"$ref":"definitions/event_batch.json"}], "properties": { "limited": { "type": "boolean", diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 2290b0700a8..c63f3432de6 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -95,7 +95,7 @@ paths: description: |- Updates to rooms. properties: - joined: + join: title: Joined Rooms type: object description: |- @@ -104,19 +104,6 @@ paths: title: Joined Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "definitions/event.json" state: title: State type: object @@ -127,7 +114,7 @@ paths: ``timeline``, if ``since`` is not given, or ``full_state`` is true). allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/state_batch.json" timeline: title: Timeline type: object @@ -145,7 +132,7 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" - invited: + invite: title: Invited Rooms type: object description: |- @@ -171,36 +158,23 @@ paths: delta against the archived ``state`` not the ``invite_state``. allOf: - - $ref: "definitions/event_batch.json" - archived: - title: Archived rooms + - $ref: "definitions/state_batch.json" + leave: + title: Left rooms type: object description: |- The rooms that the user has left or been banned from. additionalProperties: - title: Archived Room + title: Left Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "definitions/event.json" state: title: State type: object description: |- The state updates for the room up to the start of the timeline. allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/state_batch.json" timeline: title: Timeline type: object @@ -230,46 +204,43 @@ paths: ] }, "rooms": { - "joined": { + "join": { "!726s6s6q:example.com": { - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "unsigned": { - "prev_content": {"membership": "invite"} - }, - "origin_server_ts": 1417731086795 - }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": 124524, "txn_id": "1234"}, - "content": { - "body": "I am a fish", - "msgtype": "m.text" - }, - "origin_server_ts": 1417731086797 - } - }, "state": { - "events": [ - "$66697273743031:example.com" - ] + "m.room.member": { + "@alice:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795, + "event_id": "$66697273743031:example.com" + } + } }, "timeline": { "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" + { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "prev_content": {"membership": "invite"}, + "origin_server_ts": 1417731086795, + "event_id": "$7365636s6r6432:example.com": + }, + { + "sender": "@alice:example.com", + "type": "m.room.message", + "age": 124524, + "txn_id": "1234", + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797, + "event_id": "$74686972643033:example.com" + } ], "limited": true, "prev_batch": "t34-23535_0_0" @@ -277,6 +248,7 @@ paths: "ephemeral": { "events": [ { + "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } @@ -284,26 +256,30 @@ paths: } } }, - "invited": { + "invite": { "!696r7674:example.com": { "invite_state": { - "events": [ - { + "m.room.name": { + "": { "sender": "@alice:example.com", "type": "m.room.name", "state_key": "", - "content": {"name": "My Room Name"} - }, - { + "content": {"name": "My Room Name"}, + "event_id": "$asdkgjrsfg2314375:example.com", + + } + }, + "m.room.member": { + "@bob:example.com": { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@bob:example.com", - "content": {"membership": "invite"} - } - ] + "content": {"membership": "invite"}, + "event_id": "$257kasjdg315324akhg:example.com", + } } } }, - "archived": {} + "leave": {} } } diff --git a/specification/events.rst b/specification/events.rst index 7bf080125bf..3d069be8ad9 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -28,22 +28,19 @@ formatted for federation by: * Removing the following keys: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, - ``origin``, ``prev_state``. -* Adding an ``age`` to the ``unsigned`` object which gives the time in + ``origin``, ``prev_state``, ``unsigned``. +* Adding an ``age`` to the event object which gives the time in milliseconds that has elapsed since the event was sent. -* Adding a ``prev_content`` to the ``unsigned`` object if the event is - a ``state event`` which gives previous content of that state key. -* Adding a ``redacted_because`` to the ``unsigned`` object if the event was +* Adding ``prev_content`` and ``prev_sender`` to the event object if the event + is a ``state event``, which give the previous content and previous sender of + that state key +* Adding a ``redacted_because`` to event object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the - client requesting it. +* Adding a ``txn_id`` if the event was sent by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: -* Moving the folling keys from the ``unsigned`` object to the top level event - object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. -* Removing the ``unsigned`` object. * Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From e1b12a753ecd96227effef3612f70d9adf48c0a9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 00:17:22 +0000 Subject: [PATCH 190/989] Fix typos and missing file --- .../v2_alpha/definitions/state_batch.json | 12 ++++++++++++ api/client-server/v2_alpha/sync.yaml | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 api/client-server/v2_alpha/definitions/state_batch.json diff --git a/api/client-server/v2_alpha/definitions/state_batch.json b/api/client-server/v2_alpha/definitions/state_batch.json new file mode 100644 index 00000000000..45728f40347 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/state_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "additionalProperties": { + "type": "object", + "x-pattern": "$EVENT_TYPE", + "additionalProperties": { + "type": "object", + "x-pattern": "$STATE_KEY", + "allOf": [{"$ref": "event.json" }] + } + } +} diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c63f3432de6..198f48337aa 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -227,7 +227,7 @@ paths: "content": {"membership": "join"}, "prev_content": {"membership": "invite"}, "origin_server_ts": 1417731086795, - "event_id": "$7365636s6r6432:example.com": + "event_id": "$7365636s6r6432:example.com" }, { "sender": "@alice:example.com", @@ -265,8 +265,7 @@ paths: "type": "m.room.name", "state_key": "", "content": {"name": "My Room Name"}, - "event_id": "$asdkgjrsfg2314375:example.com", - + "event_id": "$asdkgjrsfg2314375:example.com" } }, "m.room.member": { @@ -275,7 +274,8 @@ paths: "type": "m.room.member", "state_key": "@bob:example.com", "content": {"membership": "invite"}, - "event_id": "$257kasjdg315324akhg:example.com", + "event_id": "$257kasjdg315324akhg:example.com" + } } } } From 96be7ff24140ec80937db48da300c6f4e05473e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Nov 2015 15:00:31 +0000 Subject: [PATCH 191/989] Support more nesting --- templating/matrix_templates/units.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index adb7f427c43..e5a7c319f60 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -122,10 +122,12 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "x-pattern", "string" ) value_type = "{%s: %s}" % (key, nested_object[0]["title"]) + value_id = "%s: %s" % (key, nested_object[0]["title"]) if not nested_object[0].get("no-table"): tables += nested_object else: - value_type = "{string: %s}" % prop_val + value_type = "{string: %s}" % (prop_val,) + value_id = "string: %s" % (prop_val,) else: nested_object = get_json_schema_object_fields( props[key_name], @@ -133,6 +135,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals include_parents=include_parents, ) value_type = "{%s}" % nested_object[0]["title"] + value_id = "%s" % (nested_object[0]["title"],) if not nested_object[0].get("no-table"): tables += nested_object @@ -145,12 +148,14 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals include_parents=include_parents, ) value_type = "[%s]" % nested_object[0]["title"] + value_id = "%s" % (nested_object[0]["title"],) tables += nested_object else: value_type = props[key_name]["items"]["type"] if isinstance(value_type, list): value_type = " or ".join(value_type) value_type = "[%s]" % value_type + value_id = "%s" % (value_type,) array_enums = props[key_name]["items"].get("enum") if array_enums: if len(array_enums) > 1: @@ -164,6 +169,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals ) else: value_type = props[key_name]["type"] + value_id = props[key_name]["type"] if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" @@ -184,6 +190,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals fields["rows"].append({ "key": key_name, "type": value_type, + "id": value_id, "required": required, "desc": desc, "req_str": "**Required.** " if required else "" @@ -313,10 +320,16 @@ def _load_swagger_meta(self, filepath, api, group_name): if req_tables > 1: for table in req_tables[1:]: - nested_key_name = [ - s["key"] for s in req_tables[0]["rows"] if - s["type"] == ("{%s}" % (table["title"],)) - ][0] + nested_key_name = { + "key": s["key"] + for rtable in req_tables + for s in rtable["rows"] + if s["id"] == table["title"] + }.get("key", None) + + if nested_key_name is None: + raise Exception("Failed to find table for %r" % (table["title"],)) + for row in table["rows"]: row["key"] = "%s.%s" % (nested_key_name, row["key"]) From 8648f86032bea6c7576fa92a4bd8ea8e0fdd9793 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 16 Nov 2015 15:08:37 +0000 Subject: [PATCH 192/989] Moar spaces --- templating/matrix_templates/templates/http-api.tmpl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index d7258e98cf7..4a9491f3f6c 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -13,17 +13,17 @@ Request format: {% if (endpoint.req_param_by_loc | length) %} -=========================================== ================= =========================================== - Parameter Value Description -=========================================== ================= =========================================== +=============================================================== ================= =========================================== + Parameter Value Description +=============================================================== ================= =========================================== {% for loc in endpoint.req_param_by_loc -%} *{{loc}} parameters* ---------------------------------------------------------------------------------------------------------- +----------------------------------------------------------------------------------------------------------------------------- {% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} +{{param.key}}{{param.type|indent(64-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(82)}} {% endfor -%} {% endfor -%} -=========================================== ================= =========================================== +=============================================================== ================= =========================================== {% else %} `No parameters` {% endif %} From 36af793f05992d0bef5ef325c303a753f16f24e0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 15:30:22 +0000 Subject: [PATCH 193/989] s/full object/full event/ --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 9cc10141f88..127a3572192 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -612,7 +612,7 @@ attempting to join. The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some of the information that the joining server will need. Despite its name, this -object is not a full object; notably it does not need to be hashed or signed by +object is not a full event; notably it does not need to be hashed or signed by the assisting resident. The required fields are: ==================== ======== ============ From 923f05e5541f354c2003acf77dbe4e2c3d6f90ec Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 15:34:11 +0000 Subject: [PATCH 194/989] More consistency around 'resident homeserver' --- specification/server_server_api.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 127a3572192..4c315edcb66 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -613,7 +613,7 @@ The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some of the information that the joining server will need. Despite its name, this object is not a full event; notably it does not need to be hashed or signed by -the assisting resident. The required fields are: +the resident homeserver. The required fields are: ==================== ======== ============ Key Type Description @@ -625,9 +625,9 @@ the assisting resident. The required fields are: ``content`` Object The event content ``depth`` Integer (this field must be present but is ignored; it may be 0) -``event_id`` String A new event ID specified by the assisting - resident -``origin`` String The name of the assisting resident homeserver +``event_id`` String A new event ID specified by the resident + homeserver +``origin`` String The name of the resident homeserver ``origin_server_ts`` Integer A timestamp added by the resident homeserver ``prev_events`` List An event-reference list containing the immediate predecessor events @@ -665,12 +665,12 @@ algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` fields. To complete the join handshake, the joining server must now submit this new -event to an assisting resident, by using the ``send_join`` endpoint. This is +event to an resident homeserver, by using the ``send_join`` endpoint. This is invoked using the room ID and the event ID of the new member event. -The assisting resident then accepts this event into the room's event graph, and -responds to the joining server with the full set of state for the newly-joined -room. This is returned as a two-element list, whose first element is the +The resident homeserver then accepts this event into the room's event graph, +and responds to the joining server with the full set of state for the newly- +joined room. This is returned as a two-element list, whose first element is the integer 200, and whose second element contains the following keys: ============== ===== ============ @@ -919,6 +919,6 @@ Querying directory information:: The list of join candidates is a list of server names that are likely to hold the given room; these are servers that the requesting server may wish to use as -assisting resident servers as part of the remote join handshake. This list may -or may not include the server answering the query. +resident servers as part of the remote join handshake. This list may or may not +include the server answering the query. From e2eb0c7ad9c0424450e9e14c3ee24d65d93e45de Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Nov 2015 16:04:23 +0000 Subject: [PATCH 195/989] Add new request options --- api/client-server/v1/search.yaml | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index cc2990564a8..5549ab2ccf6 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -39,6 +39,14 @@ paths: ], "search_term": "martians and men" } + }, + "order_by": "recent", + "groupings": { + "group_by": [ + { + "key": "room_id" + } + ] } } properties: @@ -68,6 +76,52 @@ paths: description: |- The filter to apply to search results. This has the same format as v2 filter API. + order_by: + title: "Ordering" + type: string + enum: ["recent", "rank"] + description: "The order in which to search for results." + event_context: + title: "Event Context" + type: object + description: |- + Configures whether any context for the events + returned are included in the response. + properties: + before_limit: + type: integer + title: "Before limit" + description: |- + How many events before the result are + returned. + after_limit: + type: integer + title: "After limit" + description: |- + How many events after the result are + returned. + groupings: + type: object + title: Groupings + description: |- + Requests the server returns results with information + about how they fit into groups the specified groups. + properties: + group_by: + type: array + title: Groups + description: List of groups to request. + items: + type: object + title: Group + description: Configuration for group. + properties: + key: + type: string + title: Group Key + description: |- + Key that defines the group. + enum: ["room_id", "sender"] required: ["search_term"] required: ["search_categories"] responses: From 122c082fcf0ef14cb93e09d1b8d2b7469866c786 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:29:43 +0000 Subject: [PATCH 196/989] Comment about origin servers of invites having subsequently left the room --- specification/server_server_api.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 4c315edcb66..20026890dd6 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -600,7 +600,9 @@ request the room ID and join candidates. This is covered in more detail on the directory server documentation, below. In the case of a new user joining a room as a result of a received invite, the joining user's homeserver could optimise this step away by picking the origin server of that invite message as -the join candidate. +the join candidate. However, the joining server should be aware that the origin +server of the invite might since have left the room, so should be prepared to +fall back on the regular join flow if this optimisation fails. Once the joining server has the room ID and the join candidates, it then needs to obtain enough information about the room to fill in the required fields of From 6cbfba70113e302985c4f93af181141e7b0c84f1 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:33:26 +0000 Subject: [PATCH 197/989] 'auth_events' is a List, not a String --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 20026890dd6..6c66b3d9c23 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -621,7 +621,7 @@ the resident homeserver. The required fields are: Key Type Description ==================== ======== ============ ``type`` String The value ``m.room.member`` -``auth_events`` String An event-reference list containing the +``auth_events`` List An event-reference list containing the authorization events that would allow this member to join ``content`` Object The event content From 769c5285ab75243d247afd2b6bdb806224d0309d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:33:57 +0000 Subject: [PATCH 198/989] Add API for setting client config --- api/client-server/v2_alpha/client-config.yaml | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 api/client-server/v2_alpha/client-config.yaml diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/client-config.yaml new file mode 100644 index 00000000000..91be1686a81 --- /dev/null +++ b/api/client-server/v2_alpha/client-config.yaml @@ -0,0 +1,105 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server Client Config API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/config/{configType}": + put: + summary: Set some config for the user. + description: |- + Set some config for the client. This config is only visible to the user + that set the config. The config will be synced to clients in the + top-level ``private_user_data``. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to set config for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: configType + required: true + description: |- + The type of the config to set. Custom types should be namespaced to + avoid clashes. + x-example: "org.example.custom.config" + - in: body + name: content + required: true + description: |- + The content of the config + schema: + type: object + example: |- + {"custom_config_key": "custom_config_value"} + responses: + 200: + description: + The config was successfully added. + "/user/{userId}/rooms/{roomId}/config/{configType}": + put: + summary: Set some config for the user. + description: |- + Set some config for the client on a given room. This config is only + visible to the user that set the config. The config will be synced to + clients in the per-room ``private_user_data``. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to set config for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to set config on. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: configType + required: true + description: |- + The type of the config to set. Custom types should be namespaced to + avoid clashes. + x-example: "org.example.custom.room.config" + - in: body + name: content + required: true + description: |- + The content of the config + schema: + type: object + example: |- + {"custom_config_key": "custom_config_value"} + responses: + 200: + description: + The config was successfully added. From 22b3159a3983a98762961ff5374deb3eff6c6d99 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:34:39 +0000 Subject: [PATCH 199/989] Add examples of v1 initialSync and v2 /sync returning the client config --- api/client-server/v1/sync.yaml | 24 ++++++++++++++++++++---- api/client-server/v2_alpha/sync.yaml | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b52908a99ec..57a1558373c 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -131,6 +131,14 @@ paths: "type": "m.presence" } ], + "private_user_data": [ + { + "type": "org.example.custom.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ], "rooms": [ { "membership": "join", @@ -246,10 +254,18 @@ paths: } ], "visibility": "private", - "private_user_data": [{ - "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} - }] + "private_user_data": [ + { + "type": "m.tag", + "content": {"tags": {"work": {"order": 1}}} + }, + { + "type": "org.example.custom.room.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ] } ] } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 4c1a7601bad..e180c5ff0ba 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -215,6 +215,16 @@ paths: } ] }, + "private_user_data": { + "events": [ + { + "type": "org.example.custom.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ] + }, "rooms": { "joined": { "!726s6s6q:example.com": { @@ -271,6 +281,12 @@ paths: { "type": "m.tags", "content": {"tags": {"work": {"order": 1}}} + }, + { + "type": "org.example.custom.room.config", + "content": { + "custom_config_key": "custom_config_value" + } } ] } From c3769ef75ca039ad2bfe2532a17ad979c631c71c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:49:32 +0000 Subject: [PATCH 200/989] Add a module for client config to the specification --- specification/modules/client_config.rst | 26 +++++++++++++++++++++++++ specification/targets.yaml | 1 + 2 files changed, 27 insertions(+) create mode 100644 specification/modules/client_config.rst diff --git a/specification/modules/client_config.rst b/specification/modules/client_config.rst new file mode 100644 index 00000000000..837f174d72b --- /dev/null +++ b/specification/modules/client_config.rst @@ -0,0 +1,26 @@ +Client Config +============= + +.. _module:client_config: + +Clients can store their config on their homeserver. This config will be synced +between different devices and can persist across installations on a particular +device. + +The config may be either global or scoped to a particular rooms. + +Events +------ + +The client recieves the config as a event in the ``private_user_data`` sections +of a v2 /sync. + +These events can also be received in a v1 /events response or in the +``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +events appearing in v1 /events will have a ``room_id`` with the room +the tags are for. + +Client Behaviour +---------------- + +{{v2_client_config_http_api}} diff --git a/specification/targets.yaml b/specification/targets.yaml index 2c6a8f2051e..b6ac6a65f2e 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -26,6 +26,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/third_party_invites.rst - modules/search.rst - modules/tags.rst + - modules/client_config.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 83168813939ea1dc4c5a778781cb12e1f8eb9d87 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 16:50:54 +0000 Subject: [PATCH 201/989] txn_id field in events is called transaction_id --- api/client-server/v2_alpha/definitions/event.json | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 6a01c688522..5a8f52f6fb5 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -43,7 +43,7 @@ "type": "string", "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." }, - "txn_id": { + "transaction_id": { "type": "string", "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 2290b0700a8..d23929df36b 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -253,7 +253,7 @@ paths: "$74686972643033:example.com": { "sender": "@alice:example.com", "type": "m.room.message", - "unsigned": {"age": 124524, "txn_id": "1234"}, + "unsigned": {"age": 124524, "transaction_id": "1234"}, "content": { "body": "I am a fish", "msgtype": "m.text" From 233e8486bc41e4ad87fc8e06a57bbcd4b8579974 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:50:58 +0000 Subject: [PATCH 202/989] Wording fix - objects contain keys, not list elements directly --- specification/server_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6c66b3d9c23..cc9426e2b0f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -673,7 +673,8 @@ invoked using the room ID and the event ID of the new member event. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the newly- joined room. This is returned as a two-element list, whose first element is the -integer 200, and whose second element contains the following keys: +integer 200, and whose second element is an object which contains the +following keys: ============== ===== ============ Key Type Description From 0db055b4ea22f4d29064728eaaaf2c37a2f9d926 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 16:54:57 +0000 Subject: [PATCH 203/989] Fix another reference to 'txn_id' --- specification/events.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/events.rst b/specification/events.rst index 7bf080125bf..9d0be55f8fd 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -35,8 +35,8 @@ formatted for federation by: a ``state event`` which gives previous content of that state key. * Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the - client requesting it. +* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent + by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: From 299af673da3d4a698dc943aaabe3c2e961f2da49 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:04:19 +0000 Subject: [PATCH 204/989] Specify how ordering of tags is supposed to work --- api/client-server/v1/rooms.yaml | 2 +- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- api/client-server/v2_alpha/tags.yaml | 4 ++-- specification/modules/tags.rst | 5 +++++ 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 1cce76f30f9..bafcb98b69d 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,7 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b52908a99ec..8050ede39bb 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index ec34c085c5d..7a597f9e424 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -291,7 +291,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} } ] } diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml index 009e6000ce8..91945dca27c 100644 --- a/api/client-server/v2_alpha/tags.yaml +++ b/api/client-server/v2_alpha/tags.yaml @@ -55,7 +55,7 @@ paths: application/json: |- { "tags": { - "work": {"order": 1}, + "work": {"order": "1"}, "pinned": {} } } @@ -97,7 +97,7 @@ paths: schema: type: object example: |- - {"order": 1} + {"order": "1"} responses: 200: description: diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 3d45975c42c..15982a0af02 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -21,6 +21,11 @@ the tags are for. Each tag has an associated JSON object with information about the tag, e.g how to order the rooms with a given tag. +Ordering information is given under the ``order`` key as a string. The string +are compared lexicographically by unicode codepoint to determine which should +displayed first. So a tag with an ``order`` key of ``"apples"`` would appear +before a tag with an ``order`` key of ``"oranges"``. + {{m_tag_event}} Client Behaviour From 48f35e15cba959e28aeb897e3339e5d6dfbfddc7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:08:59 +0000 Subject: [PATCH 205/989] describe how to order rooms that don't have an order in their tags --- specification/modules/tags.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 15982a0af02..56d216e7c50 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -23,10 +23,13 @@ to order the rooms with a given tag. Ordering information is given under the ``order`` key as a string. The string are compared lexicographically by unicode codepoint to determine which should -displayed first. So a tag with an ``order`` key of ``"apples"`` would appear -before a tag with an ``order`` key of ``"oranges"``. +displayed first. So a room with a tag with an ``order`` key of ``"apples"`` +would appear before a room with a tag with an ``order`` key of ``"oranges"``. +If a room has a tag without an ``order`` key then it should appear after the +rooms with that tag that have an ``order`` key. -{{m_tag_event}} + +{{m_tag_event} Client Behaviour ---------------- From e7fbe6f13b12797817eed767cd7306498dcddcec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:13:02 +0000 Subject: [PATCH 206/989] Limit the size of a tag --- specification/modules/tags.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 56d216e7c50..4d88c77154c 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -28,6 +28,7 @@ would appear before a room with a tag with an ``order`` key of ``"oranges"``. If a room has a tag without an ``order`` key then it should appear after the rooms with that tag that have an ``order`` key. +The name of a tag MUST not exceed 255 bytes. {{m_tag_event} From 25769493b11cd95ef73b5cfe936971a8bc7565ea Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:14:41 +0000 Subject: [PATCH 207/989] Fix template --- specification/modules/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 4d88c77154c..5fe6345f44c 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -30,7 +30,7 @@ rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. -{{m_tag_event} +{{m_tag_event}} Client Behaviour ---------------- From 5e0e4a851869abd81709deb2677c74657a9ce9dd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Nov 2015 17:22:56 +0000 Subject: [PATCH 208/989] Document group and context response keys --- api/client-server/v1/search.yaml | 89 +++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index 5549ab2ccf6..e453c523219 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -161,15 +161,92 @@ paths: this result matches the search. Higher is closer. result: - type: object - title: Event - description: The event that matched. - allOf: - - "$ref": "core-event-schema/room_event.json" + type: object + title: Event + description: The event that matched. + allOf: + - "$ref": "core-event-schema/room_event.json" + context: + type: object + title: Event Context + description: Context for result, if requested. + properties: + start: + type: string + title: Start Token + description: |- + Pagination token for the start of the chunk + end: + type: string + title: End Token + description: |- + Pagination token for the end of the chunk + events_before: + type: array + title: Events Before + description: Events just before the result. + items: + type: object + title: Event + allOf: + - "$ref": "core-event-schema/room_event.json" + events_after: + type: array + title: Events After + descriptions: Events just after the result. + items: + type: object + title: Event + allOf: + - "$ref": "core-event-schema/room_event.json" + groups: + type: object + title: Groups + description: Any groups that were requseted. + additionalProperties: + type: object + title: Group Key + description: The results for a given group. + additionalProperties: + type: object + title: Group Value + description: |- + The results for a particular group value. + properties: + next_batch: + type: string + title: Next Batch in Group + description: |- + Token that can be used to get the next + batch of results if exists. + order: + type: integer + title: Group Order + description: |- + Key that can be used to order different + groups. + results: + type: array + description: |- + Which results are in this group. + items: + type: string + title: Result Event ID examples: application/json: |- { "search_categories": { + "groups": { + "room_id": { + "!qPewotXpIctQySfjSy:localhost": { + "order": 1, + "next_batch": "BdgFsdfHSf-dsFD", + "results": [ + "$144429830826TWwbB:localhost" + ] + } + } + }, "room_events": { "count": 24, "results": { @@ -178,7 +255,7 @@ paths: "result": { "age": 526228296, "content": { - "body": "Test content", + "body": "Test content martians and men", "msgtype": "m.text" }, "event_id": "$144429830826TWwbB:localhost", From c77b22778fab74d7bae2ebc25b1d635d17e8afb3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:48:41 +0000 Subject: [PATCH 209/989] Add some documentation on names of tags --- specification/modules/tags.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 5fe6345f44c..681d33d71d3 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -30,6 +30,16 @@ rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. +The name of a tag should be human readable. When displaying tags for a room a +client should display this human readable name. When adding a tag for a room +a client may offer a list to choose from that includes all the tags that the +user has previously set on any of their rooms. + +Two special names are listed in the specification: + +* ``m.favourite`` +* ``m.lowpriority`` + {{m_tag_event}} Client Behaviour From 8991c4fa78c5a3854145a5f3d63e21aa98738bfa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Nov 2015 18:00:48 +0000 Subject: [PATCH 210/989] Fix schema and example --- api/client-server/v1/search.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index e453c523219..e9598ac3c5e 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -193,7 +193,7 @@ paths: events_after: type: array title: Events After - descriptions: Events just after the result. + description: Events just after the result. items: type: object title: Event @@ -236,18 +236,18 @@ paths: application/json: |- { "search_categories": { - "groups": { - "room_id": { - "!qPewotXpIctQySfjSy:localhost": { - "order": 1, - "next_batch": "BdgFsdfHSf-dsFD", - "results": [ - "$144429830826TWwbB:localhost" - ] - } - } - }, "room_events": { + "groups": { + "room_id": { + "!qPewotXpIctQySfjSy:localhost": { + "order": 1, + "next_batch": "BdgFsdfHSf-dsFD", + "results": [ + "$144429830826TWwbB:localhost" + ] + } + } + }, "count": 24, "results": { "$144429830826TWwbB:localhost": { From 03a0377c76861eb1c9af503b718b31508dcdd7dc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 20:07:42 +0000 Subject: [PATCH 211/989] Plans for end-to-end in matrix --- drafts/markjh_end_to_end.rst | 142 +++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 drafts/markjh_end_to_end.rst diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst new file mode 100644 index 00000000000..1699f79bd6d --- /dev/null +++ b/drafts/markjh_end_to_end.rst @@ -0,0 +1,142 @@ +Goals of Key-Distribution in Matrix +=================================== + +* No Central Authority: Users should not need to trust a central authority + when determining the authenticity of keys. + +* Easy to Add New Devices: It should be easy for a user to start using a + new device. + +* Possible to discover MITM: It should be possible for a user to determine if + they are being MITM. + +* Lost Devices: It should be possible for a user to recover if they lose all + their devices. + +* No Copying Keys: Keys should be per device and shouldn't leave the device + they were created on. + +A Possible Mechanism for Key Distribution +========================================= + +Basic API for setting up keys on a server: + +https://github.com/matrix-org/matrix-doc/pull/24 + +Client shouldn't trust the keys unless they have been verified, e.g by +comparing fingerprints. + +If a user adds a new device it should some yet to be specified protocol +communicate with an old device and obtain a cross-signature from the old +device for its public key. + +The new device can then present the cross-signed key to all the devices +that the user is in conversations with. Those devices should then include +the new device into those conversations. + +If the user cannot cross-sign the new key, e.g. because their old device +is lost or stolen. Then they will need to reauthenticate their conversations +out of band, e.g by comparing fingerprints. + + +Goals of End-to-end encryption in Matrix +======================================== + +* Access to Chat History: Users should be able to see the history of a + conversation on a new device. User should be able to control who can + see their chat history and how much of the chat history they can see. + +* Forward Secrecy of Discarded Chat History: Users should be able to discard + history from their device, once they have discarded the history it should be + impossible for an adversary to recover that history. + +* Forward Secrecy of Future Messages: Users should be able to recover from + disclosure of the chat history on their device. + +* Deniablity of Chat History: It should not be possible to prove to a third + party that a given user sent a message. + +* Authenticity of Chat History: It should be possible to prove amoungst + the members of a chat that a message sent by a user was authored by that + user. + + +Bonus Goals: + +* Traffic Analysis: It would be nice if the protocol was resilient to traffic + or metadata analysis. However it's not something we want to persue if it + harms the usability of the protocol. It might be cool if there was a + way for the user to could specify the trade off between performance and + resilience to traffic analysis that they wanted. + + +A Possible Design for Group Chat using Olm +========================================== + +Protecting the secrecy of history +--------------------------------- + +Each message sent by a client has a 32-bit counter. This counter increments +by one for each message sent by the client. This counter is used to advance a +ratchet. The ratchet is split into a vector four 256-bit values, +:math:`R_{n,j}` for :math:`j \in {0,1,2,3}`. The ratchet can be advanced as +follows: + +.. math:: + \begin{align} + R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ + R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_1\left(R_{(i-1),3}\right) + \end{align} + +Where :math:`H_1` and :math:`H_2` are different hash functions. For example +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and +:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. + +So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. +Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. +Every :math:`2^8` iterations :math:`R_{n,3}` is reseeded from :math:`R_{n,2}`. + +This scheme allows the ratchet to be advanced an arbitrary amount forwards +while needing only 1024 hash computations. + +This the value of the ratchet is hashed to generate the keys used to encrypt +each mesage. + +A client can decrypt chat history onwards from the earliest value of the +ratchet it is aware of. But cannot decrypt history from before that point +without reversing the hash function. + +This allows a client to share its ability to decrypt chat history with another +from a point in the conversation onwards by giving a copy of the ratchet at +that point in the conversation. + +A client can discard history by advancing a ratchet to beyond the last message +they want to discard and then forgetting all previous values of the ratchet. + +Proving and denying the authenticity of history +----------------------------------------------- + +Client sign the messages they send using a Ed25519 key generated per +conversation. That key, along with the ratchet key, is distributed +to other clients using 1:1 olm ratchets. Those 1:1 ratchets are started using +Triple Diffie-Hellman which provides authenticity of the messages to the +participants and deniability of the messages to third parties. Therefore +any keys shared over those keys inherit the same levels of deniability and +authenticity. + +Protecting the secrecy of future messages +----------------------------------------- + +A client would need to generate new keys if it wanted to prevent access to +messages beyond a given point in the conversation. It must generate new keys +whenever someone leaves the room. It should generate new keys periodically +anyway. + +The frequency of key generation in a large room may need to be restricted to +keep the frequency of messages broadcast over the individual 1:1 channels +low. From ffed14a0cd5693131b6f44a0d67788699804fd12 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 10:02:02 +0000 Subject: [PATCH 212/989] Add next_batch token --- api/client-server/v1/search.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index e9598ac3c5e..34b3c150faa 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -218,7 +218,7 @@ paths: title: Next Batch in Group description: |- Token that can be used to get the next - batch of results if exists. + batch of results in the group, if exists. order: type: integer title: Group Order @@ -232,6 +232,12 @@ paths: items: type: string title: Result Event ID + next_batch: + type: string + title: Next Batch + description: |- + Token that can be used to get the next batch of + results, if exists. examples: application/json: |- { From 373c6c827056124a98a70b8a75387e633c53127f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 11:56:05 +0000 Subject: [PATCH 213/989] Add ordering, pagination and grouping comments --- specification/modules/search.rst | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 089953d2256..f44bb585d10 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -40,6 +40,41 @@ The value of ``count`` may not match the number of results. For example due to the search query matching 1000s of results and the server truncating the response. +Ordering +-------- + +The client can specify the ordering that the server returns results in. The two +allowed orderings are: + +- ``rank``, which returns the most relevant results first. +- ``recent``, which returns the most recent results first. + +The default ordering is ``rank``. + +Groups +------ + +The client can request that the results are returned along with grouping +information, e.g. grouped by ``room_id``. In this case the response will +contain a group entry for each distinct value of ``room_id``. Each group entry +contains at least a list of the ``event_ids`` that are in that group, as well +as potentially other metadata about the group. + +Pagination +---------- + +The server may return a ``next_batch`` key at various places in the response. +These are used to paginate the results. To fetch more results, the client +should send the *same* request to the server with a ``next_batch`` query +parameter set to that of the token. + +The scope of the pagination is defined depending on where the ``next_batch`` +token was returned. For example, using a token inside a group will return more +results from within that group. + +A server need not support pagination, even if there are more matching results. + + Security considerations ----------------------- The server must only return results that the user has permission to see. From 8201eaa042875feb69389f14262356fd3cfa73ed Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 17 Nov 2015 15:31:10 +0000 Subject: [PATCH 214/989] Swaggerify /rooms/:room_id/leave --- .../v1/{membership.yaml => inviting.yaml} | 51 +------------- api/client-server/v1/joining.yaml | 68 +++++++++++++++++++ api/client-server/v1/leaving.yaml | 57 ++++++++++++++++ specification/client_server_api.rst | 32 ++------- 4 files changed, 132 insertions(+), 76 deletions(-) rename api/client-server/v1/{membership.yaml => inviting.yaml} (61%) create mode 100644 api/client-server/v1/joining.yaml create mode 100644 api/client-server/v1/leaving.yaml diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/inviting.yaml similarity index 61% rename from api/client-server/v1/membership.yaml rename to api/client-server/v1/inviting.yaml index c089e154a9a..0a028710313 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/inviting.yaml @@ -1,6 +1,6 @@ swagger: '2.0' info: - title: "Matrix Client-Server v1 Room Membership API" + title: "Matrix Client-Server v1 Room Joining API" version: "1.0.0" host: localhost:8008 schemes: @@ -18,55 +18,6 @@ securityDefinitions: name: access_token in: query paths: - "/rooms/{roomId}/join": - post: - summary: Start the requesting user participating in a particular room. - description: |- - This API starts a user participating in a particular room, if that user - is allowed to participate in that room. After this call, the client is - allowed to see all current state events in the room, and all subsequent - events associated with the room until the user leaves the room. - - After a user has joined a room, the room will appear as an entry in the - response of the |initialSync| API. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room identifier or room alias to join. - required: true - x-example: "#monkeys:matrix.org" - responses: - 200: - description: |- - The room has been joined. - - The joined room ID must be returned in the ``room_id`` field. - examples: - application/json: |- - {"room_id": "!d41d8cd:matrix.org"} - schema: - type: object - 403: - description: |- - You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: - - - The room is invite-only and the user was not invited. - - The user has been banned from the room. - examples: - application/json: |- - {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} - 429: - description: This request was rate-limited. - schema: - "$ref": "definitions/error.yaml" - x-alias: - canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" - aliases: - - /join/{roomId} - # With an extra " " to disambiguate from the 3pid invite endpoint # The extra space makes it sort first for what I'm sure is a good reason. "/rooms/{roomId}/invite ": diff --git a/api/client-server/v1/joining.yaml b/api/client-server/v1/joining.yaml new file mode 100644 index 00000000000..20fdbd65450 --- /dev/null +++ b/api/client-server/v1/joining.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Inviting API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/join": + post: + summary: Start the requesting user participating in a particular room. + description: |- + This API starts a user participating in a particular room, if that user + is allowed to participate in that room. After this call, the client is + allowed to see all current state events in the room, and all subsequent + events associated with the room until the user leaves the room. + + After a user has joined a room, the room will appear as an entry in the + response of the |initialSync| API. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to join. + required: true + x-example: "#monkeys:matrix.org" + responses: + 200: + description: |- + The room has been joined. + + The joined room ID must be returned in the ``room_id`` field. + examples: + application/json: |- + {"room_id": "!d41d8cd:matrix.org"} + schema: + type: object + 403: + description: |- + You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: + + - The room is invite-only and the user was not invited. + - The user has been banned from the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + x-alias: + canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" + aliases: + - /join/{roomId} diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/v1/leaving.yaml new file mode 100644 index 00000000000..376b2060c17 --- /dev/null +++ b/api/client-server/v1/leaving.yaml @@ -0,0 +1,57 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Leaving API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/leave": + post: + summary: Stop the requesting user participating in a particular room. + description: |- + This API stops a user participating in a particular room. + + If the user was already in the room, they will no longer be able to see + new events in the room. If the room requires an invite to join, they + will need to be re-invited before they can re-join. + + If the user was invited to the room, but had not joined, this call + serves to reject the invite. + + The user will still be allowed to retrieve history from the room which + they were previously allowed to see. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to leave. + required: true + x-example: "!nkl290a:matrix.org" + responses: + 200: + description: |- + The room has been left. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 05b6ff3c28a..ba5578bcb55 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -870,42 +870,22 @@ following values: ``invite`` This room can only be joined if you were invited. -{{membership_http_api}} +{{inviting_http_api}} + +{{joining_http_api}} Leaving rooms ~~~~~~~~~~~~~ -.. TODO-spec - HS deleting rooms they are no longer a part of. Not implemented. - - This is actually Very Tricky. If all clients a HS is serving leave a room, - the HS will no longer get any new events for that room, because the servers - who get the events are determined on the *membership list*. There should - probably be a way for a HS to lurk on a room even if there are 0 of their - members in the room. - - Grace period before deletion? - - Under what conditions should a room NOT be purged? - - A user can leave a room to stop receiving events for that room. A user must have been invited to or have joined the room before they are eligible to leave the room. Leaving a room to which the user has been invited rejects the invite. +Once a user leaves a room, it will no longer appear on the |initialSync|_ API. Whether or not they actually joined the room, if the room is an "invite-only" room they will need to be re-invited before they can re-join -the room. To leave a room, a request should be made to -|/rooms//leave|_ with:: - - {} - -Alternatively, the membership state for this user in this room can be modified -directly by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "leave" - } +the room. -See the `Room events`_ section for more information on ``m.room.member``. Once a -user has left a room, that room will no longer appear on the |initialSync|_ API. -If all members in a room leave, that room becomes eligible for deletion. +{{leaving_http_api}} Banning users in a room ~~~~~~~~~~~~~~~~~~~~~~~ From 6763317e64645945224b7c68ac385ed055fe80c3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 17 Nov 2015 10:33:46 -0500 Subject: [PATCH 215/989] Specify /rooms/:room_id/forget --- api/client-server/v1/leaving.yaml | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/v1/leaving.yaml index 376b2060c17..e81d812b5de 100644 --- a/api/client-server/v1/leaving.yaml +++ b/api/client-server/v1/leaving.yaml @@ -55,3 +55,38 @@ paths: description: This request was rate-limited. schema: "$ref": "definitions/error.yaml" + "/rooms/{roomId}/forget": + post: + summary: Stop the requesting user remembering about a particular room. + description: |- + This API stops a user remembering about a particular room. + + In general, history is a first class citizen in Matrix. After this API + is called, however, a user will no longer be able to retrieve history + for this room. If all users on a homeserver forget a room, the room is + eligible for deletion from that homeserver. + + If the user is currently joined to the room, they will implicitly leave + the room as part of this API call. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to forget. + required: true + x-example: "!au1ba7o:matrix.org" + responses: + 200: + description: |- + The room has been forgotten. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" From 6653362f315d1379d1dda248dfd299c8c3c8530d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Nov 2015 15:15:21 +0000 Subject: [PATCH 216/989] Unflatten 'unsigned' It turns out that flattening 'unsigned' comes with too many downsides. Let's stick with the status quo. --- .../v2_alpha/definitions/event.json | 45 ++++++++++--------- specification/events.rst | 19 ++++---- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 3a15357aa5e..5a8f52f6fb5 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -2,34 +2,16 @@ "type": "object", "title": "Event", "properties": { - "age": { - "type": "integer", - "format": "int64", - "description": "Time in milliseconds since the event was sent." - }, "content": { "type": "object", "title": "EventContent", "description": "The content of this event. The fields in this object will vary depending on the type of event." }, - "event_id": { - "type": "string", - "description": "Globally unique identifier for this event." - }, "origin_server_ts": { "type": "integer", "format": "int64", "description": "Timestamp in milliseconds on originating homeserver when this event was sent." }, - "prev_content": { - "title": "EventContent", - "type": "object", - "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "prev_sender": { - "type": "string", - "description": "Optional. The ``sender`` of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there was no previous event for this state, this key will be missing." - }, "sender": { "type": "string", "description": "The MXID of the user who sent this event." @@ -42,9 +24,30 @@ "type": "string", "description": "The type of event." }, - "txn_id": { - "type": "string", - "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + "unsigned": { + "type": "object", + "title": "Unsigned", + "description": "Information about this event which was not sent by the originating homeserver", + "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "replaces_state": { + "type": "string", + "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "transaction_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + } + } } } } diff --git a/specification/events.rst b/specification/events.rst index eb48ca17d1d..5a003115556 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -28,20 +28,23 @@ formatted for federation by: * Removing the following keys: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, - ``origin``, ``prev_state``, ``unsigned``. -* Adding an ``age`` to the event object which gives the time in + ``origin``, ``prev_state``. +* Adding an ``age`` to the ``unsigned`` object which gives the time in milliseconds that has elapsed since the event was sent. -* Adding ``prev_content`` and ``prev_sender`` to the event object if the event - is a ``state event``, which give the previous content and previous sender of - that state key -* Adding a ``redacted_because`` to event object if the event was +* Adding ``prev_content`` and ``prev_sender`` to the ``unsigned`` object if the + event is a ``state event``, which give the previous content and previous + sender of that state key +* Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the event object if the event was sent by the client - requesting it. +* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent + by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: +* Moving the folling keys from the ``unsigned`` object to the top level event + object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. +* Removing the ``unsigned`` object. * Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From fcbb985073e82368894a41285dab86feb53daa21 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Nov 2015 15:27:26 +0000 Subject: [PATCH 217/989] s/private_user_data/account_data/ --- api/client-server/v1/rooms.yaml | 4 ++-- api/client-server/v1/sync.yaml | 4 ++-- api/client-server/v2_alpha/sync.yaml | 4 ++-- specification/modules/tags.rst | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index bafcb98b69d..ff5587a902f 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -312,7 +312,7 @@ paths: } ], "visibility": "private", - "private_user_data": [{ + "account_data": [{ "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} }] @@ -375,7 +375,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - private_user_data: + account_data: type: array description: |- The private data that this user has attached to this room. diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 8050ede39bb..973ccc6e38d 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -246,7 +246,7 @@ paths: } ], "visibility": "private", - "private_user_data": [{ + "account_data": [{ "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} }] @@ -336,7 +336,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - private_user_data: + account_data: type: array description: |- The private data that this user has attached to diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 7a597f9e424..28358f55805 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -139,7 +139,7 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" - private_user_data: + account_data: title: Private User Data type: object description: |- @@ -287,7 +287,7 @@ paths: } ] }, - "private_user_data": { + "account_data": { "events": [ { "type": "m.tags", diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 681d33d71d3..bb467bc0583 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -11,10 +11,10 @@ Events ------ The tags on a room are received as single ``m.tag`` event in the -``private_user_data`` section of a room in a v2 /sync. +``account_data`` section of a room in a v2 /sync. The ``m.tag`` can also be received in a v1 /events response or in the -``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +``account_data`` section of a room in v1 /initialSync. ``m.tag`` events appearing in v1 /events will have a ``room_id`` with the room the tags are for. From 40f7eab73f1a3055c3c4874b8d122f6b9c57e167 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Nov 2015 15:44:18 +0000 Subject: [PATCH 218/989] s/private_user_data/account_data/ --- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/client-config.yaml | 4 ++-- api/client-server/v2_alpha/sync.yaml | 2 +- specification/modules/client_config.rst | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b8608171ce5..58cd5544336 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -131,7 +131,7 @@ paths: "type": "m.presence" } ], - "private_user_data": [ + "account_data": [ { "type": "org.example.custom.config", "content": { diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/client-config.yaml index 91be1686a81..76b6219ee9c 100644 --- a/api/client-server/v2_alpha/client-config.yaml +++ b/api/client-server/v2_alpha/client-config.yaml @@ -24,7 +24,7 @@ paths: description: |- Set some config for the client. This config is only visible to the user that set the config. The config will be synced to clients in the - top-level ``private_user_data``. + top-level ``account_data``. security: - accessToken: [] parameters: @@ -63,7 +63,7 @@ paths: description: |- Set some config for the client on a given room. This config is only visible to the user that set the config. The config will be synced to - clients in the per-room ``private_user_data``. + clients in the per-room ``account_data``. security: - accessToken: [] parameters: diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index e2016d68d67..f5c99508ee3 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -233,7 +233,7 @@ paths: } ] }, - "private_user_data": { + "account_data": { "events": [ { "type": "org.example.custom.config", diff --git a/specification/modules/client_config.rst b/specification/modules/client_config.rst index 837f174d72b..581a5828194 100644 --- a/specification/modules/client_config.rst +++ b/specification/modules/client_config.rst @@ -12,11 +12,11 @@ The config may be either global or scoped to a particular rooms. Events ------ -The client recieves the config as a event in the ``private_user_data`` sections +The client recieves the config as a event in the ``account_data`` sections of a v2 /sync. These events can also be received in a v1 /events response or in the -``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +``account_data`` section of a room in v1 /initialSync. ``m.tag`` events appearing in v1 /events will have a ``room_id`` with the room the tags are for. From d7d59d78e1dca06407dea20ad875f64569844ad3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Nov 2015 16:17:29 +0000 Subject: [PATCH 219/989] /sync: Put state dict back to being a list Turning the state into a dict-of-dicts caused more pain than it solved. Put it back to a list. --- .../v2_alpha/definitions/state_batch.json | 12 ------- api/client-server/v2_alpha/sync.yaml | 31 ++++++++----------- 2 files changed, 13 insertions(+), 30 deletions(-) delete mode 100644 api/client-server/v2_alpha/definitions/state_batch.json diff --git a/api/client-server/v2_alpha/definitions/state_batch.json b/api/client-server/v2_alpha/definitions/state_batch.json deleted file mode 100644 index 45728f40347..00000000000 --- a/api/client-server/v2_alpha/definitions/state_batch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "additionalProperties": { - "type": "object", - "x-pattern": "$EVENT_TYPE", - "additionalProperties": { - "type": "object", - "x-pattern": "$STATE_KEY", - "allOf": [{"$ref": "event.json" }] - } - } -} diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 198f48337aa..0c9d6186f9b 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -114,7 +114,7 @@ paths: ``timeline``, if ``since`` is not given, or ``full_state`` is true). allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -158,7 +158,7 @@ paths: delta against the archived ``state`` not the ``invite_state``. allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" leave: title: Left rooms type: object @@ -174,7 +174,7 @@ paths: description: |- The state updates for the room up to the start of the timeline. allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -207,8 +207,8 @@ paths: "join": { "!726s6s6q:example.com": { "state": { - "m.room.member": { - "@alice:example.com": { + "events": [ + { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@alice:example.com", @@ -216,7 +216,7 @@ paths: "origin_server_ts": 1417731086795, "event_id": "$66697273743031:example.com" } - } + ] }, "timeline": { "events": [ @@ -248,7 +248,6 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } @@ -259,24 +258,20 @@ paths: "invite": { "!696r7674:example.com": { "invite_state": { - "m.room.name": { - "": { + "events": [ + { "sender": "@alice:example.com", "type": "m.room.name", "state_key": "", - "content": {"name": "My Room Name"}, - "event_id": "$asdkgjrsfg2314375:example.com" - } - }, - "m.room.member": { - "@bob:example.com": { + "content": {"name": "My Room Name"} + }, + { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@bob:example.com", - "content": {"membership": "invite"}, - "event_id": "$257kasjdg315324akhg:example.com" + "content": {"membership": "invite"} } - } + ] } } }, From 5aeaa42a5039bd0435f66be0a9787cb3efc24985 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 18 Nov 2015 18:41:25 -0500 Subject: [PATCH 220/989] Changelog for 0.3.0 --- CHANGELOG.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6e3198ef996..4916c771ff2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,35 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. +Specification changes in v0.3.0 (2015-11-18) +============================================ + +Many sections have been filled in, or added to reflect long-standing status +quos. Several API endpoints have also been converted to Swagger. Several +presentation improvements (e.g. syntax highlighting) have also been added. + +Additions: + - Extensive narrative on instant messaging + - Documentation of event sending + - Documentation of state event retrieval + - Redaction algorithm + - Algorithm for calculating display names for rooms and users + - New endpoint: ``/publicRooms`` + - New event: ``m.room.avatar`` + - invite_room_state on ``m.room.member`` events + - New section: Third party invitations + - Ability to reject invitations + - New API: Search + - Guest access to rooms + - Descriptions of server-server API endpoints for joining rooms + - Sample expected crypto output + - Several security warnings + + Modifications: + - Fixed errors in parameters of /login + - Significant clarification and improvements to description of pagination + - Changes to v2_alpha/sync API (Note: this API is still not stable) + Specification changes in v0.2.0 (2015-10-02) ============================================ @@ -38,4 +67,4 @@ Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. - Restructure the format of Event information. Add more information. -- Restructure the format of the Client-Server HTTP APIs. \ No newline at end of file +- Restructure the format of the Client-Server HTTP APIs. From 1a1a7d87dcd1f8ada30592cae7b4da86b8b2468f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 18 Nov 2015 18:53:20 -0500 Subject: [PATCH 221/989] Revert "Changelog for 0.3.0" This reverts commit 5aeaa42a5039bd0435f66be0a9787cb3efc24985. --- CHANGELOG.rst | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4916c771ff2..6e3198ef996 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,35 +6,6 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. -Specification changes in v0.3.0 (2015-11-18) -============================================ - -Many sections have been filled in, or added to reflect long-standing status -quos. Several API endpoints have also been converted to Swagger. Several -presentation improvements (e.g. syntax highlighting) have also been added. - -Additions: - - Extensive narrative on instant messaging - - Documentation of event sending - - Documentation of state event retrieval - - Redaction algorithm - - Algorithm for calculating display names for rooms and users - - New endpoint: ``/publicRooms`` - - New event: ``m.room.avatar`` - - invite_room_state on ``m.room.member`` events - - New section: Third party invitations - - Ability to reject invitations - - New API: Search - - Guest access to rooms - - Descriptions of server-server API endpoints for joining rooms - - Sample expected crypto output - - Several security warnings - - Modifications: - - Fixed errors in parameters of /login - - Significant clarification and improvements to description of pagination - - Changes to v2_alpha/sync API (Note: this API is still not stable) - Specification changes in v0.2.0 (2015-10-02) ============================================ @@ -67,4 +38,4 @@ Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. - Restructure the format of Event information. Add more information. -- Restructure the format of the Client-Server HTTP APIs. +- Restructure the format of the Client-Server HTTP APIs. \ No newline at end of file From 05c00926641a7a1156b2f239efc153b3aa39e9eb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 19 Nov 2015 10:36:27 +0000 Subject: [PATCH 222/989] s/config/account_data/ --- .../{client-config.yaml => account-data.yaml} | 44 +++++++++---------- .../{client_config.rst => account_data.rst} | 4 +- specification/targets.yaml | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) rename api/client-server/v2_alpha/{client-config.yaml => account-data.yaml} (60%) rename specification/modules/{client_config.rst => account_data.rst} (91%) diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/account-data.yaml similarity index 60% rename from api/client-server/v2_alpha/client-config.yaml rename to api/client-server/v2_alpha/account-data.yaml index 76b6219ee9c..98134d6feb5 100644 --- a/api/client-server/v2_alpha/client-config.yaml +++ b/api/client-server/v2_alpha/account-data.yaml @@ -18,12 +18,12 @@ securityDefinitions: name: access_token in: query paths: - "/user/{userId}/config/{configType}": + "/user/{userId}/account_data/{type}": put: - summary: Set some config for the user. + summary: Set some account_data for the user. description: |- - Set some config for the client. This config is only visible to the user - that set the config. The config will be synced to clients in the + Set some account_data for the client. This config is only visible to the user + that set the account_data. The config will be synced to clients in the top-level ``account_data``. security: - accessToken: [] @@ -33,36 +33,36 @@ paths: name: userId required: true description: |- - The id of the user to set config for. The access token must be + The id of the user to set account_data for. The access token must be authorized to make requests for this user id. x-example: "@alice:example.com" - in: path type: string - name: configType + name: type required: true description: |- - The type of the config to set. Custom types should be namespaced to + The type of the account_data to set. Custom types should be namespaced to avoid clashes. x-example: "org.example.custom.config" - in: body name: content required: true description: |- - The content of the config + The content of the account_data schema: type: object example: |- - {"custom_config_key": "custom_config_value"} + {"custom_account_data_key": "custom_config_value"} responses: 200: description: - The config was successfully added. - "/user/{userId}/rooms/{roomId}/config/{configType}": + The account_data was successfully added. + "/user/{userId}/rooms/{roomId}/account_data/{type}": put: - summary: Set some config for the user. + summary: Set some account_data for the user. description: |- - Set some config for the client on a given room. This config is only - visible to the user that set the config. The config will be synced to + Set some account_data for the client on a given room. This config is only + visible to the user that set the account_data. The config will be synced to clients in the per-room ``account_data``. security: - accessToken: [] @@ -72,7 +72,7 @@ paths: name: userId required: true description: |- - The id of the user to set config for. The access token must be + The id of the user to set account_data for. The access token must be authorized to make requests for this user id. x-example: "@alice:example.com" - in: path @@ -80,26 +80,26 @@ paths: name: roomId required: true description: |- - The id of the room to set config on. + The id of the room to set account_data on. x-example: "!726s6s6q:example.com" - in: path type: string - name: configType + name: type required: true description: |- - The type of the config to set. Custom types should be namespaced to - avoid clashes. + The type of the account_data to set. Custom types should be + namespaced to avoid clashes. x-example: "org.example.custom.room.config" - in: body name: content required: true description: |- - The content of the config + The content of the account_data schema: type: object example: |- - {"custom_config_key": "custom_config_value"} + {"custom_account_data_key": "custom_account_data_value"} responses: 200: description: - The config was successfully added. + The account_data was successfully added. diff --git a/specification/modules/client_config.rst b/specification/modules/account_data.rst similarity index 91% rename from specification/modules/client_config.rst rename to specification/modules/account_data.rst index 581a5828194..fc0e2e68794 100644 --- a/specification/modules/client_config.rst +++ b/specification/modules/account_data.rst @@ -1,7 +1,7 @@ Client Config ============= -.. _module:client_config: +.. _module:account_data: Clients can store their config on their homeserver. This config will be synced between different devices and can persist across installations on a particular @@ -23,4 +23,4 @@ the tags are for. Client Behaviour ---------------- -{{v2_client_config_http_api}} +{{v2_account_data_http_api}} diff --git a/specification/targets.yaml b/specification/targets.yaml index 418515c3e68..afb7f8d91fa 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -27,7 +27,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/search.rst - modules/guest_access.rst - modules/tags.rst - - modules/client_config.rst + - modules/account_data.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 032ee75537fd8d67db0e7e930ada4cea6d428faf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 19 Nov 2015 10:42:22 +0000 Subject: [PATCH 223/989] Update specification wording to match s/config/account_data/ --- specification/modules/account_data.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index fc0e2e68794..784bc874f1e 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -3,16 +3,17 @@ Client Config .. _module:account_data: -Clients can store their config on their homeserver. This config will be synced -between different devices and can persist across installations on a particular -device. +Clients can store custom config data for their account on their homeserver. +This account data will be synced between different devices and can persist +across installations on a particular device. Users may only view the account +data for their own account -The config may be either global or scoped to a particular rooms. +The account_data may be either global or scoped to a particular rooms. Events ------ -The client recieves the config as a event in the ``account_data`` sections +The client recieves the account data as events in the ``account_data`` sections of a v2 /sync. These events can also be received in a v1 /events response or in the From 9ad64b02d183e72b6548d7a36f9b96949f624e70 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 15:41:15 -0500 Subject: [PATCH 224/989] speculator: guard against concurrent git commands --- scripts/speculator/main.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 97e67c8c2fd..4472164c03c 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -23,6 +23,7 @@ import ( "path" "strconv" "strings" + "sync" "syscall" "time" @@ -136,17 +137,26 @@ func writeError(w http.ResponseWriter, code int, err error) { } type server struct { + mu sync.Mutex // Must be locked around any git command on matrixDocCloneURL matrixDocCloneURL string } +func (s *server) updateBase() error { + s.mu.Lock() + defer s.mu.Unlock() + return gitFetch(s.matrixDocCloneURL) +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = gitFetch(s.matrixDocCloneURL) + err = s.updateBase() if err != nil { return } + s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) + s.mu.Unlock() if err != nil { return } @@ -164,7 +174,10 @@ func (s *server) getSHAOf(ref string) (string, error) { cmd.Dir = path.Join(s.matrixDocCloneURL) var b bytes.Buffer cmd.Stdout = &b - if err := cmd.Run(); err != nil { + s.mu.Lock() + err := cmd.Run() + s.mu.Unlock() + if err != nil { return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return strings.TrimSpace(b.String()), nil @@ -174,7 +187,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - if err := gitFetch(s.matrixDocCloneURL); err != nil { + if err := s.updateBase(); err != nil { writeError(w, 500, err) return } @@ -383,7 +396,7 @@ func main() { if err != nil { log.Fatal(err) } - s := server{masterCloneDir} + s := server{matrixDocCloneURL: masterCloneDir} http.HandleFunc("/spec/", forceHTML(s.serveSpec)) http.HandleFunc("/diff/rst/", s.serveRSTDiff) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) From 757b0bcd12269b0a9905efee7520e61abc81ec7e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:08:37 -0500 Subject: [PATCH 225/989] Try to build continuserv and speculator --- jenkins.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jenkins.sh b/jenkins.sh index 0b217e58a87..b2aa848911c 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -7,3 +7,8 @@ set -ex (cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) + +if which go >/dev/null 2>/dev/null; then + (cd scripts/continuserv && go build) + (cd scripts/speculator && go build) +fi From dd53847211c57553180efca9d843c1c8741d6de3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:11:19 -0500 Subject: [PATCH 226/989] Include command stderr in error text --- scripts/speculator/main.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 4472164c03c..00a59eeaa58 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -70,13 +70,15 @@ const ( func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - cmd := exec.Command("git", "clone", url, directory) + if err := os.MkdirAll(directory, 0755); err != nil { + return "", fmt.Errorf("error making directory %s: %v", directory, err) + } + args := []string{"clone", url, directory} if shared { - cmd.Args = append(cmd.Args, "--shared") + args = append(args, "--shared") } - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("error cloning repo: %v", err) + if err := runGitCommand(directory, args); err != nil { + return "", err } return directory, nil } @@ -92,8 +94,10 @@ func gitFetch(path string) error { func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path + var b bytes.Buffer + cmd.Stderr = &b if err := cmd.Run(); err != nil { - return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) + return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String()) } return nil } From 8872e17f9355a47f9b7a2822cfd1303d8e56e2bc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:14:22 -0500 Subject: [PATCH 227/989] Fall back to last known HEAD sha if fetch fails --- scripts/speculator/main.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 00a59eeaa58..380cd2bc990 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -87,10 +87,6 @@ func gitCheckout(path, sha string) error { return runGitCommand(path, []string{"checkout", sha}) } -func gitFetch(path string) error { - return runGitCommand(path, []string{"fetch"}) -} - func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path @@ -148,7 +144,7 @@ type server struct { func (s *server) updateBase() error { s.mu.Lock() defer s.mu.Unlock() - return gitFetch(s.matrixDocCloneURL) + return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } // generateAt generates spec from repo at sha. @@ -191,16 +187,12 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - if err := s.updateBase(); err != nil { + if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return + } else { + sha = headSha } - originHead, err := s.getSHAOf("origin/master") - if err != nil { - writeError(w, 500, err) - return - } - sha = originHead } else { pr, err := lookupPullRequest(*req.URL, "/spec") if err != nil { @@ -237,6 +229,25 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { specCache.Add(sha, b) } +// lookupHeadSHA looks up what origin/master's HEAD SHA is. +// It attempts to `git fetch` before doing so. +// If this fails, it may still return a stale sha, but will also return an error. +func (s *server) lookupHeadSHA() (sha string, retErr error) { + retErr = s.updateBase() + if retErr != nil { + log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr) + } + originHead, err := s.getSHAOf("origin/master") + if err != nil { + retErr = err + } + sha = originHead + if retErr != nil && originHead != "" { + log.Printf("Successfully fell back to possibly stale sha: %s", sha) + } + return +} + func checkAuth(pr *PullRequest) error { if !pr.User.IsTrusted() { return fmt.Errorf("%q is not a trusted pull requester", pr.User.Login) From 6f1d00097be403998e5490845835ecb23d2c741c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:15:13 -0500 Subject: [PATCH 228/989] Only bother trying to fetch if we need to --- scripts/speculator/main.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 380cd2bc990..4972210eb98 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -147,12 +147,20 @@ func (s *server) updateBase() error { return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } +func (s *server) knowsAbout(sha string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = s.updateBase() - if err != nil { - return + if !s.knowsAbout(sha) { + err = s.updateBase() + if err != nil { + return + } } s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) From 858674477111bfc80adb299ad73715a98099c296 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:41:58 -0500 Subject: [PATCH 229/989] Add anchors to spec This is currently done by a script on the prod serving machine. We might as well keep the matrix.org spec and dev spec as similar as possible. --- scripts/gendoc.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index ed36a5a7826..28a115284e7 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -238,6 +238,18 @@ def rst2html(i, o): ) +def addAnchors(path): + with open(path, "r") as f: + lines = f.readlines() + + replacement = replacement = r'

\n\1' + with open(path, "w") as f: + for line in lines: + line = re.sub(r'()', replacement, line.rstrip()) + line = re.sub(r'(
)', replacement, line.rstrip()) + f.write(line + "\n") + + def run_through_template(input, set_verbose): tmpfile = './tmp/output' try: @@ -387,6 +399,7 @@ def main(target_name, keep_intermediates): shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") + addAnchors("gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: cleanup_env() From e0410330484257e6d0c8da481d8d7e31a2b05053 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 17:17:50 -0500 Subject: [PATCH 230/989] Rename file --- scripts/{matrix-org-gendoc.sh => add-matrix-org-stylings.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{matrix-org-gendoc.sh => add-matrix-org-stylings.sh} (100%) diff --git a/scripts/matrix-org-gendoc.sh b/scripts/add-matrix-org-stylings.sh similarity index 100% rename from scripts/matrix-org-gendoc.sh rename to scripts/add-matrix-org-stylings.sh From ca3a9e3562492964e314d7f9173d320281fbddfd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 17:18:33 -0500 Subject: [PATCH 231/989] exec gendoc outside of script --- scripts/add-matrix-org-stylings.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index 5716786d2fe..b02cb306ff3 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -35,8 +35,6 @@ if [ ! -f $FOOTER ]; then exit 1 fi -python gendoc.py - perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header #' gen/specification.html gen/howtos.html From da93317a7874096c2c13974542e2f2272e5d2a60 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 18:13:40 -0500 Subject: [PATCH 232/989] Take dir not files as args --- scripts/add-matrix-org-stylings.sh | 43 +++++++++--------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index b02cb306ff3..83888346c4e 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -1,39 +1,20 @@ -#! /bin/bash +#!/bin/bash -eu -if [ -z "$1" ]; then - echo "Expected /includes/head.html file as 1st arg." - exit 1 -fi - -if [ -z "$2" ]; then - echo "Expected /includes/nav.html file as 2nd arg." - exit 1 +if [[ $# != 1 || ! -d $1 ]]; then + echo >&2 "Usage: $0 include_dir" + exit 1 fi -if [ -z "$3" ]; then - echo "Expected /includes/footer.html file as 3rd arg." - exit 1 -fi - - -HEADER=$1 -NAV_BAR=$2 -FOOTER=$3 +HEADER="$1/head.html" +NAV_BAR="$1/nav.html" +FOOTER="$1/footer.html" -if [ ! -f $HEADER ]; then - echo $HEADER " does not exist" +for f in "${HEADER}"/{head,nav,footer}.html; do + if [[ ! -e "${f}" ]]; then + echo >&2 "Need ${f} to exist" exit 1 -fi - -if [ ! -f $NAV_BAR ]; then - echo $NAV_BAR " does not exist" - exit 1 -fi - -if [ ! -f $FOOTER ]; then - echo $FOOTER " does not exist" - exit 1 -fi + fi +done perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header From 4ac85997f52303b5fa75241f9bc50cdc227af3fa Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 18:16:02 -0500 Subject: [PATCH 233/989] Fix check --- scripts/add-matrix-org-stylings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index 83888346c4e..ef4e1014ac3 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -9,7 +9,7 @@ HEADER="$1/head.html" NAV_BAR="$1/nav.html" FOOTER="$1/footer.html" -for f in "${HEADER}"/{head,nav,footer}.html; do +for f in "$1"/{head,nav,footer}.html; do if [[ ! -e "${f}" ]]; then echo >&2 "Need ${f} to exist" exit 1 From 46870da57d03d4f682fd754e6374d06c4d7a94d9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Nov 2015 15:29:36 +0000 Subject: [PATCH 234/989] Add optional profile info and state in search response --- api/client-server/v1/search.yaml | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml index 34b3c150faa..a5cc9422e89 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/v1/search.yaml @@ -100,6 +100,19 @@ paths: description: |- How many events after the result are returned. + include_profile: + type: boolean + title: "Return profile information" + description: |- + Requests that the server returns the + historic profile information for the users + that sent the events that were returned. + include_state: + type: boolean + title: Include current state + description: |- + Requests the server return the current state for + each room returned. groupings: type: object title: Groupings @@ -181,6 +194,22 @@ paths: title: End Token description: |- Pagination token for the end of the chunk + profile_info: + type: object + title: Profile Information + description: |- + The historic profile information of the + users that sent the events returned. + additionalProperties: + type: object + title: User Profile + properties: + displayname: + type: string + title: Display name + avatar_url: + type: string + title: Avatar Url events_before: type: array title: Events Before @@ -199,6 +228,17 @@ paths: title: Event allOf: - "$ref": "core-event-schema/room_event.json" + state: + type: object + title: Current state + additionalProperties: + type: array + title: Room State + items: + type: object + title: Event + allOf: + - "$ref": "core-event-schema/room_event.json" groups: type: object title: Groups From 181d3f976d4f236c34ba0e86f58f729690d82e34 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 13:01:20 +0000 Subject: [PATCH 235/989] Initial proposal for websockets support. --- drafts/websockets.rst | 285 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 drafts/websockets.rst diff --git a/drafts/websockets.rst b/drafts/websockets.rst new file mode 100644 index 00000000000..bd0ff081dbe --- /dev/null +++ b/drafts/websockets.rst @@ -0,0 +1,285 @@ +WebSockets API +============== + +Introduction +------------ +This document is a proposal for a WebSockets-based client-server API. It is not +intended to replace the REST API, but rather to complement it and provide an +alternative interface for certain operations. + +The primary goal is to offer a more efficient interface than the REST API: by +using a bidirectional protocol such as WebSockets we can avoid the overheads +involved in long-polling (SSL negotiation, HTTP headers, etc). In doing so we +will reduce the latency between server and client by allowing the server to +send events as soon as they arrive, rather than having to wait for a poll from +the client. + +Handshake +--------- +1. Instead of calling ``/sync``, the client makes a websocket request to + ``/_matrix/client/rN/stream``, passing the query parameters ``access_token`` + and ``since``, and optionally ``filter`` - all of which have the same + meaning as for ``/sync``. + + * The client sets the ``Sec-WebSocket-Protocol`` to ``m.json``. (Servers may + offer alternative encodings; at present only the JSON encoding is + specified but in future we will specify alternative encodings.) + +#. The server returns the websocket handshake; the socket is then connected. + +If the server does not return a valid websocket handshake, this indicates that +the server or an intermediate proxy does not support WebSockets. In this case, +the client should fall back to polling the ``/sync`` REST endpoint. + +Example +~~~~~~~ + +Client request: + +.. code:: http + + GET /_matrix/client/v2_alpha/stream?access_token=123456&since=s72594_4483_1934 HTTP/1.1 + Host: matrix.org + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw== + Sec-WebSocket-Protocol: m.json + Sec-WebSocket-Version: 13 + Origin: https://matrix.org + +Server response: + +.. code:: http + + HTTP/1.1 101 Switching Protocols + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk= + Sec-WebSocket-Protocol: m.json + + +Update Notifications +-------------------- +Once the socket is connected, the server begins streaming updates over the +websocket. The server sends Update notifications about new messages or state +changes. To make it easy for clients to parse, Update notifications have the +same structure as the response to ``/sync``: an object with the following +members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +next_batch string The batch token to supply in the ``since`` param of + the next /sync request. This is not required for + streaming of events over the WebSocket, but is + provided so that clients can reconnect if the + socket is disconnected. +presence Presence The updates to the presence status of other users. +rooms Rooms Updates to rooms. +============= ========== =================================================== + +Example +~~~~~~~ +Message from the server: + +.. code:: json + + { + "next_batch": "s72595_4483_1934", + "presence": { + "events": [] + }, + "rooms": { + "join": {}, + "invite": {}, + "leave": {} + } + } + + +Client-initiated operations +--------------------------- + +The client can perform certain operations by sending a websocket message to +the server. Such a "Request" message should be a JSON-encoded object with +the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string A unique identifier for this request +method string Specifies the name of the operation to be + performed; see below for available operations +param object The parameters for the requested operation. +============= ========== =================================================== + +The server responds to a client Request with a Response message. This is a +JSON-encoded object with the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string The same as the value in the corresponding Request + object. The presence of the ``id`` field + distinguishes a Response message from an Update + notification. +result object On success, the results of the request. +error object On error, an object giving the resons for the + error. This has the same structure as the "standard + error response" for the Matrix API: an object with + the fields ``errcode`` and ``error``. +============= ========== =================================================== + +Request methods +~~~~~~~~~~~~~~~ +It is not intended that all operations which are available via the REST API +will be available via the WebSockets API, but a few simple, common operations +will be exposed. The initial operations will be as follows. + +``ping`` +^^^^^^^^ +This is a no-op which clients may use to keep their connection alive. + +The request ``params`` and the response ``result`` should be empty. + +``send`` +^^^^^^^^ +Send a message event to a room. The parameters are as follows: + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to send the event to +event_type string **Required.** The type of event to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + +The ``id`` from the Request message is used as the transaction ID by the +server. + +``state`` +^^^^^^^^^ +Update the state on a room. + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to set the state in +event_type string **Required.** The type of event to send. +state_key string **Required.** The state_key for the state to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + + +Example +~~~~~~~ +Client request: + +.. code:: json + + { + "id": "12345", + "method": "send", + "params": { + "room_id": "!d41d8cd:matrix.org", + "event_type": "m.room.message", + "content": { + "msgtype": "m.text", + "body": "hello" + } + } + } + +Server response: + +.. code:: json + + { + "id": "12345", + "result": { + "event_id": "$66697273743031:matrix.org" + } + } + +Alternative server response, in case of error: + +.. code:: json + + { + "id": "12345", + "error": { + "errcode": "M_MISSING_PARAM", + "error": "Missing parameter: event_type" + } + } + + +Rationale +--------- +Alternatives to WebSockets include HTTP/2, CoAP, and simply rolling our own +protocol over raw TCP sockets. However, the need to implement browser-based +clients essentially reduces our choice to WebSockets. HTTP/2 streams will +probably provide an interesting alternative in the future, but current browsers +do not appear to give javascript applications low-level access to the protocol. + +Concerning the continued use of the JSON encoding: we prefer to focus on the +transition to WebSockets initially. Replacing JSON with a compact +representation such as CBOR, MessagePack, or even just compressed JSON will be +a likely extension for the future. The support for negotiation of subprotocols +within WebSockets should make this a simple transition once time permits. + +The number of methods available for client requests is deliberately limited, as +each method requires code to be written to map it onto the equivalent REST +implementation. Some REST methods - for instance, user registration and login - +would be pointless to expose via WebSockets. It is likely, however, that we +will increate the number of methods available via the WebSockets API as it +becomes clear which would be most useful. + +Open questions +-------------- + +Throttling +~~~~~~~~~~ +At least in v2 sync, clients are inherently self-throttling - if they do not +poll quickly enough, events will be dropped from the next result. This proposal +raises the possibility that events will be produced more quickly than they can +be sent to the client; backlogs will build up on the server and/or in the +intermediate network, which will not only lead to high latency on events being +delivered, but will lead to responses to client requests also being delayed. + +We may need to implement some sort of throttling mechanism by which the server +can start to drop events. The difficulty is in knowing when to start dropping +events. A few ideas: + +* Use websocket pings to measure the RTT; if it starts to increase, start + dropping events. But this requires knowledge of the base RTT, and a useful + model of what constitutes an excessive increase. + +* Have the client acknowledge each batch of events, and use a window to ensure + the number of outstanding batches is limited. This is annoying as it requires + the client to have to acknowledge batches - and it's not clear what the right + window size is: we want a big window for long fat networks (think of mobile + clients), but a small one for one with lower latency. + +* Start dropping events if the server's TCP buffer starts filling up. This has + the advantage of delegating the congestion-detection to TCP (which already + has a number of algorithms to deal with it, to greater or lesser + effectiveness), but relies on homeservers being hosted on OSes which use + sensible TCP congestion-avoidance algorithms, and more critically, an ability + to read the fill level of the TCP send buffer. From 5ccc39b850a13874774782f2454f5f136f6e9e1d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 20 Nov 2015 18:45:09 +0000 Subject: [PATCH 236/989] Say that type is an event type --- api/client-server/v2_alpha/account-data.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/account-data.yaml b/api/client-server/v2_alpha/account-data.yaml index 98134d6feb5..b634dddc55d 100644 --- a/api/client-server/v2_alpha/account-data.yaml +++ b/api/client-server/v2_alpha/account-data.yaml @@ -41,8 +41,8 @@ paths: name: type required: true description: |- - The type of the account_data to set. Custom types should be namespaced to - avoid clashes. + The event type of the account_data to set. Custom types should be + namespaced to avoid clashes. x-example: "org.example.custom.config" - in: body name: content @@ -87,7 +87,7 @@ paths: name: type required: true description: |- - The type of the account_data to set. Custom types should be + The event type of the account_data to set. Custom types should be namespaced to avoid clashes. x-example: "org.example.custom.room.config" - in: body From e045f28b44137303acccc833ffb486008c46be06 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:20:54 +0000 Subject: [PATCH 237/989] Pull out constant for permissions Also, drop permissions from 0755 to 0700 --- scripts/speculator/main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 4972210eb98..cfe0f4ee72b 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -64,13 +64,14 @@ func (u *User) IsTrusted() bool { } const ( - pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" - matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + permissionsOwnerFull = 0700 ) func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - if err := os.MkdirAll(directory, 0755); err != nil { + if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil { return "", fmt.Errorf("error making directory %s: %v", directory, err) } args := []string{"clone", url, directory} From 866fa582763279063856ea89c3459e4eeb5ae2b7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:22:53 +0000 Subject: [PATCH 238/989] Rename --- scripts/speculator/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index cfe0f4ee72b..e18743834a9 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -148,7 +148,8 @@ func (s *server) updateBase() error { return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } -func (s *server) knowsAbout(sha string) bool { +// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL. +func (s *server) canCheckout(sha string) bool { s.mu.Lock() defer s.mu.Unlock() return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil @@ -157,7 +158,7 @@ func (s *server) knowsAbout(sha string) bool { // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - if !s.knowsAbout(sha) { + if !s.canCheckout(sha) { err = s.updateBase() if err != nil { return From c432396079645052c8a9c60eb5be822889d25117 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:26:32 +0000 Subject: [PATCH 239/989] Add comment --- scripts/speculator/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e18743834a9..0c9a2bfa7f5 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -197,6 +197,8 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { + // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. + // This is to deal with cases like where github is down but we still want to serve the spec. if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return From 6a6cbd9d24a7cbef9db04a57b6d273e09d51ddf1 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:28:58 +0000 Subject: [PATCH 240/989] Always try to build continuserv & speculator on jenkins --- jenkins.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jenkins.sh b/jenkins.sh index b2aa848911c..f5ed3b15a6c 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -8,7 +8,11 @@ set -ex (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) -if which go >/dev/null 2>/dev/null; then - (cd scripts/continuserv && go build) - (cd scripts/speculator && go build) -fi +: ${GOPATH:=${WORKSPACE}/.gopath} +mkdir -p "${GOPATH}" +export GOPATH +go get github.com/hashicorp/golang-lru +go get gopkg.in/fsnotify.v1 + +(cd scripts/continuserv && go build) +(cd scripts/speculator && go build) From 4e42aab245514a0d7373e23b282c1a284b100dec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 25 Nov 2015 15:13:00 +0000 Subject: [PATCH 241/989] Fix up the multi-hash ratchet thing --- drafts/markjh_end_to_end.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst index 1699f79bd6d..07390b5e506 100644 --- a/drafts/markjh_end_to_end.rst +++ b/drafts/markjh_end_to_end.rst @@ -84,18 +84,22 @@ follows: .. math:: \begin{align} - R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ - R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,0} &= H_0\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,2} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,3} &= H_3\left(R_{2^24(i-1),0}\right) \\ R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ - R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ - R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ - R_{i,3} &= H_1\left(R_{(i-1),3}\right) + R_{2^16n,3} &= H_3\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_3\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_3\left(R_{(i-1),3}\right) \end{align} -Where :math:`H_1` and :math:`H_2` are different hash functions. For example -:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and -:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. +Where :math:`H_0`, :math:`H_1`, :math:`H_2`, and :math:`H_3` +are different hash functions. For example +:math:`H_0` could be :math:`HMAC\left(X,\text{"\textbackslash x00"}\right)` and +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)`. So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. From ec31c0f5184fcf932981b854d770e90011e8fd24 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 12:04:37 +0000 Subject: [PATCH 242/989] speculator: allow styling like matrix.org --- scripts/speculator/main.go | 47 ++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 0c9a2bfa7f5..74731ff63a9 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -54,9 +54,11 @@ type User struct { } var ( - port = flag.Int("port", 9000, "Port on which to listen for HTTP") - allowedMembers map[string]bool - specCache *lru.Cache // string -> []byte + port = flag.Int("port", 9000, "Port on which to listen for HTTP") + includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org") + allowedMembers map[string]bool + specCache *lru.Cache // string -> []byte + styledSpecCache *lru.Cache // string -> []byte ) func (u *User) IsTrusted() bool { @@ -196,6 +198,13 @@ func (s *server) getSHAOf(ref string) (string, error) { func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string + var styleLikeMatrixDotOrg = req.URL.Query().Get("matrixdotorgstyle") != "" + + if styleLikeMatrixDotOrg && *includesDir == "" { + writeError(w, 500, fmt.Errorf("Cannot style like matrix.org - no include dir specified")) + return + } + if strings.ToLower(req.URL.Path) == "/spec/head" { // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. // This is to deal with cases like where github is down but we still want to serve the spec. @@ -220,7 +229,13 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { } sha = pr.Head.SHA } - if cached, ok := specCache.Get(sha); ok { + + var cache = specCache + if styleLikeMatrixDotOrg { + cache = styledSpecCache + } + + if cached, ok := cache.Get(sha); ok { w.Write(cached.([]byte)) return } @@ -232,13 +247,24 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } + if styleLikeMatrixDotOrg { + cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) + cmd.Dir = path.Join(dst, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) + return + } + } + b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) if err != nil { writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) return } w.Write(b) - specCache.Add(sha, b) + cache.Add(sha, b) } // lookupHeadSHA looks up what origin/master's HEAD SHA is. @@ -384,6 +410,10 @@ func listPulls(w http.ResponseWriter, req *http.Request) { pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number) } s += `` + if *includesDir != "" { + s += `` + } + io.WriteString(w, s) } @@ -448,7 +478,10 @@ func serveText(s string) func(http.ResponseWriter, *http.Request) { } func initCache() error { - c, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) - specCache = c + c1, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + specCache = c1 + + c2, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + styledSpecCache = c2 return err } From c25a806cef3f5129141449d21fbb888fffa6b266 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 15:03:39 +0000 Subject: [PATCH 243/989] Fix and include /directory api docs --- api/client-server/v1/directory.yaml | 48 ++++++++++++++++++++++++++--- specification/client_server_api.rst | 24 +++------------ 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/api/client-server/v1/directory.yaml b/api/client-server/v1/directory.yaml index c70b9f6ba5f..4966a9201ee 100644 --- a/api/client-server/v1/directory.yaml +++ b/api/client-server/v1/directory.yaml @@ -29,6 +29,7 @@ paths: name: roomAlias description: The room alias to set. required: true + x-example: "#monkeys:matrix.org" - in: body name: roomInfo description: Information about this room alias. @@ -39,11 +40,18 @@ paths: room_id: type: string description: The room ID to set. + example: |- + { + "room_id": "!abnjk1jdasj98:capuchins.com" + } responses: 200: description: The mapping was created. + examples: + application/json: |- + {} schema: - type: object # empty json object + type: object get: summary: Get the room ID corresponding to this room alias. parameters: @@ -52,6 +60,7 @@ paths: name: roomAlias description: The room alias. required: true + x-example: "#monkeys:matrix.org" responses: 200: description: The room ID and other information for this alias. @@ -67,10 +76,38 @@ paths: items: type: string description: A server which is aware of this room ID. + examples: + application/json: |- + { + "room_id": "!abnjk1jdasj98:capuchins.com", + "servers": [ + "capuchins.com", + "matrix.org", + "another.com" + ] + } 404: description: There is no mapped room ID for this room alias. + examples: + application/json: |- + { + "errcode": "M_NOT_FOUND", + "error": "Room ID !abnjk1jdasj98:capuchins.com not found." + } + 409: + description: A room alias with that name already exists. + examples: + application/json: |- + { + "errcode": "M_UNKNOWN", + "error": "Room alias #monkeys:matrix.org already exists." + } delete: summary: Remove a mapping of room alias to room ID. + description: |- + Remove a mapping of room alias to room ID. + + Servers may choose to implement additional access control checks here, for instance that room aliases can only be deleted by their creator or a server administrator. security: - accessToken: [] parameters: @@ -79,9 +116,12 @@ paths: name: roomAlias description: The room alias to remove. required: true + x-example: "#monkeys:matrix.org" responses: 200: - description: The mapping was removed. + description: The mapping was deleted. + examples: + application/json: |- + {} schema: - type: object # empty json object - + type: object diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index ba5578bcb55..245a9eab071 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -783,19 +783,9 @@ client has to use the the following API. Room aliases ~~~~~~~~~~~~ -.. NOTE:: - This section is a work in progress. - -Room aliases can be created by sending a ``PUT /directory/room/``:: - - { - "room_id": - } -They can be deleted by sending a ``DELETE /directory/room/`` with -no content. Only some privileged users may be able to delete room aliases, e.g. -server admins, the creator of the room alias, etc. This specification does not -outline the privilege level required for deleting room aliases. +Servers may host aliases for rooms with human-friendly names. Aliases take the +form ``#friendlyname:server.name``. As room aliases are scoped to a particular home server domain name, it is likely that a home server will reject attempts to maintain aliases on other @@ -812,17 +802,11 @@ appears to have a room alias of ``#alias:example.com``, this SHOULD be checked to make sure that the room's ID matches the ``room_id`` returned from the request. -Room aliases can be checked in the same way they are resolved; by sending a -``GET /directory/room/``:: - - { - "room_id": , - "servers": [ , , ] - } - Home servers can respond to resolve requests for aliases on other domains than their own by using the federation API to ask other domain name home servers. +{{directory_http_api}} + Permissions ~~~~~~~~~~~ From 5e30b5b8d74cdfbd764175fd735b3c39d652453e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 16:46:29 +0000 Subject: [PATCH 244/989] Remove POST version of /send PUT should always be used. --- api/client-server/v1/room_send.yaml | 47 --------------------- specification/modules/instant_messaging.rst | 8 +--- 2 files changed, 2 insertions(+), 53 deletions(-) diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/v1/room_send.yaml index 9c9273dfe5c..3ab1ac40d8a 100644 --- a/api/client-server/v1/room_send.yaml +++ b/api/client-server/v1/room_send.yaml @@ -76,50 +76,3 @@ paths: type: string description: |- A unique identifier for the event. - "/rooms/{roomId}/send/{eventType}": - post: - summary: Send a message event to the given room. - description: |- - This endpoint can be used to send a message event to a room; however - the lack of a transaction ID means that it is possible to cause message - duplication if events are resent on error, so it is preferable to use - `PUT /_matrix/client/api/v1/rooms/{roomId}/send/{eventType}/{txnId}`_. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room to send the event to. - required: true - x-example: "!636q39766251:example.com" - - in: path - type: string - name: eventType - description: The type of event to send. - required: true - x-example: "m.room.message" - - in: body - name: body - schema: - type: object - example: |- - { - "msgtype": "m.text", - "body": "hello" - } - responses: - 200: - description: "An ID for the sent event." - examples: - application/json: |- - { - "event_id": "YUwRidLecu" - } - schema: - type: object - properties: - event_id: - type: string - description: |- - A unique identifier for the event. diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index a58c762f387..cd38500144a 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -62,10 +62,8 @@ resulting ``mxc://`` URI can then be used in the ``url`` key. Recommendations when sending messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use -``PUT`` requests with `transaction IDs`_ to make requests idempotent. This -ensures that messages are sent exactly once even under poor network conditions. -Clients SHOULD retry requests using an exponential-backoff algorithm for a +In the event of send failure, clients SHOULD retry requests using an +exponential-backoff algorithm for a certain amount of time T. It is recommended that T is no longer than 5 minutes. After this time, the client should stop retrying and mark the message as "unsent". Users should be able to manually resend unsent messages. @@ -78,8 +76,6 @@ reduce the impact of head-of-line blocking, clients should use a queue per room rather than a global queue, as ordering is only relevant within a single room rather than between rooms. -.. _`transaction IDs`: `sect:txn_ids`_ - Local echo ~~~~~~~~~~ From d39494b6dfda679bc4d38a96efb1715428f35279 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 26 Nov 2015 16:55:12 +0000 Subject: [PATCH 245/989] Fix typo in sync example --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c3d120b4547..6f2f1412d2f 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -264,7 +264,7 @@ paths: "account_data": { "events": [ { - "type": "m.tags", + "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} } ] From 8d415367573bce44a3a9f2f663157fa08e054bf9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 11:37:24 +0000 Subject: [PATCH 246/989] Address kegan's comments Minor fixes to the e2e spec as raiseds by kegan --- specification/41_end_to_end_encryption.rst | 27 ++++++++++++---------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 023a568482c..c5fa4528e01 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -67,32 +67,35 @@ Messaging Algorithm Names ~~~~~~~~~~~~~~~~~~~~~~~~~ Messaging algorithm names use the extensible naming scheme used throughout this -specification. Algorithm names that start with "m." are reserved for algorithms +specification. Algorithm names that start with `m.` are reserved for algorithms defined by this specification. Implementations wanting to experiment with new algorithms are encouraged to pick algorithm names that start with their domain to reduce the risk of collisions. -The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm +Algorithm names should be short and meaningful, and should list the primitives +used by the algorithm so that it is easier to see if the algorithm is using a +broken primitive. + +The name `m.olm.v1.curve25519-aes-sha2` corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. -Algorithm names should be short and meaningful. A name of "m.olm.v1" is too -short. However a name of -"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256" -is too long despite giving a more precise description of the algorithm. - -Algorithm names should list the primitives used by the algorithm so that it -easier to see if the algorithm is using a broken primitive. +A name of `m.olm.v1` is too short: it gives no information about the primitives +in use, and is difficult to extend for different primitives. However a name of +`m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256` +is too long despite giving a more precise description of the algorithm: it adds +to the data transfer overhead and sacrifices clarity for human readers without +adding any useful extra information. Key Algorithms ~~~~~~~~~~~~~~ -The name "ed25519" corresponds to the Ed25519 signature algorithm. The key is +The name `ed25519` corresponds to the Ed25519 signature algorithm. The key is a Base64 encoded 32-byte Ed25519 public key. -The name "curve25519" corresponds to the Curve25519 ECDH algorithm. The key is +The name `curve25519` corresponds to the Curve25519 ECDH algorithm. The key is a Base64 encoded 32-byte Curve25519 public key. Client Behaviour @@ -149,7 +152,7 @@ Downloading Keys ~~~~~~~~~~~~~~~~ Keys are downloaded as a collection of signed JSON objects. There -will be a JSON object per device per user. If one of the user's +will be one JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their homeserver must synthesise a JSON object without any device keys for that device. From 2aa4773cc15acfdfa86be396267b1745cbe73ee6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 12:01:03 +0000 Subject: [PATCH 247/989] Make the speculator serve up errors as plain text ... so that they are legible. --- scripts/speculator/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 74731ff63a9..abe77c74db3 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -135,6 +135,7 @@ func generate(dir string) error { } func writeError(w http.ResponseWriter, code int, err error) { + w.Header().Set("Content-Type", "text/plain") w.WriteHeader(code) io.WriteString(w, fmt.Sprintf("%v\n", err)) } From 2dbb8ba56c35bf4e914d0c8105bf8db14e4c09ff Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 12:53:03 +0000 Subject: [PATCH 248/989] Fix title levels make the title decoration consistent with the rest of the spec --- specification/modules/end_to_end_encryption.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index cb770680fa4..3ec9368adbe 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -285,7 +285,7 @@ Encrypted messages are sent in the form. Using Olm -######### ++++++++++ Devices that support olm must include "m.olm.v1.curve25519-aes-sha2" in their list of supported chat algorithms, must list a Curve25519 device key, and From 0b1ba70a320a7f4fe90e34edf4e3352d5e770433 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 13:33:58 +0000 Subject: [PATCH 249/989] fix rst markup `` > ` --- .../modules/end_to_end_encryption.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 3ec9368adbe..8b38ab9382c 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -69,24 +69,24 @@ Messaging Algorithm Names ~~~~~~~~~~~~~~~~~~~~~~~~~ Messaging algorithm names use the extensible naming scheme used throughout this -specification. Algorithm names that start with `m.` are reserved for algorithms -defined by this specification. Implementations wanting to experiment with new -algorithms are encouraged to pick algorithm names that start with their -domain to reduce the risk of collisions. +specification. Algorithm names that start with ``m.`` are reserved for +algorithms defined by this specification. Implementations wanting to experiment +with new algorithms are encouraged to pick algorithm names that start with +their domain to reduce the risk of collisions. Algorithm names should be short and meaningful, and should list the primitives used by the algorithm so that it is easier to see if the algorithm is using a broken primitive. -The name `m.olm.v1.curve25519-aes-sha2` corresponds to version 1 of the Olm +The name ``m.olm.v1.curve25519-aes-sha2`` corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. -A name of `m.olm.v1` is too short: it gives no information about the primitives +A name of ``m.olm.v1`` is too short: it gives no information about the primitives in use, and is difficult to extend for different primitives. However a name of -`m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256` +``m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256`` is too long despite giving a more precise description of the algorithm: it adds to the data transfer overhead and sacrifices clarity for human readers without adding any useful extra information. @@ -94,10 +94,10 @@ adding any useful extra information. Key Algorithms ~~~~~~~~~~~~~~ -The name `ed25519` corresponds to the Ed25519 signature algorithm. The key is +The name ``ed25519`` corresponds to the Ed25519 signature algorithm. The key is a Base64 encoded 32-byte Ed25519 public key. -The name `curve25519` corresponds to the Curve25519 ECDH algorithm. The key is +The name ``curve25519`` corresponds to the Curve25519 ECDH algorithm. The key is a Base64 encoded 32-byte Curve25519 public key. Client Behaviour From ad4d8ae7a611bdd13167c358c0447e8fad089fd8 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 13:50:24 +0000 Subject: [PATCH 250/989] Move client-server intro to client-server section --- specification/client_server_api.rst | 109 ++++++++++++++++++++++++++++ specification/intro.rst | 109 ---------------------------- 2 files changed, 109 insertions(+), 109 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index ba5578bcb55..402ce75318f 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -17,6 +17,115 @@ shortly. Documentation for the old `V1 authentication <../attic/v1_registration_login.rst>`_ is still available separately. +API Standards +------------- + +.. TODO + Need to specify any HMAC or access_token lifetime/ratcheting tricks + We need to specify capability negotiation for extensible transports + +The mandatory baseline for communication in Matrix is exchanging JSON objects +over HTTP APIs. HTTPS is mandated as the baseline for server-server +(federation) communication. HTTPS is recommended for client-server +communication, although HTTP may be supported as a fallback to support basic +HTTP clients. More efficient optional transports for client-server +communication will in future be supported as optional extensions - e.g. a +packed binary encoding over stream-cipher encrypted TCP socket for +low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all +API calls use a Content-Type of ``application/json``. In addition, all strings +MUST be encoded as UTF-8. Clients are authenticated using opaque +``access_token`` strings (see `Client Authentication`_ for details), passed as a +query string parameter on all requests. + +Any errors which occur at the Matrix API level MUST return a "standard error +response". This is a JSON object which looks like:: + + { + "errcode": "", + "error": "" + } + +The ``error`` string will be a human-readable error message, usually a sentence +explaining what went wrong. The ``errcode`` string will be a unique string +which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error +codes should have their namespace first in ALL CAPS, followed by a single _ to +ease separating the namespace from the error code. For example, if there was a +custom namespace ``com.mydomain.here``, and a +``FORBIDDEN`` code, the error code should look like +``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the +error, but the keys ``error`` and ``errcode`` MUST always be present. + +Some standard error codes are below: + +:``M_FORBIDDEN``: + Forbidden access, e.g. joining a room without permission, failed login. + +:``M_UNKNOWN_TOKEN``: + The access token specified was not recognised. + +:``M_BAD_JSON``: + Request contained valid JSON, but it was malformed in some way, e.g. missing + required keys, invalid values for keys. + +:``M_NOT_JSON``: + Request did not contain valid JSON. + +:``M_NOT_FOUND``: + No resource was found for this request. + +:``M_LIMIT_EXCEEDED``: + Too many requests have been sent in a short period of time. Wait a while then + try again. + +Some requests have unique error codes: + +:``M_USER_IN_USE``: + Encountered when trying to register a user ID which has been taken. + +:``M_ROOM_IN_USE``: + Encountered when trying to create a room which has been taken. + +:``M_BAD_PAGINATION``: + Encountered when specifying bad pagination query parameters. + +.. _sect:txn_ids: + +The Client-Server API typically uses ``HTTP POST`` to submit requests. This +means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to +make requests idempotent. In order to use a ``PUT``, paths should be suffixed +with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which +identifies the request, and is scoped to a given Client (identified by that +client's ``access_token``). Crucially, it **only** serves to identify new +requests from retransmits. After the request has finished, the ``{txnId}`` +value should be changed (how is not specified; a monotonically increasing +integer is recommended). It is preferable to use ``HTTP PUT`` to make sure +requests to send messages do not get sent more than once should clients need to +retransmit requests. + +Valid requests look like:: + + POST /some/path/here?access_token=secret + { + "key": "This is a post." + } + + PUT /some/path/here/11?access_token=secret + { + "key": "This is a put with a txnId of 11." + } + +In contrast, these are invalid requests:: + + POST /some/path/here/11?access_token=secret + { + "key": "This is a post, but it has a txnId." + } + + PUT /some/path/here?access_token=secret + { + "key": "This is a put but it is missing a txnId." + } + Client Authentication --------------------- Most API endpoints require the user to identify themselves by presenting diff --git a/specification/intro.rst b/specification/intro.rst index 64a211083ff..6ff7946cffd 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -358,112 +358,3 @@ dedicated API. The API is symmetrical to managing Profile data. Would it really be overengineered to use the same API for both profile & private user data, but with different ACLs? -API Standards -------------- - -.. TODO - Need to specify any HMAC or access_token lifetime/ratcheting tricks - We need to specify capability negotiation for extensible transports - -The mandatory baseline for communication in Matrix is exchanging JSON objects -over HTTP APIs. HTTPS is mandated as the baseline for server-server -(federation) communication. HTTPS is recommended for client-server -communication, although HTTP may be supported as a fallback to support basic -HTTP clients. More efficient optional transports for client-server -communication will in future be supported as optional extensions - e.g. a -packed binary encoding over stream-cipher encrypted TCP socket for -low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all -API calls use a Content-Type of ``application/json``. In addition, all strings -MUST be encoded as UTF-8. Clients are authenticated using opaque -``access_token`` strings (see `Client Authentication`_ for details), passed as a -query string parameter on all requests. - -Any errors which occur at the Matrix API level MUST return a "standard error -response". This is a JSON object which looks like:: - - { - "errcode": "", - "error": "" - } - -The ``error`` string will be a human-readable error message, usually a sentence -explaining what went wrong. The ``errcode`` string will be a unique string -which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error -codes should have their namespace first in ALL CAPS, followed by a single _ to -ease separating the namespace from the error code. For example, if there was a -custom namespace ``com.mydomain.here``, and a -``FORBIDDEN`` code, the error code should look like -``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the -error, but the keys ``error`` and ``errcode`` MUST always be present. - -Some standard error codes are below: - -:``M_FORBIDDEN``: - Forbidden access, e.g. joining a room without permission, failed login. - -:``M_UNKNOWN_TOKEN``: - The access token specified was not recognised. - -:``M_BAD_JSON``: - Request contained valid JSON, but it was malformed in some way, e.g. missing - required keys, invalid values for keys. - -:``M_NOT_JSON``: - Request did not contain valid JSON. - -:``M_NOT_FOUND``: - No resource was found for this request. - -:``M_LIMIT_EXCEEDED``: - Too many requests have been sent in a short period of time. Wait a while then - try again. - -Some requests have unique error codes: - -:``M_USER_IN_USE``: - Encountered when trying to register a user ID which has been taken. - -:``M_ROOM_IN_USE``: - Encountered when trying to create a room which has been taken. - -:``M_BAD_PAGINATION``: - Encountered when specifying bad pagination query parameters. - -.. _sect:txn_ids: - -The Client-Server API typically uses ``HTTP POST`` to submit requests. This -means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to -make requests idempotent. In order to use a ``PUT``, paths should be suffixed -with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which -identifies the request, and is scoped to a given Client (identified by that -client's ``access_token``). Crucially, it **only** serves to identify new -requests from retransmits. After the request has finished, the ``{txnId}`` -value should be changed (how is not specified; a monotonically increasing -integer is recommended). It is preferable to use ``HTTP PUT`` to make sure -requests to send messages do not get sent more than once should clients need to -retransmit requests. - -Valid requests look like:: - - POST /some/path/here?access_token=secret - { - "key": "This is a post." - } - - PUT /some/path/here/11?access_token=secret - { - "key": "This is a put with a txnId of 11." - } - -In contrast, these are invalid requests:: - - POST /some/path/here/11?access_token=secret - { - "key": "This is a post, but it has a txnId." - } - - PUT /some/path/here?access_token=secret - { - "key": "This is a put but it is missing a txnId." - } - From f0c99a6925a4d5ff38fcc563a50ec280b6895631 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 15:03:22 +0000 Subject: [PATCH 251/989] Generate API docs from swagger --- api/client-server/api-docs | 50 - api/client-server/api-docs-content | 119 --- api/client-server/api-docs-directory | 101 -- api/client-server/api-docs-events | 247 ----- api/client-server/api-docs-login | 120 --- api/client-server/api-docs-presence | 164 ---- api/client-server/api-docs-profile | 122 --- api/client-server/api-docs-registration | 120 --- api/client-server/api-docs-rooms | 1128 ----------------------- scripts/add-matrix-org-stylings.sh | 8 +- scripts/generate-http-docs.sh | 27 + 11 files changed, 32 insertions(+), 2174 deletions(-) delete mode 100644 api/client-server/api-docs delete mode 100644 api/client-server/api-docs-content delete mode 100644 api/client-server/api-docs-directory delete mode 100644 api/client-server/api-docs-events delete mode 100644 api/client-server/api-docs-login delete mode 100644 api/client-server/api-docs-presence delete mode 100644 api/client-server/api-docs-profile delete mode 100644 api/client-server/api-docs-registration delete mode 100644 api/client-server/api-docs-rooms create mode 100755 scripts/generate-http-docs.sh diff --git a/api/client-server/api-docs b/api/client-server/api-docs deleted file mode 100644 index 9c8c606654b..00000000000 --- a/api/client-server/api-docs +++ /dev/null @@ -1,50 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "apis": [ - { - "path": "-login", - "description": "Login operations" - }, - { - "path": "-registration", - "description": "Registration operations" - }, - { - "path": "-rooms", - "description": "Room operations" - }, - { - "path": "-profile", - "description": "Profile operations" - }, - { - "path": "-presence", - "description": "Presence operations" - }, - { - "path": "-events", - "description": "Event operations" - }, - { - "path": "-directory", - "description": "Directory operations" - }, - { - "path": "-content", - "description": "Content repository operations" - } - ], - "authorizations": { - "token": { - "scopes": [] - } - }, - "info": { - "title": "Matrix Client-Server API Reference", - "description": "This contains the client-server API for the reference implementation of the home server", - "termsOfServiceUrl": "http://matrix.org", - "license": "Apache 2.0", - "licenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.html" - } -} diff --git a/api/client-server/api-docs-content b/api/client-server/api-docs-content deleted file mode 100644 index a0a31da077d..00000000000 --- a/api/client-server/api-docs-content +++ /dev/null @@ -1,119 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix", - "resourcePath": "/media/v1/", - "apis": [ - { - "path": "/media/v1/upload", - "operations": [ - { - "method": "POST", - "summary": "Upload some content to the content repository.", - "type": "ContentUploadResponse", - "nickname": "upload_content", - "parameters": [ - { - "name": "body", - "description": "The file to upload.", - "required": true, - "type": "file", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/media/v1/download/{serverName}/{mediaId}", - "operations": [ - { - "method": "GET", - "summary": "Get the content stored at this address.", - "type": "file", - "nickname": "download_content", - "parameters": [ - { - "name": "serverName", - "description": "The serverName from the mxc:/// URI (the authority component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "mediaId", - "description": "The mediaId from the mxc:/// URI (the path component).", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/media/v1/thumbnail/{serverName}/{mediaId}", - "operations": [ - { - "method": "GET", - "summary": "Get a thumbnail of the content stored at this address.", - "type": "file", - "nickname": "thumbnail_content", - "parameters": [ - { - "name": "serverName", - "description": "The serverName from the mxc:/// URI (the authority component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "mediaId", - "description": "The mediaId from the mxc:/// URI (the path component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "width", - "description": "The desired width of the thumbnail.", - "required": false, - "type": "integer", - "paramType": "query" - }, - { - "name": "height", - "description": "The desired height of the thumbnail.", - "required": false, - "type": "integer", - "paramType": "query" - }, - { - "name": "method", - "description": "The desired resizing method.", - "enum": [ - "crop", - "scale" - ], - "required": false, - "type": "string", - "paramType": "query" - } - ] - } - ] - } - ], - "models": { - "ContentUploadResponse": { - "id": "ContentUploadResponse", - "properties": { - "content_uri": { - "type": "string", - "description": "The mxc:// URI where this content is stored. This is of the form 'mxc://{serverName}/{mediaId}'", - "required": true - } - } - } - } -} diff --git a/api/client-server/api-docs-directory b/api/client-server/api-docs-directory deleted file mode 100644 index 5dda5806587..00000000000 --- a/api/client-server/api-docs-directory +++ /dev/null @@ -1,101 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/directory", - "produces": [ - "application/json" - ], - "apis": [ - { - "path": "/directory/room/{roomAlias}", - "operations": [ - { - "method": "GET", - "summary": "Get the room ID corresponding to this room alias.", - "notes": "Volatile: This API is likely to change.", - "type": "DirectoryResponse", - "nickname": "get_room_id_for_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "PUT", - "summary": "Create a new mapping from room alias to room ID.", - "notes": "Volatile: This API is likely to change.", - "type": "void", - "nickname": "add_room_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias to set.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The room ID to set.", - "required": true, - "type": "RoomAliasRequest", - "paramType": "body" - } - ] - }, - { - "method": "DELETE", - "summary": "Removes a mapping of room alias to room ID.", - "notes": "Only privileged users can perform this action.", - "type": "void", - "nickname": "remove_room_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias to remove.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "DirectoryResponse": { - "id": "DirectoryResponse", - "properties": { - "room_id": { - "type": "string", - "description": "The fully-qualified room ID.", - "required": true - }, - "servers": { - "type": "array", - "items": { - "$ref": "string" - }, - "description": "A list of servers that know about this room.", - "required": true - } - } - }, - "RoomAliasRequest": { - "id": "RoomAliasRequest", - "properties": { - "room_id": { - "type": "string", - "description": "The room ID to map the alias to.", - "required": true - } - } - } - } -} diff --git a/api/client-server/api-docs-events b/api/client-server/api-docs-events deleted file mode 100644 index 1bdb9b034a4..00000000000 --- a/api/client-server/api-docs-events +++ /dev/null @@ -1,247 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/events", - "produces": [ - "application/json" - ], - "apis": [ - { - "path": "/events", - "operations": [ - { - "method": "GET", - "summary": "Listen on the event stream", - "notes": "This can only be done by the logged in user. This will block until an event is received, or until the timeout is reached.", - "type": "PaginationChunk", - "nickname": "get_event_stream", - "parameters": [ - { - "name": "from", - "description": "The token to stream from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "timeout", - "description": "The maximum time in milliseconds to wait for an event.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ], - - "responseMessages": [ - { - "code": 400, - "message": "Bad pagination token." - } - ] - }, - { - "path": "/events/{eventId}", - "operations": [ - { - "method": "GET", - "summary": "Get information about a single event.", - "notes": "Get information about a single event.", - "type": "Event", - "nickname": "get_event", - "parameters": [ - { - "name": "eventId", - "description": "The event ID to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Event not found." - } - ] - } - ] - }, - { - "path": "/initialSync", - "operations": [ - { - "method": "GET", - "summary": "Get this user's current state.", - "notes": "Get this user's current state.", - "type": "InitialSyncResponse", - "nickname": "initial_sync", - "parameters": [ - { - "name": "limit", - "description": "The maximum number of messages to return for each room.", - "type": "integer", - "paramType": "query", - "required": false - } - ] - } - ] - }, - { - "path": "/publicRooms", - "operations": [ - { - "method": "GET", - "summary": "Get a list of publicly visible rooms.", - "type": "PublicRoomsPaginationChunk", - "nickname": "get_public_room_list" - } - ] - } - ], - "models": { - "PaginationChunk": { - "id": "PaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - }, - "chunk": { - "type": "array", - "description": "An array of events.", - "required": true, - "items": { - "$ref": "Event" - } - } - } - }, - "Event": { - "id": "Event", - "properties": { - "event_id": { - "type": "string", - "description": "An ID which uniquely identifies this event.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The room in which this event occurred.", - "required": true - } - } - }, - "PublicRoomInfo": { - "id": "PublicRoomInfo", - "properties": { - "aliases": { - "type": "array", - "description": "A list of room aliases for this room.", - "items": { - "$ref": "string" - } - }, - "name": { - "type": "string", - "description": "The name of the room, as given by the m.room.name state event." - }, - "room_id": { - "type": "string", - "description": "The room ID for this public room.", - "required": true - }, - "topic": { - "type": "string", - "description": "The topic of this room, as given by the m.room.topic state event." - } - } - }, - "PublicRoomsPaginationChunk": { - "id": "PublicRoomsPaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - }, - "chunk": { - "type": "array", - "description": "A list of public room data.", - "required": true, - "items": { - "$ref": "PublicRoomInfo" - } - } - } - }, - "InitialSyncResponse": { - "id": "InitialSyncResponse", - "properties": { - "end": { - "type": "string", - "description": "A streaming token which can be used with /events to continue from this snapshot of data.", - "required": true - }, - "presence": { - "type": "array", - "description": "A list of presence events.", - "items": { - "$ref": "Event" - }, - "required": false - }, - "rooms": { - "type": "array", - "description": "A list of initial sync room data.", - "required": false, - "items": { - "$ref": "InitialSyncRoomData" - } - } - } - }, - "InitialSyncRoomData": { - "id": "InitialSyncRoomData", - "properties": { - "membership": { - "type": "string", - "description": "This user's membership state in this room.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The ID of this room.", - "required": true - }, - "messages": { - "type": "PaginationChunk", - "description": "The most recent messages for this room, governed by the limit parameter.", - "required": false - }, - "state": { - "type": "array", - "description": "A list of state events representing the current state of the room.", - "required": false, - "items": { - "$ref": "Event" - } - } - } - } - } -} diff --git a/api/client-server/api-docs-login b/api/client-server/api-docs-login deleted file mode 100644 index d6f8d84f29e..00000000000 --- a/api/client-server/api-docs-login +++ /dev/null @@ -1,120 +0,0 @@ -{ - "apiVersion": "1.0.0", - "apis": [ - { - "operations": [ - { - "method": "GET", - "nickname": "get_login_info", - "notes": "All login stages MUST be mentioned if there is >1 login type.", - "summary": "Get the login mechanism to use when logging in.", - "type": "LoginFlows" - }, - { - "method": "POST", - "nickname": "submit_login", - "notes": "If this is part of a multi-stage login, there MUST be a 'session' key.", - "parameters": [ - { - "description": "A login submission", - "name": "body", - "paramType": "body", - "required": true, - "type": "LoginSubmission" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad login type" - }, - { - "code": 400, - "message": "Missing JSON keys" - } - ], - "summary": "Submit a login action.", - "type": "LoginResult" - } - ], - "path": "/login" - } - ], - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "consumes": [ - "application/json" - ], - "models": { - "LoginFlows": { - "id": "LoginFlows", - "properties": { - "flows": { - "description": "A list of valid login flows.", - "type": "array", - "items": { - "$ref": "LoginInfo" - } - } - } - }, - "LoginInfo": { - "id": "LoginInfo", - "properties": { - "stages": { - "description": "Multi-stage login only: An array of all the login types required to login.", - "items": { - "$ref": "string" - }, - "type": "array" - }, - "type": { - "description": "The login type that must be used when logging in.", - "type": "string" - } - } - }, - "LoginResult": { - "id": "LoginResult", - "properties": { - "access_token": { - "description": "The access token for this user's login if this is the final stage of the login process.", - "type": "string" - }, - "user_id": { - "description": "The user's fully-qualified user ID.", - "type": "string" - }, - "next": { - "description": "Multi-stage login only: The next login type to submit.", - "type": "string" - }, - "session": { - "description": "Multi-stage login only: The session token to send when submitting the next login type.", - "type": "string" - } - } - }, - "LoginSubmission": { - "id": "LoginSubmission", - "properties": { - "type": { - "description": "The type of login being submitted.", - "type": "string" - }, - "session": { - "description": "Multi-stage login only: The session token from an earlier login stage.", - "type": "string" - }, - "_login_type_defined_keys_": { - "description": "Keys as defined by the specified login type, e.g. \"user\", \"password\"" - } - } - } - }, - "produces": [ - "application/json" - ], - "resourcePath": "/login", - "swaggerVersion": "1.2" -} - diff --git a/api/client-server/api-docs-presence b/api/client-server/api-docs-presence deleted file mode 100644 index 6b22446024e..00000000000 --- a/api/client-server/api-docs-presence +++ /dev/null @@ -1,164 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/presence", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "apis": [ - { - "path": "/presence/{userId}/status", - "operations": [ - { - "method": "PUT", - "summary": "Update this user's presence state.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "update_presence", - "parameters": [ - { - "name": "body", - "description": "The new presence state", - "required": true, - "type": "PresenceUpdate", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose presence to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get this user's presence state.", - "notes": "Get this user's presence state.", - "type": "PresenceUpdate", - "nickname": "get_presence", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/presence/list/{userId}", - "operations": [ - { - "method": "GET", - "summary": "Retrieve a list of presences for all of this user's friends.", - "notes": "", - "type": "array", - "items": { - "$ref": "Presence" - }, - "nickname": "get_presence_list", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence list to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "POST", - "summary": "Add or remove users from this presence list.", - "notes": "Add or remove users from this presence list.", - "type": "void", - "nickname": "modify_presence_list", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence list is being modified.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The modifications to make to this presence list.", - "required": true, - "type": "PresenceListModifications", - "paramType": "body" - } - ] - } - ] - } - ], - "models": { - "PresenceUpdate": { - "id": "PresenceUpdate", - "properties": { - "presence": { - "type": "string", - "description": "Enum: The presence state.", - "enum": [ - "offline", - "unavailable", - "online", - "free_for_chat" - ] - }, - "status_msg": { - "type": "string", - "description": "The user-defined message associated with this presence state." - } - }, - "subTypes": [ - "Presence" - ] - }, - "Presence": { - "id": "Presence", - "properties": { - "last_active_ago": { - "type": "integer", - "format": "int64", - "description": "The last time this user performed an action on their home server." - }, - "user_id": { - "type": "string", - "description": "The fully qualified user ID" - } - } - }, - "PresenceListModifications": { - "id": "PresenceListModifications", - "properties": { - "invite": { - "type": "array", - "description": "A list of user IDs to add to the list.", - "items": { - "type": "string", - "description": "A fully qualified user ID." - } - }, - "drop": { - "type": "array", - "description": "A list of user IDs to remove from the list.", - "items": { - "type": "string", - "description": "A fully qualified user ID." - } - } - } - } - } -} diff --git a/api/client-server/api-docs-profile b/api/client-server/api-docs-profile deleted file mode 100644 index d2fccaa67d3..00000000000 --- a/api/client-server/api-docs-profile +++ /dev/null @@ -1,122 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/profile", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "apis": [ - { - "path": "/profile/{userId}/displayname", - "operations": [ - { - "method": "PUT", - "summary": "Set a display name.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "set_display_name", - "parameters": [ - { - "name": "body", - "description": "The new display name for this user.", - "required": true, - "type": "DisplayName", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose display name to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get a display name.", - "notes": "This can be done by anyone.", - "type": "DisplayName", - "nickname": "get_display_name", - "parameters": [ - { - "name": "userId", - "description": "The user whose display name to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/profile/{userId}/avatar_url", - "operations": [ - { - "method": "PUT", - "summary": "Set an avatar URL.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "set_avatar_url", - "parameters": [ - { - "name": "body", - "description": "The new avatar url for this user.", - "required": true, - "type": "AvatarUrl", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose avatar url to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get an avatar url.", - "notes": "This can be done by anyone.", - "type": "AvatarUrl", - "nickname": "get_avatar_url", - "parameters": [ - { - "name": "userId", - "description": "The user whose avatar url to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "DisplayName": { - "id": "DisplayName", - "properties": { - "displayname": { - "type": "string", - "description": "The textual display name" - } - } - }, - "AvatarUrl": { - "id": "AvatarUrl", - "properties": { - "avatar_url": { - "type": "string", - "description": "A url to an image representing an avatar." - } - } - } - } -} diff --git a/api/client-server/api-docs-registration b/api/client-server/api-docs-registration deleted file mode 100644 index 11c170c3ecb..00000000000 --- a/api/client-server/api-docs-registration +++ /dev/null @@ -1,120 +0,0 @@ -{ - "apiVersion": "1.0.0", - "apis": [ - { - "operations": [ - { - "method": "GET", - "nickname": "get_registration_info", - "notes": "All login stages MUST be mentioned if there is >1 login type.", - "summary": "Get the login mechanism to use when registering.", - "type": "RegistrationFlows" - }, - { - "method": "POST", - "nickname": "submit_registration", - "notes": "If this is part of a multi-stage registration, there MUST be a 'session' key.", - "parameters": [ - { - "description": "A registration submission", - "name": "body", - "paramType": "body", - "required": true, - "type": "RegistrationSubmission" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad login type" - }, - { - "code": 400, - "message": "Missing JSON keys" - } - ], - "summary": "Submit a registration action.", - "type": "RegistrationResult" - } - ], - "path": "/register" - } - ], - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "consumes": [ - "application/json" - ], - "models": { - "RegistrationFlows": { - "id": "RegistrationFlows", - "properties": { - "flows": { - "description": "A list of valid registration flows.", - "type": "array", - "items": { - "$ref": "RegistrationInfo" - } - } - } - }, - "RegistrationInfo": { - "id": "RegistrationInfo", - "properties": { - "stages": { - "description": "Multi-stage registration only: An array of all the login types required to registration.", - "items": { - "$ref": "string" - }, - "type": "array" - }, - "type": { - "description": "The first login type that must be used when logging in.", - "type": "string" - } - } - }, - "RegistrationResult": { - "id": "RegistrationResult", - "properties": { - "access_token": { - "description": "The access token for this user's registration if this is the final stage of the registration process.", - "type": "string" - }, - "user_id": { - "description": "The user's fully-qualified user ID.", - "type": "string" - }, - "next": { - "description": "Multi-stage registration only: The next registration type to submit.", - "type": "string" - }, - "session": { - "description": "Multi-stage registration only: The session token to send when submitting the next registration type.", - "type": "string" - } - } - }, - "RegistrationSubmission": { - "id": "RegistrationSubmission", - "properties": { - "type": { - "description": "The type of registration being submitted.", - "type": "string" - }, - "session": { - "description": "Multi-stage registration only: The session token from an earlier registration stage.", - "type": "string" - }, - "_registration_type_defined_keys_": { - "description": "Keys as defined by the specified registration type, e.g. \"user\", \"password\"" - } - } - } - }, - "produces": [ - "application/json" - ], - "resourcePath": "/register", - "swaggerVersion": "1.2" -} - diff --git a/api/client-server/api-docs-rooms b/api/client-server/api-docs-rooms deleted file mode 100644 index 0d3d9fbcb91..00000000000 --- a/api/client-server/api-docs-rooms +++ /dev/null @@ -1,1128 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/rooms", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "authorizations": { - "token": [] - }, - "apis": [ - { - "path": "/rooms/{roomId}/send/{eventType}", - "operations": [ - { - "method": "POST", - "summary": "Send a generic non-state event to this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_non_state_event", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The event contents", - "required": true, - "type": "EventContent", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "eventType", - "description": "The type of event to send.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/{eventType}/{stateKey}", - "operations": [ - { - "method": "PUT", - "summary": "Send a generic state event to this room.", - "notes": "The state key can be omitted, such that you can PUT to /rooms/{roomId}/state/{eventType}. The state key defaults to a 0 length string in this case.", - "type": "void", - "nickname": "send_state_event", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The event contents", - "required": true, - "type": "EventContent", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "eventType", - "description": "The type of event to send.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "stateKey", - "description": "An identifier used to specify clobbering semantics. State events with the same (roomId, eventType, stateKey) will be replaced.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/send/m.room.message", - "operations": [ - { - "method": "POST", - "summary": "Send a message in this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_message", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The message contents", - "required": true, - "type": "Message", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.topic", - "operations": [ - { - "method": "PUT", - "summary": "Set the topic for this room.", - "notes": "Set the topic for this room.", - "type": "void", - "nickname": "set_topic", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The topic contents", - "required": true, - "type": "Topic", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the topic in.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get the topic for this room.", - "notes": "Get the topic for this room.", - "type": "Topic", - "nickname": "get_topic", - "parameters": [ - { - "name": "roomId", - "description": "The room to get topic in.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Topic not found." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.name", - "operations": [ - { - "method": "PUT", - "summary": "Set the name of this room.", - "notes": "Set the name of this room.", - "type": "void", - "nickname": "set_room_name", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The name contents", - "required": true, - "type": "RoomName", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the name of.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get the room's name.", - "notes": "", - "type": "RoomName", - "nickname": "get_room_name", - "parameters": [ - { - "name": "roomId", - "description": "The room to get the name of.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Name not found." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.power_levels", - "operations": [ - { - "method": "PUT", - "summary": "Set the power levels for this room.", - "notes": "This has to be set atomically. The levels set will clobber what was previously there.", - "type": "void", - "nickname": "set_room_power_levels", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The power levels", - "required": true, - "type": "RoomPowerLevels", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the power levels for.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/send/m.room.message.feedback", - "operations": [ - { - "method": "POST", - "summary": "Send feedback to a message.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_feedback", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The feedback contents", - "required": true, - "type": "Feedback", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the feedback in.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad feedback type." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/invite", - "operations": [ - { - "method": "POST", - "summary": "Invite a user to this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "invite", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The user to invite.", - "required": true, - "type": "InviteRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/join", - "operations": [ - { - "method": "POST", - "summary": "Join this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "join_room", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room to join.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "required": true, - "type": "JoinRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/leave", - "operations": [ - { - "method": "POST", - "summary": "Leave this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "leave", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room to leave.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "required": true, - "type": "LeaveRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/ban", - "operations": [ - { - "method": "POST", - "summary": "Ban a user in the room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}. The caller must have the required power level to do this operation.", - "type": "void", - "nickname": "ban", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room which has the user to ban.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The user to ban.", - "required": true, - "type": "BanRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.member/{userId}", - "operations": [ - { - "method": "PUT", - "summary": "Change the membership state for a user in a room.", - "notes": "Change the membership state for a user in a room.", - "type": "void", - "nickname": "set_membership", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The new membership state", - "required": true, - "type": "Member", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose membership is being changed.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "No membership key." - }, - { - "code": 400, - "message": "Bad membership value." - }, - { - "code": 403, - "message": "When inviting: You are not in the room." - }, - { - "code": 403, - "message": "When inviting: is already in the room." - }, - { - "code": 403, - "message": "When joining: Cannot force another user to join." - }, - { - "code": 403, - "message": "When joining: You are not invited to this room." - } - ] - }, - { - "method": "GET", - "summary": "Get the membership state of a user in a room.", - "notes": "Get the membership state of a user in a room.", - "type": "Member", - "nickname": "get_membership", - "parameters": [ - { - "name": "userId", - "description": "The user whose membership state you want to get.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Member not found." - } - ] - } - ] - }, - { - "path": "/join/{roomAliasOrId}", - "operations": [ - { - "method": "POST", - "summary": "Join a room via a room alias or room ID.", - "notes": "This endpoint, unlike /rooms/{roomId}/join allows the client to join a room by it's alias. This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "JoinRoomInfo", - "nickname": "join", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomAliasOrId", - "description": "The room alias or room ID to join.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad room alias." - } - ] - } - ] - }, - { - "path": "/createRoom", - "operations": [ - { - "method": "POST", - "summary": "Create a room.", - "notes": "Create a room. This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "RoomInfo", - "nickname": "create_room", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The desired configuration for the room.", - "required": true, - "type": "RoomConfig", - "paramType": "body" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Body must be JSON." - }, - { - "code": 400, - "message": "Room alias already taken." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/messages", - "operations": [ - { - "method": "GET", - "summary": "Get a list of messages for this room.", - "notes": "Get a list of messages for this room.", - "type": "MessagePaginationChunk", - "nickname": "get_messages", - "parameters": [ - { - "name": "roomId", - "description": "The room to get messages in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "from", - "description": "The token to start getting results from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "to", - "description": "The token to stop getting results at.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "limit", - "description": "The maximum number of messages to return.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/members", - "operations": [ - { - "method": "GET", - "summary": "Get a list of members for this room.", - "notes": "Get a list of members for this room.", - "type": "MemberPaginationChunk", - "nickname": "get_members", - "parameters": [ - { - "name": "roomId", - "description": "The room to get a list of members from.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "from", - "description": "The token to start getting results from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "to", - "description": "The token to stop getting results at.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "limit", - "description": "The maximum number of members to return.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state", - "operations": [ - { - "method": "GET", - "summary": "Get a list of all the current state events for this room.", - "notes": "This is equivalent to the events returned under the 'state' key for this room in /initialSync.", - "type": "array", - "items": { - "$ref": "Event" - }, - "nickname": "get_state_events", - "parameters": [ - { - "name": "roomId", - "description": "The room to get a list of current state events from.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/typing/{userId}", - "operations": [ - { - "method": "PUT", - "summary": "Inform the server that this user is typing.", - "notes": "Only the authorised user can send typing notifications.", - "type": "void", - "nickname": "put_typing", - "parameters": [ - { - "name": "roomId", - "description": "The room to send the m.typing event into.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "userId", - "description": "The user ID of the typer.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The typing information.", - "required": true, - "type": "Typing", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/initialSync", - "operations": [ - { - "method": "GET", - "summary": "Get all the current information for this room, including messages and state events.", - "type": "InitialSyncRoomData", - "nickname": "get_room_sync_data", - "parameters": [ - { - "name": "roomId", - "description": "The room to get information for.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "Topic": { - "id": "Topic", - "properties": { - "topic": { - "type": "string", - "description": "The topic text" - } - } - }, - "RoomName": { - "id": "RoomName", - "properties": { - "name": { - "type": "string", - "description": "The human-readable name for the room. Can contain spaces." - } - } - }, - "UserPowerLevels": { - "id": "UserPowerLevels", - "properties": { - "__user_id__": { - "type": "integer", - "description": "The power level for __user_id__" - } - } - }, - "EventPowerLevels": { - "id": "UserPowerLevels", - "properties": { - "__event_type__": { - "type": "integer", - "description": "The power level required in order to send __event_type__ events." - } - } - }, - "RoomPowerLevels": { - "id": "RoomPowerLevels", - "properties": { - "ban": { - "type": "integer", - "description": "The minimum level required in order to ban someone." - }, - "kick": { - "type": "integer", - "description": "The minimum level required in order to kick someone." - }, - "redact": { - "type": "integer", - "description": "The minimum level required in order to redact a message." - }, - "users_default": { - "type": "integer", - "description": "The default power level of a user who is not in the 'users' list." - }, - "state_default": { - "type": "integer", - "description": "The default power level required in order to send state events." - }, - "events_default": { - "type": "integer", - "description": "The default power level required in order to send non-state events." - }, - "users": { - "type": "UserPowerLevels", - "description": "Mappings of user ID to power level." - }, - "events": { - "type": "EventPowerLevels", - "description": "Mappings of event type to power level." - } - } - }, - "Message": { - "id": "Message", - "properties": { - "msgtype": { - "type": "string", - "description": "The type of message being sent, e.g. \"m.text\"", - "required": true - }, - "_msgtype_defined_keys_": { - "description": "Additional keys as defined by the msgtype, e.g. \"body\"" - } - } - }, - "Feedback": { - "id": "Feedback", - "properties": { - "target_event_id": { - "type": "string", - "description": "The event ID being acknowledged.", - "required": true - }, - "type": { - "type": "string", - "description": "The type of feedback. Either 'delivered' or 'read'.", - "required": true - } - } - }, - "Member": { - "id": "Member", - "properties": { - "membership": { - "type": "string", - "description": "Enum: The membership state of this member.", - "enum": [ - "invite", - "join", - "leave", - "ban" - ] - } - } - }, - "RoomInfo": { - "id": "RoomInfo", - "properties": { - "room_id": { - "type": "string", - "description": "The allocated room ID.", - "required": true - }, - "room_alias": { - "type": "string", - "description": "The alias for the room.", - "required": false - } - } - }, - "JoinRoomInfo": { - "id": "JoinRoomInfo", - "properties": { - "room_id": { - "type": "string", - "description": "The room ID joined, if joined via a room alias only.", - "required": true - } - } - }, - "Typing": { - "id": "Typing", - "properties": { - "typing": { - "type": "boolean", - "description": "True if the user is currently typing.", - "required": true - }, - "timeout": { - "type": "integer", - "description": "The length of time until the user should be treated as no longer typing, in milliseconds. Can be omitted if they are no longer typing.", - "required": true - } - } - }, - "RoomConfig": { - "id": "RoomConfig", - "properties": { - "visibility": { - "type": "string", - "description": "Enum: The room visibility. The room_alias_name is required if the visibility is public; without it the room will remain private.", - "required": false, - "enum": [ - "public", - "private" - ] - }, - "room_alias_name": { - "type": "string", - "description": "Localpart of the alias to give the new room. The home server will attach its domain name after this.", - "required": false - }, - "name": { - "type": "string", - "description": "Sets the name of the room. Send a m.room.name event after creating the room with the 'name' key specified.", - "required": false - }, - "topic": { - "type": "string", - "description": "Sets the topic for the room. Send a m.room.topic event after creating the room with the 'topic' key specified.", - "required": false - }, - "invite": { - "type": "array", - "description": "The list of user IDs to invite. Sends m.room.member events after creating the room.", - "items": { - "$ref": "string" - }, - "required": false - } - } - }, - "PaginationRequest": { - "id": "PaginationRequest", - "properties": { - "from": { - "type": "string", - "description": "The token to start getting results from." - }, - "to": { - "type": "string", - "description": "The token to stop getting results at." - }, - "limit": { - "type": "integer", - "description": "The maximum number of entries to return." - } - } - }, - "PaginationChunk": { - "id": "PaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - } - }, - "subTypes": [ - "MessagePaginationChunk" - ] - }, - "MessagePaginationChunk": { - "id": "MessagePaginationChunk", - "properties": { - "chunk": { - "type": "array", - "description": "A list of message events.", - "items": { - "$ref": "MessageEvent" - }, - "required": true - } - } - }, - "MemberPaginationChunk": { - "id": "MemberPaginationChunk", - "properties": { - "chunk": { - "type": "array", - "description": "A list of member events.", - "items": { - "$ref": "MemberEvent" - }, - "required": true - } - } - }, - "Event": { - "id": "Event", - "properties": { - "event_id": { - "type": "string", - "description": "An ID which uniquely identifies this event. This is automatically set by the server.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The room in which this event occurred. This is automatically set by the server.", - "required": true - }, - "type": { - "type": "string", - "description": "The event type.", - "required": true - } - }, - "subTypes": [ - "MessageEvent" - ] - }, - "EventId": { - "id": "EventId", - "properties": { - "event_id": { - "type": "string", - "description": "The allocated event ID for this event.", - "required": true - } - } - }, - "EventContent": { - "id": "EventContent", - "properties": { - "__event_content_keys__": { - "type": "string", - "description": "Event-specific content keys and values.", - "required": false - } - } - }, - "MessageEvent": { - "id": "MessageEvent", - "properties": { - "content": { - "type": "Message" - } - } - }, - "MemberEvent": { - "id": "MemberEvent", - "properties": { - "content": { - "type": "Member" - } - } - }, - "InviteRequest": { - "id": "InviteRequest", - "properties": { - "user_id": { - "type": "string", - "description": "The fully-qualified user ID." - } - } - }, - "JoinRequest": { - "id": "JoinRequest", - "properties": {} - }, - "LeaveRequest": { - "id": "LeaveRequest", - "properties": {} - }, - "BanRequest": { - "id": "BanRequest", - "properties": { - "user_id": { - "type": "string", - "description": "The fully-qualified user ID." - }, - "reason": { - "type": "string", - "description": "The reason for the ban." - } - } - }, - "InitialSyncRoomData": { - "id": "InitialSyncRoomData", - "properties": { - "membership": { - "type": "string", - "description": "This user's membership state in this room.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The ID of this room.", - "required": true - }, - "messages": { - "type": "MessagePaginationChunk", - "description": "The most recent messages for this room, governed by the limit parameter.", - "required": false - }, - "state": { - "type": "array", - "description": "A list of state events representing the current state of the room.", - "required": false, - "items": { - "$ref": "Event" - } - }, - "presence": { - "type": "array", - "description": "A list of m.presence events representing the current presence state of the room members.", - "required": false, - "items": { - "$ref": "Event" - } - } - } - } - } -} diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index ef4e1014ac3..9a9256b2fdc 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -16,9 +16,11 @@ for f in "$1"/{head,nav,footer}.html; do fi done +files=gen/*.html + perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header -#' gen/specification.html gen/howtos.html +#' ${files} perl -MFile::Slurp -pi -e 'BEGIN { $nav = read_file("'$NAV_BAR'") } s##
@@ -27,7 +29,7 @@ perl -MFile::Slurp -pi -e 'BEGIN { $nav = read_file("'$NAV_BAR'") } s## <
-#' gen/specification.html gen/howtos.html +#' ${files} perl -MFile::Slurp -pi -e 'BEGIN { $footer = read_file("'$FOOTER'") } s##
@@ -39,4 +41,4 @@ perl -MFile::Slurp -pi -e 'BEGIN { $footer = read_file("'$FOOTER'") } s## $footer
- #' gen/specification.html gen/howtos.html + #' ${files} diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh new file mode 100755 index 00000000000..0a2bc5f6902 --- /dev/null +++ b/scripts/generate-http-docs.sh @@ -0,0 +1,27 @@ +#!/bin/bash -eu + +# This script generates an HTML page containing all of the client-server API docs. +# It takes all of the swagger YAML files for the client-server API, and turns +# them into API docs, with none of the narrative found in the rst files which +# normally wrap these API docs. + +cd "$(dirname $0)" + +mkdir -p tmp gen + +cat >tmp/http_apis <> tmp/http_apis +done + +(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis) +rst2html.py --stylesheet-path=$(echo css/*.css | tr ' ' ',') gen/http_apis > gen/http_apis.html From 7f2813354daecf30bf1a14fa715f88a3511ed5b6 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 15:42:21 +0000 Subject: [PATCH 252/989] Swaggerify /ban --- api/client-server/v1/banning.yaml | 76 +++++++++++++++++++++++++++++ specification/client_server_api.rst | 2 + 2 files changed, 78 insertions(+) create mode 100644 api/client-server/v1/banning.yaml diff --git a/api/client-server/v1/banning.yaml b/api/client-server/v1/banning.yaml new file mode 100644 index 00000000000..e841050f9f6 --- /dev/null +++ b/api/client-server/v1/banning.yaml @@ -0,0 +1,76 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Banning API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/ban": + post: + summary: Ban a user in the room. + description: |- + Ban a user in the room. If the user is currently in the room, also kick them. + + When a user is banned from a room, they may not join it until they are unbanned. + + The caller must have the required power level in order to perform this operation. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) from which the user should be banned. + required: true + x-example: "!e42d8c:matrix.org" + - in: body + name: body + required: true + schema: + type: object + example: |- + { + "reason": "Telling unfunny jokes", + "user_id": "@cheeky_monkey:matrix.org" + } + properties: + user_id: + type: string + description: The fully qualified user ID of the user being banned. + reason: + type: string + description: The reason the user has been banned. + required: ["user_id"] + responses: + 200: + description: The user has been kicked and banned from the room. + examples: + application/json: |- + {} + schema: + type: object + 403: + description: |- + You do not have permission to ban the user from the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + + - The banner is not currently in the room. + - The banner's power level is insufficient to ban users from the room. + examples: + application/json: |- + { + "errcode": "M_FORBIDDEN", + "error": "You do not have a high enough power level to ban from this room." + } diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 402ce75318f..58d0f1b2032 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -983,6 +983,8 @@ following values: {{joining_http_api}} +{{banning_http_api}} + Leaving rooms ~~~~~~~~~~~~~ A user can leave a room to stop receiving events for that room. A user must From 3951785f19b2b6c38980bcf2748479ce4ba5ef86 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 16:23:31 +0000 Subject: [PATCH 253/989] Fix alias path --- api/client-server/v1/joining.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v1/joining.yaml b/api/client-server/v1/joining.yaml index 20fdbd65450..b6d2df18eb9 100644 --- a/api/client-server/v1/joining.yaml +++ b/api/client-server/v1/joining.yaml @@ -65,4 +65,4 @@ paths: x-alias: canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" aliases: - - /join/{roomId} + - /_matrix/client/api/v1/join/{roomId} From e171acf01ffecc4a931712928ffca024cbdade8a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 14:21:32 +0000 Subject: [PATCH 254/989] Split spec into page-per-section --- scripts/gendoc.py | 52 +++++++++++++------ specification/intro.rst | 10 ++++ .../modules/typing_notifications.rst | 6 +-- specification/targets.yaml | 11 +++- 4 files changed, 57 insertions(+), 22 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 28a115284e7..9b87cb30ee4 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -276,6 +276,12 @@ def run_through_template(input, set_verbose): raise +def get_build_targets(targets_listing): + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + return all_targets["targets"].keys() + + """ Extract and resolve groups for the given target in the given targets listing. Args: @@ -386,21 +392,34 @@ def cleanup_env(): shutil.rmtree("./tmp") -def main(target_name, keep_intermediates): +def main(requested_target_name, keep_intermediates): prepare_env() - log("Building spec [target=%s]" % target_name) - target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename="tmp/templated_spec.rst") - run_through_template("tmp/templated_spec.rst", VERBOSE) - fix_relative_titles( - target=target, filename="tmp/templated_spec.rst", - out_filename="tmp/full_spec.rst" - ) - shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this - rst2html("tmp/full_spec.rst", "gen/specification.html") - addAnchors("gen/specification.html") - rst2html("tmp/howto.rst", "gen/howtos.html") + log("Building spec [target=%s]" % requested_target_name) + + targets = [requested_target_name] + if requested_target_name == "all": + targets = get_build_targets("../specification/targets.yaml") + + for target_name in targets: + templated_file = "tmp/templated_%s.rst" % (target_name,) + rst_file = "tmp/spec_%s.rst" % (target_name,) + html_file = "gen/%s.html" % (target_name,) + + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename=templated_file) + run_through_template(templated_file, VERBOSE) + fix_relative_titles( + target=target, filename=templated_file, + out_filename=rst_file, + ) + rst2html(rst_file, html_file) + addAnchors(html_file) + + if requested_target_name == "all": + shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") + run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this + rst2html("tmp/howto.rst", "gen/howtos.html") + if not keep_intermediates: cleanup_env() @@ -414,8 +433,9 @@ def main(target_name, keep_intermediates): help="Do not delete intermediate files. They will be found in tmp/" ) parser.add_argument( - "--target", "-t", default="main", - help="Specify the build target to build from specification/targets.yaml" + "--target", "-t", default="all", + help="Specify the build target to build from specification/targets.yaml. " + + "The value 'all' will build all of the targets therein." ) parser.add_argument( "--verbose", "-v", action="store_true", diff --git a/specification/intro.rst b/specification/intro.rst index 6ff7946cffd..8c08bf24998 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -8,6 +8,16 @@ https://github.com/matrix-org/matrix-doc using https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}} +APIs +~~~~ +The following APIs are documented in this specification: + +- `Client-Server API `_ for writing Matrix clients. +- `Server-Server API `_ for writing servers which can federate with Matrix. +- `Application Service API `_ for writing privileged plugins to servers. + +There are also some `appendices `_. + Changelog ~~~~~~~~~ {{spec_changelog}} diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index d2253632718..da383e73176 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -5,10 +5,8 @@ Typing Notifications Users may wish to be informed when another user is typing in a room. This can be achieved using typing notifications. These are ephemeral events scoped to a -``room_id``. This means they do not form part of the `Event Graph`_ but still -have a ``room_id`` key. - -.. _Event Graph: `sect:event-graph`_ +``room_id``. This means they do not form part of the +`Event Graph `_ but still have a ``room_id`` key. Events ------ diff --git a/specification/targets.yaml b/specification/targets.yaml index 8e6a2ce00b2..cd0f6f4139a 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,16 +1,23 @@ targets: - main: # arbitrary name to identify this build target + index: files: # the sort order of files to cat - intro.rst + client_server: + files: - client_server_api.rst - { 1: events.rst } - { 1: event_signing.rst } - modules.rst - { 1: feature_profiles.rst } - { 1: "group:modules" } # reference a group of files + application_service: + files: - application_service_api.rst + server_server: + files: - server_server_api.rst - - identity_servers.rst + appendices: + files: - appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: From 30ed918633cf47ceb2d5dac793b1e3a3e9a0d80f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 14:53:58 +0000 Subject: [PATCH 255/989] speculator: Allow spec viewing for multi-page spec --- scripts/speculator/main.go | 94 +++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index abe77c74db3..3736672f499 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -57,8 +57,8 @@ var ( port = flag.Int("port", 9000, "Port on which to listen for HTTP") includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org") allowedMembers map[string]bool - specCache *lru.Cache // string -> []byte - styledSpecCache *lru.Cache // string -> []byte + specCache *lru.Cache // string -> map[string][]byte filename -> contents + styledSpecCache *lru.Cache // string -> map[string][]byte filename -> contents ) func (u *User) IsTrusted() bool { @@ -105,10 +105,7 @@ func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) { if !strings.HasPrefix(url.Path, pathPrefix+"/") { return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) } - prNumber := url.Path[len(pathPrefix)+1:] - if strings.Contains(prNumber, "/") { - return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) - } + prNumber := strings.Split(url.Path[len(pathPrefix)+1:], "/")[0] resp, err := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber)) defer resp.Body.Close() @@ -196,6 +193,17 @@ func (s *server) getSHAOf(ref string) (string, error) { return strings.TrimSpace(b.String()), nil } +func extractPath(path, base string) string { + // Assume exactly one flat directory + max := strings.Count(base, "/") + 2 + parts := strings.SplitN(path, "/", max) + + if len(parts) < max || parts[max-1] == "" { + return "index.html" + } + return parts[max-1] +} + func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string @@ -206,7 +214,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } - if strings.ToLower(req.URL.Path) == "/spec/head" { + if strings.HasPrefix(strings.ToLower(req.URL.Path), "/spec/head") { // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. // This is to deal with cases like where github is down but we still want to serve the spec. if headSha, err := s.lookupHeadSHA(); headSha == "" { @@ -236,36 +244,59 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { cache = styledSpecCache } + var pathToContent map[string][]byte + if cached, ok := cache.Get(sha); ok { - w.Write(cached.([]byte)) - return - } + pathToContent = cached.(map[string][]byte) + } else { + dst, err := s.generateAt(sha) + defer os.RemoveAll(dst) + if err != nil { + writeError(w, 500, err) + return + } - dst, err := s.generateAt(sha) - defer os.RemoveAll(dst) - if err != nil { - writeError(w, 500, err) - return - } + if styleLikeMatrixDotOrg { + cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) + cmd.Dir = path.Join(dst, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) + return + } + } - if styleLikeMatrixDotOrg { - cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) - cmd.Dir = path.Join(dst, "scripts") - var b bytes.Buffer - cmd.Stderr = &b - if err := cmd.Run(); err != nil { - writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) - return + fis, err := ioutil.ReadDir(path.Join(dst, "scripts", "gen")) + if err != nil { + writeError(w, 500, fmt.Errorf("Error reading directory: %v", err)) + } + pathToContent = make(map[string][]byte) + for _, fi := range fis { + b, err := ioutil.ReadFile(path.Join(dst, "scripts", "gen", fi.Name())) + if err != nil { + writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) + return + } + pathToContent[fi.Name()] = b } + cache.Add(sha, pathToContent) } - b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) - if err != nil { - writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) + requestedPath := extractPath(req.URL.Path, "/spec/pr") + if b, ok := pathToContent[requestedPath]; ok { + w.Write(b) return } - w.Write(b) - cache.Add(sha, b) + if requestedPath == "index.html" { + // Fall back to single-page spec for old PRs + if b, ok := pathToContent["specification.html"]; ok { + w.Write(b) + return + } + } + w.WriteHeader(404) + w.Write([]byte("Not found")) } // lookupHeadSHA looks up what origin/master's HEAD SHA is. @@ -322,7 +353,7 @@ func (s *server) serveRSTDiff(w http.ResponseWriter, req *http.Request) { return } - diffCmd := exec.Command("diff", "-u", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst")) + diffCmd := exec.Command("diff", "-r", "-u", path.Join(base, "scripts", "tmp"), path.Join(head, "scripts", "tmp")) var diff bytes.Buffer diffCmd.Stdout = &diff if err := ignoreExitCodeOne(diffCmd.Run()); err != nil { @@ -366,7 +397,8 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) { return } - cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", "specification.html"), path.Join(head, "scripts", "gen", "specification.html")) + requestedPath := extractPath(req.URL.Path, "/diff/spec/pr") + cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath)) var b bytes.Buffer cmd.Stdout = &b if err := cmd.Run(); err != nil { From b479b54cd887d0fd49a33458a77a51adaffc45bf Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 18:52:58 +0000 Subject: [PATCH 256/989] Add tables of contents to individual specs --- specification/application_service_api.rst | 3 +++ specification/client_server_api.rst | 3 +++ specification/server_server_api.rst | 3 +++ 3 files changed, 9 insertions(+) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index cf2f9d57ffd..bdf8a52eaf3 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -12,6 +12,9 @@ irrespective of the underlying homeserver implementation. Add in Client-Server services? Overview of bots? Seems weird to be in the spec given it is VERY implementation specific. +.. contents:: Table of Contents +.. sectnum:: + Application Services -------------------- Application services are passive and can only observe events from a given diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 36ed2290c30..134c12c3584 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -17,6 +17,9 @@ shortly. Documentation for the old `V1 authentication <../attic/v1_registration_login.rst>`_ is still available separately. +.. contents:: Table of Contents +.. sectnum:: + API Standards ------------- diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index cc9426e2b0f..012c2ac9693 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -41,6 +41,9 @@ EDUs and PDUs are further wrapped in an envelope called a Transaction, which is transferred from the origin to the destination home server using an HTTPS PUT request. +.. contents:: Table of Contents +.. sectnum:: + Server Discovery ---------------- From 6c66bfc75589392b61d67e1a37c3f11b66a5cf7e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 30 Nov 2015 11:22:57 +0000 Subject: [PATCH 257/989] Flatten out v1 and v2_alpha directories As a side effect, I got rid of all of the horrible symlinks and just put in all of the proper relative paths. Because the horrible symlinks were horrible. --- .../{v2_alpha => }/account-data.yaml | 0 .../{v1 => }/application_service.yaml | 0 api/client-server/{v1 => }/banning.yaml | 0 api/client-server/{v1 => }/content-repo.yaml | 0 api/client-server/{v1 => }/create_room.yaml | 0 .../{v1 => }/definitions/error.yaml | 0 .../{v2_alpha => }/definitions/event.json | 0 .../definitions/event_batch.json | 0 .../definitions/event_filter.json | 0 .../{v1 => }/definitions/push_condition.json | 0 .../{v1 => }/definitions/push_rule.json | 0 .../{v1 => }/definitions/push_ruleset.json | 0 .../definitions/room_event_filter.json | 2 +- .../definitions/sync_filter.json | 8 ++-- .../definitions/timeline_batch.json | 2 +- api/client-server/{v1 => }/directory.yaml | 0 api/client-server/{v2_alpha => }/filter.yaml | 0 api/client-server/{v1 => }/guest_events.yaml | 2 +- api/client-server/{v1 => }/inviting.yaml | 0 api/client-server/{v1 => }/joining.yaml | 0 api/client-server/{v1 => }/leaving.yaml | 0 .../{v1 => }/list_public_rooms.yaml | 0 api/client-server/{v1 => }/login.yaml | 0 .../{v1 => }/message_pagination.yaml | 0 .../{v1/sync.yaml => old_sync.yaml} | 14 +++--- api/client-server/{v1 => }/presence.yaml | 2 +- api/client-server/{v1 => }/profile.yaml | 0 api/client-server/{v1 => }/push_notifier.yaml | 0 api/client-server/{v1 => }/pusher.yaml | 0 api/client-server/{v1 => }/pushrules.yaml | 0 .../{v2_alpha => }/receipts.yaml | 0 .../{v2_alpha => }/registration.yaml | 0 api/client-server/{v1 => }/room_send.yaml | 0 api/client-server/{v1 => }/room_state.yaml | 0 api/client-server/{v1 => }/rooms.yaml | 10 ++--- api/client-server/{v1 => }/search.yaml | 2 +- api/client-server/{v2_alpha => }/sync.yaml | 0 api/client-server/{v2_alpha => }/tags.yaml | 0 .../{v1 => }/third_party_membership.yaml | 0 api/client-server/{v1 => }/typing.yaml | 0 api/client-server/v1/core-event-schema | 1 - api/client-server/v1/v1-event-schema | 1 - .../v2_alpha/definitions/definitions | 1 - .../v2_alpha/definitions/error.yaml | 10 ----- api/client-server/{v1 => }/voip.yaml | 0 event-schemas/README.md | 2 +- event-schemas/check.sh | 6 +-- event-schemas/examples/{v1 => }/m.call.answer | 0 .../examples/{v1 => }/m.call.candidates | 0 event-schemas/examples/{v1 => }/m.call.hangup | 0 event-schemas/examples/{v1 => }/m.call.invite | 0 event-schemas/examples/{v1 => }/m.presence | 0 event-schemas/examples/{v1 => }/m.receipt | 0 .../examples/{v1 => }/m.room.aliases | 0 event-schemas/examples/{v1 => }/m.room.avatar | 0 .../examples/{v1 => }/m.room.canonical_alias | 0 event-schemas/examples/{v1 => }/m.room.create | 0 .../examples/{v1 => }/m.room.guest_access | 0 .../{v1 => }/m.room.history_visibility | 0 .../examples/{v1 => }/m.room.join_rules | 0 event-schemas/examples/{v1 => }/m.room.member | 0 .../{v1 => }/m.room.member#invite_room_state | 0 .../{v1 => }/m.room.member#third_party_invite | 0 .../examples/{v1 => }/m.room.message#m.audio | 0 .../examples/{v1 => }/m.room.message#m.emote | 0 .../examples/{v1 => }/m.room.message#m.file | 0 .../examples/{v1 => }/m.room.message#m.image | 0 .../{v1 => }/m.room.message#m.location | 0 .../examples/{v1 => }/m.room.message#m.notice | 0 .../examples/{v1 => }/m.room.message#m.text | 0 .../examples/{v1 => }/m.room.message#m.video | 0 .../examples/{v1 => }/m.room.message.feedback | 0 event-schemas/examples/{v1 => }/m.room.name | 0 .../examples/{v1 => }/m.room.power_levels | 0 .../examples/{v1 => }/m.room.redaction | 0 .../{v1 => }/m.room.third_party_invite | 0 event-schemas/examples/{v1 => }/m.room.topic | 0 event-schemas/examples/{v1 => }/m.tag | 0 event-schemas/examples/{v1 => }/m.typing | 0 .../{v1 => }/core-event-schema/event.json | 0 .../msgtype_infos/image_info.json | 0 .../core-event-schema/room_event.json | 2 +- .../core-event-schema/state_event.json | 2 +- event-schemas/schema/{v1 => }/m.call.answer | 0 .../schema/{v1 => }/m.call.candidates | 0 event-schemas/schema/{v1 => }/m.call.hangup | 0 event-schemas/schema/{v1 => }/m.call.invite | 0 event-schemas/schema/{v1 => }/m.presence | 0 event-schemas/schema/{v1 => }/m.receipt | 0 event-schemas/schema/{v1 => }/m.room.aliases | 0 event-schemas/schema/{v1 => }/m.room.avatar | 0 .../schema/{v1 => }/m.room.canonical_alias | 0 event-schemas/schema/{v1 => }/m.room.create | 0 .../schema/{v1 => }/m.room.guest_access | 0 .../schema/{v1 => }/m.room.history_visibility | 0 .../schema/{v1 => }/m.room.join_rules | 0 event-schemas/schema/{v1 => }/m.room.member | 0 event-schemas/schema/{v1 => }/m.room.message | 0 .../schema/{v1 => }/m.room.message#m.audio | 0 .../schema/{v1 => }/m.room.message#m.emote | 0 .../schema/{v1 => }/m.room.message#m.file | 0 .../schema/{v1 => }/m.room.message#m.image | 0 .../schema/{v1 => }/m.room.message#m.location | 0 .../schema/{v1 => }/m.room.message#m.notice | 0 .../schema/{v1 => }/m.room.message#m.text | 0 .../schema/{v1 => }/m.room.message#m.video | 0 .../schema/{v1 => }/m.room.message.feedback | 0 event-schemas/schema/{v1 => }/m.room.name | 0 .../schema/{v1 => }/m.room.power_levels | 0 .../schema/{v1 => }/m.room.redaction | 0 .../schema/{v1 => }/m.room.third_party_invite | 0 event-schemas/schema/{v1 => }/m.room.topic | 0 event-schemas/schema/{v1 => }/m.tag | 0 event-schemas/schema/{v1 => }/m.typing | 0 .../v1/core-event-schema/core-event-schema | 1 - event-schemas/schema/v1/v1-event-schema | 1 - jenkins.sh | 2 +- scripts/generate-http-docs.sh | 2 +- specification/client_server_api.rst | 6 +-- specification/modules/account_data.rst | 2 +- specification/modules/receipts.rst | 2 +- specification/modules/tags.rst | 2 +- templating/matrix_templates/units.py | 44 +++++++------------ 123 files changed, 52 insertions(+), 77 deletions(-) rename api/client-server/{v2_alpha => }/account-data.yaml (100%) rename api/client-server/{v1 => }/application_service.yaml (100%) rename api/client-server/{v1 => }/banning.yaml (100%) rename api/client-server/{v1 => }/content-repo.yaml (100%) rename api/client-server/{v1 => }/create_room.yaml (100%) rename api/client-server/{v1 => }/definitions/error.yaml (100%) rename api/client-server/{v2_alpha => }/definitions/event.json (100%) rename api/client-server/{v2_alpha => }/definitions/event_batch.json (100%) rename api/client-server/{v2_alpha => }/definitions/event_filter.json (100%) rename api/client-server/{v1 => }/definitions/push_condition.json (100%) rename api/client-server/{v1 => }/definitions/push_rule.json (100%) rename api/client-server/{v1 => }/definitions/push_ruleset.json (100%) rename api/client-server/{v2_alpha => }/definitions/room_event_filter.json (92%) rename api/client-server/{v2_alpha => }/definitions/sync_filter.json (84%) rename api/client-server/{v2_alpha => }/definitions/timeline_batch.json (87%) rename api/client-server/{v1 => }/directory.yaml (100%) rename api/client-server/{v2_alpha => }/filter.yaml (100%) rename api/client-server/{v1 => }/guest_events.yaml (97%) rename api/client-server/{v1 => }/inviting.yaml (100%) rename api/client-server/{v1 => }/joining.yaml (100%) rename api/client-server/{v1 => }/leaving.yaml (100%) rename api/client-server/{v1 => }/list_public_rooms.yaml (100%) rename api/client-server/{v1 => }/login.yaml (100%) rename api/client-server/{v1 => }/message_pagination.yaml (100%) rename api/client-server/{v1/sync.yaml => old_sync.yaml} (96%) rename api/client-server/{v1 => }/presence.yaml (98%) rename api/client-server/{v1 => }/profile.yaml (100%) rename api/client-server/{v1 => }/push_notifier.yaml (100%) rename api/client-server/{v1 => }/pusher.yaml (100%) rename api/client-server/{v1 => }/pushrules.yaml (100%) rename api/client-server/{v2_alpha => }/receipts.yaml (100%) rename api/client-server/{v2_alpha => }/registration.yaml (100%) rename api/client-server/{v1 => }/room_send.yaml (100%) rename api/client-server/{v1 => }/room_state.yaml (100%) rename api/client-server/{v1 => }/rooms.yaml (97%) rename api/client-server/{v1 => }/search.yaml (97%) rename api/client-server/{v2_alpha => }/sync.yaml (100%) rename api/client-server/{v2_alpha => }/tags.yaml (100%) rename api/client-server/{v1 => }/third_party_membership.yaml (100%) rename api/client-server/{v1 => }/typing.yaml (100%) delete mode 120000 api/client-server/v1/core-event-schema delete mode 120000 api/client-server/v1/v1-event-schema delete mode 120000 api/client-server/v2_alpha/definitions/definitions delete mode 100644 api/client-server/v2_alpha/definitions/error.yaml rename api/client-server/{v1 => }/voip.yaml (100%) rename event-schemas/examples/{v1 => }/m.call.answer (100%) rename event-schemas/examples/{v1 => }/m.call.candidates (100%) rename event-schemas/examples/{v1 => }/m.call.hangup (100%) rename event-schemas/examples/{v1 => }/m.call.invite (100%) rename event-schemas/examples/{v1 => }/m.presence (100%) rename event-schemas/examples/{v1 => }/m.receipt (100%) rename event-schemas/examples/{v1 => }/m.room.aliases (100%) rename event-schemas/examples/{v1 => }/m.room.avatar (100%) rename event-schemas/examples/{v1 => }/m.room.canonical_alias (100%) rename event-schemas/examples/{v1 => }/m.room.create (100%) rename event-schemas/examples/{v1 => }/m.room.guest_access (100%) rename event-schemas/examples/{v1 => }/m.room.history_visibility (100%) rename event-schemas/examples/{v1 => }/m.room.join_rules (100%) rename event-schemas/examples/{v1 => }/m.room.member (100%) rename event-schemas/examples/{v1 => }/m.room.member#invite_room_state (100%) rename event-schemas/examples/{v1 => }/m.room.member#third_party_invite (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.audio (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.emote (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.file (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.image (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.location (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.notice (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.text (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.video (100%) rename event-schemas/examples/{v1 => }/m.room.message.feedback (100%) rename event-schemas/examples/{v1 => }/m.room.name (100%) rename event-schemas/examples/{v1 => }/m.room.power_levels (100%) rename event-schemas/examples/{v1 => }/m.room.redaction (100%) rename event-schemas/examples/{v1 => }/m.room.third_party_invite (100%) rename event-schemas/examples/{v1 => }/m.room.topic (100%) rename event-schemas/examples/{v1 => }/m.tag (100%) rename event-schemas/examples/{v1 => }/m.typing (100%) rename event-schemas/schema/{v1 => }/core-event-schema/event.json (100%) rename event-schemas/schema/{v1 => }/core-event-schema/msgtype_infos/image_info.json (100%) rename event-schemas/schema/{v1 => }/core-event-schema/room_event.json (93%) rename event-schemas/schema/{v1 => }/core-event-schema/state_event.json (93%) rename event-schemas/schema/{v1 => }/m.call.answer (100%) rename event-schemas/schema/{v1 => }/m.call.candidates (100%) rename event-schemas/schema/{v1 => }/m.call.hangup (100%) rename event-schemas/schema/{v1 => }/m.call.invite (100%) rename event-schemas/schema/{v1 => }/m.presence (100%) rename event-schemas/schema/{v1 => }/m.receipt (100%) rename event-schemas/schema/{v1 => }/m.room.aliases (100%) rename event-schemas/schema/{v1 => }/m.room.avatar (100%) rename event-schemas/schema/{v1 => }/m.room.canonical_alias (100%) rename event-schemas/schema/{v1 => }/m.room.create (100%) rename event-schemas/schema/{v1 => }/m.room.guest_access (100%) rename event-schemas/schema/{v1 => }/m.room.history_visibility (100%) rename event-schemas/schema/{v1 => }/m.room.join_rules (100%) rename event-schemas/schema/{v1 => }/m.room.member (100%) rename event-schemas/schema/{v1 => }/m.room.message (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.audio (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.emote (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.file (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.image (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.location (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.notice (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.text (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.video (100%) rename event-schemas/schema/{v1 => }/m.room.message.feedback (100%) rename event-schemas/schema/{v1 => }/m.room.name (100%) rename event-schemas/schema/{v1 => }/m.room.power_levels (100%) rename event-schemas/schema/{v1 => }/m.room.redaction (100%) rename event-schemas/schema/{v1 => }/m.room.third_party_invite (100%) rename event-schemas/schema/{v1 => }/m.room.topic (100%) rename event-schemas/schema/{v1 => }/m.tag (100%) rename event-schemas/schema/{v1 => }/m.typing (100%) delete mode 120000 event-schemas/schema/v1/core-event-schema/core-event-schema delete mode 120000 event-schemas/schema/v1/v1-event-schema diff --git a/api/client-server/v2_alpha/account-data.yaml b/api/client-server/account-data.yaml similarity index 100% rename from api/client-server/v2_alpha/account-data.yaml rename to api/client-server/account-data.yaml diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/application_service.yaml similarity index 100% rename from api/client-server/v1/application_service.yaml rename to api/client-server/application_service.yaml diff --git a/api/client-server/v1/banning.yaml b/api/client-server/banning.yaml similarity index 100% rename from api/client-server/v1/banning.yaml rename to api/client-server/banning.yaml diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/content-repo.yaml similarity index 100% rename from api/client-server/v1/content-repo.yaml rename to api/client-server/content-repo.yaml diff --git a/api/client-server/v1/create_room.yaml b/api/client-server/create_room.yaml similarity index 100% rename from api/client-server/v1/create_room.yaml rename to api/client-server/create_room.yaml diff --git a/api/client-server/v1/definitions/error.yaml b/api/client-server/definitions/error.yaml similarity index 100% rename from api/client-server/v1/definitions/error.yaml rename to api/client-server/definitions/error.yaml diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/definitions/event.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event.json rename to api/client-server/definitions/event.json diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/definitions/event_batch.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event_batch.json rename to api/client-server/definitions/event_batch.json diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/definitions/event_filter.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event_filter.json rename to api/client-server/definitions/event_filter.json diff --git a/api/client-server/v1/definitions/push_condition.json b/api/client-server/definitions/push_condition.json similarity index 100% rename from api/client-server/v1/definitions/push_condition.json rename to api/client-server/definitions/push_condition.json diff --git a/api/client-server/v1/definitions/push_rule.json b/api/client-server/definitions/push_rule.json similarity index 100% rename from api/client-server/v1/definitions/push_rule.json rename to api/client-server/definitions/push_rule.json diff --git a/api/client-server/v1/definitions/push_ruleset.json b/api/client-server/definitions/push_ruleset.json similarity index 100% rename from api/client-server/v1/definitions/push_ruleset.json rename to api/client-server/definitions/push_ruleset.json diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/definitions/room_event_filter.json similarity index 92% rename from api/client-server/v2_alpha/definitions/room_event_filter.json rename to api/client-server/definitions/room_event_filter.json index 86375781c08..02e5d0e019a 100644 --- a/api/client-server/v2_alpha/definitions/room_event_filter.json +++ b/api/client-server/definitions/room_event_filter.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref": "definitions/event_filter.json"}], + "allOf": [{"$ref": "event_filter.json"}], "properties": { "rooms": { "type": "array", diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/definitions/sync_filter.json similarity index 84% rename from api/client-server/v2_alpha/definitions/sync_filter.json rename to api/client-server/definitions/sync_filter.json index 0cd6a798631..defc318a6d5 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/definitions/sync_filter.json @@ -7,24 +7,24 @@ "state": { "description": "The state events to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] }, "timeline": { "description": "The message and state update events to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] }, "ephemeral": { "description": "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] } } }, "presence": { "description": "The presence updates to include.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "event_filter.json"}] }, "event_format": { "description": diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/definitions/timeline_batch.json similarity index 87% rename from api/client-server/v2_alpha/definitions/timeline_batch.json rename to api/client-server/definitions/timeline_batch.json index f27a5746c5a..6f7e714a61b 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/definitions/timeline_batch.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref":"definitions/event_batch.json"}], + "allOf": [{"$ref":"event_batch.json"}], "properties": { "limited": { "type": "boolean", diff --git a/api/client-server/v1/directory.yaml b/api/client-server/directory.yaml similarity index 100% rename from api/client-server/v1/directory.yaml rename to api/client-server/directory.yaml diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/filter.yaml similarity index 100% rename from api/client-server/v2_alpha/filter.yaml rename to api/client-server/filter.yaml diff --git a/api/client-server/v1/guest_events.yaml b/api/client-server/guest_events.yaml similarity index 97% rename from api/client-server/v1/guest_events.yaml rename to api/client-server/guest_events.yaml index bbb5799af23..671b355af82 100644 --- a/api/client-server/v1/guest_events.yaml +++ b/api/client-server/guest_events.yaml @@ -98,6 +98,6 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." diff --git a/api/client-server/v1/inviting.yaml b/api/client-server/inviting.yaml similarity index 100% rename from api/client-server/v1/inviting.yaml rename to api/client-server/inviting.yaml diff --git a/api/client-server/v1/joining.yaml b/api/client-server/joining.yaml similarity index 100% rename from api/client-server/v1/joining.yaml rename to api/client-server/joining.yaml diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/leaving.yaml similarity index 100% rename from api/client-server/v1/leaving.yaml rename to api/client-server/leaving.yaml diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/list_public_rooms.yaml similarity index 100% rename from api/client-server/v1/list_public_rooms.yaml rename to api/client-server/list_public_rooms.yaml diff --git a/api/client-server/v1/login.yaml b/api/client-server/login.yaml similarity index 100% rename from api/client-server/v1/login.yaml rename to api/client-server/login.yaml diff --git a/api/client-server/v1/message_pagination.yaml b/api/client-server/message_pagination.yaml similarity index 100% rename from api/client-server/v1/message_pagination.yaml rename to api/client-server/message_pagination.yaml diff --git a/api/client-server/v1/sync.yaml b/api/client-server/old_sync.yaml similarity index 96% rename from api/client-server/v1/sync.yaml rename to api/client-server/old_sync.yaml index 58cd5544336..6dbe7e5ea3e 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/old_sync.yaml @@ -84,7 +84,7 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." "/initialSync": @@ -285,7 +285,7 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" rooms: type: array items: @@ -304,7 +304,7 @@ paths: title: "InviteEvent" description: "The invite event if ``membership`` is ``invite``" allOf: - - "$ref": "v1-event-schema/m.room.member" + - "$ref": "../../event-schemas/schema/m.room.member" messages: type: object title: PaginationChunk @@ -332,7 +332,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -345,7 +345,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -361,7 +361,7 @@ paths: title: Event type: object allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" required: ["room_id", "membership"] required: ["end", "rooms", "presence"] 404: @@ -398,6 +398,6 @@ paths: } schema: allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" 404: description: The event was not found or you do not have permission to read this event. diff --git a/api/client-server/v1/presence.yaml b/api/client-server/presence.yaml similarity index 98% rename from api/client-server/v1/presence.yaml rename to api/client-server/presence.yaml index 5684398b01b..33df17d23f2 100644 --- a/api/client-server/v1/presence.yaml +++ b/api/client-server/presence.yaml @@ -205,4 +205,4 @@ paths: type: object title: PresenceEvent allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" diff --git a/api/client-server/v1/profile.yaml b/api/client-server/profile.yaml similarity index 100% rename from api/client-server/v1/profile.yaml rename to api/client-server/profile.yaml diff --git a/api/client-server/v1/push_notifier.yaml b/api/client-server/push_notifier.yaml similarity index 100% rename from api/client-server/v1/push_notifier.yaml rename to api/client-server/push_notifier.yaml diff --git a/api/client-server/v1/pusher.yaml b/api/client-server/pusher.yaml similarity index 100% rename from api/client-server/v1/pusher.yaml rename to api/client-server/pusher.yaml diff --git a/api/client-server/v1/pushrules.yaml b/api/client-server/pushrules.yaml similarity index 100% rename from api/client-server/v1/pushrules.yaml rename to api/client-server/pushrules.yaml diff --git a/api/client-server/v2_alpha/receipts.yaml b/api/client-server/receipts.yaml similarity index 100% rename from api/client-server/v2_alpha/receipts.yaml rename to api/client-server/receipts.yaml diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/registration.yaml similarity index 100% rename from api/client-server/v2_alpha/registration.yaml rename to api/client-server/registration.yaml diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/room_send.yaml similarity index 100% rename from api/client-server/v1/room_send.yaml rename to api/client-server/room_send.yaml diff --git a/api/client-server/v1/room_state.yaml b/api/client-server/room_state.yaml similarity index 100% rename from api/client-server/v1/room_state.yaml rename to api/client-server/room_state.yaml diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/rooms.yaml similarity index 97% rename from api/client-server/v1/rooms.yaml rename to api/client-server/rooms.yaml index ff5587a902f..f7654f145b1 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/rooms.yaml @@ -173,7 +173,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" 403: description: > You aren't a member of the room and weren't previously a @@ -355,7 +355,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -368,7 +368,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -383,7 +383,7 @@ paths: title: Event type: object allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" required: ["room_id"] 403: description: > @@ -453,7 +453,7 @@ paths: title: MemberEvent type: object allOf: - - "$ref": "v1-event-schema/m.room.member" + - "$ref": "../../event-schemas/schema/m.room.member" 403: description: > You aren't a member of the room and weren't previously a diff --git a/api/client-server/v1/search.yaml b/api/client-server/search.yaml similarity index 97% rename from api/client-server/v1/search.yaml rename to api/client-server/search.yaml index cc2990564a8..3da9486f392 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/search.yaml @@ -111,7 +111,7 @@ paths: title: Event description: The event that matched. allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" examples: application/json: |- { diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/sync.yaml similarity index 100% rename from api/client-server/v2_alpha/sync.yaml rename to api/client-server/sync.yaml diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/tags.yaml similarity index 100% rename from api/client-server/v2_alpha/tags.yaml rename to api/client-server/tags.yaml diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/third_party_membership.yaml similarity index 100% rename from api/client-server/v1/third_party_membership.yaml rename to api/client-server/third_party_membership.yaml diff --git a/api/client-server/v1/typing.yaml b/api/client-server/typing.yaml similarity index 100% rename from api/client-server/v1/typing.yaml rename to api/client-server/typing.yaml diff --git a/api/client-server/v1/core-event-schema b/api/client-server/v1/core-event-schema deleted file mode 120000 index 045aecb02ec..00000000000 --- a/api/client-server/v1/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -v1-event-schema/core-event-schema \ No newline at end of file diff --git a/api/client-server/v1/v1-event-schema b/api/client-server/v1/v1-event-schema deleted file mode 120000 index 7a0d0326b45..00000000000 --- a/api/client-server/v1/v1-event-schema +++ /dev/null @@ -1 +0,0 @@ -../../../event-schemas/schema/v1 \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/definitions b/api/client-server/v2_alpha/definitions/definitions deleted file mode 120000 index 945c9b46d68..00000000000 --- a/api/client-server/v2_alpha/definitions/definitions +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/error.yaml b/api/client-server/v2_alpha/definitions/error.yaml deleted file mode 100644 index 20312ae4d6f..00000000000 --- a/api/client-server/v2_alpha/definitions/error.yaml +++ /dev/null @@ -1,10 +0,0 @@ -type: object -description: A Matrix-level Error -properties: - errcode: - type: string - description: An error code. - error: - type: string - description: A human-readable error message. -required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/v1/voip.yaml b/api/client-server/voip.yaml similarity index 100% rename from api/client-server/v1/voip.yaml rename to api/client-server/voip.yaml diff --git a/event-schemas/README.md b/event-schemas/README.md index 1030a04c531..950c3018188 100644 --- a/event-schemas/README.md +++ b/event-schemas/README.md @@ -7,7 +7,7 @@ resolved correctly. For basic CLI testing, we recommend and have verified they work with the Node.js package [z-schema](https://github.com/zaggino/z-schema): ``` $ npm install -g z-schema - $ z-schema schema/v1/m.room.message examples/v1/m.room.message_m.text + $ z-schema schema/m.room.message examples/m.room.message_m.text schema validation passed json #1 validation passed ``` diff --git a/event-schemas/check.sh b/event-schemas/check.sh index a6d03b5a44e..3d411768c2c 100755 --- a/event-schemas/check.sh +++ b/event-schemas/check.sh @@ -6,17 +6,17 @@ if ! which z-schema; then exit 1 fi -find schema/v1/m.* | while read line +find schema/m.* | while read line do split_path=(${line///// }) event_type=(${split_path[2]}) echo "Checking $event_type" echo "--------------------" # match exact name or exact name with a # - find examples/v1 -name $event_type -o -name "$event_type#*" | while read exline + find examples -name $event_type -o -name "$event_type#*" | while read exline do echo " against $exline" # run z-schema: because of bash -e if this fails we bail with exit code 1 - z-schema schema/v1/$event_type $exline + z-schema schema/$event_type $exline done done diff --git a/event-schemas/examples/v1/m.call.answer b/event-schemas/examples/m.call.answer similarity index 100% rename from event-schemas/examples/v1/m.call.answer rename to event-schemas/examples/m.call.answer diff --git a/event-schemas/examples/v1/m.call.candidates b/event-schemas/examples/m.call.candidates similarity index 100% rename from event-schemas/examples/v1/m.call.candidates rename to event-schemas/examples/m.call.candidates diff --git a/event-schemas/examples/v1/m.call.hangup b/event-schemas/examples/m.call.hangup similarity index 100% rename from event-schemas/examples/v1/m.call.hangup rename to event-schemas/examples/m.call.hangup diff --git a/event-schemas/examples/v1/m.call.invite b/event-schemas/examples/m.call.invite similarity index 100% rename from event-schemas/examples/v1/m.call.invite rename to event-schemas/examples/m.call.invite diff --git a/event-schemas/examples/v1/m.presence b/event-schemas/examples/m.presence similarity index 100% rename from event-schemas/examples/v1/m.presence rename to event-schemas/examples/m.presence diff --git a/event-schemas/examples/v1/m.receipt b/event-schemas/examples/m.receipt similarity index 100% rename from event-schemas/examples/v1/m.receipt rename to event-schemas/examples/m.receipt diff --git a/event-schemas/examples/v1/m.room.aliases b/event-schemas/examples/m.room.aliases similarity index 100% rename from event-schemas/examples/v1/m.room.aliases rename to event-schemas/examples/m.room.aliases diff --git a/event-schemas/examples/v1/m.room.avatar b/event-schemas/examples/m.room.avatar similarity index 100% rename from event-schemas/examples/v1/m.room.avatar rename to event-schemas/examples/m.room.avatar diff --git a/event-schemas/examples/v1/m.room.canonical_alias b/event-schemas/examples/m.room.canonical_alias similarity index 100% rename from event-schemas/examples/v1/m.room.canonical_alias rename to event-schemas/examples/m.room.canonical_alias diff --git a/event-schemas/examples/v1/m.room.create b/event-schemas/examples/m.room.create similarity index 100% rename from event-schemas/examples/v1/m.room.create rename to event-schemas/examples/m.room.create diff --git a/event-schemas/examples/v1/m.room.guest_access b/event-schemas/examples/m.room.guest_access similarity index 100% rename from event-schemas/examples/v1/m.room.guest_access rename to event-schemas/examples/m.room.guest_access diff --git a/event-schemas/examples/v1/m.room.history_visibility b/event-schemas/examples/m.room.history_visibility similarity index 100% rename from event-schemas/examples/v1/m.room.history_visibility rename to event-schemas/examples/m.room.history_visibility diff --git a/event-schemas/examples/v1/m.room.join_rules b/event-schemas/examples/m.room.join_rules similarity index 100% rename from event-schemas/examples/v1/m.room.join_rules rename to event-schemas/examples/m.room.join_rules diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/m.room.member similarity index 100% rename from event-schemas/examples/v1/m.room.member rename to event-schemas/examples/m.room.member diff --git a/event-schemas/examples/v1/m.room.member#invite_room_state b/event-schemas/examples/m.room.member#invite_room_state similarity index 100% rename from event-schemas/examples/v1/m.room.member#invite_room_state rename to event-schemas/examples/m.room.member#invite_room_state diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/m.room.member#third_party_invite similarity index 100% rename from event-schemas/examples/v1/m.room.member#third_party_invite rename to event-schemas/examples/m.room.member#third_party_invite diff --git a/event-schemas/examples/v1/m.room.message#m.audio b/event-schemas/examples/m.room.message#m.audio similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.audio rename to event-schemas/examples/m.room.message#m.audio diff --git a/event-schemas/examples/v1/m.room.message#m.emote b/event-schemas/examples/m.room.message#m.emote similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.emote rename to event-schemas/examples/m.room.message#m.emote diff --git a/event-schemas/examples/v1/m.room.message#m.file b/event-schemas/examples/m.room.message#m.file similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.file rename to event-schemas/examples/m.room.message#m.file diff --git a/event-schemas/examples/v1/m.room.message#m.image b/event-schemas/examples/m.room.message#m.image similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.image rename to event-schemas/examples/m.room.message#m.image diff --git a/event-schemas/examples/v1/m.room.message#m.location b/event-schemas/examples/m.room.message#m.location similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.location rename to event-schemas/examples/m.room.message#m.location diff --git a/event-schemas/examples/v1/m.room.message#m.notice b/event-schemas/examples/m.room.message#m.notice similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.notice rename to event-schemas/examples/m.room.message#m.notice diff --git a/event-schemas/examples/v1/m.room.message#m.text b/event-schemas/examples/m.room.message#m.text similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.text rename to event-schemas/examples/m.room.message#m.text diff --git a/event-schemas/examples/v1/m.room.message#m.video b/event-schemas/examples/m.room.message#m.video similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.video rename to event-schemas/examples/m.room.message#m.video diff --git a/event-schemas/examples/v1/m.room.message.feedback b/event-schemas/examples/m.room.message.feedback similarity index 100% rename from event-schemas/examples/v1/m.room.message.feedback rename to event-schemas/examples/m.room.message.feedback diff --git a/event-schemas/examples/v1/m.room.name b/event-schemas/examples/m.room.name similarity index 100% rename from event-schemas/examples/v1/m.room.name rename to event-schemas/examples/m.room.name diff --git a/event-schemas/examples/v1/m.room.power_levels b/event-schemas/examples/m.room.power_levels similarity index 100% rename from event-schemas/examples/v1/m.room.power_levels rename to event-schemas/examples/m.room.power_levels diff --git a/event-schemas/examples/v1/m.room.redaction b/event-schemas/examples/m.room.redaction similarity index 100% rename from event-schemas/examples/v1/m.room.redaction rename to event-schemas/examples/m.room.redaction diff --git a/event-schemas/examples/v1/m.room.third_party_invite b/event-schemas/examples/m.room.third_party_invite similarity index 100% rename from event-schemas/examples/v1/m.room.third_party_invite rename to event-schemas/examples/m.room.third_party_invite diff --git a/event-schemas/examples/v1/m.room.topic b/event-schemas/examples/m.room.topic similarity index 100% rename from event-schemas/examples/v1/m.room.topic rename to event-schemas/examples/m.room.topic diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/m.tag similarity index 100% rename from event-schemas/examples/v1/m.tag rename to event-schemas/examples/m.tag diff --git a/event-schemas/examples/v1/m.typing b/event-schemas/examples/m.typing similarity index 100% rename from event-schemas/examples/v1/m.typing rename to event-schemas/examples/m.typing diff --git a/event-schemas/schema/v1/core-event-schema/event.json b/event-schemas/schema/core-event-schema/event.json similarity index 100% rename from event-schemas/schema/v1/core-event-schema/event.json rename to event-schemas/schema/core-event-schema/event.json diff --git a/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json b/event-schemas/schema/core-event-schema/msgtype_infos/image_info.json similarity index 100% rename from event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json rename to event-schemas/schema/core-event-schema/msgtype_infos/image_info.json diff --git a/event-schemas/schema/v1/core-event-schema/room_event.json b/event-schemas/schema/core-event-schema/room_event.json similarity index 93% rename from event-schemas/schema/v1/core-event-schema/room_event.json rename to event-schemas/schema/core-event-schema/room_event.json index d5413f8a0e5..80f7d26514f 100644 --- a/event-schemas/schema/v1/core-event-schema/room_event.json +++ b/event-schemas/schema/core-event-schema/room_event.json @@ -3,7 +3,7 @@ "title": "Room Event", "description": "In addition to the Event fields, Room Events MUST have the following additional field.", "allOf":[{ - "$ref": "core-event-schema/event.json" + "$ref": "event.json" }], "properties": { "room_id": { diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/core-event-schema/state_event.json similarity index 93% rename from event-schemas/schema/v1/core-event-schema/state_event.json rename to event-schemas/schema/core-event-schema/state_event.json index 5809cf7f6b8..cfcb212c56b 100644 --- a/event-schemas/schema/v1/core-event-schema/state_event.json +++ b/event-schemas/schema/core-event-schema/state_event.json @@ -3,7 +3,7 @@ "title": "State Event", "description": "In addition to the Room Event fields, State Events have the following additional fields.", "allOf":[{ - "$ref": "core-event-schema/room_event.json" + "$ref": "room_event.json" }], "properties": { "state_key": { diff --git a/event-schemas/schema/v1/m.call.answer b/event-schemas/schema/m.call.answer similarity index 100% rename from event-schemas/schema/v1/m.call.answer rename to event-schemas/schema/m.call.answer diff --git a/event-schemas/schema/v1/m.call.candidates b/event-schemas/schema/m.call.candidates similarity index 100% rename from event-schemas/schema/v1/m.call.candidates rename to event-schemas/schema/m.call.candidates diff --git a/event-schemas/schema/v1/m.call.hangup b/event-schemas/schema/m.call.hangup similarity index 100% rename from event-schemas/schema/v1/m.call.hangup rename to event-schemas/schema/m.call.hangup diff --git a/event-schemas/schema/v1/m.call.invite b/event-schemas/schema/m.call.invite similarity index 100% rename from event-schemas/schema/v1/m.call.invite rename to event-schemas/schema/m.call.invite diff --git a/event-schemas/schema/v1/m.presence b/event-schemas/schema/m.presence similarity index 100% rename from event-schemas/schema/v1/m.presence rename to event-schemas/schema/m.presence diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/m.receipt similarity index 100% rename from event-schemas/schema/v1/m.receipt rename to event-schemas/schema/m.receipt diff --git a/event-schemas/schema/v1/m.room.aliases b/event-schemas/schema/m.room.aliases similarity index 100% rename from event-schemas/schema/v1/m.room.aliases rename to event-schemas/schema/m.room.aliases diff --git a/event-schemas/schema/v1/m.room.avatar b/event-schemas/schema/m.room.avatar similarity index 100% rename from event-schemas/schema/v1/m.room.avatar rename to event-schemas/schema/m.room.avatar diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/m.room.canonical_alias similarity index 100% rename from event-schemas/schema/v1/m.room.canonical_alias rename to event-schemas/schema/m.room.canonical_alias diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/m.room.create similarity index 100% rename from event-schemas/schema/v1/m.room.create rename to event-schemas/schema/m.room.create diff --git a/event-schemas/schema/v1/m.room.guest_access b/event-schemas/schema/m.room.guest_access similarity index 100% rename from event-schemas/schema/v1/m.room.guest_access rename to event-schemas/schema/m.room.guest_access diff --git a/event-schemas/schema/v1/m.room.history_visibility b/event-schemas/schema/m.room.history_visibility similarity index 100% rename from event-schemas/schema/v1/m.room.history_visibility rename to event-schemas/schema/m.room.history_visibility diff --git a/event-schemas/schema/v1/m.room.join_rules b/event-schemas/schema/m.room.join_rules similarity index 100% rename from event-schemas/schema/v1/m.room.join_rules rename to event-schemas/schema/m.room.join_rules diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/m.room.member similarity index 100% rename from event-schemas/schema/v1/m.room.member rename to event-schemas/schema/m.room.member diff --git a/event-schemas/schema/v1/m.room.message b/event-schemas/schema/m.room.message similarity index 100% rename from event-schemas/schema/v1/m.room.message rename to event-schemas/schema/m.room.message diff --git a/event-schemas/schema/v1/m.room.message#m.audio b/event-schemas/schema/m.room.message#m.audio similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.audio rename to event-schemas/schema/m.room.message#m.audio diff --git a/event-schemas/schema/v1/m.room.message#m.emote b/event-schemas/schema/m.room.message#m.emote similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.emote rename to event-schemas/schema/m.room.message#m.emote diff --git a/event-schemas/schema/v1/m.room.message#m.file b/event-schemas/schema/m.room.message#m.file similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.file rename to event-schemas/schema/m.room.message#m.file diff --git a/event-schemas/schema/v1/m.room.message#m.image b/event-schemas/schema/m.room.message#m.image similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.image rename to event-schemas/schema/m.room.message#m.image diff --git a/event-schemas/schema/v1/m.room.message#m.location b/event-schemas/schema/m.room.message#m.location similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.location rename to event-schemas/schema/m.room.message#m.location diff --git a/event-schemas/schema/v1/m.room.message#m.notice b/event-schemas/schema/m.room.message#m.notice similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.notice rename to event-schemas/schema/m.room.message#m.notice diff --git a/event-schemas/schema/v1/m.room.message#m.text b/event-schemas/schema/m.room.message#m.text similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.text rename to event-schemas/schema/m.room.message#m.text diff --git a/event-schemas/schema/v1/m.room.message#m.video b/event-schemas/schema/m.room.message#m.video similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.video rename to event-schemas/schema/m.room.message#m.video diff --git a/event-schemas/schema/v1/m.room.message.feedback b/event-schemas/schema/m.room.message.feedback similarity index 100% rename from event-schemas/schema/v1/m.room.message.feedback rename to event-schemas/schema/m.room.message.feedback diff --git a/event-schemas/schema/v1/m.room.name b/event-schemas/schema/m.room.name similarity index 100% rename from event-schemas/schema/v1/m.room.name rename to event-schemas/schema/m.room.name diff --git a/event-schemas/schema/v1/m.room.power_levels b/event-schemas/schema/m.room.power_levels similarity index 100% rename from event-schemas/schema/v1/m.room.power_levels rename to event-schemas/schema/m.room.power_levels diff --git a/event-schemas/schema/v1/m.room.redaction b/event-schemas/schema/m.room.redaction similarity index 100% rename from event-schemas/schema/v1/m.room.redaction rename to event-schemas/schema/m.room.redaction diff --git a/event-schemas/schema/v1/m.room.third_party_invite b/event-schemas/schema/m.room.third_party_invite similarity index 100% rename from event-schemas/schema/v1/m.room.third_party_invite rename to event-schemas/schema/m.room.third_party_invite diff --git a/event-schemas/schema/v1/m.room.topic b/event-schemas/schema/m.room.topic similarity index 100% rename from event-schemas/schema/v1/m.room.topic rename to event-schemas/schema/m.room.topic diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/m.tag similarity index 100% rename from event-schemas/schema/v1/m.tag rename to event-schemas/schema/m.tag diff --git a/event-schemas/schema/v1/m.typing b/event-schemas/schema/m.typing similarity index 100% rename from event-schemas/schema/v1/m.typing rename to event-schemas/schema/m.typing diff --git a/event-schemas/schema/v1/core-event-schema/core-event-schema b/event-schemas/schema/v1/core-event-schema/core-event-schema deleted file mode 120000 index 945c9b46d68..00000000000 --- a/event-schemas/schema/v1/core-event-schema/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/event-schemas/schema/v1/v1-event-schema b/event-schemas/schema/v1/v1-event-schema deleted file mode 120000 index 945c9b46d68..00000000000 --- a/event-schemas/schema/v1/v1-event-schema +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/jenkins.sh b/jenkins.sh index f5ed3b15a6c..abafefa8eb1 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -5,7 +5,7 @@ set -ex (cd event-schemas/ && ./check_examples.py) (cd api && ./check_examples.py) (cd scripts && ./gendoc.py -v) -(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") +(cd api && npm install && node validator.js -s "client-server") (cd event-schemas/ && ./check.sh) : ${GOPATH:=${WORKSPACE}/.gopath} diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh index 0a2bc5f6902..b3894a2c2a9 100755 --- a/scripts/generate-http-docs.sh +++ b/scripts/generate-http-docs.sh @@ -18,7 +18,7 @@ This contains the client-server API for the reference implementation of the home EOF -for f in ../api/client-server/*/*.yaml; do +for f in ../api/client-server/*.yaml; do f="$(basename "${f/.yaml/_http_api}")" echo "{{${f/-/_}}}" >> tmp/http_apis done diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 134c12c3584..f43daa7bdd1 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -468,7 +468,7 @@ This section refers to API Version 2. These API calls currently use the prefix .. _User-Interactive Authentication: `sect:auth-api`_ -{{v2_registration_http_api}} +{{registration_http_api}} Old V1 API docs: |register|_ @@ -769,9 +769,9 @@ When the client first logs in, they will need to initially synchronise with their home server. This is achieved via the initial sync API described below. This API also returns an ``end`` token which can be used with the event stream. -{{sync_http_api}} +{{old_sync_http_api}} -{{v2_sync_http_api}} +{{sync_http_api}} Getting events for a room diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index 784bc874f1e..f3fc72b626f 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -24,4 +24,4 @@ the tags are for. Client Behaviour ---------------- -{{v2_account_data_http_api}} +{{account_data_http_api}} diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index a8ad3cd3451..702a7275360 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -52,7 +52,7 @@ dismissing a notification in order for the event to count as "read". A client can update the markers for its user by interacting with the following HTTP APIs. -{{v2_receipts_http_api}} +{{receipts_http_api}} Server behaviour ---------------- diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index bb467bc0583..f8c28c554c2 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -45,4 +45,4 @@ Two special names are listed in the specification: Client Behaviour ---------------- -{{v2_tags_http_api}} +{{tags_http_api}} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 75a12a5ba3b..069940820d7 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -17,11 +17,10 @@ import urllib import yaml -V1_CLIENT_API = "../api/client-server/v1" -V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" -V1_EVENT_SCHEMA = "../event-schemas/schema/v1" -V2_CLIENT_API = "../api/client-server/v2_alpha" -CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" +HTTP_APIS = "../api/client-server" +V1_EVENT_EXAMPLES = "../event-schemas/examples" +V1_EVENT_SCHEMA = "../event-schemas/schema" +CORE_EVENT_SCHEMA = "../event-schemas/schema/core-event-schema" CHANGELOG = "../CHANGELOG.rst" TARGETS = "../specification/targets.yaml" @@ -549,30 +548,21 @@ def _load_swagger_meta(self, filepath, api, group_name): } def load_swagger_apis(self): - paths = [ - V1_CLIENT_API, V2_CLIENT_API - ] apis = {} - for path in paths: - is_v2 = (path == V2_CLIENT_API) - if not os.path.exists(V2_CLIENT_API): - self.log("Skipping v2 apis: %s does not exist." % V2_CLIENT_API) + path = HTTP_APIS + for filename in os.listdir(path): + if not filename.endswith(".yaml"): continue - for filename in os.listdir(path): - if not filename.endswith(".yaml"): - continue - self.log("Reading swagger API: %s" % filename) - filepath = os.path.join(path, filename) - with open(filepath, "r") as f: - # strip .yaml - group_name = filename[:-5].replace("-", "_") - if is_v2: - group_name = "v2_" + group_name - api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta( - filepath, api, group_name - ) - apis[group_name] = api + self.log("Reading swagger API: %s" % filename) + filepath = os.path.join(path, filename) + with open(filepath, "r") as f: + # strip .yaml + group_name = filename[:-5].replace("-", "_") + api = yaml.load(f.read()) + api["__meta"] = self._load_swagger_meta( + filepath, api, group_name + ) + apis[group_name] = api return apis def load_common_event_fields(self): From c9c433bc15ede21a806900503df5d9a935984faa Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 30 Nov 2015 14:31:24 +0000 Subject: [PATCH 258/989] Make section depths consistent --- specification/intro.rst | 26 +++++++++++++------------- specification/targets.yaml | 8 ++++---- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/specification/intro.rst b/specification/intro.rst index 8c08bf24998..211a3562a5a 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -9,7 +9,7 @@ https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}} APIs -~~~~ +---- The following APIs are documented in this specification: - `Client-Server API `_ for writing Matrix clients. @@ -19,7 +19,7 @@ The following APIs are documented in this specification: There are also some `appendices `_. Changelog -~~~~~~~~~ +--------- {{spec_changelog}} For a full changelog, see @@ -29,7 +29,7 @@ https://github.com/matrix-org/matrix-doc/blob/master/CHANGELOG.rst .. sectnum:: Introduction -============ +------------ .. WARNING:: The Matrix specification is still evolving: the APIs are not yet frozen and this document is in places a work in progress or stale. We have made every @@ -103,10 +103,10 @@ reliably and persistently pushed from A to B in an inter-operable and federated manner. Overview -======== +-------- Architecture ------------- +~~~~~~~~~~~~ Matrix defines APIs for synchronising extensible JSON objects known as "events" between compatible clients, servers and services. Clients are @@ -158,7 +158,7 @@ a long-lived GET request. Users -~~~~~ ++++++ Each client is associated with a user account, which is identified in Matrix using a unique "User ID". This ID is namespaced to the homeserver which @@ -173,7 +173,7 @@ this user. The ``domain`` of a user ID is the domain of the homeserver. - Need to specify precise grammar for Matrix IDs Events -~~~~~~ +++++++ All data exchanged over Matrix is expressed as an "event". Typically each client action (e.g. sending a message) correlates with exactly one event. Each event @@ -188,7 +188,7 @@ of a "Room". .. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions Event Graphs -~~~~~~~~~~~~ +++++++++++++ .. _sect:event-graph: @@ -212,7 +212,7 @@ of its parents. The root event should have a depth of 1. Thus if one event is before another, then it must have a strictly smaller depth. Room structure -~~~~~~~~~~~~~~ +++++++++++++++ A room is a conceptual place where users can send and receive events. Events are sent to a room, and all participants in that room with sufficient access will @@ -291,7 +291,7 @@ from the other servers participating in a room. Room Aliases -++++++++++++ +^^^^^^^^^^^^ Each room can also have multiple "Room Aliases", which look like:: @@ -327,7 +327,7 @@ that are in the room that can be used to join via. |________________________________| Identity -~~~~~~~~ +++++++++ Users in Matrix are identified via their matrix user ID (MXID). However, existing 3rd party ID namespaces can also be used in order to identify Matrix @@ -347,7 +347,7 @@ user IDs using 3PIDs. Profiles -~~~~~~~~ +++++++++ Users may publish arbitrary key/value data associated with their account - such as a human readable display name, a profile photo URL, contact information @@ -358,7 +358,7 @@ as a human readable display name, a profile photo URL, contact information names allowed to be? Private User Data -~~~~~~~~~~~~~~~~~ ++++++++++++++++++ Users may also store arbitrary private key/value data in their account - such as client preferences, or server configuration settings which lack any other diff --git a/specification/targets.yaml b/specification/targets.yaml index cdbfa62e573..310870a0a33 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -7,9 +7,9 @@ targets: - client_server_api.rst - { 1: events.rst } - { 1: event_signing.rst } - - modules.rst - - { 1: feature_profiles.rst } - - { 1: "group:modules" } # reference a group of files + - { 1: modules.rst } + - { 2: feature_profiles.rst } + - { 2: "group:modules" } # reference a group of files application_service: files: - application_service_api.rst @@ -37,7 +37,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/account_data.rst -title_styles: ["=", "-", "~", "+", "^", "`"] +title_styles: ["=", "-", "~", "+", "^", "`", "@"] # The templating system doesn't know the right title style to use when generating # RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title From 79244e8065e989e409177180a43e8ea312d32739 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 30 Nov 2015 16:40:48 +0000 Subject: [PATCH 259/989] Remove redundant node event schema checker The python one does the same --- event-schemas/check.sh | 22 ---------------------- event-schemas/check_examples.py | 2 +- jenkins.sh | 1 - 3 files changed, 1 insertion(+), 24 deletions(-) delete mode 100755 event-schemas/check.sh diff --git a/event-schemas/check.sh b/event-schemas/check.sh deleted file mode 100755 index 3d411768c2c..00000000000 --- a/event-schemas/check.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e -# Runs z-schema over all of the schema files (looking for matching examples) - -if ! which z-schema; then - echo >&2 "Need to install z-schema; run: sudo npm install -g z-schema" - exit 1 -fi - -find schema/m.* | while read line -do - split_path=(${line///// }) - event_type=(${split_path[2]}) - echo "Checking $event_type" - echo "--------------------" - # match exact name or exact name with a # - find examples -name $event_type -o -name "$event_type#*" | while read exline - do - echo " against $exline" - # run z-schema: because of bash -e if this fails we bail with exit code 1 - z-schema schema/$event_type $exline - done -done diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index b10b2d0efae..5a40940716a 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#!/usr/bin/env python import sys import json diff --git a/jenkins.sh b/jenkins.sh index abafefa8eb1..c58ad47348d 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -6,7 +6,6 @@ set -ex (cd api && ./check_examples.py) (cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server") -(cd event-schemas/ && ./check.sh) : ${GOPATH:=${WORKSPACE}/.gopath} mkdir -p "${GOPATH}" From c6e0322a9e557c8fafa7d6f1fc894f253e5a2cbc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 13:53:03 +0000 Subject: [PATCH 260/989] Swaggerify /account --- api/client-server/administrative_contact.yaml | 157 ++++++++++++++++++ specification/client_server_api.rst | 53 +----- 2 files changed, 160 insertions(+), 50 deletions(-) create mode 100644 api/client-server/administrative_contact.yaml diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml new file mode 100644 index 00000000000..68fd0fbaec4 --- /dev/null +++ b/api/client-server/administrative_contact.yaml @@ -0,0 +1,157 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Account Administrative Contact API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/account/password": + post: + summary: Changes a user's password. + description: |- + This API endpoint uses the User-Interactive Authentication API. + An access token should be submitted to this endpoint if the client has + an active session. + The Home Server may change the flows available depending on whether a + valid access token is provided. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "new_password": "ihatebananas" + } + properties: + new_password: + type: string + description: The new password for the account. + required: ["new_password"] + responses: + 200: + description: The password has been changed. + examples: + application/json: "{}" + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/account/3pid": + get: + summary: Gets a list of a user's third party identifiers. + description: |- + Gets a list of the third party identifiers that the homeserver has + associated with the user's account. + + This is *not* the same as the list of third party identifiers bound to + the user's Matrix ID in Identity Servers. + + Identifiers in this list may be used by the Home Server as, for example, + identifiers that it will accept to reset the user's account password. + security: + - accessToken: [] + responses: + 200: + description: The lookup was successful. + examples: + application/json: |- + { + "threepids": [ + { + "medium": "email", + "address": "monkey@banana.island" + } + ] + } + schema: + type: object + properties: + threepids: + type: array + items: + type: object + title: Third party identifier + properties: + medium: + type: string + description: The medium of the third party identifier. + enum: ["email"] + address: + type: string + description: The third party identifier address. + post: + summary: Adds contact information to the user's account. + description: Adds contact information to the user's account. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + properties: + threePidCreds: + title: "ThreePidCredentials" + type: object + description: The third party credentials to associate with the account. + properties: + client_secret: + type: string + description: The client secret used in the session with the Identity Server. + id_server: + type: string + description: The Identity Server to use. + sid: + type: string + description: The session identifier given by the Identity Server. + required: ["client_secret", "id_server", "sid"] + bind: + type: boolean + description: |- + Whether the home server should also bind this third party + identifier to the account's Matrix ID with the passed identity + server. Default: ``false``. + x-example: true + required: ["threePidCreds"] + example: |- + { + "threePidCreds": { + "id_server": "matrix.org", + "sid": "abc123987", + "client_secret": "d0n'tT3ll" + }, + "bind": false + } + responses: + 200: + description: The addition was successful. + examples: + application/json: "{}" + schema: + type: object + 403: + description: The credentials could not be verified with the identity server. + examples: + application/json: |- + { + "errcode": "M_THREEPID_AUTH_FAILED", + "error": "The third party credentials could not be verified by the identity server." + } diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index f43daa7bdd1..67973c6b3eb 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -510,58 +510,11 @@ database. Adding Account Administrative Contact Information +++++++++++++++++++++++++++++++++++++++++++++++++ -Request:: - - POST $V2PREFIX/account/3pid - -Used to add contact information to the user's account. - -The body of the POST request is a JSON object containing: - -threePidCreds - An object containing contact information. -bind - Optional. A boolean indicating whether the Home Server should also bind this - third party identifier to the account's matrix ID with the Identity Server. If - supplied and true, the Home Server must bind the 3pid accordingly. - -The contact information object comprises: - -id_server - The colon-separated hostname and port of the Identity Server used to - authenticate the third party identifier. If the port is the default, it and the - colon should be omitted. -sid - The session ID given by the Identity Server -client_secret - The client secret used in the session with the Identity Server. - -On success, the empty JSON object is returned. - -May also return error codes: - -M_THREEPID_AUTH_FAILED - If the credentials provided could not be verified with the ID Server. - -Fetching Currently Associated Contact Information -+++++++++++++++++++++++++++++++++++++++++++++++++ -Request:: - - GET $V2PREFIX/account/3pid - -This returns a list of third party identifiers that the Home Server has -associated with the user's account. This is *not* the same as the list of third -party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers -in this list may be used by the Home Server as, for example, identifiers that it -will accept to reset the user's account password. -Returns a JSON object with the key ``threepids`` whose contents is an array of -objects with the following keys: +A homeserver may keep some contact information for administrative use. +This is independent of any information kept by any Identity Servers. -medium - The medium of the 3pid (eg, ``email``) -address - The textual address of the 3pid, eg. the email address +{{administrative_contact_http_api}} Pagination ---------- From c4eaf7458ff8489697aae8683256c2d8f4bcee80 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 14:19:31 +0000 Subject: [PATCH 261/989] 302 to spec/head/index.html rather than serving it on /spec/head Otherwise relative links are broken --- scripts/speculator/main.go | 46 +++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 3736672f499..c32eec94111 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -193,15 +193,31 @@ func (s *server) getSHAOf(ref string) (string, error) { return strings.TrimSpace(b.String()), nil } -func extractPath(path, base string) string { - // Assume exactly one flat directory - max := strings.Count(base, "/") + 2 +// extractPath extracts the file path within the gen directory which should be served for the request. +// Returns one of (file to serve, path to redirect to). +// path is the actual path being requested, e.g. "/spec/head/client_server.html". +// base is the base path of the handler, including a trailing slash, before the PR number, e.g. "/spec/". +func extractPath(path, base string) (string, string) { + // Assumes exactly one flat directory + + // Count slashes in /spec/head/client_server.html + // base is /spec/ + // +1 for the PR number - /spec/head + // +1 for the path-part after the slash after the PR number + max := strings.Count(base, "/") + 1 parts := strings.SplitN(path, "/", max) - if len(parts) < max || parts[max-1] == "" { - return "index.html" + if len(parts) < max { + // Path is base/pr - redirect to base/pr/index.html + return "", path + "/index.html" } - return parts[max-1] + if parts[max-1] == "" { + // Path is base/pr/ - serve index.html + return "index.html", "" + } + + // Path is base/pr/file.html - serve file + return parts[max-1], "" } func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { @@ -283,7 +299,11 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { cache.Add(sha, pathToContent) } - requestedPath := extractPath(req.URL.Path, "/spec/pr") + requestedPath, redirect := extractPath(req.URL.Path, "/spec/") + if redirect != "" { + s.redirectTo(w, req, redirect) + return + } if b, ok := pathToContent[requestedPath]; ok { w.Write(b) return @@ -299,6 +319,12 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { w.Write([]byte("Not found")) } +func (s *server) redirectTo(w http.ResponseWriter, req *http.Request, path string) { + req.URL.Path = path + w.Header().Set("Location", req.URL.String()) + w.WriteHeader(302) +} + // lookupHeadSHA looks up what origin/master's HEAD SHA is. // It attempts to `git fetch` before doing so. // If this fails, it may still return a stale sha, but will also return an error. @@ -397,7 +423,11 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) { return } - requestedPath := extractPath(req.URL.Path, "/diff/spec/pr") + requestedPath, redirect := extractPath(req.URL.Path, "/diff/spec/") + if redirect != "" { + s.redirectTo(w, req, redirect) + return + } cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath)) var b bytes.Buffer cmd.Stdout = &b From a4668c1d8c69124e029b456bc6b7cf13dfac0385 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 14:21:20 +0000 Subject: [PATCH 262/989] Fix typo --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index c32eec94111..602cdb577e3 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -204,7 +204,7 @@ func extractPath(path, base string) (string, string) { // base is /spec/ // +1 for the PR number - /spec/head // +1 for the path-part after the slash after the PR number - max := strings.Count(base, "/") + 1 + max := strings.Count(base, "/") + 2 parts := strings.SplitN(path, "/", max) if len(parts) < max { From 23b2497743903d47a7c2b7ebc6e53d4527050eb8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Dec 2015 15:31:35 +0000 Subject: [PATCH 263/989] Expand on where pagination tokens can be returned --- specification/modules/search.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index f44bb585d10..8227128a156 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -72,6 +72,11 @@ The scope of the pagination is defined depending on where the ``next_batch`` token was returned. For example, using a token inside a group will return more results from within that group. +The currently supported locations for the ``next_batch`` token are: + +- ``search_categories..next_batch`` +- ``search_categories..groups...next_batch`` + A server need not support pagination, even if there are more matching results. From 5e8bb34f362147b4a5e7056e425f55e6a6555beb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Dec 2015 15:33:13 +0000 Subject: [PATCH 264/989] Change the results dict to a list --- api/client-server/search.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index a9eaa0857a3..c0ca0016737 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -160,10 +160,10 @@ paths: type: number description: Total number of results found results: - type: object + type: array title: Results description: Mapping of event_id to result. - additionalProperties: + items: type: object title: Result description: The result object. @@ -295,8 +295,8 @@ paths: } }, "count": 24, - "results": { - "$144429830826TWwbB:localhost": { + "results": [ + { "rank": 0.00424866, "result": { "age": 526228296, @@ -311,7 +311,7 @@ paths: "user_id": "@test:localhost" } } - } + ] } } } From ca7ef1b5363d788b9fd56743a37a669338b4b4c9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Dec 2015 15:34:25 +0000 Subject: [PATCH 265/989] Update Results description --- api/client-server/search.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index c0ca0016737..698c6448203 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -162,7 +162,7 @@ paths: results: type: array title: Results - description: Mapping of event_id to result. + description: List of results in the requested order. items: type: object title: Result From ebed3b60bd68ef8a258ec9ebbd866477283297c9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 15:58:15 +0000 Subject: [PATCH 266/989] Add trailing slashes to links --- scripts/speculator/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 602cdb577e3..1c2780e38ab 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -469,12 +469,12 @@ func listPulls(w http.ResponseWriter, req *http.Request) { } s := "
    " for _, pull := range pulls { - s += fmt.Sprintf(`
  • %d: %s: %s: spec spec diff rst diff
  • `, + s += fmt.Sprintf(`
  • %d: %s: %s: spec spec diff rst diff
  • `, pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number) } - s += `
` + s += `` if *includesDir != "" { - s += `` + s += `` } io.WriteString(w, s) From 681c25820602b543264eb81f341284f6d4671ab8 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 16:56:28 +0000 Subject: [PATCH 267/989] Fix turnServer endpoint --- api/client-server/voip.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/voip.yaml b/api/client-server/voip.yaml index 5fdf1ca7a99..629f2c177b3 100644 --- a/api/client-server/voip.yaml +++ b/api/client-server/voip.yaml @@ -18,7 +18,7 @@ securityDefinitions: name: access_token in: query paths: - "/turnServer": + "/voip/turnServer": get: summary: Obtain TURN server credentials. description: |- From f6229f649a88a2ec365f270896d9917ab6e86244 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 16:58:40 +0000 Subject: [PATCH 268/989] Remove confusing wording --- .../modules/typing_notifications.rst | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index da383e73176..8be1ac287be 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -34,28 +34,6 @@ to inform the server that the user has stopped typing. {{typing_http_api}} -Server behaviour ----------------- - -Servers MUST emit typing EDUs in a different form to ``m.typing`` events which -are shown to clients. This form looks like:: - - { - "type": "m.typing", - "content": { - "room_id": "!room-id-here:matrix.org", - "user_id": "@user-id-here:matrix.org", - "typing": true/false - } - } - -This does not contain timing information so it is up to originating homeservers -to ensure they eventually send "stop" notifications. - -.. TODO - ((This will eventually need addressing, as part of the wider typing/presence - timer addition work)) - Security considerations ----------------------- From 97fd1fdd627c45522cf888ec5d858c067933b02c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 17:02:23 +0000 Subject: [PATCH 269/989] Embed client and server release numbers Note that this also removes the changelog - I'm going to re-add the changelog differently soon. --- scripts/gendoc.py | 35 +++++++++++++++++++++---- scripts/generate-http-docs.sh | 13 ++++++++- specification/intro.rst | 20 ++++---------- templating/build.py | 22 ++++++++++++++-- templating/matrix_templates/sections.py | 4 --- templating/matrix_templates/units.py | 21 +++++---------- 6 files changed, 73 insertions(+), 42 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 9b87cb30ee4..801c06bbf1f 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -250,7 +250,7 @@ def addAnchors(path): f.write(line + "\n") -def run_through_template(input, set_verbose): +def run_through_template(input, set_verbose, substitutions): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: @@ -260,6 +260,9 @@ def run_through_template(input, set_verbose): "-o", "../scripts/tmp", "../scripts/"+input ] + for k, v in substitutions.items(): + args.append("--substitution=%s=%s" % (k, v)) + if set_verbose: args.insert(2, "-v") log("EXEC: %s" % " ".join(args)) @@ -392,7 +395,7 @@ def cleanup_env(): shutil.rmtree("./tmp") -def main(requested_target_name, keep_intermediates): +def main(requested_target_name, keep_intermediates, substitutions): prepare_env() log("Building spec [target=%s]" % requested_target_name) @@ -407,7 +410,7 @@ def main(requested_target_name, keep_intermediates): target = get_build_target("../specification/targets.yaml", target_name) build_spec(target=target, out_filename=templated_file) - run_through_template(templated_file, VERBOSE) + run_through_template(templated_file, VERBOSE, substitutions) fix_relative_titles( target=target, filename=templated_file, out_filename=rst_file, @@ -417,13 +420,21 @@ def main(requested_target_name, keep_intermediates): if requested_target_name == "all": shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this + run_through_template("tmp/howto.rst", False, substitutions) # too spammy to mark -v on this rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: cleanup_env() +def extract_major(s): + major_version = s + match = re.match("^(r\d)+(\.\d+)?$", s) + if match: + major_version = match.group(1) + return major_version + + if __name__ == '__main__': parser = ArgumentParser( "gendoc.py - Generate the Matrix specification as HTML to the gen/ folder." @@ -441,9 +452,23 @@ def main(requested_target_name, keep_intermediates): "--verbose", "-v", action="store_true", help="Turn on verbose mode." ) + parser.add_argument( + "--client_release", "-c", action="store", default="unstable", + help="The client-server release tag to generate, e.g. r1.2" + ) + parser.add_argument( + "--server_release", "-s", action="store", default="unstable", + help="The server-server release tag to generate, e.g. r1.2" + ) args = parser.parse_args() if not args.target: parser.print_help() sys.exit(1) VERBOSE = args.verbose - main(args.target, args.nodelete) + substitutions = { + "%CLIENT_RELEASE_LABEL%": args.client_release, + "%CLIENT_MAJOR_VERSION%": extract_major(args.client_release), + "%SERVER_RELEASE_LABEL%": args.server_release, + "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), + } + main(args.target, args.nodelete, substitutions) diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh index b3894a2c2a9..48c01bdbdc6 100755 --- a/scripts/generate-http-docs.sh +++ b/scripts/generate-http-docs.sh @@ -4,6 +4,17 @@ # It takes all of the swagger YAML files for the client-server API, and turns # them into API docs, with none of the narrative found in the rst files which # normally wrap these API docs. +# +# Optionally takes one positional argument, the label of the release, e.g. r1.2. +# This falls back to "unstable" if unspecified. + +if [[ $# == 1 ]]; then + client_release=$1 +else + client_release="unstable" +fi + +client_major_version="$(echo "${client_release}" | perl -pe 's/^(r\d+)\..*$/$1/g')" cd "$(dirname $0)" @@ -23,5 +34,5 @@ for f in ../api/client-server/*.yaml; do echo "{{${f/-/_}}}" >> tmp/http_apis done -(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis) +(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis --substitution=%CLIENT_RELEASE_LABEL%="${client_release}" --substitution=%CLIENT_MAJOR_VERSION%="${client_major_version}") rst2html.py --stylesheet-path=$(echo css/*.css | tr ' ' ',') gen/http_apis > gen/http_apis.html diff --git a/specification/intro.rst b/specification/intro.rst index 211a3562a5a..eb6f98fc321 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -1,30 +1,20 @@ Matrix Specification ==================== -Version: {{spec_version}} ------------------------------ This specification has been generated from https://github.com/matrix-org/matrix-doc using https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of -revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}} +revision ``{{git_version}}`` - +https://github.com/matrix-org/matrix-doc/tree/{{git_rev}}. -APIs ----- The following APIs are documented in this specification: -- `Client-Server API `_ for writing Matrix clients. -- `Server-Server API `_ for writing servers which can federate with Matrix. -- `Application Service API `_ for writing privileged plugins to servers. +- `Client-Server API `_ version %CLIENT_RELEASE_LABEL% for writing Matrix clients. +- `Server-Server API `_ version %SERVER_RELEASE_LABEL% for writing servers which can federate with Matrix. +- `Application Service API `_ version %CLIENT_RELEASE_LABEL% for writing privileged plugins to servers. There are also some `appendices `_. -Changelog ---------- -{{spec_changelog}} - -For a full changelog, see -https://github.com/matrix-org/matrix-doc/blob/master/CHANGELOG.rst - .. contents:: Table of Contents .. sectnum:: diff --git a/templating/build.py b/templating/build.py index 10fb5aea2e2..f08c2201990 100755 --- a/templating/build.py +++ b/templating/build.py @@ -44,6 +44,7 @@ import json import logging import os +import re import sys from textwrap import TextWrapper @@ -56,7 +57,7 @@ def check_unaccessed(name, store): log("Found %s unused %s keys." % (len(unaccessed_keys), name)) log(unaccessed_keys) -def main(input_module, file_stream=None, out_dir=None, verbose=False): +def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}): if out_dir and not os.path.exists(out_dir): os.makedirs(out_dir) @@ -167,6 +168,12 @@ def colwidth(key, default): temp = Template(temp_str) log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) + + # Do these substitutions outside of the ordinary templating system because + # we want them to apply to things like the underlying swagger used to + # generate the templates, not just the top-level sections. + for old, new in substitutions.items(): + output = output.replace(old, new) with open( os.path.join(out_dir, os.path.basename(file_stream.name)), "w" ) as f: @@ -209,6 +216,10 @@ def log(line): "--verbose", "-v", action="store_true", help="Turn on verbose mode." ) + parser.add_argument( + "--substitution", action="append", + help="Substitutions to apply to the generated output, of form NEEDLE=REPLACEMENT." + ) args = parser.parse_args() if args.verbose: @@ -226,7 +237,14 @@ def log(line): parser.print_help() sys.exit(1) + substitutions = {} + for substitution in args.substitution: + parts = substitution.split("=", 1) + if len(parts) != 2: + raise Exception("Invalid substitution") + substitutions[parts[0]] = parts[1] + main( args.input, file_stream=args.file, out_dir=args.out_directory, - verbose=args.verbose + substitutions=substitutions, verbose=args.verbose ) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 78aabca761c..d285bb61f93 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -15,10 +15,6 @@ def render_git_version(self): def render_git_rev(self): return self.units.get("git_version")["revision"] - def render_spec_version(self): - spec_meta = self.units.get("spec_meta") - return spec_meta["version"] - def render_spec_changelog(self): spec_meta = self.units.get("spec_meta") return spec_meta["changelog"] diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 069940820d7..feae0aae576 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -18,8 +18,8 @@ import yaml HTTP_APIS = "../api/client-server" -V1_EVENT_EXAMPLES = "../event-schemas/examples" -V1_EVENT_SCHEMA = "../event-schemas/schema" +EVENT_EXAMPLES = "../event-schemas/examples" +EVENT_SCHEMA = "../event-schemas/schema" CORE_EVENT_SCHEMA = "../event-schemas/schema/core-event-schema" CHANGELOG = "../CHANGELOG.rst" TARGETS = "../specification/targets.yaml" @@ -605,7 +605,7 @@ def load_common_event_fields(self): return event_types def load_event_examples(self): - path = V1_EVENT_EXAMPLES + path = EVENT_EXAMPLES examples = {} for filename in os.listdir(path): if not filename.startswith("m."): @@ -622,7 +622,7 @@ def load_event_examples(self): return examples def load_event_schemas(self): - path = V1_EVENT_SCHEMA + path = EVENT_SCHEMA schemata = {} for filename in os.listdir(path): @@ -714,7 +714,6 @@ def load_event_schemas(self): def load_spec_meta(self): path = CHANGELOG title_part = None - version = None changelog_lines = [] with open(path, "r") as f: prev_line = None @@ -738,19 +737,11 @@ def load_spec_meta(self): break changelog_lines.append(line) - # parse out version from title - for word in title_part.split(): - if re.match("^v[0-9\.]+$", word): - version = word[1:] # strip the 'v' - - self.log("Version: %s Title part: %s Changelog line count: %s" % ( - version, title_part, len(changelog_lines) + self.log("Title part: %s Changelog line count: %s" % ( + title_part, len(changelog_lines) )) - if not version or len(changelog_lines) == 0: - raise Exception("Failed to parse CHANGELOG.rst") return { - "version": version, "changelog": "".join(changelog_lines) } From aa4ed10821220c0f1b43541ea61d611f1993908d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 17:23:58 +0000 Subject: [PATCH 270/989] Unify v1 and v2 event schemas --- api/client-server/guest_events.yaml | 2 +- api/client-server/old_sync.yaml | 19 ++++---- api/client-server/presence.yaml | 3 +- api/client-server/rooms.yaml | 31 ++++++------- api/client-server/search.yaml | 2 +- event-schemas/examples/m.call.answer | 2 +- event-schemas/examples/m.call.candidates | 2 +- event-schemas/examples/m.call.hangup | 2 +- event-schemas/examples/m.call.invite | 2 +- event-schemas/examples/m.room.aliases | 2 +- event-schemas/examples/m.room.avatar | 2 +- event-schemas/examples/m.room.canonical_alias | 2 +- event-schemas/examples/m.room.create | 2 +- event-schemas/examples/m.room.guest_access | 2 +- .../examples/m.room.history_visibility | 2 +- event-schemas/examples/m.room.join_rules | 2 +- event-schemas/examples/m.room.member | 2 +- .../examples/m.room.member#invite_room_state | 2 +- .../examples/m.room.member#third_party_invite | 2 +- event-schemas/examples/m.room.message#m.audio | 4 +- event-schemas/examples/m.room.message#m.emote | 2 +- event-schemas/examples/m.room.message#m.file | 4 +- event-schemas/examples/m.room.message#m.image | 2 +- .../examples/m.room.message#m.location | 2 +- .../examples/m.room.message#m.notice | 2 +- event-schemas/examples/m.room.message#m.text | 2 +- event-schemas/examples/m.room.message#m.video | 4 +- .../examples/m.room.message.feedback | 2 +- event-schemas/examples/m.room.name | 2 +- event-schemas/examples/m.room.power_levels | 2 +- event-schemas/examples/m.room.redaction | 2 +- event-schemas/examples/m.room.topic | 2 +- event-schemas/examples/m.typing | 2 +- .../schema/core-event-schema/room_event.json | 34 +++++++++++---- specification/events.rst | 43 ++----------------- 35 files changed, 87 insertions(+), 109 deletions(-) diff --git a/api/client-server/guest_events.yaml b/api/client-server/guest_events.yaml index 671b355af82..4d7a957d69f 100644 --- a/api/client-server/guest_events.yaml +++ b/api/client-server/guest_events.yaml @@ -74,7 +74,7 @@ paths: "origin_server_ts": 1432804485886, "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "type": "m.room.message", - "user_id": "@bob:localhost" + "sender": "@bob:localhost" } ] } diff --git a/api/client-server/old_sync.yaml b/api/client-server/old_sync.yaml index 6dbe7e5ea3e..fe76e3bc5f8 100644 --- a/api/client-server/old_sync.yaml +++ b/api/client-server/old_sync.yaml @@ -60,7 +60,7 @@ paths: "origin_server_ts": 1432804485886, "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "type": "m.room.message", - "user_id": "@bob:localhost" + "sender": "@bob:localhost" } ] } @@ -154,7 +154,7 @@ paths: "origin_server_ts": 1432804485886, "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "type": "m.room.message", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" }, { "age": 343511809, @@ -166,7 +166,7 @@ paths: "origin_server_ts": 1432804487480, "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "type": "m.room.message", - "user_id": "@bob:localhost" + "sender": "@bob:localhost" } ], "end": "s3456_9_0", @@ -184,13 +184,12 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "", "type": "m.room.join_rules", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" }, { "age": 6547561012, "content": { "avatar_url": "mxc://localhost/fzysBrHpPEeTGANCVLXWXNMI#auto", - "displayname": null, "membership": "join" }, "event_id": "$1426600438280zExKY:localhost", @@ -199,7 +198,7 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "@alice:localhost", "type": "m.room.member", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" }, { "age": 7148267200, @@ -211,7 +210,7 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "", "type": "m.room.create", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" }, { "age": 1622568720, @@ -226,7 +225,7 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "@bob:localhost", "type": "m.room.member", - "user_id": "@bob:localhost" + "sender": "@bob:localhost" }, { "age": 7148267004, @@ -250,7 +249,7 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "", "type": "m.room.power_levels", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" } ], "visibility": "private", @@ -392,7 +391,7 @@ paths: "msgtype": "m.text" }, "room_id:": "!wfgy43Sg4a:matrix.org", - "user_id": "@bob:matrix.org", + "sender": "@bob:matrix.org", "event_id": "$asfDuShaf7Gafaw:matrix.org", "type": "m.room.message" } diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index 33df17d23f2..6ac204dd5e4 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -85,8 +85,7 @@ paths: application/json: |- { "presence": "unavailable", - "last_active_ago": 420845, - "status_msg": null + "last_active_ago": 420845 } schema: type: object diff --git a/api/client-server/rooms.yaml b/api/client-server/rooms.yaml index f7654f145b1..cc3aecc4bb3 100644 --- a/api/client-server/rooms.yaml +++ b/api/client-server/rooms.yaml @@ -92,13 +92,12 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.join_rules", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 6547561012, "content": { "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", - "displayname": null, "membership": "join" }, "event_id": "$1426600438280zExKY:example.com", @@ -107,7 +106,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 7148267200, @@ -119,7 +118,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.create", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 1622568720, @@ -134,7 +133,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", - "user_id": "@bob:example.com" + "sender": "@bob:example.com" }, { "age": 7148267004, @@ -158,7 +157,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.power_levels", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" } ] schema: @@ -212,7 +211,7 @@ paths: "origin_server_ts": 1432804485886, "room_id": "!636q39766251:example.com", "type": "m.room.message", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 343511809, @@ -224,7 +223,7 @@ paths: "origin_server_ts": 1432804487480, "room_id": "!636q39766251:example.com", "type": "m.room.message", - "user_id": "@bob:example.com" + "sender": "@bob:example.com" } ], "end": "s3456_9_0", @@ -242,13 +241,12 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.join_rules", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 6547561012, "content": { "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", - "displayname": null, "membership": "join" }, "event_id": "$1426600438280zExKY:example.com", @@ -257,7 +255,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 7148267200, @@ -269,7 +267,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.create", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 1622568720, @@ -284,7 +282,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", - "user_id": "@bob:example.com" + "sender": "@bob:example.com" }, { "age": 7148267004, @@ -308,7 +306,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.power_levels", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" } ], "visibility": "private", @@ -416,7 +414,6 @@ paths: "age": 6547561012, "content": { "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", - "displayname": null, "membership": "join" }, "event_id": "$1426600438280zExKY:example.com", @@ -425,7 +422,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 1622568720, @@ -440,7 +437,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", - "user_id": "@bob:example.com" + "sender": "@bob:example.com" } ] } diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index 3da9486f392..8a8c3f5a068 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -131,7 +131,7 @@ paths: "origin_server_ts": 1444298308034, "room_id": "!qPewotXpIctQySfjSy:localhost", "type": "m.room.message", - "user_id": "@test:localhost" + "sender": "@test:localhost" } } } diff --git a/event-schemas/examples/m.call.answer b/event-schemas/examples/m.call.answer index 0301fb3b5f3..f7d1443942a 100644 --- a/event-schemas/examples/m.call.answer +++ b/event-schemas/examples/m.call.answer @@ -13,5 +13,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.call.answer", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.call.candidates b/event-schemas/examples/m.call.candidates index 389d227181b..8e6849bb1c9 100644 --- a/event-schemas/examples/m.call.candidates +++ b/event-schemas/examples/m.call.candidates @@ -15,5 +15,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.call.candidates", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.call.hangup b/event-schemas/examples/m.call.hangup index 52ddad1620e..42e1f346ccb 100644 --- a/event-schemas/examples/m.call.hangup +++ b/event-schemas/examples/m.call.hangup @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.call.hangup", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.call.invite b/event-schemas/examples/m.call.invite index 2df2592951c..974a5b4cdbe 100644 --- a/event-schemas/examples/m.call.invite +++ b/event-schemas/examples/m.call.invite @@ -13,5 +13,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.call.invite", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.aliases b/event-schemas/examples/m.room.aliases index 07b4b330634..ca87510e3ae 100644 --- a/event-schemas/examples/m.room.aliases +++ b/event-schemas/examples/m.room.aliases @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.aliases", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.avatar b/event-schemas/examples/m.room.avatar index 9fb1189c1ef..2080d96ef40 100644 --- a/event-schemas/examples/m.room.avatar +++ b/event-schemas/examples/m.room.avatar @@ -14,5 +14,5 @@ "type": "m.room.avatar", "state_key": "", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.canonical_alias b/event-schemas/examples/m.room.canonical_alias index 0203a851edd..59df586d9d3 100644 --- a/event-schemas/examples/m.room.canonical_alias +++ b/event-schemas/examples/m.room.canonical_alias @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.canonical_alias", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.create b/event-schemas/examples/m.room.create index a3598853881..34dabb53678 100644 --- a/event-schemas/examples/m.room.create +++ b/event-schemas/examples/m.room.create @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.create", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.guest_access b/event-schemas/examples/m.room.guest_access index 8ad4d294d5e..c636ff395ed 100644 --- a/event-schemas/examples/m.room.guest_access +++ b/event-schemas/examples/m.room.guest_access @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEG:localhost", "type": "m.room.guest_access", "room_id": "!Cuyf34gef24u:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.history_visibility b/event-schemas/examples/m.room.history_visibility index fcc3f881b29..6fedc5dc6a5 100644 --- a/event-schemas/examples/m.room.history_visibility +++ b/event-schemas/examples/m.room.history_visibility @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.history_visibility", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.join_rules b/event-schemas/examples/m.room.join_rules index f22ad97e9e5..39e14fc5d30 100644 --- a/event-schemas/examples/m.room.join_rules +++ b/event-schemas/examples/m.room.join_rules @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.join_rules", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.member b/event-schemas/examples/m.room.member index e2ca56685e8..133cad96b9d 100644 --- a/event-schemas/examples/m.room.member +++ b/event-schemas/examples/m.room.member @@ -26,5 +26,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.member", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.member#invite_room_state b/event-schemas/examples/m.room.member#invite_room_state index e2ca56685e8..133cad96b9d 100644 --- a/event-schemas/examples/m.room.member#invite_room_state +++ b/event-schemas/examples/m.room.member#invite_room_state @@ -26,5 +26,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.member", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.member#third_party_invite b/event-schemas/examples/m.room.member#third_party_invite index 2457302ac89..1bae2e9fdcf 100644 --- a/event-schemas/examples/m.room.member#third_party_invite +++ b/event-schemas/examples/m.room.member#third_party_invite @@ -21,5 +21,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.member", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.audio b/event-schemas/examples/m.room.message#m.audio index b57adc9a2d9..367eb954061 100644 --- a/event-schemas/examples/m.room.message#m.audio +++ b/event-schemas/examples/m.room.message#m.audio @@ -14,5 +14,5 @@ "origin_server_ts": 1432735824653, "room_id": "!jEsUZKDJdhlrceRyVU:localhost", "type": "m.room.message", - "user_id": "@example:localhost" -} \ No newline at end of file + "sender": "@example:localhost" +} diff --git a/event-schemas/examples/m.room.message#m.emote b/event-schemas/examples/m.room.message#m.emote index bd9848315d8..4280928ee42 100644 --- a/event-schemas/examples/m.room.message#m.emote +++ b/event-schemas/examples/m.room.message#m.emote @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.file b/event-schemas/examples/m.room.message#m.file index 7ac36c9036d..e52c3a9479f 100644 --- a/event-schemas/examples/m.room.message#m.file +++ b/event-schemas/examples/m.room.message#m.file @@ -14,5 +14,5 @@ "origin_server_ts": 1432735824653, "room_id": "!jEsUZKDJdhlrceRyVU:localhost", "type": "m.room.message", - "user_id": "@example:localhost" -} \ No newline at end of file + "sender": "@example:localhost" +} diff --git a/event-schemas/examples/m.room.message#m.image b/event-schemas/examples/m.room.message#m.image index ba405a1ae62..91e72be264f 100644 --- a/event-schemas/examples/m.room.message#m.image +++ b/event-schemas/examples/m.room.message#m.image @@ -15,5 +15,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.location b/event-schemas/examples/m.room.message#m.location index 54198acbf4d..fcbeb97e4c9 100644 --- a/event-schemas/examples/m.room.message#m.location +++ b/event-schemas/examples/m.room.message#m.location @@ -16,5 +16,5 @@ "origin_server_ts": 1432735824653, "room_id": "!jEsUZKDJdhlrceRyVU:localhost", "type": "m.room.message", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.notice b/event-schemas/examples/m.room.message#m.notice index 2e5fff97f2e..978c67e6b0e 100644 --- a/event-schemas/examples/m.room.message#m.notice +++ b/event-schemas/examples/m.room.message#m.notice @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.text b/event-schemas/examples/m.room.message#m.text index d877ee6416f..e00c7aa5663 100644 --- a/event-schemas/examples/m.room.message#m.text +++ b/event-schemas/examples/m.room.message#m.text @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.video b/event-schemas/examples/m.room.message#m.video index e3da9077021..576d80de4ed 100644 --- a/event-schemas/examples/m.room.message#m.video +++ b/event-schemas/examples/m.room.message#m.video @@ -23,5 +23,5 @@ "origin_server_ts": 1432735824653, "room_id": "!jEsUZKDJdhlrceRyVU:localhost", "type": "m.room.message", - "user_id": "@example:localhost" -} \ No newline at end of file + "sender": "@example:localhost" +} diff --git a/event-schemas/examples/m.room.message.feedback b/event-schemas/examples/m.room.message.feedback index 6282305b0c4..16fe0ee0975 100644 --- a/event-schemas/examples/m.room.message.feedback +++ b/event-schemas/examples/m.room.message.feedback @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message.feedback", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.name b/event-schemas/examples/m.room.name index 636119b8b69..87db2008ed1 100644 --- a/event-schemas/examples/m.room.name +++ b/event-schemas/examples/m.room.name @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.name", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.power_levels b/event-schemas/examples/m.room.power_levels index 8278597af59..42ef5a5d6ea 100644 --- a/event-schemas/examples/m.room.power_levels +++ b/event-schemas/examples/m.room.power_levels @@ -20,5 +20,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.power_levels", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.redaction b/event-schemas/examples/m.room.redaction index 2e8260eabf8..5f4aae1e265 100644 --- a/event-schemas/examples/m.room.redaction +++ b/event-schemas/examples/m.room.redaction @@ -8,5 +8,5 @@ "type": "m.room.redaction", "room_id": "!Cuyf34gef24t:localhost", "redacts": "!fukweghifu23:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.topic b/event-schemas/examples/m.room.topic index 8a469122c13..65daa987219 100644 --- a/event-schemas/examples/m.room.topic +++ b/event-schemas/examples/m.room.topic @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.topic", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.typing b/event-schemas/examples/m.typing index bd53f6fb6b3..1d2c517b912 100644 --- a/event-schemas/examples/m.typing +++ b/event-schemas/examples/m.typing @@ -4,4 +4,4 @@ "content": { "user_ids": ["@alice:matrix.org", "@bob:example.com"] } -} \ No newline at end of file +} diff --git a/event-schemas/schema/core-event-schema/room_event.json b/event-schemas/schema/core-event-schema/room_event.json index 80f7d26514f..09cb2478242 100644 --- a/event-schemas/schema/core-event-schema/room_event.json +++ b/event-schemas/schema/core-event-schema/room_event.json @@ -1,23 +1,41 @@ { "type": "object", "title": "Room Event", - "description": "In addition to the Event fields, Room Events MUST have the following additional field.", + "description": "In addition to the Event fields, Room Events may have the following additional fields.", "allOf":[{ "$ref": "event.json" }], "properties": { - "room_id": { + "event_id": { "type": "string", - "description": "The ID of the room associated with this event." + "description": "Required. The globally unique event identifier." }, - "event_id": { + "room_id": { "type": "string", - "description": "The globally unique event identifier." + "description": "Required. The ID of the room associated with this event." }, - "user_id": { + "sender": { "type": "string", - "description": "Contains the fully-qualified ID of the user who *sent* this event." + "description": "Required. Contains the fully-qualified ID of the user who *sent* this event." + }, + "unsigned": { + "type": "object", + "description": "Contains optional extra information about the event.", + "properties": { + "age": { + "type": "integer", + "description": "The time in milliseconds that has elapsed since the event was sent" + }, + "redacted_because": { + "type": "string", + "description": "The reason this event was redacted, if it was redacted" + }, + "transaction_id": { + "type": "string", + "description": "The client-supplied transaction ID, if the client being given the event is the same one which sent it." + } + } } }, - "required": ["room_id"] + "required": ["event_id", "room_id", "sender"] } diff --git a/specification/events.rst b/specification/events.rst index 5a003115556..92d52d22861 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -5,6 +5,9 @@ All communication in Matrix is expressed in the form of data objects called Events. These are the fundamental building blocks common to the client-server, server-server and application-service APIs, and are described below. +Note that the structure of these events may be different than those in the +server-server API. + {{common_event_fields}} {{common_room_event_fields}} @@ -12,51 +15,13 @@ server-server and application-service APIs, and are described below. {{common_state_event_fields}} -Differences between /v1 and /v2 events --------------------------------------- - -There are a few differences between how events are formatted for sending -between servers over federation and how they are formatted for sending between -a server and its clients. - -Additionally there are a few differences between the format of events in the -responses to client APIs with a /v1 prefix and responses APIs with a /v2 -prefix. - -Events in responses for APIs with the /v2 prefix are generated from an event -formatted for federation by: - -* Removing the following keys: - ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, - ``origin``, ``prev_state``. -* Adding an ``age`` to the ``unsigned`` object which gives the time in - milliseconds that has elapsed since the event was sent. -* Adding ``prev_content`` and ``prev_sender`` to the ``unsigned`` object if the - event is a ``state event``, which give the previous content and previous - sender of that state key -* Adding a ``redacted_because`` to the ``unsigned`` object if the event was - redacted which gives the event that redacted it. -* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent - by the client requesting it. - -Events in responses for APIs with the /v1 prefix are generated from an event -formatted for the /v2 prefix by: - -* Moving the folling keys from the ``unsigned`` object to the top level event - object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. -* Removing the ``unsigned`` object. -* Rename the ``sender`` key to ``user_id``. -* If the event was an ``m.room.member`` with ``membership`` set to ``invite`` - then adding a ``invite_room_state`` key to the top level event object. - - Size limits ----------- The total size of any event MUST NOT exceed 65 KB. There are additional restrictions on sizes per key: -- ``user_id`` MUST NOT exceed 255 bytes (including domain). +- ``sender`` MUST NOT exceed 255 bytes (including domain). - ``room_id`` MUST NOT exceed 255 bytes. - ``state_key`` MUST NOT exceed 255 bytes. - ``type`` MUST NOT exceed 255 bytes. From 7f07da4b507f6d0c20a8b17cab23276d9d050d7f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 17:29:09 +0000 Subject: [PATCH 271/989] Remove obsolete documentation --- api/client-server/definitions/event.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/client-server/definitions/event.json b/api/client-server/definitions/event.json index 5a8f52f6fb5..98aac21e93e 100644 --- a/api/client-server/definitions/event.json +++ b/api/client-server/definitions/event.json @@ -39,10 +39,6 @@ "type": "object", "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." }, - "replaces_state": { - "type": "string", - "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, "transaction_id": { "type": "string", "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." From b946d398864343180ca96837f56916d726bbb8cb Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 2 Dec 2015 01:25:07 +0000 Subject: [PATCH 272/989] typo --- specification/modules/search.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 089953d2256..655b1ed3957 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -34,7 +34,7 @@ The supported keys to search over are: The search will *not* include rooms that are end to end encrypted. The results include a ``rank`` key that can be used to sort the results by -revelancy. The higher the ``rank`` the more relevant the result is. +relevancy. The higher the ``rank`` the more relevant the result is. The value of ``count`` may not match the number of results. For example due to the search query matching 1000s of results and the server truncating the From 1ce33579270f86ef2c85601a5eb6423fdd009d9f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 2 Dec 2015 14:53:52 +0000 Subject: [PATCH 273/989] continuserv: serve multiple paths --- scripts/continuserv/main.go | 44 ++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 14d0c5bf51c..a1295077f1d 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -26,7 +26,7 @@ var ( port = flag.Int("port", 8000, "Port on which to serve HTTP") mu sync.Mutex // Prevent multiple updates in parallel. - toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + toServe atomic.Value // Always contains a bytesOrErr. May be stale unless wg is zero. wgMu sync.Mutex // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel. wg sync.WaitGroup // Indicates how many updates are pending. @@ -116,14 +116,29 @@ func serve(w http.ResponseWriter, req *http.Request) { wgMu.Lock() wg.Wait() wgMu.Unlock() - b := toServe.Load().(bytesOrErr) - if b.err != nil { + + file := req.URL.Path + if file[0] == '/' { + file = file[1:] + } + if file == "" { + file = "index.html" + } + m := toServe.Load().(bytesOrErr) + if m.err != nil { w.Header().Set("Content-Type", "text/plain") - w.Write([]byte(b.err.Error())) - } else { + w.Write([]byte(m.err.Error())) + return + } + b, ok := m.bytes[file] + if ok { w.Header().Set("Content-Type", "text/html") - w.Write([]byte(b.bytes)) + w.Write([]byte(b)) + return } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(404) + w.Write([]byte("Not found")) } func populateOnce(dir string) { @@ -139,12 +154,21 @@ func populateOnce(dir string) { toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())}) return } - specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) + fis, err := ioutil.ReadDir(path.Join(dir, "scripts", "gen")) if err != nil { - toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) + toServe.Store(bytesOrErr{nil, err}) return } - toServe.Store(bytesOrErr{specBytes, nil}) + files := make(map[string][]byte) + for _, fi := range fis { + bytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", fi.Name())) + if err != nil { + toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) + return + } + files[fi.Name()] = bytes + } + toServe.Store(bytesOrErr{files, nil}) } func doPopulate(ch chan struct{}, dir string) { @@ -173,6 +197,6 @@ func exists(path string) bool { } type bytesOrErr struct { - bytes []byte + bytes map[string][]byte // filename -> contents err error } From e072d215cf466e76552a6eb53dc9e1cea8a4838c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 2 Dec 2015 15:27:25 +0000 Subject: [PATCH 274/989] Specify redaction --- api/client-server/redaction.yaml | 82 +++++++++++++++++++++++++++++ specification/client_server_api.rst | 17 +++--- 2 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 api/client-server/redaction.yaml diff --git a/api/client-server/redaction.yaml b/api/client-server/redaction.yaml new file mode 100644 index 00000000000..d30ef8c164f --- /dev/null +++ b/api/client-server/redaction.yaml @@ -0,0 +1,82 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server message redaction API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/redact/{eventId}/{txnId}": + put: + summary: Strips all non-integrity-critical information out of an event. + description: |- + Strips all information out of an event which isn't critical to the + integrity of the server-side representation of the room. + + This cannot be undone. + + Users may redact their own events, and any user with a power level + greater than or equal to the `redact` power level of the room may + redact events there. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room from which to redact the event. + required: true + x-example: "!637q39766251:example.com" + - in: path + type: string + name: eventId + description: The ID of the event to redact + required: true + x-example: "bai2b1i9:matrix.org" + - in: path + name: txnId + type: string + description: |- + The transaction ID for this event. Clients should generate a + unique ID; it will be used by the server to ensure idempotency of requests. + required: true + x-example: "37" + - in: body + name: body + schema: + type: object + example: |- + { + "reason": "Indecent material" + } + properties: + reason: + type: string + description: The reason for the event being redacted. + responses: + 200: + description: "An ID for the redaction event." + examples: + application/json: |- + { + "event_id": "YUwQidLecu" + } + schema: + type: object + properties: + event_id: + type: string + description: |- + A unique identifier for the event. diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 67973c6b3eb..4c24c3503a8 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -793,10 +793,6 @@ owners to delete the offending content from the databases. Events that have been redacted include a ``redacted_because`` key whose value is the event that caused it to be redacted, which may include a reason. -.. TODO - Currently, only room admins can redact events by sending a ``m.room.redaction`` - event, but server admins also need to be able to redact events by a similar - mechanism. Upon receipt of a redaction event, the server should strip off any keys not in the following list: @@ -819,13 +815,20 @@ one of the following event types: ``kick``, ``redact``, ``state_default``, ``users``, ``users_default``. - ``m.room.aliases`` allows key ``aliases`` -.. TODO - Need to update m.room.power_levels to reflect new power levels formatting - The redaction event should be added under the key ``redacted_because``. When a client receives a redaction event it should change the redacted event in the same way a server does. +Events +++++++ + +{{m_room_redaction_event}} + +Client behaviour +++++++++++++++++ + +{{redaction_http_api}} + Rooms ----- From 2f3a00fe34b2414484ab2c2265c571d84db9328b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 2 Dec 2015 19:23:33 +0000 Subject: [PATCH 275/989] Consistently spell homeserver as homeserver --- api/client-server/administrative_contact.yaml | 6 +- api/client-server/create_room.yaml | 2 +- api/client-server/inviting.yaml | 2 +- api/client-server/login.yaml | 2 +- api/client-server/pusher.yaml | 2 +- api/client-server/registration.yaml | 4 +- api/client-server/third_party_membership.yaml | 4 +- attic/v1_registration_login.rst | 48 ++++++++-------- ...ient_federated_versioning_design_notes.rst | 8 +-- drafts/as-http-api.rst | 42 +++++++------- drafts/erikj_federation.rst | 6 +- drafts/general_api.rst | 54 +++++++++--------- drafts/human-id-rules.rst | 14 ++--- drafts/model/presence.rst | 12 ++-- drafts/model/profiles.rst | 16 +++--- drafts/model/room-join-workflow.rst | 12 ++-- drafts/model/rooms.rst | 20 +++---- drafts/model/third-party-id.rst | 2 +- scripts/generate-http-docs.sh | 2 +- specification/application_service_api.rst | 14 ++--- specification/client_server_api.rst | 56 +++++++++---------- specification/intro.rst | 4 +- specification/modules/presence.rst | 4 +- specification/modules/push.rst | 2 +- specification/server_server_api.rst | 50 ++++++++--------- .../examples/application-services.rst | 8 +-- .../guides/2015-08-10-client-server.rst | 16 +++--- .../guides/2015-08-21-application_services.md | 18 +++--- supporting-docs/howtos/client-server.rst | 16 +++--- .../jsfiddles/create_room_send_msg/demo.html | 2 +- .../jsfiddles/create_room_send_msg/demo.js | 2 +- .../howtos/jsfiddles/event_stream/demo.html | 2 +- .../howtos/jsfiddles/event_stream/demo.js | 2 +- .../howtos/jsfiddles/example_app/demo.html | 2 +- .../howtos/jsfiddles/example_app/demo.js | 4 +- .../howtos/jsfiddles/register_login/demo.html | 2 +- .../howtos/jsfiddles/register_login/demo.js | 4 +- .../jsfiddles/room_memberships/demo.html | 2 +- .../howtos/jsfiddles/room_memberships/demo.js | 2 +- 39 files changed, 235 insertions(+), 235 deletions(-) diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml index 68fd0fbaec4..7e3f8bcea9e 100644 --- a/api/client-server/administrative_contact.yaml +++ b/api/client-server/administrative_contact.yaml @@ -25,7 +25,7 @@ paths: This API endpoint uses the User-Interactive Authentication API. An access token should be submitted to this endpoint if the client has an active session. - The Home Server may change the flows available depending on whether a + The homeserver may change the flows available depending on whether a valid access token is provided. security: - accessToken: [] @@ -64,7 +64,7 @@ paths: This is *not* the same as the list of third party identifiers bound to the user's Matrix ID in Identity Servers. - Identifiers in this list may be used by the Home Server as, for example, + Identifiers in this list may be used by the homeserver as, for example, identifiers that it will accept to reset the user's account password. security: - accessToken: [] @@ -126,7 +126,7 @@ paths: bind: type: boolean description: |- - Whether the home server should also bind this third party + Whether the homeserver should also bind this third party identifier to the account's Matrix ID with the passed identity server. Default: ``false``. x-example: true diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 051c4b84380..20c00f6c5c0 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -57,7 +57,7 @@ paths: description: |- The desired room alias **local part**. If this is included, a room alias will be created and mapped to the newly created - room. The alias will belong on the *same* home server which + room. The alias will belong on the *same* homeserver which created the room. For example, if this was set to "foo" and sent to the homeserver "example.com" the complete room alias would be ``#foo:example.com``. diff --git a/api/client-server/inviting.yaml b/api/client-server/inviting.yaml index 0a028710313..d9b0ad1097a 100644 --- a/api/client-server/inviting.yaml +++ b/api/client-server/inviting.yaml @@ -38,7 +38,7 @@ paths: Only users currently in a particular room can invite other users to join that room. - If the user was invited to the room, the home server will append a + If the user was invited to the room, the homeserver will append a ``m.room.member`` event to the room. .. _third party invites section: `invite-by-third-party-id-endpoint`_ diff --git a/api/client-server/login.yaml b/api/client-server/login.yaml index bbab46dfa8b..f2e3d0446f5 100644 --- a/api/client-server/login.yaml +++ b/api/client-server/login.yaml @@ -81,7 +81,7 @@ paths: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint. home_server: type: string - description: The hostname of the Home Server on which the account has been registered. + description: The hostname of the homeserver on which the account has been registered. 400: description: |- Part of the request was invalid. For example, the login type may not be recognised. diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 8c243f2bcbd..ed6c4f910a2 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -114,7 +114,7 @@ paths: description: |- If true, the homeserver should add another pusher with the given pushkey and App ID in addition to any others with - different user IDs. Otherwise, the Home Server must remove any + different user IDs. Otherwise, the homeserver must remove any other pushers with the same App ID and pushkey for different users. The default is ``false``. required: ['profile_tag', 'kind', 'app_id', 'app_display_name', diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index 681654ae72e..7bb49f2cc4e 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -94,7 +94,7 @@ paths: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint. home_server: type: string - description: The hostname of the Home Server on which the account has been registered. + description: The hostname of the homeserver on which the account has been registered. 400: description: |- Part of the request was invalid. This may include one of the following error codes: @@ -107,7 +107,7 @@ paths: including after authentication if the requested user ID was registered whilst the client was performing authentication. - Home Servers MUST perform the relevant checks and return these codes before + Homeservers MUST perform the relevant checks and return these codes before performing `User-Interactive Authentication`_, although they may also return them after authentication is completed if, for example, the requested user ID was registered whilst the client was performing authentication. diff --git a/api/client-server/third_party_membership.yaml b/api/client-server/third_party_membership.yaml index a10d616722d..e9c4d2f23aa 100644 --- a/api/client-server/third_party_membership.yaml +++ b/api/client-server/third_party_membership.yaml @@ -39,7 +39,7 @@ paths: join that room. If the identity server did know the Matrix user identifier for the - third party identifier, the home server will append a ``m.room.member`` + third party identifier, the homeserver will append a ``m.room.member`` event to the room. If the identity server does not know a Matrix user identifier for the @@ -62,7 +62,7 @@ paths: - The matrix user ID who invited them to the room - If a token is requested from the identity server, the home server will + If a token is requested from the identity server, the homeserver will append a ``m.room.third_party_invite`` event to the room. .. _joining rooms section: `invite-by-user-id-endpoint`_ diff --git a/attic/v1_registration_login.rst b/attic/v1_registration_login.rst index 4b80600b8a1..1364319bb6e 100644 --- a/attic/v1_registration_login.rst +++ b/attic/v1_registration_login.rst @@ -1,17 +1,17 @@ Registration and Login ---------------------- -Clients must register with a home server in order to use Matrix. After +Clients must register with a homeserver in order to use Matrix. After registering, the client will be given an access token which must be used in ALL -requests to that home server as a query parameter 'access_token'. +requests to that homeserver as a query parameter 'access_token'. If the client has already registered, they need to be able to login to their -account. The home server may provide many different ways of logging in, such as +account. The homeserver may provide many different ways of logging in, such as user/password auth, login via a social network (OAuth2), login by confirming a token sent to their email address, etc. This specification does not define how -home servers should authorise their users who want to login to their existing +homeservers should authorise their users who want to login to their existing accounts, but instead defines the standard interface which implementations -should follow so that ANY client can login to ANY home server. Clients login +should follow so that ANY client can login to ANY homeserver. Clients login using the |login|_ API. Clients register using the |register|_ API. Registration follows the same general procedure as login, but the path requests are sent to and the details contained in them are different. @@ -26,7 +26,7 @@ In order to determine up-front what the server's requirements are, the client can request from the server a complete description of all of its acceptable flows of the registration or login process. It can then inspect the list of returned flows looking for one for which it believes it can complete all of the -required stages, and perform it. As each home server may have different ways of +required stages, and perform it. As each homeserver may have different ways of logging in, the client needs to know how they should login. All distinct login stages MUST have a corresponding ``type``. A ``type`` is a namespaced string which details the mechanism for logging in. @@ -64,12 +64,12 @@ ID and a new access token MUST be returned:: "access_token": "abcdef0123456789" } -The ``user_id`` key is particularly useful if the home server wishes to support +The ``user_id`` key is particularly useful if the homeserver wishes to support localpart entry of usernames (e.g. "user" rather than "@user:matrix.org"), as the client may not be able to determine its ``user_id`` in this case. -If the flow has multiple stages to it, the home server may wish to create a -session to store context between requests. If a home server responds with a +If the flow has multiple stages to it, the homeserver may wish to create a +session to store context between requests. If a homeserver responds with a ``session`` key to a request, clients MUST submit it in subsequent requests until the flow is completed:: @@ -99,7 +99,7 @@ To respond to this type, reply with:: "password": "" } -The home server MUST respond with either new credentials, the next stage of the +The homeserver MUST respond with either new credentials, the next stage of the login process, or a standard error response. Captcha-based @@ -123,7 +123,7 @@ To respond to this type, reply with:: Recaptcha.get_challenge(); Recaptcha.get_response(); -The home server MUST respond with either new credentials, the next stage of the +The homeserver MUST respond with either new credentials, the next stage of the login process, or a standard error response. OAuth2-based @@ -146,24 +146,24 @@ The server MUST respond with:: "uri": } -The home server acts as a 'confidential' client for the purposes of OAuth2. If +The homeserver acts as a 'confidential' client for the purposes of OAuth2. If the uri is a ``sevice selection URI``, it MUST point to a webpage which prompts the user to choose which service to authorize with. On selection of a service, this MUST link through to an ``Authorization Request URI``. If there is only 1 -service which the home server accepts when logging in, this indirection can be +service which the homeserver accepts when logging in, this indirection can be skipped and the "uri" key can be the ``Authorization Request URI``. The client then visits the ``Authorization Request URI``, which then shows the OAuth2 Allow/Deny prompt. Hitting 'Allow' returns the ``redirect URI`` with the -auth code. Home servers can choose any path for the ``redirect URI``. The +auth code. Homeservers can choose any path for the ``redirect URI``. The client should visit the ``redirect URI``, which will then finish the OAuth2 -login process, granting the home server an access token for the chosen service. -When the home server gets this access token, it verifies that the cilent has +login process, granting the homeserver an access token for the chosen service. +When the homeserver gets this access token, it verifies that the cilent has authorised with the 3rd party, and can now complete the login. The OAuth2 ``redirect URI`` (with auth code) MUST respond with either new credentials, the next stage of the login process, or a standard error response. -For example, if a home server accepts OAuth2 from Google, it would return the +For example, if a homeserver accepts OAuth2 from Google, it would return the Authorization Request URI for Google:: { @@ -171,7 +171,7 @@ Authorization Request URI for Google:: client_id=CLIENT_ID&redirect_uri=REDIRECT_URI&scope=photos" } -The client then visits this URI and authorizes the home server. The client then +The client then visits this URI and authorizes the homeserver. The client then visits the REDIRECT_URI with the auth code= query parameter which returns:: { @@ -195,7 +195,7 @@ To respond to this type, reply with:: "email": "" } -After validating the email address, the home server MUST send an email +After validating the email address, the homeserver MUST send an email containing an authentication code and return:: { @@ -212,7 +212,7 @@ code:: "code": "" } -The home server MUST respond to this with either new credentials, the next +The homeserver MUST respond to this with either new credentials, the next stage of the login process, or a standard error response. Email-based (url) @@ -231,7 +231,7 @@ To respond to this type, reply with:: "email": "" } -After validating the email address, the home server MUST send an email +After validating the email address, the homeserver MUST send an email containing an authentication URL and return:: { @@ -247,7 +247,7 @@ client should perform another request:: "session": "" } -The home server MUST respond to this with either new credentials, the next +The homeserver MUST respond to this with either new credentials, the next stage of the login process, or a standard error response. A common client implementation will be to periodically poll until the link is @@ -264,7 +264,7 @@ Email-based (identity server) Prior to submitting this, the client should authenticate with an identity server. After authenticating, the session information should be submitted to -the home server. +the homeserver. To respond to this type, reply with:: @@ -293,7 +293,7 @@ of a previous login stage:: "next": "" } -If a home server implements N-factor authentication, it MUST respond with all +If a homeserver implements N-factor authentication, it MUST respond with all ``stages`` when initially queried for their login requirements:: { diff --git a/drafts/ancient_federated_versioning_design_notes.rst b/drafts/ancient_federated_versioning_design_notes.rst index ffda60633f2..036989ebf15 100644 --- a/drafts/ancient_federated_versioning_design_notes.rst +++ b/drafts/ancient_federated_versioning_design_notes.rst @@ -1,11 +1,11 @@ -Versioning is, like, hard for backfilling backwards because of the number of Home Servers involved. +Versioning is, like, hard for backfilling backwards because of the number of homeservers involved. -The way we solve this is by doing versioning as an acyclic directed graph of PDUs. For backfilling purposes, this is done on a per context basis. +The way we solve this is by doing versioning as an acyclic directed graph of PDUs. For backfilling purposes, this is done on a per context basis. When we send a PDU we include all PDUs that have been received for that context that hasn't been subsequently listed in a later PDU. The trivial case is a simple list of PDUs, e.g. A <- B <- C. However, if two servers send out a PDU at the same to, both B and C would point at A - a later PDU would then list both B and C. Problems with opaque version strings: - - How do you do clustering without mandating that a cluster can only have one transaction in flight to a given remote home server at a time. + - How do you do clustering without mandating that a cluster can only have one transaction in flight to a given remote homeserver at a time. If you have multiple transactions sent at once, then you might drop one transaction, receive another with a version that is later than the dropped transaction and which point ARGH WE LOST A TRANSACTION. - - How do you do backfilling? A version string defines a point in a stream w.r.t. a single home server, not a point in the context. + - How do you do backfilling? A version string defines a point in a stream w.r.t. a single homeserver, not a point in the context. We only need to store the ends of the directed graph, we DO NOT need to do the whole one table of nodes and one of edges. diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 2c8f8e37ca1..4f8d6bd4844 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -13,9 +13,9 @@ Application Services HTTP API .. sectnum:: -Application Service -> Home Server +Application Service -> Homeserver ---------------------------------- -This contains home server APIs which are used by the application service. +This contains homeserver APIs which are used by the application service. Registration API ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -34,7 +34,7 @@ Output: Side effects: - The HS will start delivering events to the URL base specified if this 200s. API called when: - - The application service wants to register with a brand new home server. + - The application service wants to register with a brand new homeserver. Notes: - An application service can state whether they should be the only ones who can manage a specified namespace. This is referred to as an "exclusive" @@ -100,7 +100,7 @@ Notes: Unregister API ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -This API unregisters a previously registered AS from the home server. +This API unregisters a previously registered AS from the homeserver. Inputs: - AS token @@ -122,9 +122,9 @@ API called when: } -Home Server -> Application Service +Homeserver -> Application Service ---------------------------------- -This contains application service APIs which are used by the home server. +This contains application service APIs which are used by the homeserver. User Query ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~ @@ -152,9 +152,9 @@ Notes: - This is deemed more flexible than alternative methods (e.g. returning a JSON blob with the user's display name and get the HS to provision the user). Retry notes: - - The home server cannot respond to the client's request until the response to + - The homeserver cannot respond to the client's request until the response to this API is obtained from the AS. - - Recommended that home servers try a few times then time out, returning a + - Recommended that homeservers try a few times then time out, returning a 408 Request Timeout to the client. :: @@ -199,9 +199,9 @@ Notes: style JSON blob and get the HS to provision the room). It also means that the AS knows the room ID -> alias mapping. Retry notes: - - The home server cannot respond to the client's request until the response to + - The homeserver cannot respond to the client's request until the response to this API is obtained from the AS. - - Recommended that home servers try a few times then time out, returning a + - Recommended that homeservers try a few times then time out, returning a 408 Request Timeout to the client. :: @@ -236,13 +236,13 @@ Data flows: :: Typical - HS ---> AS : Home server sends events with transaction ID T. + HS ---> AS : Homeserver sends events with transaction ID T. <--- : AS sends back 200 OK. AS ACK Lost - HS ---> AS : Home server sends events with transaction ID T. + HS ---> AS : Homeserver sends events with transaction ID T. <-/- : AS 200 OK is lost. - HS ---> AS : Home server retries with the same transaction ID of T. + HS ---> AS : Homeserver retries with the same transaction ID of T. <--- : AS sends back 200 OK. If the AS had processed these events already, it can NO-OP this request (and it knows if it is the same events based on the transacton ID). @@ -253,15 +253,15 @@ Retry notes: - Since ASes by definition cannot alter the traffic being passed to it (unlike say, a Policy Server), these requests can be done in parallel to general HS processing; the HS doesn't need to block whilst doing this. - - Home servers should use exponential backoff as their retry algorithm. - - Home servers MUST NOT alter (e.g. add more) events they were going to + - Homeservers should use exponential backoff as their retry algorithm. + - Homeservers MUST NOT alter (e.g. add more) events they were going to send within that transaction ID on retries, as the AS may have already processed the events. Ordering notes: - The events sent to the AS should be linearised, as they are from the event stream. - - The home server will need to maintain a queue of transactions to send to + - The homeserver will need to maintain a queue of transactions to send to the AS. :: @@ -336,7 +336,7 @@ Notes: Server admin style permissions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The home server needs to give the application service *full control* over its +The homeserver needs to give the application service *full control* over its namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as create/delete any user in its namespace. No additional API changes need to be @@ -451,15 +451,15 @@ Examples IRC ~~~ Pre-conditions: - - Server admin stores the AS token "T_a" on the home server. - - Home server has a token "T_h". - - Home server has the domain "hsdomain.com" + - Server admin stores the AS token "T_a" on the homeserver. + - Homeserver has a token "T_h". + - Homeserver has the domain "hsdomain.com" 1. Application service registration :: - AS -> HS: Registers itself with the home server + AS -> HS: Registers itself with the homeserver POST /register { url: "https://someapp.com/matrix", diff --git a/drafts/erikj_federation.rst b/drafts/erikj_federation.rst index 0644f27105b..ce089def7c5 100644 --- a/drafts/erikj_federation.rst +++ b/drafts/erikj_federation.rst @@ -237,7 +237,7 @@ Domain specific string ``room_id`` A domain specific string with prefix ``!`` that is static across all events in a graph and uniquely identifies it. The ``domain`` should be that of the - home server that created the room (i.e., the server that generated the + homeserver that created the room (i.e., the server that generated the first ``m.room.create`` event). ``sender`` @@ -246,7 +246,7 @@ Domain specific string User Id A domain specific string with prefix ``@`` representing a user account. The - ``domain`` is the home server of the user and is the server used to contact + ``domain`` is the homeserver of the user and is the server used to contact the user. Joining a room @@ -280,7 +280,7 @@ can then process the join event itself. Inviting a user --------------- -To invite a remote user to a room we need their home server to sign the invite +To invite a remote user to a room we need their homeserver to sign the invite event. This is done by sending the event to the remote server, which then signs the event, before distributing the invite to other servers. diff --git a/drafts/general_api.rst b/drafts/general_api.rst index ae2446d63d3..baad39017db 100644 --- a/drafts/general_api.rst +++ b/drafts/general_api.rst @@ -61,7 +61,7 @@ This version will change the path prefix for HTTP: - Version 2: ``/_matrix/client/v2`` Note the lack of the ``api`` segment. This is for consistency between other -home server path prefixes. +homeserver path prefixes. Terminology: - ``Chunk token`` : An opaque string which can be used to return another chunk @@ -169,16 +169,16 @@ Outputs: ``content`` key. Deleted message events are ``m.room.redaction`` events. - New position in the stream. (chunk token) State Events Ordering Notes: - - Home servers may receive state events over federation that are superceded by - state events previously sent to the client. The home server *cannot* send + - Homeservers may receive state events over federation that are superceded by + state events previously sent to the client. The homeserver *cannot* send these events to the client else they would end up erroneously clobbering the superceding state event. - - As a result, the home server reserves the right to omit sending state events + - As a result, the homeserver reserves the right to omit sending state events which are known to be superceded already. - This may result in missed *state* events. However, the state of the room will always be eventually consistent. Message Events Ordering Notes: - - Home servers may receive message events over federation that happened a long + - Homeservers may receive message events over federation that happened a long time ago. The client may or may not be interested in these message events. - For clients which do not store scrollback for a room (they discard events after processing them), this is not a problem as they only care about the @@ -191,11 +191,11 @@ Message Events Ordering Notes: - The event, when it comes down the stream, will indicate which event it comes after. Rejected events: - - A home server may find out via federation that it should not have accepted + - A homeserver may find out via federation that it should not have accepted an event (e.g. to send a message/state event in a room). For example, it may - send an event to another home server and receive an auth event stating + send an event to another homeserver and receive an auth event stating that the event should not have been sent. - - If this happens, the home server will send a ``m.room.redaction`` for the + - If this happens, the homeserver will send a ``m.room.redaction`` for the event in question. This will be a local server event (not shared with other servers). - If the event was a state event, it will synthesise a new state event to @@ -206,7 +206,7 @@ Unknown rooms: - You could receive events for rooms you are unaware of (e.g. you didn't do an initial sync, or your HS lost its database and is told from another HS that they are in this room). How do you handle this? - - The simplest option would be to redo the initial sync with a filter on the + - The simplest option would be to redo the initial sync with a filter on the room ID you're unaware of. This would retrieve the room state so you can display the room. What data flows does it address: @@ -291,7 +291,7 @@ Scrollback API ``[Draft]`` but as a purely informational display thing it would be nice. Additional Inputs: - - flag to say if the home server should do a backfill over federation + - flag to say if the homeserver should do a backfill over federation Additional Outputs: - whether there are more events on the local HS / over federation. What data flows does it address: @@ -313,7 +313,7 @@ Additional Outputs: Room Alias API ``[Draft]`` -------------------------- This provides mechanisms for creating and removing room aliases for a room on a -home server. Typically, any user in a room can make an alias for that room. The +homeserver. Typically, any user in a room can make an alias for that room. The alias creator (or anyone in the room?) can delete that alias. Server admins can also delete any alias on their server. @@ -323,7 +323,7 @@ Inputs: - Room Alias Output: - Room ID - - List of home servers to join via. + - List of homeservers to join via. Mapping a room to an alias: @@ -334,7 +334,7 @@ Inputs: Output: - Room alias Notes: - - The home server may add restrictions e.g. the user must be in the room. + - The homeserver may add restrictions e.g. the user must be in the room. Deleting a mapping: @@ -347,11 +347,11 @@ Output: Published room list API ``[Draft]`` ----------------------------------- -This provides mechanisms for searching for published rooms on a home server. +This provides mechanisms for searching for published rooms on a homeserver. Inputs: - Search text (e.g. room alias/name/topic to search on) - - Home server to search on (this may just be the URL hit for HTTP) + - Homeserver to search on (this may just be the URL hit for HTTP) - Any existing pagination token, can be missing if this is the first hit. - Limit for pagination Output: @@ -378,7 +378,7 @@ Notes: User Profile API ``[Draft]`` ---------------------------- -Every user on a home server has a profile. This profile is effectively a +Every user on a homeserver has a profile. This profile is effectively a key-value store scoped to a user ID. It can include an ``avatar_url``, ``displayname`` and other metadata. Updates to a profile should propagate to other interested users. @@ -435,7 +435,7 @@ This had a number of problems associated with it: flicker. - Name/avatar changes created more ``m.room.member`` events which meant they needed to be included in the auth chains for federation. This - created long auth chains which is suboptimal since home servers need + created long auth chains which is suboptimal since homeservers need to store the auth chains forever. These problems can be resolved by creating an ``m.room.member.profile`` @@ -448,7 +448,7 @@ However, this introduces its own set of problems, namely flicker. The client would receive the ``m.room.member`` event first, followed by the ``m.room.member.profile`` event, which could cause a flicker. In addition, federation may not send both events in a single transaction, -resulting in missing information on the receiving home server. +resulting in missing information on the receiving homeserver. For federation, these problems can be resolved by sending the ``m.room.member`` event as they are in v1 (with ``displayname`` and @@ -457,7 +457,7 @@ they cannot be in the ``unsigned`` part of the event. The receiving home server will then extract these keys and create a server-generated ``m.room.member.profile`` event. To avoid confusion with duplicate information, the ``avatar_url`` and ``displayname`` keys should be -removed from the ``m.room.member`` event by the receiving home server. +removed from the ``m.room.member`` event by the receiving homeserver. When a client requests these events (either from the event stream or from an initial sync), the server will send the generated ``m.room.member.profile`` event under the ``unsigned.profile`` key of the @@ -840,17 +840,17 @@ information per device to all other users just redirects the union problem to the client, which will commonly be presenting this information as an icon alongside the user. -When a client hits the event stream, the home server can treat the user as +When a client hits the event stream, the homeserver can treat the user as "online". This behaviour should be able to be overridden to avoid flicker during connection losses when the client is appear offline (e.g. device is appear offline > goes into a tunnel > server times out > device regains connection and hits the event stream forcing the device online before the "appear offline" state can be set). When the client has not hit the event -stream for a certain period of time, the home server can treat the user as +stream for a certain period of time, the homeserver can treat the user as "offline". The user can also set a global *per-user* appear offline flag. The user should also be able to set their presence state via a direct API, -without having to hit the event stream. The home server will set a timer when +without having to hit the event stream. The homeserver will set a timer when the connection ends, after which it will set that device to offline. As the idle flag and online state is determined per device, there needs to be a @@ -970,12 +970,12 @@ the hashes the same is the best as that means clients do not need to request the capabilities for the given hash. On first signup, the client will attempt to send the hash and be most likely -refused by the home server as it does not know the full capability set for that +refused by the homeserver as it does not know the full capability set for that hash. The client will then have to upload the full capability set to the home server. The client will then be able to send the hash as normal. When a client receives a hash, the client will either recognise the hash or -will have to request the capability set from their home server: +will have to request the capability set from their homeserver: Inputs: - Hash @@ -1070,7 +1070,7 @@ Main use cases for ``updates``: - Call signalling (child events are ICE candidates, answer to the offer, and termination) - *Local* Delivery/Read receipts : "Local" means they are not shared with other - users on the same home server or via federation but *are* shared between + users on the same homeserver or via federation but *are* shared between clients for the same user; useful for push notifications, read count markers, etc. This is done to avoid the ``n^2`` problem for sending receipts, where the vast majority of traffic tends towards sending more receipts. @@ -1168,11 +1168,11 @@ Events (breaking changes; event version 2) ``[Draft]`` when dealing with custom event types. E.g. ``_custom.event`` would allow anything in the state key, ``_@custom.event`` would only allow user IDs in the state key, etc. -- s/user_id/sender/g given that home servers can send events, not just users. +- s/user_id/sender/g given that homeservers can send events, not just users. Server-generated events ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Home servers may want to send events to their local clients or to other home +Homeservers may want to send events to their local clients or to other home servers e.g. for server status notifications. These events look like regular events but have a server domain name as the diff --git a/drafts/human-id-rules.rst b/drafts/human-id-rules.rst index 914a9a42320..d0c2ee2caf0 100644 --- a/drafts/human-id-rules.rst +++ b/drafts/human-id-rules.rst @@ -13,9 +13,9 @@ phishing/spoofing of IDs, commonly known as a homograph attack. Web browers encountered this problem when International Domain Names were introduced. A variety of checks were put in place in order to protect users. If an address failed the check, the raw punycode would be displayed to -disambiguate the address. Similar checks are performed by home servers in +disambiguate the address. Similar checks are performed by homeservers in Matrix. However, Matrix does not use punycode representations, and so does not -show raw punycode on a failed check. Instead, home servers must outright reject +show raw punycode on a failed check. Instead, homeservers must outright reject these misleading IDs. Types of human-readable IDs @@ -48,12 +48,12 @@ Checks Rejecting --------- -- Home servers MUST reject room aliases which do not pass the check, both on +- Homeservers MUST reject room aliases which do not pass the check, both on GETs and PUTs. -- Home servers MUST reject user ID localparts which do not pass the check, both +- Homeservers MUST reject user ID localparts which do not pass the check, both on creation and on events. -- Any home server whose domain does not pass this check, MUST use their punycode - domain name instead of the IDN, to prevent other home servers rejecting you. +- Any homeserver whose domain does not pass this check, MUST use their punycode + domain name instead of the IDN, to prevent other homeservers rejecting you. - Error code is ``M_FAILED_HUMAN_ID_CHECK``. (generic enough for both failing due to homograph attacks, and failing due to including ``:`` s, etc) - Error message MAY go into further information about which characters were @@ -74,7 +74,7 @@ Other considerations - High security: Outright rejection of the ID at the point of creation / receiving event. Point of creation rejection is preferable to avoid the ID entering the system in the first place. However, malicious HSes can just - allow the ID. Hence, other home servers must reject them if they see them in + allow the ID. Hence, other homeservers must reject them if they see them in events. Client never sees the problem ID, provided the HS is correctly implemented. - High security decided; client doesn't need to worry about it, no additional diff --git a/drafts/model/presence.rst b/drafts/model/presence.rst index bd60cba0fcd..c2c88737bf7 100644 --- a/drafts/model/presence.rst +++ b/drafts/model/presence.rst @@ -5,14 +5,14 @@ A simple implementation of presence messaging has the ability to cause a large amount of Internet traffic relating to presence updates. In order to minimise the impact of such a feature, the following observations can be made: - * There is no point in a Home Server polling status for peers in a user's + * There is no point in a homeserver polling status for peers in a user's presence list if the user has no clients connected that care about it. * It is highly likely that most presence subscriptions will be symmetric - a given user watching another is likely to in turn be watched by that user. * It is likely that most subscription pairings will be between users who share - at least one Room in common, and so their Home Servers are actively + at least one Room in common, and so their homeservers are actively exchanging message PDUs or transactions relating to that Room. * Presence update messages do not need realtime guarantees. It is acceptable to @@ -25,7 +25,7 @@ promise to send them when required. Rather than actively polling for the current state all the time, HSes can rely on their relative stability to only push updates when required. -A Home Server should not rely on the longterm validity of this presence +A homeserver should not rely on the longterm validity of this presence information, however, as this would not cover such cases as a user's server crashing and thus failing to inform their peers that users it used to host are no longer available online. Therefore, each promise of future updates should @@ -33,7 +33,7 @@ carry with a timeout value (whether explicit in the message, or implicit as some defined default in the protocol), after which the receiving HS should consider the information potentially stale and request it again. -However, because of the likelihood that two home servers are exchanging messages +However, because of the likelihood that two homeservers are exchanging messages relating to chat traffic in a room common to both of them, the ongoing receipt of these messages can be taken by each server as an implicit notification that the sending server is still up and running, and therefore that no status changes @@ -98,7 +98,7 @@ The data model presented here puts the following requirements on the APIs: Client-Server ------------- -Requests that a client can make to its Home Server +Requests that a client can make to its homeserver * get/set current presence state Basic enumeration + ability to set a custom piece of text @@ -128,7 +128,7 @@ Requests that a client can make to its Home Server Server-Server ------------- -Requests that Home Servers make to others +Requests that homeservers make to others * request permission to add a user to presence list diff --git a/drafts/model/profiles.rst b/drafts/model/profiles.rst index 15f65feb962..f81e115eb2c 100644 --- a/drafts/model/profiles.rst +++ b/drafts/model/profiles.rst @@ -9,7 +9,7 @@ Overview ======== Internally within Synapse users are referred to by an opaque ID, which consists -of some opaque localpart combined with the domain name of their home server. +of some opaque localpart combined with the domain name of their homeserver. Obviously this does not yield a very nice user experience; users would like to see readable names for other users that are in some way meaningful to them. Additionally, users like to be able to publish "profile" details to inform other @@ -59,7 +59,7 @@ servers should be accounted for here.]] Visibility Permissions ====================== -A home server implementation could offer the ability to set permissions on +A homeserver implementation could offer the ability to set permissions on limited visibility of those fields. When another user requests access to the target user's profile, their own identity should form part of that request. The HS implementation can then decide which fields to make available to the @@ -130,12 +130,12 @@ namespace to allocate names into. It would also be nice from a user experience perspective if the profile that a given name links to can also declare that name as part of its metadata. Furthermore as a security and consistency perspective it would be nice if each -end (the directory server and the user's home server) check the validity of the +end (the directory server and the user's homeserver) check the validity of the mapping in some way. This needs investigation from a security perspective to ensure against spoofing. One such model may be that the user starts by declaring their intent to use a -given user name link to their home server, which then contacts the directory +given user name link to their homeserver, which then contacts the directory service. At some point later (maybe immediately for "public open FCFS servers", maybe after some kind of human intervention for verification) the DS decides to honour this link, and includes it in its served output. It should also tell the @@ -170,7 +170,7 @@ balancing choice on behalf of the user who would choose, or not, to make use of such a feature to publish their information. Additionally, unless some form of strong end-to-end user-based encryption is -used, a user of ACLs for information privacy has to trust other home servers not +used, a user of ACLs for information privacy has to trust other homeservers not to lie about the identify of the user requesting access to the Profile. @@ -182,7 +182,7 @@ The data model presented here puts the following requirements on the APIs: Client-Server ------------- -Requests that a client can make to its Home Server +Requests that a client can make to its homeserver * get/set my Display Name This should return/take a simple "text/plain" field @@ -207,7 +207,7 @@ TODO(paul): At some later stage we should consider the API for: Server-Server ------------- -Requests that Home Servers make to others +Requests that homeservers make to others * get a user's Display Name / Avatar @@ -221,7 +221,7 @@ Requests that Home Servers make to others Room Event PDU Types -------------------- -Events that are pushed from Home Servers to other Home Servers or clients. +Events that are pushed from homeservers to other homeservers or clients. * user Display Name change diff --git a/drafts/model/room-join-workflow.rst b/drafts/model/room-join-workflow.rst index c321a64fabf..ff469740c45 100644 --- a/drafts/model/room-join-workflow.rst +++ b/drafts/model/room-join-workflow.rst @@ -8,7 +8,7 @@ Discovery ========= To join a room, a user has to discover the room by some mechanism in order to -obtain the (opaque) Room ID and a candidate list of likely home servers that +obtain the (opaque) Room ID and a candidate list of likely homeservers that contain it. Sending an Invitation @@ -21,7 +21,7 @@ The inviter's HS sets the membership status of the invitee to "invited" in the "m.members" state key by sending a state update PDU. The HS then broadcasts this PDU among the existing members in the usual way. An invitation message is also sent to the invited user, containing the Room ID and the PDU ID of this -invitation state change and potentially a list of some other home servers to use +invitation state change and potentially a list of some other homeservers to use to accept the invite. The user's client can then choose to display it in some way to alert the user. @@ -34,7 +34,7 @@ Directory Service Alternatively, the user may discover the channel via a directory service; either by performing a name lookup, or some kind of browse or search acitivty. However -this is performed, the end result is that the user's home server requests the +this is performed, the end result is that the user's homeserver requests the Room ID and candidate list from the directory service. [[TODO(paul): At present, no API has been designed or described for this @@ -44,14 +44,14 @@ directory service]] Joining ======= -Once the ID and home servers are obtained, the user can then actually join the +Once the ID and homeservers are obtained, the user can then actually join the room. Accepting an Invite ------------------- If a user has received and accepted an invitation to join a room, the invitee's -home server can now send an invite acceptance message to a chosen candidate +homeserver can now send an invite acceptance message to a chosen candidate server from the list given in the invitation, citing also the PDU ID of the invitation as "proof" of their invite. (This is required as due to late message propagation it could be the case that the acceptance is received before the @@ -85,7 +85,7 @@ can instead post a "knock" message, which informs other members of the room that the would-be joiner wishes to become a member and sets their membership value to "knocked". If any of them wish to accept this, they can then send an invitation in the usual way described above. Knowing that the user has already knocked and -expressed an interest in joining, the invited user's home server should +expressed an interest in joining, the invited user's homeserver should immediately accept that invitation on the user's behalf, and go on to join the room in the usual way. diff --git a/drafts/model/rooms.rst b/drafts/model/rooms.rst index 07c5e71f91a..2f57f2bc4bf 100644 --- a/drafts/model/rooms.rst +++ b/drafts/model/rooms.rst @@ -18,19 +18,19 @@ users, and other management and miscellaneous metadata), and a message history. Room Identity and Naming ======================== -Rooms can be arbitrarily created by any user on any home server; at which point -the home server will sign the message that creates the channel, and the +Rooms can be arbitrarily created by any user on any homeserver; at which point +the homeserver will sign the message that creates the channel, and the fingerprint of this signature becomes the strong persistent identify of the -room. This now identifies the room to any home server in the network regardless +room. This now identifies the room to any homeserver in the network regardless of its original origin. This allows the identify of the room to outlive any particular server. Subject to appropriate permissions [to be discussed later], any current member of a room can invite others to join it, can post messages that become part of its history, and can change the persistent state of the room (including its current set of permissions). -Home servers can provide a directory service, allowing a lookup from a +Homeservers can provide a directory service, allowing a lookup from a convenient human-readable form of room label to a room ID. This mapping is -scoped to the particular home server domain and so simply represents that server +scoped to the particular homeserver domain and so simply represents that server administrator's opinion of what room should take that label; it does not have to be globally replicated and does not form part of the stored state of that room. @@ -156,7 +156,7 @@ m.public_history to be a member of the room. m.archive_servers - For "public" rooms with public history, gives a list of home servers that + For "public" rooms with public history, gives a list of homeservers that should be included in message distribution to the room, even if no users on that server are present. These ensure that a public room can still persist even if no users are currently members of it. This list should be consulted by @@ -179,7 +179,7 @@ m.topic Room Creation Templates ======================= -A client (or maybe home server?) could offer a few templates for the creation of +A client (or maybe homeserver?) could offer a few templates for the creation of new rooms. For example, for a simple private one-to-one chat the channel could assign the creator a power-level of 1, requiring a level of 1 to invite, and needing an invite before members can join. An invite is then sent to the other @@ -215,7 +215,7 @@ permissions to allow this direct messaging. Between any given pair of user IDs that wish to exchange private messages, there will exist a single shared Room, created lazily by either side. These rooms will -need a certain amount of special handling in both home servers and display on +need a certain amount of special handling in both homeservers and display on clients, but as much as possible should be treated by the lower layers of code the same as other rooms. @@ -226,7 +226,7 @@ clients should display these in a special way too as the room name is not important; instead it should distinguish them on the Display Name of the other party. -Home Servers will need a client-API option to request setting up a new user-user +Homeservers will need a client-API option to request setting up a new user-user chat room, which will then need special handling within the server. It will create a new room with the following @@ -260,7 +260,7 @@ history with each other simultaneously create a room and invite the other to it. This is called a "glare" situation. There are two possible ideas for how to resolve this: - * Each Home Server should persist the mapping of (user ID pair) to room ID, so + * Each homeserver should persist the mapping of (user ID pair) to room ID, so that duplicate requests can be suppressed. On receipt of a room creation request that the HS thinks there already exists a room for, the invitation to join can be rejected if: diff --git a/drafts/model/third-party-id.rst b/drafts/model/third-party-id.rst index 1f8138ddf7f..838a6799388 100644 --- a/drafts/model/third-party-id.rst +++ b/drafts/model/third-party-id.rst @@ -66,7 +66,7 @@ Privacy A User may publish the association between their phone number and Matrix User ID on the Identity Server without publishing the number in their Profile hosted on -their Home Server. +their homeserver. Identity Servers should refrain from publishing reverse mappings and should take steps, such as rate limiting, to prevent attackers enumerating the space of diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh index 48c01bdbdc6..41a652962e2 100755 --- a/scripts/generate-http-docs.sh +++ b/scripts/generate-http-docs.sh @@ -24,7 +24,7 @@ cat >tmp/http_apis < Application Service API +Homeserver -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pushing events @@ -101,13 +101,13 @@ events. Each list of events includes a transaction ID, which works as follows: :: Typical - HS ---> AS : Home server sends events with transaction ID T. + HS ---> AS : Homeserver sends events with transaction ID T. <--- : AS sends back 200 OK. AS ACK Lost - HS ---> AS : Home server sends events with transaction ID T. + HS ---> AS : Homeserver sends events with transaction ID T. <-/- : AS 200 OK is lost. - HS ---> AS : Home server retries with the same transaction ID of T. + HS ---> AS : Homeserver retries with the same transaction ID of T. <--- : AS sends back 200 OK. If the AS had processed these events already, it can NO-OP this request (and it knows if it is the same events based on the transaction ID). @@ -145,7 +145,7 @@ this request (e.g. to join a room alias). HTTP APIs +++++++++ -This contains application service APIs which are used by the home server. All +This contains application service APIs which are used by the homeserver. All application services MUST implement these APIs. These APIs are defined below. {{application_service_http_api}} @@ -158,7 +158,7 @@ Client-Server v2 API Extensions Application services can utilise a more powerful version of the client-server API by identifying itself as an application service to the -home server. +homeserver. Identity assertion ++++++++++++++++++ @@ -212,7 +212,7 @@ Server admin style permissions .. _sect:asapi-permissions: -The home server needs to give the application service *full control* over its +The homeserver needs to give the application service *full control* over its namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as create/delete any user in its namespace. No additional API changes need to be diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 67973c6b3eb..9edf7e5563b 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -147,12 +147,12 @@ User-Interactive Authentication API This section refers to API Version 2. Some API endpoints such as ``login`` or ``register`` require authentication that -interacts with the user. The home server may provide many different ways of +interacts with the user. The homeserver may provide many different ways of authenticating, such as user/password auth, login via a social network (OAuth2), login by confirming a token sent to their email address, etc. This specification -does not define how home servers should authorise their users but instead +does not define how homeservers should authorise their users but instead defines the standard interface which implementations should follow so that ANY -client can login to ANY home server. +client can login to ANY homeserver. The process takes the form of one or more stages, where at each stage the client submits a set of data for a given stage type and awaits a response from the @@ -168,9 +168,9 @@ more than one stage to implement n-factor auth. When all stages are complete, authentication is complete and the API call succeeds. To establish what flows a server supports for an endpoint, a client sends the request with no authentication. A request to an endpoint that uses User-Interactive -Authentication never succeeds without auth. Home Servers may allow requests that +Authentication never succeeds without auth. Homeservers may allow requests that don't require auth by offering a stage with only the ``m.login.dummy`` auth -type. The home server returns a response with HTTP status 401 and a JSON object +type. The homeserver returns a response with HTTP status 401 and a JSON object as follows:: { @@ -208,7 +208,7 @@ does this by resubmitting the same request with the the addition of an 'auth' key in the object that it submits. This dictionary contains a ``type`` key whose value is the name of the stage type that the client is attempting to complete. It must also contains a ``session`` key with the value of the session key given -by the home server, if one was given. It also contains other keys dependent on +by the homeserver, if one was given. It also contains other keys dependent on the stage type being attempted. For example, if the client is attempting to complete login type ``example.type.foo``, it might submit something like this:: @@ -222,7 +222,7 @@ complete login type ``example.type.foo``, it might submit something like this:: } } -If the home server deems the authentication attempt to be successful but still +If the homeserver deems the authentication attempt to be successful but still requires more stages to be completed, it returns HTTP status 401 along with the same object as when no authentication was attempted, with the addition of the ``completed`` key which is an array of stage type the client has completed @@ -246,7 +246,7 @@ successfully:: "session": "xxxxxx" } -If the home server decides the attempt was unsuccessful, it returns an error +If the homeserver decides the attempt was unsuccessful, it returns an error message in the standard format:: { @@ -258,7 +258,7 @@ Individual stages may require more than one request to complete, in which case the response will be as if the request was unauthenticated with the addition of any other keys as defined by the login type. -If the client has completed all stages of a flow, the home server performs the +If the client has completed all stages of a flow, the homeserver performs the API call and returns the result as normal. Some authentication types may be completed by means other than through the @@ -389,16 +389,16 @@ OAuth2-based ``uri``: Authorization Request URI OR service selection URI. Both contain an encoded ``redirect URI``. -The home server acts as a 'confidential' client for the purposes of OAuth2. If +The homeserver acts as a 'confidential' client for the purposes of OAuth2. If the uri is a ``service selection URI``, it MUST point to a webpage which prompts the user to choose which service to authorize with. On selection of a service, this MUST link through to an ``Authorization Request URI``. If there is only one -service which the home server accepts when logging in, this indirection can be +service which the homeserver accepts when logging in, this indirection can be skipped and the "uri" key can be the ``Authorization Request URI``. The client then visits the ``Authorization Request URI``, which then shows the OAuth2 Allow/Deny prompt. Hitting 'Allow' redirects to the ``redirect URI`` with -the auth code. Home servers can choose any path for the ``redirect URI``. Once +the auth code. Homeservers can choose any path for the ``redirect URI``. Once the OAuth flow has completed, the client retries the request with the session only, as above. @@ -412,7 +412,7 @@ Email-based (identity server) Prior to submitting this, the client should authenticate with an identity server. After authenticating, the session information should be submitted to -the home server. +the homeserver. To respond to this type, reply with an auth dict as follows:: @@ -450,12 +450,12 @@ Clients cannot be expected to be able to know how to process every single login type. If a client does not know how to handle a given login type, it can direct the user to a web browser with the URL of a fallback page which will allow the user to complete that login step out-of-band in their web browser. The URL it -should open is the Home Server base URL plus prefix, plus:: +should open is the homeserver base URL plus prefix, plus:: /auth//fallback/web?session= Where ``stage type`` is the type name of the stage it is attempting and -``session id`` is the ID of the session given by the home server. +``session id`` is the ID of the session given by the homeserver. This MUST return an HTML page which can perform this authentication stage. This page must attempt to call the JavaScript function ``window.onAuthDone`` when @@ -494,7 +494,7 @@ Request:: This API endpoint uses the User-Interactive Authentication API. An access token should be submitted to this endpoint if the client has an active session. The -Home Server may change the flows available depending on whether a valid access +homeserver may change the flows available depending on whether a valid access token is provided. The body of the POST request is a JSON object containing: @@ -505,7 +505,7 @@ new_password On success, an empty JSON object is returned. The error code M_NOT_FOUND is returned if the user authenticated with a third -party identifier but the Home Server could not find a matching account in its +party identifier but the homeserver could not find a matching account in its database. Adding Account Administrative Contact Information @@ -699,7 +699,7 @@ namespaced for each application and reduces the risk of clashes. Syncing ~~~~~~~ -Clients receive new events by "long-polling" the home server via the events API. +Clients receive new events by "long-polling" the homeserver via the events API. This involves specifying a timeout in the request which will hold open the HTTP connection for a short period of time waiting for new events, returning early if an event occurs. Only the events API supports long-polling. @@ -719,7 +719,7 @@ last request left off. Multiple events can be returned per long-poll. Do we ever support streaming requests? Why not websockets? When the client first logs in, they will need to initially synchronise with -their home server. This is achieved via the initial sync API described below. +their homeserver. This is achieved via the initial sync API described below. This API also returns an ``end`` token which can be used with the event stream. {{old_sync_http_api}} @@ -831,7 +831,7 @@ Rooms Creation ~~~~~~~~ -The home server will create an ``m.room.create`` event when a room is created, +The homeserver will create an ``m.room.create`` event when a room is created, which serves as the root of the event graph for this room. This event also has a ``creator`` key which contains the user ID of the room creator. It will also generate several other events in order to manage permissions in this room. This @@ -852,9 +852,9 @@ Room aliases Servers may host aliases for rooms with human-friendly names. Aliases take the form ``#friendlyname:server.name``. -As room aliases are scoped to a particular home server domain name, it is -likely that a home server will reject attempts to maintain aliases on other -domain names. This specification does not provide a way for home servers to +As room aliases are scoped to a particular homeserver domain name, it is +likely that a homeserver will reject attempts to maintain aliases on other +domain names. This specification does not provide a way for homeservers to send update requests to other servers. Rooms store a *partial* list of room aliases via the ``m.room.aliases`` state @@ -867,8 +867,8 @@ appears to have a room alias of ``#alias:example.com``, this SHOULD be checked to make sure that the room's ID matches the ``room_id`` returned from the request. -Home servers can respond to resolve requests for aliases on other domains than -their own by using the federation API to ask other domain name home servers. +Homeservers can respond to resolve requests for aliases on other domains than +their own by using the federation API to ask other domain name homeservers. {{directory_http_api}} @@ -985,10 +985,10 @@ values. This change is conveyed using two separate mechanisms: values of the ``displayname`` and ``avatar_url`` keys, in addition to the required ``presence`` key containing the current presence state of the user. -Both of these should be done automatically by the home server when a user +Both of these should be done automatically by the homeserver when a user successfully changes their display name or avatar URL fields. -Additionally, when home servers emit room membership events for their own +Additionally, when homeservers emit room membership events for their own users, they should include the display name and avatar URL fields in these events so that clients already have these details to hand, and do not have to perform extra round trips to query it. @@ -998,7 +998,7 @@ Security Rate limiting ~~~~~~~~~~~~~ -Home servers SHOULD implement rate limiting to reduce the risk of being +Homeservers SHOULD implement rate limiting to reduce the risk of being overloaded. If a request is refused due to rate limiting, it should return a standard error response of the form:: diff --git a/specification/intro.rst b/specification/intro.rst index eb6f98fc321..ced721fad6f 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -140,7 +140,7 @@ a long-lived GET request. | V | V +------------------+ +------------------+ | |---------( HTTPS )--------->| | - | Home Server | | Home Server | + | homeserver | | homeserver | | |<--------( HTTPS )----------| | +------------------+ Server-Server API +------------------+ History Synchronisation @@ -227,7 +227,7 @@ They are case-sensitive. The following conceptual diagram shows an | | V | +------------------+ +------------------+ - | Home Server | | Home Server | + | homeserver | | homeserver | | matrix.org | | domain.com | +------------------+ +------------------+ | ^ diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index 3602d105bb0..4eddaeeedcf 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -61,7 +61,7 @@ recommended. Server behaviour ---------------- -Each user's home server stores a "presence list" per user. Once a user accepts +Each user's homeserver stores a "presence list" per user. Once a user accepts a presence list, both user's HSes must track the subscription. Propagating profile information @@ -73,7 +73,7 @@ automatic propagation event to occur, informing likely-interested parties of the new values. One of these change mechanisms SHOULD be via ``m.presence`` events. These events should set ``displayname`` and ``avatar_url`` to the new values along with the presence-specific keys. This SHOULD be done automatically by the -home server when a user successfully changes their display name or avatar URL. +homeserver when a user successfully changes their display name or avatar URL. .. admonition:: Rationale diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 893e948154c..de45bfe68e0 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -11,7 +11,7 @@ Push Notifications | | | | +-------------------+ | +----------------+ | | +---------------+ | | | | | | | | | | | - | Matrix Home Server+-----> Push Gateway +------> Push Provider | | + | Matrix homeserver+-----> Push Gateway +------> Push Provider | | | | | | | | | | | | +-^-----------------+ | +----------------+ | | +----+----------+ | | | | | | | diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 012c2ac9693..ba6b7d35077 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1,8 +1,8 @@ Federation API ============== -Matrix home servers use the Federation APIs (also known as server-server APIs) -to communicate with each other. Home servers use these APIs to push messages to +Matrix homeservers use the Federation APIs (also known as server-server APIs) +to communicate with each other. Homeservers use these APIs to push messages to each other in real-time, to request historic messages from each other, and to query profile and presence information about users on each other's servers. @@ -11,10 +11,10 @@ servers. These HTTPS requests are strongly authenticated using public key signatures at the TLS transport layer and using public key signatures in HTTP Authorization headers at the HTTP layer. -There are three main kinds of communication that occur between home servers: +There are three main kinds of communication that occur between homeservers: Persisted Data Units (PDUs): - These events are broadcast from one home server to any others that have + These events are broadcast from one homeserver to any others that have joined the same room (identified by Room ID). They are persisted in long-term storage and record the history of messages and state for a room. @@ -25,9 +25,9 @@ Persisted Data Units (PDUs): deliver them through third-party servers. Ephemeral Data Units (EDUs): - These events are pushed between pairs of home servers. They are not + These events are pushed between pairs of homeservers. They are not persisted and are not part of the history of a room, nor does the - receiving home server have to reply to them. + receiving homeserver have to reply to them. Queries: These are single request/response interactions between a given pair of @@ -38,7 +38,7 @@ Queries: EDUs and PDUs are further wrapped in an envelope called a Transaction, which is -transferred from the origin to the destination home server using an HTTPS PUT +transferred from the origin to the destination homeserver using an HTTPS PUT request. .. contents:: Table of Contents @@ -50,7 +50,7 @@ Server Discovery Resolving Server Names ~~~~~~~~~~~~~~~~~~~~~~ -Each matrix home server is identified by a server name consisting of a DNS name +Each matrix homeserver is identified by a server name consisting of a DNS name and an optional TLS port. .. code:: @@ -67,7 +67,7 @@ is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV record for the DNS name. If this record does not exist then the server is discovered by looking up an AAAA or A record on the DNS name and taking the default fallback port number of 8448. -Home servers may use SRV records to load balance requests between multiple TLS +Homeservers may use SRV records to load balance requests between multiple TLS endpoints or to failover to another endpoint if an endpoint fails. Retrieving Server Keys @@ -76,8 +76,8 @@ Retrieving Server Keys Version 2 +++++++++ -Each home server publishes its public keys under ``/_matrix/key/v2/server/``. -Home servers query for keys by either getting ``/_matrix/key/v2/server/`` +Each homeserver publishes its public keys under ``/_matrix/key/v2/server/``. +Homeservers query for keys by either getting ``/_matrix/key/v2/server/`` directly or by querying an intermediate notary server using a ``/_matrix/key/v2/query`` API. Intermediate notary servers query the ``/_matrix/key/v2/server/`` API on behalf of another server and sign the @@ -95,7 +95,7 @@ server by querying other servers. Publishing Keys ^^^^^^^^^^^^^^^ -Home servers publish the allowed TLS fingerprints and signing keys in a JSON +Homeservers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the server and for signing events. It contains a list of ``old_verify_keys`` @@ -114,7 +114,7 @@ certificate currently in use by the server. These fingerprints are valid until the millisecond POSIX timestamp in ``valid_until_ts``. The ``verify_keys`` can be used to sign requests and events made by the server -until the millisecond POSIX timestamp in ``valid_until_ts``. If a Home Server +until the millisecond POSIX timestamp in ``valid_until_ts``. If a homeserver receives an event with a ``origin_server_ts`` after the ``valid_until_ts`` then it should request that ``key_id`` for the originating server to check whether the key has expired. @@ -136,8 +136,8 @@ events sent by that server can still be checked. ==================== =================== ====================================== Key Type Description ==================== =================== ====================================== -``server_name`` String DNS name of the home server. -``verify_keys`` Object Public keys of the home server for +``server_name`` String DNS name of the homeserver. +``verify_keys`` Object Public keys of the homeserver for verifying digital signatures. ``old_verify_keys`` Object The public keys that the server used to use and when it stopped using them. @@ -238,14 +238,14 @@ Version 1 Version 1 of key distribution is obsolete -Home servers publish their TLS certificates and signing keys in a JSON object +Homeservers publish their TLS certificates and signing keys in a JSON object at ``/_matrix/key/v1``. ==================== =================== ====================================== Key Type Description ==================== =================== ====================================== -``server_name`` String DNS name of the home server. -``verify_keys`` Object Public keys of the home server for +``server_name`` String DNS name of the homeserver. +``verify_keys`` Object Public keys of the homeserver for verifying digital signatures. ``signatures`` Object Digital signatures for this object signed using the ``verify_keys``. @@ -278,9 +278,9 @@ Transactions .. WARNING:: This section may be misleading or inaccurate. -The transfer of EDUs and PDUs between home servers is performed by an exchange +The transfer of EDUs and PDUs between homeservers is performed by an exchange of Transaction messages, which are encoded as JSON objects, passed over an HTTP -PUT request. A Transaction is meaningful only to the pair of home servers that +PUT request. A Transaction is meaningful only to the pair of homeservers that exchanged it; they are not globally-meaningful. Each transaction has: @@ -445,7 +445,7 @@ EDUs EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of "previous" IDs. The only mandatory fields for these are the type, origin and -destination home server names, and the actual nested content. +destination homeserver names, and the actual nested content. ======================== ============ ========================================= Key Type Description @@ -531,7 +531,7 @@ To make a query:: Query args: as specified by the individual query types Response: JSON encoding of a response object -Performs a single query request on the receiving home server. The Query Type +Performs a single query request on the receiving homeserver. The Query Type part of the path specifies the kind of query being made, and its query arguments have a meaning specific to that kind of query. The response is a JSON-encoded object whose meaning also depends on the kind of query. @@ -791,9 +791,9 @@ because HTTP services like Matrix are often deployed behind load balancers that handle the TLS and these load balancers make it difficult to check TLS client certificates. -A home server may provide a TLS client certificate and the receiving home server +A homeserver may provide a TLS client certificate and the receiving homeserver may check that the client certificate matches the certificate of the origin -home server. +homeserver. Server-Server Authorization --------------------------- @@ -905,7 +905,7 @@ Querying profile information:: If the query contains the optional ``field`` key, it should give the name of a result field. If such is present, then the result should contain only a field of that name, with no others present. If not, the result should contain as much -of the user's profile as the home server has available and can make public. +of the user's profile as the homeserver has available and can make public. Directory --------- diff --git a/supporting-docs/examples/application-services.rst b/supporting-docs/examples/application-services.rst index bde4531bf5e..fae3613d797 100644 --- a/supporting-docs/examples/application-services.rst +++ b/supporting-docs/examples/application-services.rst @@ -6,15 +6,15 @@ This file contains examples of some application service IRC Bridge ---------- Pre-conditions: - - Server admin stores the AS token "T_a" on the home server. - - Home server has a token "T_h". - - Home server has the domain "hsdomain.com" + - Server admin stores the AS token "T_a" on the homeserver. + - Homeserver has a token "T_h". + - Homeserver has the domain "hsdomain.com" 1. Application service registration :: - AS -> HS: Registers itself with the home server + AS -> HS: Registers itself with the homeserver POST /register { url: "https://someapp.com/matrix", diff --git a/supporting-docs/guides/2015-08-10-client-server.rst b/supporting-docs/guides/2015-08-10-client-server.rst index 2800584ca02..fe6b464ac9a 100644 --- a/supporting-docs/guides/2015-08-10-client-server.rst +++ b/supporting-docs/guides/2015-08-10-client-server.rst @@ -19,11 +19,11 @@ How to use the client-server API The git version of this document is {% project_version %} This guide focuses on how the client-server APIs *provided by the reference -home server* can be used. Since this is specific to a home server +homeserver* can be used. Since this is specific to a homeserver implementation, there may be variations in relation to registering/logging in which are not covered in extensive detail in this guide. -If you haven't already, get a home server up and running on +If you haven't already, get a homeserver up and running on ``http://localhost:8008``. @@ -56,8 +56,8 @@ if you forget the ``access_token``. Implementation note: The matrix specification does not enforce how users register with a server. It just specifies the URL path and absolute minimum -keys. The reference home server uses a username/password to authenticate user, -but other home servers may use different methods. This is why you need to +keys. The reference homeserver uses a username/password to authenticate user, +but other homeservers may use different methods. This is why you need to specify the ``type`` of method. Login @@ -82,13 +82,13 @@ The aim when logging in is to get an access token for your existing user ID:: "user_id": "@example:localhost" } -Implementation note: Different home servers may implement different methods for +Implementation note: Different homeservers may implement different methods for logging in to an existing account. In order to check that you know how to login -to this home server, you must perform a ``GET`` first and make sure you +to this homeserver, you must perform a ``GET`` first and make sure you recognise the login type. If you do not know how to login, you can ``GET /login/fallback`` which will return a basic webpage which you can use to -login. The reference home server implementation support username/password login, -but other home servers may support different login methods (e.g. OAuth2). +login. The reference homeserver implementation support username/password login, +but other homeservers may support different login methods (e.g. OAuth2). Communicating diff --git a/supporting-docs/guides/2015-08-21-application_services.md b/supporting-docs/guides/2015-08-21-application_services.md index dd532d42347..e034e154b53 100644 --- a/supporting-docs/guides/2015-08-21-application_services.md +++ b/supporting-docs/guides/2015-08-21-application_services.md @@ -6,7 +6,7 @@ categories: guides # Application services -Application services are distinct modules which which sit alongside a home server providing arbitrary extensible functionality decoupled from the home server implementation. Just like the rest of Matrix, they communicate via HTTP using JSON. Application services function in a very similar way to traditional clients, but they are given much more power than a normal client. They can reserve entire namespaces of room aliases and user IDs for their own purposes. They can silently monitor events in rooms, or any events directed at any user ID. This power allows application services to have extremely useful abilities which overall enhance the end user experience. +Application services are distinct modules which which sit alongside a homeserver providing arbitrary extensible functionality decoupled from the homeserver implementation. Just like the rest of Matrix, they communicate via HTTP using JSON. Application services function in a very similar way to traditional clients, but they are given much more power than a normal client. They can reserve entire namespaces of room aliases and user IDs for their own purposes. They can silently monitor events in rooms, or any events directed at any user ID. This power allows application services to have extremely useful abilities which overall enhance the end user experience. | @@ -47,17 +47,17 @@ At present, the IRC application service is in beta, and is being run on #matrix | # What Application services can do for you -Application services have enormous potential for creating new and exciting ways to transform and enhance the core Matrix protocol. For example, you could aggregate information from multiple rooms into a summary room, or create throwaway virtual user accounts to proxy messages for a fixed user ID on-the-fly. As you may expect, all of this power assumes a high degree of trust between application services and home servers. Only home server admins can allow an application service to link up with their home server, and the application service is in no way federated to other home servers. You can think of application services as additional logic on the home server itself, without messing around with the book-keeping that home servers have to do. This makes adding useful functionality very easy. +Application services have enormous potential for creating new and exciting ways to transform and enhance the core Matrix protocol. For example, you could aggregate information from multiple rooms into a summary room, or create throwaway virtual user accounts to proxy messages for a fixed user ID on-the-fly. As you may expect, all of this power assumes a high degree of trust between application services and homeservers. Only homeserver admins can allow an application service to link up with their homeserver, and the application service is in no way federated to other homeservers. You can think of application services as additional logic on the homeserver itself, without messing around with the book-keeping that homeservers have to do. This makes adding useful functionality very easy. | ### Example -The application service (AS) API itself uses webhooks to communicate from the home server to the AS: +The application service (AS) API itself uses webhooks to communicate from the homeserver to the AS: -- Room Alias Query API : The home server hits a URL on your application server to see if a room alias exists. -- User Query API : The home server hits a URL on your application server to see if a user ID exists. -- Push API : The home server hits a URL on your application server to notify you of new events for your users and rooms. +- Room Alias Query API : The homeserver hits a URL on your application server to see if a room alias exists. +- User Query API : The homeserver hits a URL on your application server to see if a user ID exists. +- Push API : The homeserver hits a URL on your application server to notify you of new events for your users and rooms. A very basic application service may want to log all messages in rooms which have an alias starting with "#logged_" (side note: logging won't work if these rooms are using end-to-end encryption). @@ -85,7 +85,7 @@ Set your new application service running on port 5000 with: python app_service.py -The home server needs to know that the application service exists before it will send requests to it. This is done via a registration YAML file which is specified in Synapse's main config file e.g. homeserver.yaml. The server admin needs to add the application service registration configuration file as an entry to this file. +The homeserver needs to know that the application service exists before it will send requests to it. This is done via a registration YAML file which is specified in Synapse's main config file e.g. homeserver.yaml. The server admin needs to add the application service registration configuration file as an entry to this file. # homeserver.yaml app_service_config_files: @@ -115,7 +115,7 @@ NB: Note the "-" at the start; this indicates a list element. The registration f - exclusive: false regex: "#logged_.*" -**You will need to restart the home server after editing the config file before it will take effect.** +**You will need to restart the homeserver after editing the config file before it will take effect.** | @@ -138,6 +138,6 @@ This makes the application service lazily create a room with the requested alias | -Application services are powerful components which extend the functionality of home servers, but they are limited. They can only ever function in a "passive" way. For example, you cannot implement an application service which censors swear words in rooms, because there is no way to prevent the event from being sent. Aside from the fact that censoring will not work when using end-to-end encryption, all federated home servers would also need to reject the event in order to stop developing an inconsistent event graph. To "actively" monitor events, another component called a "Policy Server" is required, which is beyond the scope of this post.  Also, Application Services can result in a performance bottleneck, as all events on the homeserver must be ordered and sent to the registered application services.  If you are bridging huge amounts of traffic, you may be better off having your bridge directly talk the Server-Server federation API rather than the simpler Application Service API. +Application services are powerful components which extend the functionality of homeservers, but they are limited. They can only ever function in a "passive" way. For example, you cannot implement an application service which censors swear words in rooms, because there is no way to prevent the event from being sent. Aside from the fact that censoring will not work when using end-to-end encryption, all federated homeservers would also need to reject the event in order to stop developing an inconsistent event graph. To "actively" monitor events, another component called a "Policy Server" is required, which is beyond the scope of this post.  Also, Application Services can result in a performance bottleneck, as all events on the homeserver must be ordered and sent to the registered application services.  If you are bridging huge amounts of traffic, you may be better off having your bridge directly talk the Server-Server federation API rather than the simpler Application Service API. I hope this demonstrates how easy it is to create an application service, along with a few ideas of the kinds of things you can do with them. Obvious uses include build protocol bridges, search engines, invisible bots, etc. For more information on the AS HTTP API, check out the new Application Service API section in the spec, or the raw drafts and spec in https://github.com/matrix-org/matrix-doc/. diff --git a/supporting-docs/howtos/client-server.rst b/supporting-docs/howtos/client-server.rst index 3bed5a9f418..e5a85d41b26 100644 --- a/supporting-docs/howtos/client-server.rst +++ b/supporting-docs/howtos/client-server.rst @@ -11,11 +11,11 @@ How to use the client-server API The git version of this document is ``{{git_version}}`` This guide focuses on how the client-server APIs *provided by the reference -home server* can be used. Since this is specific to a home server +homeserver* can be used. Since this is specific to a homeserver implementation, there may be variations in relation to registering/logging in which are not covered in extensive detail in this guide. -If you haven't already, get a home server up and running on +If you haven't already, get a homeserver up and running on ``http://localhost:8008``. @@ -48,8 +48,8 @@ if you forget the ``access_token``. Implementation note: The matrix specification does not enforce how users register with a server. It just specifies the URL path and absolute minimum -keys. The reference home server uses a username/password to authenticate user, -but other home servers may use different methods. This is why you need to +keys. The reference homeserver uses a username/password to authenticate user, +but other homeservers may use different methods. This is why you need to specify the ``type`` of method. Login @@ -74,13 +74,13 @@ The aim when logging in is to get an access token for your existing user ID:: "user_id": "@example:localhost" } -Implementation note: Different home servers may implement different methods for +Implementation note: Different homeservers may implement different methods for logging in to an existing account. In order to check that you know how to login -to this home server, you must perform a ``GET`` first and make sure you +to this homeserver, you must perform a ``GET`` first and make sure you recognise the login type. If you do not know how to login, you can ``GET /login/fallback`` which will return a basic webpage which you can use to -login. The reference home server implementation support username/password login, -but other home servers may support different login methods (e.g. OAuth2). +login. The reference homeserver implementation support username/password login, +but other homeservers may support different login methods (e.g. OAuth2). Communicating diff --git a/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.html b/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.html index 088ff7ac0f1..b7e874c2c60 100644 --- a/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.html +++ b/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.html @@ -1,5 +1,5 @@
-

This room creation / message sending demo requires a home server to be running on http://localhost:8008

+

This room creation / message sending demo requires a homeserver to be running on http://localhost:8008

diff --git a/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.js b/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.js index 9c346e2f64f..c16395acb23 100644 --- a/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.js +++ b/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.js @@ -19,7 +19,7 @@ $('.login').live('click', function() { showLoggedIn(data); }, error: function(err) { - var errMsg = "To try this, you need a home server running!"; + var errMsg = "To try this, you need a homeserver running!"; var errJson = $.parseJSON(err.responseText); if (errJson) { errMsg = JSON.stringify(errJson); diff --git a/supporting-docs/howtos/jsfiddles/event_stream/demo.html b/supporting-docs/howtos/jsfiddles/event_stream/demo.html index 7657780d28d..3de7b0813da 100644 --- a/supporting-docs/howtos/jsfiddles/event_stream/demo.html +++ b/supporting-docs/howtos/jsfiddles/event_stream/demo.html @@ -1,5 +1,5 @@
-

This event stream demo requires a home server to be running on http://localhost:8008

+

This event stream demo requires a homeserver to be running on http://localhost:8008

diff --git a/supporting-docs/howtos/jsfiddles/event_stream/demo.js b/supporting-docs/howtos/jsfiddles/event_stream/demo.js index acba8391fa3..65b118d27c1 100644 --- a/supporting-docs/howtos/jsfiddles/event_stream/demo.js +++ b/supporting-docs/howtos/jsfiddles/event_stream/demo.js @@ -58,7 +58,7 @@ $('.login').live('click', function() { showLoggedIn(data); }, error: function(err) { - var errMsg = "To try this, you need a home server running!"; + var errMsg = "To try this, you need a homeserver running!"; var errJson = $.parseJSON(err.responseText); if (errJson) { errMsg = JSON.stringify(errJson); diff --git a/supporting-docs/howtos/jsfiddles/example_app/demo.html b/supporting-docs/howtos/jsfiddles/example_app/demo.html index 7a9dffddd0f..a183f61db36 100644 --- a/supporting-docs/howtos/jsfiddles/example_app/demo.html +++ b/supporting-docs/howtos/jsfiddles/example_app/demo.html @@ -1,5 +1,5 @@