Merge commit 'a466b6797' into anoa/dinsic_release_1_21_x

* commit 'a466b6797':
  Reduce run-times of tests by advancing the reactor less (#7757)
  Update debian systemd service to use Type=notify (#8169)
  Remove remaining is_guest argument uses from get_room_data calls (#8181)
  Do not propagate typing notifications from shadow-banned users. (#8176)
  Remove unused parameter from, and add safeguard in, get_room_data (#8174)
  Add required Debian dependencies to allow docker builds on the arm platform (#8144)
  Allow running mypy directly. (#8175)
  Update the test federation client to handle streaming responses (#8130)
  Do not propagate profile changes of shadow-banned users into rooms. (#8157)
  Make SlavedIdTracker.advance have same interface as MultiWriterIDGenerator (#8171)
  Convert simple_select_one and simple_select_one_onecol to async (#8162)

CHANGES.md

@@ -1,3 +1,17 @@
+For the next release
+====================
+
+Removal warning
+---------------
+
+Some older clients used a
+[disallowed character](https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-register-email-requesttoken)
+(`:`) in the `client_secret` parameter of various endpoints. The incorrect
+behaviour was allowed for backwards compatibility, but is now being removed
+from Synapse as most users have updated their client. Further context can be
+found at [\#6766](https://github.com/matrix-org/synapse/issues/6766).
+
+
 Synapse 1.19.1 (2020-08-27)
 ===========================
 

changelog.d/7757.misc

@@ -0,0 +1 @@
+Reduce run times of some unit tests by advancing the reactor a fewer number of times.

changelog.d/8130.misc

@@ -0,0 +1 @@
+Update the test federation client to handle streaming responses.

changelog.d/8144.docker

@@ -0,0 +1 @@
+Fix builds of the Docker image on non-x86 platforms.

changelog.d/8157.feature

@@ -0,0 +1 @@
+Add support for shadow-banning users (ignoring any message send requests).

changelog.d/8162.misc

@@ -0,0 +1 @@
+ Convert various parts of the codebase to async/await.

changelog.d/8171.misc

@@ -0,0 +1 @@
+Make `SlavedIdTracker.advance` have the same interface as `MultiWriterIDGenerator`.

changelog.d/8174.misc

@@ -0,0 +1 @@
+Remove unused `is_guest` parameter from, and add safeguard to, `MessageHandler.get_room_data`.

changelog.d/8175.misc

@@ -0,0 +1 @@
+Standardize the mypy configuration.

changelog.d/8176.feature

@@ -0,0 +1 @@
+Add support for shadow-banning users (ignoring any message send requests).

changelog.d/8181.misc

@@ -0,0 +1 @@
+Remove unused `is_guest` parameter from, and add safeguard to, `MessageHandler.get_room_data`.

debian/changelog

@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.19.0ubuntu1) UNRELEASED; urgency=medium
+
+  * Use Type=notify in systemd service
+
+ -- Dexter Chua <[email protected]>  Wed, 26 Aug 2020 12:41:36 +0000
+
 matrix-synapse-py3 (1.19.1) stable; urgency=medium
 
   * New synapse release 1.19.1.

debian/matrix-synapse.service

@@ -2,7 +2,7 @@
 Description=Synapse Matrix homeserver
 
 [Service]
-Type=simple
+Type=notify
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

docker/Dockerfile

@@ -19,11 +19,16 @@ ARG PYTHON_VERSION=3.7
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
-
-
 RUN apt-get update && apt-get install -y \
     build-essential \
+    libffi-dev \
+    libjpeg-dev \
     libpq-dev \
+    libssl-dev \
+    libwebp-dev \
+    libxml++2.6-dev \
+    libxslt1-dev \
+    zlib1g-dev \
  && rm -rf /var/lib/apt/lists/*
 
 # Build dependencies that are not available as wheels, to speed up rebuilds
@@ -56,9 +61,11 @@ FROM docker.io/python:${PYTHON_VERSION}-slim
 
 RUN apt-get update && apt-get install -y \
     curl \
+    gosu \
+    libjpeg62-turbo \
     libpq5 \
+    libwebp6 \
     xmlsec1 \
-    gosu \
  && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /install /usr/local

mypy.ini

@@ -6,6 +6,55 @@ check_untyped_defs = True
 show_error_codes = True
 show_traceback = True
 mypy_path = stubs
+files =
+  synapse/api,
+  synapse/appservice,
+  synapse/config,
+  synapse/event_auth.py,
+  synapse/events/builder.py,
+  synapse/events/spamcheck.py,
+  synapse/federation,
+  synapse/handlers/auth.py,
+  synapse/handlers/cas_handler.py,
+  synapse/handlers/directory.py,
+  synapse/handlers/federation.py,
+  synapse/handlers/identity.py,
+  synapse/handlers/message.py,
+  synapse/handlers/oidc_handler.py,
+  synapse/handlers/presence.py,
+  synapse/handlers/room.py,
+  synapse/handlers/room_member.py,
+  synapse/handlers/room_member_worker.py,
+  synapse/handlers/saml_handler.py,
+  synapse/handlers/sync.py,
+  synapse/handlers/ui_auth,
+  synapse/http/server.py,
+  synapse/http/site.py,
+  synapse/logging/,
+  synapse/metrics,
+  synapse/module_api,
+  synapse/notifier.py,
+  synapse/push/pusherpool.py,
+  synapse/push/push_rule_evaluator.py,
+  synapse/replication,
+  synapse/rest,
+  synapse/server.py,
+  synapse/server_notices,
+  synapse/spam_checker_api,
+  synapse/state,
+  synapse/storage/databases/main/ui_auth.py,
+  synapse/storage/database.py,
+  synapse/storage/engines,
+  synapse/storage/state.py,
+  synapse/storage/util,
+  synapse/streams,
+  synapse/types.py,
+  synapse/util/caches/stream_change_cache.py,
+  synapse/util/metrics.py,
+  tests/replication,
+  tests/test_utils,
+  tests/rest/client/v2_alpha/test_auth.py,
+  tests/util/test_stream_change_cache.py
 
 [mypy-pymacaroons.*]
 ignore_missing_imports = True

scripts-dev/federation_client.py

@@ -21,10 +21,12 @@
 import base64
 import json
 import sys
+from typing import Any, Optional
 from urllib import parse as urlparse
 
 import nacl.signing
 import requests
+import signedjson.types
 import srvlookup
 import yaml
 from requests.adapters import HTTPAdapter
@@ -69,7 +71,9 @@ def encode_canonical_json(value):
     ).encode("UTF-8")
 
 
-def sign_json(json_object, signing_key, signing_name):
+def sign_json(
+    json_object: Any, signing_key: signedjson.types.SigningKey, signing_name: str
+) -> Any:
     signatures = json_object.pop("signatures", {})
     unsigned = json_object.pop("unsigned", None)
 
@@ -122,7 +126,14 @@ def read_signing_keys(stream):
     return keys
 
 
-def request_json(method, origin_name, origin_key, destination, path, content):
+def request(
+    method: Optional[str],
+    origin_name: str,
+    origin_key: signedjson.types.SigningKey,
+    destination: str,
+    path: str,
+    content: Optional[str],
+) -> requests.Response:
     if method is None:
         if content is None:
             method = "GET"
@@ -159,11 +170,14 @@ def request_json(method, origin_name, origin_key, destination, path, content):
     if method == "POST":
         headers["Content-Type"] = "application/json"
 
-    result = s.request(
-        method=method, url=dest, headers=headers, verify=False, data=content
+    return s.request(
+        method=method,
+        url=dest,
+        headers=headers,
+        verify=False,
+        data=content,
+        stream=True,
     )
-    sys.stderr.write("Status Code: %d\n" % (result.status_code,))
-    return result.json()
 
 
 def main():
@@ -222,7 +236,7 @@ def main():
     with open(args.signing_key_path) as f:
         key = read_signing_keys(f)[0]
 
-    result = request_json(
+    result = request(
         args.method,
         args.server_name,
         key,
@@ -231,7 +245,12 @@ def main():
         content=args.body,
     )
 
-    json.dump(result, sys.stdout)
+    sys.stderr.write("Status Code: %d\n" % (result.status_code,))
+
+    for chunk in result.iter_content():
+        # we write raw utf8 to stdout.
+        sys.stdout.buffer.write(chunk)
+
     print("")
 
 

synapse/handlers/message.py

@@ -96,12 +96,7 @@ def __init__(self, hs):
             )
 
     async def get_room_data(
-        self,
-        user_id: str,
-        room_id: str,
-        event_type: str,
-        state_key: str,
-        is_guest: bool,
+        self, user_id: str, room_id: str, event_type: str, state_key: str,
     ) -> dict:
         """ Get data from a room.
 
@@ -110,11 +105,10 @@ async def get_room_data(
             room_id
             event_type
             state_key
-            is_guest
         Returns:
             The path data content.
         Raises:
-            SynapseError if something went wrong.
+            SynapseError or AuthError if the user is not in the room
         """
         (
             membership,
@@ -131,6 +125,16 @@ async def get_room_data(
                 [membership_event_id], StateFilter.from_types([key])
             )
             data = room_state[membership_event_id].get(key)
+        else:
+            # check_user_in_room_or_world_readable, if it doesn't raise an AuthError, should
+            # only ever return a Membership.JOIN/LEAVE object
+            #
+            # Safeguard in case it returned something else
+            logger.error(
+                "Attempted to retrieve data from a room for a user that has never been in it. "
+                "This should not have happened."
+            )
+            raise SynapseError(403, "User not in room", errcode=Codes.FORBIDDEN)
 
         return data