Merge tag 'v1.33.2' into babolivier/dinsic_1.41.0

Synapse 1.33.2 (2021-05-11)
===========================

Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild.

Security advisory
-----------------

This release fixes a denial of service attack ([CVE-2021-29471](GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade.

Internal Changes
----------------

- Unpin attrs dependency. ([\#9946](matrix-org/synapse#9946))

CHANGES.md

@@ -1,3 +1,28 @@
+Synapse 1.33.2 (2021-05-11)
+===========================
+
+Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild.
+
+Security advisory
+-----------------
+
+This release fixes a denial of service attack ([CVE-2021-29471](https://github.com/matrix-org/synapse/security/advisories/GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade.
+
+Internal Changes
+----------------
+
+- Unpin attrs dependency. ([\#9946](https://github.com/matrix-org/synapse/issues/9946))
+
+
+Synapse 1.33.1 (2021-05-06)
+===========================
+
+Bugfixes
+--------
+
+- Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. ([\#9937](https://github.com/matrix-org/synapse/issues/9937))
+
+
 Synapse 1.33.0 (2021-05-05)
 ===========================
 

debian/changelog

@@ -1,3 +1,15 @@
+matrix-synapse-py3 (1.33.2) stable; urgency=medium
+
+  * New synapse release 1.33.2.
+
+ -- Synapse Packaging team <[email protected]>  Tue, 11 May 2021 11:17:59 +0100
+
+matrix-synapse-py3 (1.33.1) stable; urgency=medium
+
+  * New synapse release 1.33.1.
+
+ -- Synapse Packaging team <[email protected]>  Thu, 06 May 2021 14:06:33 +0100
+
 matrix-synapse-py3 (1.33.0) stable; urgency=medium
 
   * New synapse release 1.33.0.

synapse/__init__.py

@@ -47,7 +47,7 @@
 except ImportError:
     pass
 
-__version__ = "1.33.0"
+__version__ = "1.33.2"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/config/tls.py

@@ -17,7 +17,7 @@
 import warnings
 from datetime import datetime
 from hashlib import sha256
-from typing import List, Optional
+from typing import List, Optional, Pattern
 
 from unpaddedbase64 import encode_base64
 
@@ -124,7 +124,7 @@ def read_config(self, config: dict, config_dir_path: str, **kwargs):
             fed_whitelist_entries = []
 
         # Support globs (*) in whitelist values
-        self.federation_certificate_verification_whitelist = []  # type: List[str]
+        self.federation_certificate_verification_whitelist = []  # type: List[Pattern]
         for entry in fed_whitelist_entries:
             try:
                 entry_regex = glob_to_regex(entry.encode("ascii").decode("ascii"))

synapse/push/push_rule_evaluator.py

@@ -19,6 +19,7 @@
 
 from synapse.events import EventBase
 from synapse.types import UserID
+from synapse.util import glob_to_regex, re_word_boundary
 from synapse.util.caches.lrucache import LruCache
 
 logger = logging.getLogger(__name__)
@@ -183,7 +184,7 @@ def _contains_display_name(self, display_name: str) -> bool:
         r = regex_cache.get((display_name, False, True), None)
         if not r:
             r1 = re.escape(display_name)
-            r1 = _re_word_boundary(r1)
+            r1 = re_word_boundary(r1)
             r = re.compile(r1, flags=re.IGNORECASE)
             regex_cache[(display_name, False, True)] = r
 
@@ -212,64 +213,14 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool:
     try:
         r = regex_cache.get((glob, True, word_boundary), None)
         if not r:
-            r = _glob_to_re(glob, word_boundary)
+            r = glob_to_regex(glob, word_boundary)
             regex_cache[(glob, True, word_boundary)] = r
         return bool(r.search(value))
     except re.error:
         logger.warning("Failed to parse glob to regex: %r", glob)
         return False
 
 
-def _glob_to_re(glob: str, word_boundary: bool) -> Pattern:
-    """Generates regex for a given glob.
-
-    Args:
-        glob
-        word_boundary: Whether to match against word boundaries or entire string.
-    """
-    if IS_GLOB.search(glob):
-        r = re.escape(glob)
-
-        r = r.replace(r"\*", ".*?")
-        r = r.replace(r"\?", ".")
-
-        # handle [abc], [a-z] and [!a-z] style ranges.
-        r = GLOB_REGEX.sub(
-            lambda x: (
-                "[%s%s]" % (x.group(1) and "^" or "", x.group(2).replace(r"\\\-", "-"))
-            ),
-            r,
-        )
-        if word_boundary:
-            r = _re_word_boundary(r)
-
-            return re.compile(r, flags=re.IGNORECASE)
-        else:
-            r = "^" + r + "$"
-
-            return re.compile(r, flags=re.IGNORECASE)
-    elif word_boundary:
-        r = re.escape(glob)
-        r = _re_word_boundary(r)
-
-        return re.compile(r, flags=re.IGNORECASE)
-    else:
-        r = "^" + re.escape(glob) + "$"
-        return re.compile(r, flags=re.IGNORECASE)
-
-
-def _re_word_boundary(r: str) -> str:
-    """
-    Adds word boundary characters to the start and end of an
-    expression to require that the match occur as a whole word,
-    but do so respecting the fact that strings starting or ending
-    with non-word characters will change word boundaries.
-    """
-    # we can't use \b as it chokes on unicode. however \W seems to be okay
-    # as shorthand for [^0-9A-Za-z_].
-    return r"(^|\W)%s(\W|$)" % (r,)
-
-
 def _flatten_dict(
     d: Union[EventBase, dict],
     prefix: Optional[List[str]] = None,

synapse/python_dependencies.py

@@ -78,7 +78,8 @@
     # we use attr.validators.deep_iterable, which arrived in 19.1.0 (Note:
     # Fedora 31 only has 19.1, so if we want to upgrade we should wait until 33
     # is out in November.)
-    "attrs>=19.1.0",
+    # Note: 21.1.0 broke `/sync`, see #9936
+    "attrs>=19.1.0,!=21.1.0",
     "netaddr>=0.7.18",
     "Jinja2>=2.9",
     "bleach>=1.4.3",

synapse/util/__init__.py

@@ -15,6 +15,7 @@
 import json
 import logging
 import re
+from typing import Pattern
 
 import attr
 from frozendict import frozendict
@@ -26,6 +27,9 @@
 logger = logging.getLogger(__name__)
 
 
+_WILDCARD_RUN = re.compile(r"([\?\*]+)")
+
+
 def _reject_invalid_json(val):
     """Do not allow Infinity, -Infinity, or NaN values in JSON."""
     raise ValueError("Invalid JSON value: '%s'" % val)
@@ -158,25 +162,54 @@ def log_failure(failure, msg, consumeErrors=True):
         return failure
 
 
-def glob_to_regex(glob):
+def glob_to_regex(glob: str, word_boundary: bool = False) -> Pattern:
     """Converts a glob to a compiled regex object.
 
-    The regex is anchored at the beginning and end of the string.
-
     Args:
-        glob (str)
+        glob: pattern to match
+        word_boundary: If True, the pattern will be allowed to match at word boundaries
+           anywhere in the string. Otherwise, the pattern is anchored at the start and
+           end of the string.
 
     Returns:
-        re.RegexObject
+        compiled regex pattern
     """
-    res = ""
-    for c in glob:
-        if c == "*":
-            res = res + ".*"
-        elif c == "?":
-            res = res + "."
+
+    # Patterns with wildcards must be simplified to avoid performance cliffs
+    # - The glob `?**?**?` is equivalent to the glob `???*`
+    # - The glob `???*` is equivalent to the regex `.{3,}`
+    chunks = []
+    for chunk in _WILDCARD_RUN.split(glob):
+        # No wildcards? re.escape()
+        if not _WILDCARD_RUN.match(chunk):
+            chunks.append(re.escape(chunk))
+            continue
+
+        # Wildcards? Simplify.
+        qmarks = chunk.count("?")
+        if "*" in chunk:
+            chunks.append(".{%d,}" % qmarks)
         else:
-            res = res + re.escape(c)
+            chunks.append(".{%d}" % qmarks)
+
+    res = "".join(chunks)
 
-    # \A anchors at start of string, \Z at end of string
-    return re.compile(r"\A" + res + r"\Z", re.IGNORECASE)
+    if word_boundary:
+        res = re_word_boundary(res)
+    else:
+        # \A anchors at start of string, \Z at end of string
+        res = r"\A" + res + r"\Z"
+
+    return re.compile(res, re.IGNORECASE)
+
+
+def re_word_boundary(r: str) -> str:
+    """
+    Adds word boundary characters to the start and end of an
+    expression to require that the match occur as a whole word,
+    but do so respecting the fact that strings starting or ending
+    with non-word characters will change word boundaries.
+    """
+    # we can't use \b as it chokes on unicode. however \W seems to be okay
+    # as shorthand for [^0-9A-Za-z_].
+    return r"(^|\W)%s(\W|$)" % (r,)

tests/federation/test_federation_server.py

@@ -74,6 +74,25 @@ def test_block_ip_literals(self):
         self.assertFalse(server_matches_acl_event("[1:2::]", e))
         self.assertTrue(server_matches_acl_event("1:2:3:4", e))
 
+    def test_wildcard_matching(self):
+        e = _create_acl_event({"allow": ["good*.com"]})
+        self.assertTrue(
+            server_matches_acl_event("good.com", e),
+            "* matches 0 characters",
+        )
+        self.assertTrue(
+            server_matches_acl_event("GOOD.COM", e),
+            "pattern is case-insensitive",
+        )
+        self.assertTrue(
+            server_matches_acl_event("good.aa.com", e),
+            "* matches several characters, including '.'",
+        )
+        self.assertFalse(
+            server_matches_acl_event("ishgood.com", e),
+            "pattern does not allow prefixes",
+        )
+
 
 class StateQueryTests(unittest.FederatingHomeserverTestCase):